<commit_before>package etcd

import (
	"errors"
	"strings"

	kerrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/validation/field"
	apirequest "k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/apiserver/pkg/registry/generic"
	"k8s.io/apiserver/pkg/registry/generic/registry"
	"k8s.io/apiserver/pkg/registry/rest"

	"github.com/openshift/origin/pkg/cmd/server/bootstrappolicy"
	userapi "github.com/openshift/origin/pkg/user/apis/user"
	"github.com/openshift/origin/pkg/user/apis/user/validation"
	"github.com/openshift/origin/pkg/user/registry/user"
	"github.com/openshift/origin/pkg/util/restoptions"
)

// rest implements a RESTStorage for users against etcd
type REST struct {
	*registry.Store
}

var _ rest.StandardStorage = &REST{}

// NewREST returns a RESTStorage object that will work against users
func NewREST(optsGetter restoptions.Getter) (*REST, error) {
	store := &registry.Store{
		NewFunc:                  func() runtime.Object { return &userapi.User{} },
		NewListFunc:              func() runtime.Object { return &userapi.UserList{} },
		DefaultQualifiedResource: userapi.Resource("users"),

		CreateStrategy: user.Strategy,
		UpdateStrategy: user.Strategy,
		DeleteStrategy: user.Strategy,
	}

	options := &generic.StoreOptions{RESTOptions: optsGetter}
	if err := store.CompleteWithOptions(options); err != nil {
		return nil, err
	}

	return &REST{store}, nil
}

// Get retrieves the item from etcd.
func (r *REST) Get(ctx apirequest.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
	// "~" means the currently authenticated user
	if name == "~" {
		user, ok := apirequest.UserFrom(ctx)
		if !ok || user.GetName() == "" {
			return nil, kerrs.NewForbidden(userapi.Resource("user"), "~", errors.New("requests to ~ must be authenticated"))
		}
		name = user.GetName()

		// remove the known virtual groups from the list if they are present
		contextGroups := sets.NewString(user.GetGroups()...)
		contextGroups.Delete(bootstrappolicy.UnauthenticatedGroup, bootstrappolicy.AuthenticatedGroup)

		if reasons := validation.ValidateUserName(name, false); len(reasons) != 0 {
			// The user the authentication layer has identified cannot be a valid persisted user
			// Return an API representation of the virtual user
			return &userapi.User{ObjectMeta: metav1.ObjectMeta{Name: name}, Groups: contextGroups.List()}, nil
		}

		obj, err := r.Store.Get(ctx, name, options)
		if err == nil {
			return obj, nil
		}

		if !kerrs.IsNotFound(err) {
			return nil, err
		}

		return &userapi.User{ObjectMeta: metav1.ObjectMeta{Name: name}, Groups: contextGroups.List()}, nil
	}

	if reasons := validation.ValidateUserName(name, false); len(reasons) != 0 {
		return nil, field.Invalid(field.NewPath("metadata", "name"), name, strings.Join(reasons, ", "))
	}

	return r.Store.Get(ctx, name, options)
}
<commit_msg>Include context UID for virtual user ~<commit_after>package etcd

import (
	"errors"
	"strings"

	kerrs "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/apimachinery/pkg/util/validation/field"
	apirequest "k8s.io/apiserver/pkg/endpoints/request"
	"k8s.io/apiserver/pkg/registry/generic"
	"k8s.io/apiserver/pkg/registry/generic/registry"
	"k8s.io/apiserver/pkg/registry/rest"

	"github.com/openshift/origin/pkg/cmd/server/bootstrappolicy"
	userapi "github.com/openshift/origin/pkg/user/apis/user"
	"github.com/openshift/origin/pkg/user/apis/user/validation"
	"github.com/openshift/origin/pkg/user/registry/user"
	"github.com/openshift/origin/pkg/util/restoptions"
)

// rest implements a RESTStorage for users against etcd
type REST struct {
	*registry.Store
}

var _ rest.StandardStorage = &REST{}

// NewREST returns a RESTStorage object that will work against users
func NewREST(optsGetter restoptions.Getter) (*REST, error) {
	store := &registry.Store{
		NewFunc:                  func() runtime.Object { return &userapi.User{} },
		NewListFunc:              func() runtime.Object { return &userapi.UserList{} },
		DefaultQualifiedResource: userapi.Resource("users"),

		CreateStrategy: user.Strategy,
		UpdateStrategy: user.Strategy,
		DeleteStrategy: user.Strategy,
	}

	options := &generic.StoreOptions{RESTOptions: optsGetter}
	if err := store.CompleteWithOptions(options); err != nil {
		return nil, err
	}

	return &REST{store}, nil
}

// Get retrieves the item from etcd.
func (r *REST) Get(ctx apirequest.Context, name string, options *metav1.GetOptions) (runtime.Object, error) {
	// "~" means the currently authenticated user
	if name == "~" {
		user, ok := apirequest.UserFrom(ctx)
		if !ok || user.GetName() == "" {
			return nil, kerrs.NewForbidden(userapi.Resource("user"), "~", errors.New("requests to ~ must be authenticated"))
		}
		name = user.GetName()

		// remove the known virtual groups from the list if they are present
		contextGroups := sets.NewString(user.GetGroups()...)
		contextGroups.Delete(bootstrappolicy.UnauthenticatedGroup, bootstrappolicy.AuthenticatedGroup)

		// build a virtual user object using the context data
		virtualUser := &userapi.User{ObjectMeta: metav1.ObjectMeta{Name: name, UID: types.UID(user.GetUID())}, Groups: contextGroups.List()}

		if reasons := validation.ValidateUserName(name, false); len(reasons) != 0 {
			// The user the authentication layer has identified cannot be a valid persisted user
			// Return an API representation of the virtual user
			return virtualUser, nil
		}

		// see if the context user exists in storage
		obj, err := r.Store.Get(ctx, name, options)

		// valid persisted user
		if err == nil {
			return obj, nil
		}

		// server is broken
		if !kerrs.IsNotFound(err) {
			return nil, kerrs.NewInternalError(err)
		}

		// impersonation, remote token authn, etc
		return virtualUser, nil
	}

	// do not bother looking up users that cannot be persisted
	if reasons := validation.ValidateUserName(name, false); len(reasons) != 0 {
		return nil, field.Invalid(field.NewPath("metadata", "name"), name, strings.Join(reasons, ", "))
	}

	return r.Store.Get(ctx, name, options)
}
<|endoftext|>


<commit_before>package kafka

import (
	"fmt"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/dbus/engine"
	"github.com/funkygao/dbus/pkg/kafka"
	"github.com/funkygao/dbus/pkg/model"
	"github.com/funkygao/golib/gofmt"
	conf "github.com/funkygao/jsconf"
	log "github.com/funkygao/log4go"
)

// KafkaOutput is an Output plugin that send pack to a single specified kafka topic.
type KafkaOutput struct {
	zone, cluster, topic string
	reporter             bool

	p *kafka.Producer
}

// Init setup KafkaOutput state according to config section.
// Default kafka delivery: async WaitForAll.
func (this *KafkaOutput) Init(config *conf.Conf) {
	var err error
	this.zone, this.cluster, this.topic, err = kafka.ParseDSN(config.String("dsn", ""))
	if err != nil || this.cluster == "" || this.zone == "" || this.topic == "" {
		panic("invalid configuration: " + fmt.Sprintf("%s.%s.%s", this.zone, this.cluster, this.topic))
	}
	this.reporter = config.Bool("reporter", false)

	cf := kafka.DefaultConfig()
	cf.Sarama.Producer.Flush.Messages = config.Int("batch_size", 1024)
	cf.Sarama.Producer.RequiredAcks = sarama.RequiredAcks(config.Int("ack", int(sarama.WaitForAll)))
	switch config.String("mode", "async") {
	case "sync":
		cf.SyncMode()
	case "async":
		cf.AsyncMode()
	case "dryrun":
		cf.DryrunMode()
	default:
		panic("invalid KafkaOut mode")
	}

	// get the bootstrap broker list
	zkzone := engine.Globals().GetOrRegisterZkzone(this.zone)
	zkcluster := zkzone.NewCluster(this.cluster)
	this.p = kafka.NewProducer(config.String("name", "undefined"), zkcluster.BrokerList(), cf)
}

func (this *KafkaOutput) CleanupForRestart() bool {
	return true // yes, restart allowed
}

func (this *KafkaOutput) Run(r engine.OutputRunner, h engine.PluginHelper) error {
	this.p.SetErrorHandler(func(err *sarama.ProducerError) {
		// e,g.
		// kafka: Failed to produce message to topic dbustest: kafka server: Message was too large, server rejected it to avoid allocation error.
		// kafka server: Unexpected (unknown?) server error.
		// java.lang.OutOfMemoryError: Direct buffer memory
		row := err.Msg.Value.(*model.RowsEvent)
		log.Error("[%s.%s.%s] %s %s", this.zone, this.cluster, this.topic, err, row.MetaInfo())
	})

	this.p.SetSuccessHandler(func(msg *sarama.ProducerMessage) {
		// FIXME what if:
		// [1, 2, 3, 4, 5] sent
		// [1, 2, 4, 5] ok, [3] fails
		// then shutdown dbusd? 3 might be lost
		pack := msg.Metadata.(*engine.Packet)
		if err := r.Ack(pack); err != nil {
			row := msg.Value.(*model.RowsEvent)
			log.Error("[%s.%s.%s] {%s} %v", this.zone, this.cluster, this.topic, row, err)
		}

		// safe to recycle
		// FIXME delayed recycle will block input channel, so currently let input chan
		// larger than batch size
		pack.Recycle()
	})

	// start the producer background routines
	if err := this.p.Start(); err != nil {
		return err
	}

	defer func() {
		log.Trace("[%s.%s.%s] start draining...", this.zone, this.cluster, this.topic)

		if err := this.p.Close(); err != nil {
			log.Error("[%s.%s.%s] drain: %s", this.zone, this.cluster, this.topic, err)
		} else {
			log.Trace("[%s.%s.%s] drained ok", this.zone, this.cluster, this.topic)
		}
	}()

	tick := time.NewTicker(time.Second * 10)
	defer tick.Stop()

	var reportTimer <-chan time.Time
	if this.reporter {
		reportTimer = tick.C
	}

	var n, lastN int64
	for {
		select {
		case <-reportTimer:
			log.Trace("[%s] throughput %s/s", r.Name(), gofmt.Comma((n-lastN)/10))
			lastN = n

		case pack, ok := <-r.InChan():
			if !ok {
				log.Trace("[%s.%s.%s] yes sir!", this.zone, this.cluster, this.topic)
				return nil
			}

			row, ok := pack.Payload.(*model.RowsEvent)
			if !ok {
				pack.Recycle()

				log.Error("[%s.%s.%s] bad payload: %+v", this.zone, this.cluster, this.topic, pack.Payload)
				continue
			}

			n++

			// loop is for sync mode only: async send will never return error
			for {
				if err := this.p.Send(&sarama.ProducerMessage{
					Topic:    this.topic,
					Value:    row,
					Metadata: pack,
				}); err == nil {
					break
				} else {
					log.Error("[%s.%s.%s] %+v", this.zone, this.cluster, this.topic, err)

					time.Sleep(time.Millisecond * 500)
				}
			}
		}
	}

	return nil
}
<commit_msg>BUG FIX: KafkaOutput.Init shouldn't check brokers<commit_after>package kafka

import (
	"fmt"
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/dbus/engine"
	"github.com/funkygao/dbus/pkg/kafka"
	"github.com/funkygao/dbus/pkg/model"
	"github.com/funkygao/golib/gofmt"
	conf "github.com/funkygao/jsconf"
	log "github.com/funkygao/log4go"
)

// KafkaOutput is an Output plugin that send pack to a single specified kafka topic.
type KafkaOutput struct {
	zone, cluster, topic string
	reporter             bool

	cf *conf.Conf
}

// Init setup KafkaOutput state according to config section.
// Default kafka delivery: async WaitForAll.
func (this *KafkaOutput) Init(config *conf.Conf) {
	var err error
	this.zone, this.cluster, this.topic, err = kafka.ParseDSN(config.String("dsn", ""))
	if err != nil || this.cluster == "" || this.zone == "" || this.topic == "" {
		panic("invalid configuration: " + fmt.Sprintf("%s.%s.%s", this.zone, this.cluster, this.topic))
	}
	this.reporter = config.Bool("reporter", false)
	this.cf = config
}

func (this *KafkaOutput) CleanupForRestart() bool {
	return true // yes, restart allowed
}

func (this *KafkaOutput) Run(r engine.OutputRunner, h engine.PluginHelper) error {
	cf := kafka.DefaultConfig()
	cf.Sarama.Producer.Flush.Messages = this.cf.Int("batch_size", 1024)
	cf.Sarama.Producer.RequiredAcks = sarama.RequiredAcks(this.cf.Int("ack", int(sarama.WaitForAll)))
	switch this.cf.String("mode", "async") {
	case "sync":
		cf.SyncMode()
	case "async":
		cf.AsyncMode()
	case "dryrun":
		cf.DryrunMode()
	default:
		panic("invalid KafkaOut mode")
	}

	// get the bootstrap broker list
	zkzone := engine.Globals().GetOrRegisterZkzone(this.zone)
	zkcluster := zkzone.NewCluster(this.cluster)
	producer := kafka.NewProducer(this.cf.String("name", "undefined"), zkcluster.BrokerList(), cf)

	producer.SetErrorHandler(func(err *sarama.ProducerError) {
		// e,g.
		// kafka: Failed to produce message to topic dbustest: kafka server: Message was too large, server rejected it to avoid allocation error.
		// kafka server: Unexpected (unknown?) server error.
		// java.lang.OutOfMemoryError: Direct buffer memory
		row := err.Msg.Value.(*model.RowsEvent)
		log.Error("[%s.%s.%s] %s %s", this.zone, this.cluster, this.topic, err, row.MetaInfo())
	})

	producer.SetSuccessHandler(func(msg *sarama.ProducerMessage) {
		// FIXME what if:
		// [1, 2, 3, 4, 5] sent
		// [1, 2, 4, 5] ok, [3] fails
		// then shutdown dbusd? 3 might be lost
		pack := msg.Metadata.(*engine.Packet)
		if err := r.Ack(pack); err != nil {
			row := msg.Value.(*model.RowsEvent)
			log.Error("[%s.%s.%s] {%s} %v", this.zone, this.cluster, this.topic, row, err)
		}

		// safe to recycle
		// FIXME delayed recycle will block input channel, so currently let input chan
		// larger than batch size
		pack.Recycle()
	})

	// start the producer background routines
	if err := producer.Start(); err != nil {
		return err
	}

	defer func() {
		log.Trace("[%s.%s.%s] start draining...", this.zone, this.cluster, this.topic)

		if err := producer.Close(); err != nil {
			log.Error("[%s.%s.%s] drain: %s", this.zone, this.cluster, this.topic, err)
		} else {
			log.Trace("[%s.%s.%s] drained ok", this.zone, this.cluster, this.topic)
		}
	}()

	tick := time.NewTicker(time.Second * 10)
	defer tick.Stop()

	var reportTimer <-chan time.Time
	if this.reporter {
		reportTimer = tick.C
	}

	var n, lastN int64
	for {
		select {
		case <-reportTimer:
			log.Trace("[%s] throughput %s/s", r.Name(), gofmt.Comma((n-lastN)/10))
			lastN = n

		case pack, ok := <-r.InChan():
			if !ok {
				log.Trace("[%s.%s.%s] yes sir!", this.zone, this.cluster, this.topic)
				return nil
			}

			row, ok := pack.Payload.(*model.RowsEvent)
			if !ok {
				pack.Recycle()

				log.Error("[%s.%s.%s] bad payload: %+v", this.zone, this.cluster, this.topic, pack.Payload)
				continue
			}

			n++

			// loop is for sync mode only: async send will never return error
			for {
				if err := producer.Send(&sarama.ProducerMessage{
					Topic:    this.topic,
					Value:    row,
					Metadata: pack,
				}); err == nil {
					break
				} else {
					log.Error("[%s.%s.%s] %+v", this.zone, this.cluster, this.topic, err)

					time.Sleep(time.Millisecond * 500)
				}
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package restplugin

import (
	"git.fd.io/govpp.git"
	"github.com/ligato/vpp-agent/plugins/defaultplugins/ifplugin/vppdump"
	"github.com/unrolled/render"
	"net/http"
)

//interfaceGetHandler - used to get list of all interfaces
func (plugin *RESTAPIPlugin) interfaceGetHandler(formatter *render.Render) http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {

		plugin.Deps.Log.Info("Getting list of all interfaces")

		// create an API channel
		ch, err := plugin.Deps.GoVppmux.NewAPIChannel()
		if err != nil {
			plugin.Deps.Log.Errorf("Error: %v", err)
			formatter.JSON(w, http.StatusInternalServerError, nil)
		} else {
			res, err := vppdump.DumpInterfaces(plugin.Deps.Log, ch, nil)
			if err != nil {
				plugin.Deps.Log.Errorf("Error: %v", err)
				formatter.JSON(w, http.StatusInternalServerError, nil)
			} else {
				plugin.Deps.Log.Debug(res)
				formatter.JSON(w, http.StatusOK, res)
			}
		}
		defer ch.Close()
	}
}
<commit_msg>SPOPT-1690 - REST API for VPP<commit_after>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package restplugin

import (
	"github.com/ligato/vpp-agent/plugins/defaultplugins/ifplugin/vppdump"
	"github.com/unrolled/render"
	"net/http"
)

//interfaceGetHandler - used to get list of all interfaces
func (plugin *RESTAPIPlugin) interfaceGetHandler(formatter *render.Render) http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {

		plugin.Deps.Log.Info("Getting list of all interfaces")

		// create an API channel
		ch, err := plugin.Deps.GoVppmux.NewAPIChannel()
		if err != nil {
			plugin.Deps.Log.Errorf("Error: %v", err)
			formatter.JSON(w, http.StatusInternalServerError, nil)
		} else {
			res, err := vppdump.DumpInterfaces(plugin.Deps.Log, ch, nil)
			if err != nil {
				plugin.Deps.Log.Errorf("Error: %v", err)
				formatter.JSON(w, http.StatusInternalServerError, nil)
			} else {
				plugin.Deps.Log.Debug(res)
				formatter.JSON(w, http.StatusOK, res)
			}
		}
		defer ch.Close()
	}
}
<|endoftext|>


<commit_before>package test_utils

import (
	"log"

	"github.com/nytlabs/streamtools/st/blocks"
	"github.com/nytlabs/streamtools/st/library"
)

// this would be run once before EACH of the tests
// func (s *StreamSuite) SetUpTest(c *C) {
//   // do something
// }

func NewBlock(id, kind string) (blocks.BlockInterface, blocks.BlockChans) {

	chans := blocks.BlockChans{
		InChan:         make(chan *blocks.Msg),
		QueryChan:      make(chan *blocks.QueryMsg),
		QueryParamChan: make(chan *blocks.QueryParamMsg),
		AddChan:        make(chan *blocks.AddChanMsg),
		DelChan:        make(chan *blocks.Msg),
		ErrChan:        make(chan error),
		QuitChan:       make(chan bool),
	}

	// actual block
	newblock, ok := library.Blocks[kind]
	if !ok {
		log.Println("block", kind, "not found!")
	}
	b := newblock()
	b.Build(chans)

	return b, chans

}

func StringInSlice(stringSlice []string, term string) bool {
	termIndex := -1
	for i, value := range stringSlice {
		if term == value {
			termIndex = i
		}
	}
	if termIndex == -1 {
		return false
	}
	return true
}
<commit_msg>added break for speed<commit_after>package test_utils

import (
	"log"

	"github.com/nytlabs/streamtools/st/blocks"
	"github.com/nytlabs/streamtools/st/library"
)

// this would be run once before EACH of the tests
// func (s *StreamSuite) SetUpTest(c *C) {
//   // do something
// }

func NewBlock(id, kind string) (blocks.BlockInterface, blocks.BlockChans) {

	chans := blocks.BlockChans{
		InChan:         make(chan *blocks.Msg),
		QueryChan:      make(chan *blocks.QueryMsg),
		QueryParamChan: make(chan *blocks.QueryParamMsg),
		AddChan:        make(chan *blocks.AddChanMsg),
		DelChan:        make(chan *blocks.Msg),
		ErrChan:        make(chan error),
		QuitChan:       make(chan bool),
	}

	// actual block
	newblock, ok := library.Blocks[kind]
	if !ok {
		log.Println("block", kind, "not found!")
	}
	b := newblock()
	b.Build(chans)

	return b, chans

}

func StringInSlice(stringSlice []string, term string) bool {
	termIndex := -1
	for i, value := range stringSlice {
		if term == value {
			termIndex = i
			break
		}
	}
	if termIndex == -1 {
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package testblas

import (
	"math"
	"testing"

	"github.com/gonum/blas"
)

// throwPanic will throw unexpected panics if true, or will just report them as errors if false
const throwPanic = true

func dTolEqual(a, b float64) bool {
	m := math.Max(math.Abs(a), math.Abs(b))
	if m > 1 {
		a /= m
		b /= m
	}
	if math.Abs(a-b) < 1e-14 {
		return true
	}
	return false
}

func dSliceTolEqual(a, b []float64) bool {
	if len(a) != len(b) {
		return false
	}
	for i := range a {
		if !dTolEqual(a[i], b[i]) {
			return false
		}
	}
	return true
}

func dStridedSliceTolEqual(n int, a []float64, inca int, b []float64, incb int) bool {
	ia := 0
	ib := 0
	if inca <= 0 {
		ia = -(n - 1) * inca
	}
	if incb <= 0 {
		ib = -(n - 1) * incb
	}
	for i := 0; i < n; i++ {
		if !dTolEqual(a[ia], b[ib]) {
			return false
		}
		ia += inca
		ib += incb
	}
	return true
}

func dSliceEqual(a, b []float64) bool {
	if len(a) != len(b) {
		return false
	}
	for i := range a {
		if !(a[i] == b[i]) {
			return false
		}
	}
	return true
}

func dCopyTwoTmp(x, xTmp, y, yTmp []float64) {
	if len(x) != len(xTmp) {
		panic("x size mismatch")
	}
	if len(y) != len(yTmp) {
		panic("y size mismatch")
	}
	for i, val := range x {
		xTmp[i] = val
	}
	for i, val := range y {
		yTmp[i] = val
	}
}

// returns true if the function panics
func panics(f func()) (b bool) {
	defer func() {
		err := recover()
		if err != nil {
			b = true
		}
	}()
	f()
	return
}

func testpanics(f func(), name string, t *testing.T) {
	b := panics(f)
	if !b {
		t.Errorf("%v should panic and does not", name)
	}
}

func sliceOfSliceCopy(a [][]float64) [][]float64 {
	n := make([][]float64, len(a))
	for i := range a {
		n[i] = make([]float64, len(a[i]))
		copy(n[i], a[i])
	}
	return n
}

func sliceCopy(a []float64) []float64 {
	n := make([]float64, len(a))
	copy(n, a)
	return n
}

func flatten(a [][]float64) []float64 {
	if len(a) == 0 {
		return nil
	}
	m := len(a)
	n := len(a[0])
	s := make([]float64, m*n)
	for i := 0; i < m; i++ {
		for j := 0; j < n; j++ {
			s[i*n+j] = a[i][j]
		}
	}
	return s
}

func unflatten(a []float64, m, n int) [][]float64 {
	s := make([][]float64, m)
	for i := 0; i < m; i++ {
		s[i] = make([]float64, n)
		for j := 0; j < n; j++ {
			s[i][j] = a[i*n+j]
		}
	}
	return s
}

// flattenTriangular turns the upper or lower triangle of a dense slice of slice
// into a single slice with packed storage.
func flattenTriangular(a [][]float64, ul blas.Uplo) []float64 {
	m := len(a)
	n := len(a[0])
	if m != n {
		panic("must be square")
	}
	aFlat := make([]float64, n*(n+1)/2)
	var count int
	if ul == blas.Upper {
		for i := 0; i < m; i++ {
			for j := i; j < n; j++ {
				aFlat[count] = a[i][j]
				count++
			}
		}
		return aFlat
	}
	for i := 0; i < m; i++ {
		for j := 0; j <= i; j++ {
			aFlat[count] = a[i][j]
			count++
		}
	}
	return aFlat
}

// flattenBanded turns a dense banded slice of slice into the compact banded matrix format
func flattenBanded(a [][]float64, ku, kl int) []float64 {
	m := len(a)
	n := len(a[0])
	if ku < 0 || kl < 0 {
		panic("testblas: negative band length")
	}
	nRows := m
	nCols := (ku + kl + 1)
	aflat := make([]float64, nRows*nCols)
	for i := range aflat {
		aflat[i] = math.NaN()
	}
	// loop over the rows, and then the bands
	// elements in the ith row stay in the ith row
	// order in bands is kept
	for i := 0; i < nRows; i++ {
		min := -kl
		if i-kl < 0 {
			min = -i
		}
		max := ku
		if i+ku >= n {
			max = n - i - 1
		}
		for j := min; j <= max; j++ {
			col := kl + j
			aflat[i*nCols+col] = a[i][i+j]
		}
	}
	return aflat
}

// makeIncremented takes a slice with inc == 1 and makes an incremented version
// and adds extra values on the end
func makeIncremented(x []float64, inc int, extra int) []float64 {
	if inc == 0 {
		panic("zero inc")
	}
	absinc := inc
	if absinc < 0 {
		absinc = -inc
	}
	xcopy := make([]float64, len(x))
	if inc > 0 {
		copy(xcopy, x)
	} else {
		for i := 0; i < len(x); i++ {
			xcopy[i] = x[len(x)-i-1]
		}
	}

	// don't use NaN because it makes comparison hard
	// Do use a weird unique value for easier debugging
	counter := 100.0
	var xnew []float64
	for i, v := range xcopy {
		xnew = append(xnew, v)
		if i != len(x)-1 {
			for j := 0; j < absinc-1; j++ {
				xnew = append(xnew, counter)
				counter++
			}
		}
	}
	for i := 0; i < extra; i++ {
		xnew = append(xnew, counter)
		counter++
	}
	return xnew
}
<commit_msg>remove triangular to pull from master<commit_after>package testblas

import (
	"math"
	"testing"
)

// throwPanic will throw unexpected panics if true, or will just report them as errors if false
const throwPanic = true

func dTolEqual(a, b float64) bool {
	m := math.Max(math.Abs(a), math.Abs(b))
	if m > 1 {
		a /= m
		b /= m
	}
	if math.Abs(a-b) < 1e-14 {
		return true
	}
	return false
}

func dSliceTolEqual(a, b []float64) bool {
	if len(a) != len(b) {
		return false
	}
	for i := range a {
		if !dTolEqual(a[i], b[i]) {
			return false
		}
	}
	return true
}

func dStridedSliceTolEqual(n int, a []float64, inca int, b []float64, incb int) bool {
	ia := 0
	ib := 0
	if inca <= 0 {
		ia = -(n - 1) * inca
	}
	if incb <= 0 {
		ib = -(n - 1) * incb
	}
	for i := 0; i < n; i++ {
		if !dTolEqual(a[ia], b[ib]) {
			return false
		}
		ia += inca
		ib += incb
	}
	return true
}

func dSliceEqual(a, b []float64) bool {
	if len(a) != len(b) {
		return false
	}
	for i := range a {
		if !(a[i] == b[i]) {
			return false
		}
	}
	return true
}

func dCopyTwoTmp(x, xTmp, y, yTmp []float64) {
	if len(x) != len(xTmp) {
		panic("x size mismatch")
	}
	if len(y) != len(yTmp) {
		panic("y size mismatch")
	}
	for i, val := range x {
		xTmp[i] = val
	}
	for i, val := range y {
		yTmp[i] = val
	}
}

// returns true if the function panics
func panics(f func()) (b bool) {
	defer func() {
		err := recover()
		if err != nil {
			b = true
		}
	}()
	f()
	return
}

func testpanics(f func(), name string, t *testing.T) {
	b := panics(f)
	if !b {
		t.Errorf("%v should panic and does not", name)
	}
}

func sliceOfSliceCopy(a [][]float64) [][]float64 {
	n := make([][]float64, len(a))
	for i := range a {
		n[i] = make([]float64, len(a[i]))
		copy(n[i], a[i])
	}
	return n
}

func sliceCopy(a []float64) []float64 {
	n := make([]float64, len(a))
	copy(n, a)
	return n
}

func flatten(a [][]float64) []float64 {
	if len(a) == 0 {
		return nil
	}
	m := len(a)
	n := len(a[0])
	s := make([]float64, m*n)
	for i := 0; i < m; i++ {
		for j := 0; j < n; j++ {
			s[i*n+j] = a[i][j]
		}
	}
	return s
}

func unflatten(a []float64, m, n int) [][]float64 {
	s := make([][]float64, m)
	for i := 0; i < m; i++ {
		s[i] = make([]float64, n)
		for j := 0; j < n; j++ {
			s[i][j] = a[i*n+j]
		}
	}
	return s
}

// flattenBanded turns a dense banded slice of slice into the compact banded matrix format
func flattenBanded(a [][]float64, ku, kl int) []float64 {
	m := len(a)
	n := len(a[0])
	if ku < 0 || kl < 0 {
		panic("testblas: negative band length")
	}
	nRows := m
	nCols := (ku + kl + 1)
	aflat := make([]float64, nRows*nCols)
	for i := range aflat {
		aflat[i] = math.NaN()
	}
	// loop over the rows, and then the bands
	// elements in the ith row stay in the ith row
	// order in bands is kept
	for i := 0; i < nRows; i++ {
		min := -kl
		if i-kl < 0 {
			min = -i
		}
		max := ku
		if i+ku >= n {
			max = n - i - 1
		}
		for j := min; j <= max; j++ {
			col := kl + j
			aflat[i*nCols+col] = a[i][i+j]
		}
	}
	return aflat
}

// makeIncremented takes a slice with inc == 1 and makes an incremented version
// and adds extra values on the end
func makeIncremented(x []float64, inc int, extra int) []float64 {
	if inc == 0 {
		panic("zero inc")
	}
	absinc := inc
	if absinc < 0 {
		absinc = -inc
	}
	xcopy := make([]float64, len(x))
	if inc > 0 {
		copy(xcopy, x)
	} else {
		for i := 0; i < len(x); i++ {
			xcopy[i] = x[len(x)-i-1]
		}
	}

	// don't use NaN because it makes comparison hard
	// Do use a weird unique value for easier debugging
	counter := 100.0
	var xnew []float64
	for i, v := range xcopy {
		xnew = append(xnew, v)
		if i != len(x)-1 {
			for j := 0; j < absinc-1; j++ {
				xnew = append(xnew, counter)
				counter++
			}
		}
	}
	for i := 0; i < extra; i++ {
		xnew = append(xnew, counter)
		counter++
	}
	return xnew
}
<|endoftext|>


<commit_before>// Copyright 2016 The PIO Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package distro

import (
	"io/ioutil"
)

// DTModel returns platform model info from the Linux device tree (/proc/device-tree/model), and
// returns "unknown" on non-linux systems or if the file is missing.
func DTModel() string {
	lock.Lock()
	defer lock.Unlock()

	if dtModel == "" {
		dtModel = "unknown"
		if isLinux {
			// Read model from device tree.
			if bytes, err := ioutil.ReadFile("/proc/device-tree/model"); err == nil {
				if model := splitNull(bytes); len(model) > 0 {
					dtModel = model[0]
				}
			}
		}
	}
	return dtModel
}

// DTCompatible returns platform compatibility info from the Linux device tree
// (/proc/device-tree/compatible), and returns []{"unknown"} on non-linux systems or if the file is
// missing.
func DTCompatible() []string {
	lock.Lock()
	defer lock.Unlock()

	if dtCompatible == nil {
		dtCompatible = []string{}
		if isLinux {
			// Read compatible from device tree.
			if bytes, err := ioutil.ReadFile("/proc/device-tree/compatible"); err == nil {
				dtCompatible = splitNull(bytes)
			}
		}
	}
	return dtCompatible
}

var (
	dtModel      string   // cached /proc/device-tree/model
	dtCompatible []string // cached /proc/device-tree/compatible
)
<commit_msg>fix devtree `"unknown"` to `"<unknown>"` per conversation<commit_after>// Copyright 2016 The PIO Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package distro

import (
	"io/ioutil"
)

// DTModel returns platform model info from the Linux device tree (/proc/device-tree/model), and
// returns "unknown" on non-linux systems or if the file is missing.
func DTModel() string {
	lock.Lock()
	defer lock.Unlock()

	if dtModel == "" {
		dtModel = "<unknown>"
		if isLinux {
			// Read model from device tree.
			if bytes, err := ioutil.ReadFile("/proc/device-tree/model"); err == nil {
				if model := splitNull(bytes); len(model) > 0 {
					dtModel = model[0]
				}
			}
		}
	}
	return dtModel
}

// DTCompatible returns platform compatibility info from the Linux device tree
// (/proc/device-tree/compatible), and returns []{"unknown"} on non-linux systems or if the file is
// missing.
func DTCompatible() []string {
	lock.Lock()
	defer lock.Unlock()

	if dtCompatible == nil {
		dtCompatible = []string{}
		if isLinux {
			// Read compatible from device tree.
			if bytes, err := ioutil.ReadFile("/proc/device-tree/compatible"); err == nil {
				dtCompatible = splitNull(bytes)
			}
		}
	}
	return dtCompatible
}

var (
	dtModel      string   // cached /proc/device-tree/model
	dtCompatible []string // cached /proc/device-tree/compatible
)
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package s3

import (
	"log"
	"path"

	"camlistore.org/pkg/blob"
	"camlistore.org/pkg/blobserver"
	"golang.org/x/net/context"
)

var _ blobserver.MaxEnumerateConfig = (*s3Storage)(nil)

func (sto *s3Storage) MaxEnumerate() int { return 1000 }

// marker returns the string lexically greater than the provided s
// with the same length as s.
func nextStr(s string) string {
	if s == "" {
		return s
	}
	b := []byte(s)
	i := len(b)
	for i > 0 {
		i--
		b[i]++
		if b[i] != 0 {
			break
		}
	}
	return string(b)
}

func (sto *s3Storage) EnumerateBlobs(ctx context.Context, dest chan<- blob.SizedRef, after string, limit int) (err error) {
	defer close(dest)
	if faultEnumerate.FailErr(&err) {
		return
	}
	startAt := after
	if _, ok := blob.Parse(after); ok {
		startAt = nextStr(after)
	}
	objs, err := sto.s3Client.ListBucket(sto.bucket, sto.dirPrefix+startAt, limit)
	if err != nil {
		log.Printf("s3 ListBucket: %v", err)
		return err
	}
	for _, obj := range objs {
		dir, file := path.Split(obj.Key)
		if dir != sto.dirPrefix {
			continue
		}
		if file == after {
			continue
		}
		br, ok := blob.Parse(file)
		if !ok {
			// TODO(mpl): I've noticed that on GCS we error out for this case. Do the same here ?
			continue
		}
		select {
		case dest <- blob.SizedRef{Ref: br, Size: uint32(obj.Size)}:
		case <-ctx.Done():
			return ctx.Err()
		}
	}
	return nil
}
<commit_msg>pkg/blobserver/s3: Error when an s3 bucket contains non-blob files.<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package s3

import (
	"fmt"
	"log"
	"path"

	"camlistore.org/pkg/blob"
	"camlistore.org/pkg/blobserver"
	"golang.org/x/net/context"
)

var _ blobserver.MaxEnumerateConfig = (*s3Storage)(nil)

func (sto *s3Storage) MaxEnumerate() int { return 1000 }

// marker returns the string lexically greater than the provided s
// with the same length as s.
func nextStr(s string) string {
	if s == "" {
		return s
	}
	b := []byte(s)
	i := len(b)
	for i > 0 {
		i--
		b[i]++
		if b[i] != 0 {
			break
		}
	}
	return string(b)
}

func (sto *s3Storage) EnumerateBlobs(ctx context.Context, dest chan<- blob.SizedRef, after string, limit int) (err error) {
	defer close(dest)
	if faultEnumerate.FailErr(&err) {
		return
	}
	startAt := after
	if _, ok := blob.Parse(after); ok {
		startAt = nextStr(after)
	}
	objs, err := sto.s3Client.ListBucket(sto.bucket, sto.dirPrefix+startAt, limit)
	if err != nil {
		log.Printf("s3 ListBucket: %v", err)
		return err
	}
	for _, obj := range objs {
		dir, file := path.Split(obj.Key)
		if dir != sto.dirPrefix {
			continue
		}
		if file == after {
			continue
		}
		br, ok := blob.Parse(file)
		if !ok {
			return fmt.Errorf("non-Camlistore object named %q found in %v s3 bucket", file, sto.bucket)
		}
		select {
		case dest <- blob.SizedRef{Ref: br, Size: uint32(obj.Size)}:
		case <-ctx.Done():
			return ctx.Err()
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fileinspect

import (
	"os"
	"path/filepath"
	"sort"
	"strings"

	humanize "github.com/dustin/go-humanize"
)

// Walk walks all files in the target directory.
func Walk(targetDir string) (map[string]os.FileInfo, error) {
	rm := make(map[string]os.FileInfo)
	visit := func(path string, f os.FileInfo, err error) error {
		if f != nil {
			if !f.IsDir() {
				wd, err := os.Getwd()
				if err != nil {
					return err
				}
				rm[filepath.Join(wd, strings.Replace(path, wd, "", -1))] = f
			}
		}
		return nil
	}
	err := filepath.Walk(targetDir, visit)
	if err != nil {
		return nil, err
	}
	return rm, nil
}

// Size returns the size of target directory, in bytes.
// Same as 'du -sh $DIR'.
func Size(targetDir string) (int64, error) {
	fm, err := Walk(targetDir)
	if err != nil {
		return 0, err
	}
	var size int64
	for _, v := range fm {
		size += v.Size()
	}
	return size, nil
}

func walk(targetDir string) (map[string]os.FileInfo, error) {
	rm := make(map[string]os.FileInfo)
	visit := func(path string, f os.FileInfo, err error) error {
		if f != nil {
			if !f.IsDir() {
				if !filepath.HasPrefix(path, ".") && !strings.Contains(path, "/.") {
					wd, err := os.Getwd()
					if err != nil {
						return err
					}
					rm[filepath.Join(wd, strings.Replace(path, wd, "", -1))] = f
				}
			}
		}
		return nil
	}
	err := filepath.Walk(targetDir, visit)
	if err != nil {
		return nil, err
	}
	return rm, nil
}

type filepathSize struct {
	path    string
	size    uint64
	sizeTxt string
}

func filterByKbs(fs []filepathSize, kbLimit int) []filepathSize {
	var ns []filepathSize
	for _, v := range fs {
		if v.size > uint64(kbLimit*1024) {
			continue
		}
		ns = append(ns, v)
	}
	return ns
}

type filepathSizeSlice []filepathSize

func (f filepathSizeSlice) Len() int           { return len(f) }
func (f filepathSizeSlice) Swap(i, j int)      { f[i], f[j] = f[j], f[i] }
func (f filepathSizeSlice) Less(i, j int) bool { return f[i].size < f[j].size }

func walkDir(targetDir string) ([]filepathSize, error) {
	rm, err := walk(targetDir)
	if err != nil {
		return nil, err
	}

	var fs []filepathSize
	for k, v := range rm {
		fv := filepathSize{
			path:    k,
			size:    uint64(v.Size()),
			sizeTxt: humanize.Bytes(uint64(v.Size())),
		}
		fs = append(fs, fv)
	}
	sort.Sort(filepathSizeSlice(fs))

	return fs, nil
}
<commit_msg>Update fileinspect.go<commit_after>// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package fileinspect implements various file inspecting utils.
package fileinspect

import (
	"os"
	"path/filepath"
	"sort"
	"strings"

	humanize "github.com/dustin/go-humanize"
)

// Walk walks all files in the target directory.
func Walk(targetDir string) (map[string]os.FileInfo, error) {
	rm := make(map[string]os.FileInfo)
	visit := func(path string, f os.FileInfo, err error) error {
		if f != nil {
			if !f.IsDir() {
				wd, err := os.Getwd()
				if err != nil {
					return err
				}
				rm[filepath.Join(wd, strings.Replace(path, wd, "", -1))] = f
			}
		}
		return nil
	}
	err := filepath.Walk(targetDir, visit)
	if err != nil {
		return nil, err
	}
	return rm, nil
}

// Size returns the size of target directory, in bytes.
// Same as 'du -sh $DIR'.
func Size(targetDir string) (int64, error) {
	fm, err := Walk(targetDir)
	if err != nil {
		return 0, err
	}
	var size int64
	for _, v := range fm {
		size += v.Size()
	}
	return size, nil
}

func walk(targetDir string) (map[string]os.FileInfo, error) {
	rm := make(map[string]os.FileInfo)
	visit := func(path string, f os.FileInfo, err error) error {
		if f != nil {
			if !f.IsDir() {
				if !filepath.HasPrefix(path, ".") && !strings.Contains(path, "/.") {
					wd, err := os.Getwd()
					if err != nil {
						return err
					}
					rm[filepath.Join(wd, strings.Replace(path, wd, "", -1))] = f
				}
			}
		}
		return nil
	}
	err := filepath.Walk(targetDir, visit)
	if err != nil {
		return nil, err
	}
	return rm, nil
}

type filepathSize struct {
	path    string
	size    uint64
	sizeTxt string
}

func filterByKbs(fs []filepathSize, kbLimit int) []filepathSize {
	var ns []filepathSize
	for _, v := range fs {
		if v.size > uint64(kbLimit*1024) {
			continue
		}
		ns = append(ns, v)
	}
	return ns
}

type filepathSizeSlice []filepathSize

func (f filepathSizeSlice) Len() int           { return len(f) }
func (f filepathSizeSlice) Swap(i, j int)      { f[i], f[j] = f[j], f[i] }
func (f filepathSizeSlice) Less(i, j int) bool { return f[i].size < f[j].size }

func walkDir(targetDir string) ([]filepathSize, error) {
	rm, err := walk(targetDir)
	if err != nil {
		return nil, err
	}

	var fs []filepathSize
	for k, v := range rm {
		fv := filepathSize{
			path:    k,
			size:    uint64(v.Size()),
			sizeTxt: humanize.Bytes(uint64(v.Size())),
		}
		fs = append(fs, fv)
	}
	sort.Sort(filepathSizeSlice(fs))

	return fs, nil
}
<|endoftext|>


<commit_before>// Copyright 2015-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package kexec

import (
	"fmt"
	"os"

	"github.com/u-root/u-root/pkg/acpi"
	"golang.org/x/sys/unix"
)

// FileLoad loads the given kernel as the new kernel with the given ramfs and
// cmdline.
//
// The kexec_file_load(2) syscall is x86-64 bit only.
func FileLoad(kernel, ramfs *os.File, cmdline string) error {
	var flags int
	var ramfsfd int
	if ramfs != nil {
		ramfsfd = int(ramfs.Fd())
	} else {
		flags |= unix.KEXEC_FILE_NO_INITRAMFS
	}

	if base, _, err := acpi.GetRSDP(); err != nil {
		// Prepend the RSDP.
		cmdline = fmt.Sprintf("acpi_rsdp=%#x %s", base, cmdline)
	}

	if err := unix.KexecFileLoad(int(kernel.Fd()), ramfsfd, cmdline, flags); err != nil {
		return fmt.Errorf("sys_kexec(%d, %d, %s, %x) = %v", kernel.Fd(), ramfsfd, cmdline, flags, err)
	}
	return nil
}
<commit_msg>kexec: remove rsdp command line parameter.<commit_after>// Copyright 2015-2017 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package kexec

import (
	"fmt"
	"os"

	"golang.org/x/sys/unix"
)

// FileLoad loads the given kernel as the new kernel with the given ramfs and
// cmdline.
//
// The kexec_file_load(2) syscall is x86-64 bit only.
func FileLoad(kernel, ramfs *os.File, cmdline string) error {
	var flags int
	var ramfsfd int
	if ramfs != nil {
		ramfsfd = int(ramfs.Fd())
	} else {
		flags |= unix.KEXEC_FILE_NO_INITRAMFS
	}

	if err := unix.KexecFileLoad(int(kernel.Fd()), ramfsfd, cmdline, flags); err != nil {
		return fmt.Errorf("sys_kexec(%d, %d, %s, %x) = %v", kernel.Fd(), ramfsfd, cmdline, flags, err)
	}
	return nil
}
<|endoftext|>


<commit_before>package kubernetes

import (
	"fmt"
	"kourier/pkg/config"
	"net/http"
	"strconv"
	"sync"
	"time"

	"github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
)

const (
	gatewayLabelSelector = "app=3scale-kourier-gateway"
	httpClientTimeout    = 2 * time.Second
	gatewaySyncTimeout   = 3 * time.Second
)

var (
	inSync int
	wg     sync.WaitGroup
	mutex  sync.Mutex
)

func GetKourierGatewayPODS(kubeclient kubernetes.Interface, namespace string) (*v1.PodList, error) {
	opts := metav1.ListOptions{
		LabelSelector: gatewayLabelSelector,
	}
	pods, err := kubeclient.CoreV1().Pods(namespace).List(opts)
	if err != nil {
		return &v1.PodList{}, err
	}

	return pods, nil
}

func CheckGatewaySnapshot(gwPods *v1.PodList, snapshotID string) (bool, error) {
	var ips []string

	for _, pod := range gwPods.Items {
		if pod.Status.PodIP != "" {
			ips = append(ips, pod.Status.PodIP)
		}
	}

	if len(ips) == 0 {
		return false, nil
	}

	inSync = 0
	wg.Add(len(ips))

	// Golang http.Client has keepalive by default to true, we don't want it here, or we will be always hitting the
	// draining cluster, and, getting the previous revision.
	tr := &http.Transport{
		DisableKeepAlives: true,
	}
	client := http.Client{
		Transport: tr,
		Timeout:   httpClientTimeout,
	}

	for _, ip := range ips {

		go func() {
			defer wg.Done()

			currentSnapshot, err := getCurrentGWSnapshot(ip, client)
			if err != nil {
				logrus.Errorf("Failed getting the current GW snapshot: %s for gw: %s", err, ip)
				return
			}
			if currentSnapshot == snapshotID {
				mutex.Lock()
				inSync++
				mutex.Unlock()
			}
		}()
	}
	if waitTimeout(&wg, gatewaySyncTimeout) {
		return false, nil
	}

	return inSync == len(ips), nil
}

func getCurrentGWSnapshot(ip string, client http.Client) (string, error) {

	req, err := buildInternalKourierRequest(ip)
	if err != nil {
		return "", err
	}

	resp, err := client.Do(req)
	if err != nil {
		return "", err
	}

	if resp.StatusCode == http.StatusOK {
		return resp.Header.Get(config.InternalKourierHeader), nil
	}

	return "", fmt.Errorf("status code %d", resp.StatusCode)
}

// waitTimeout waits for the waitgroup for the specified max timeout.
// Returns true if waiting timed out.
func waitTimeout(wg *sync.WaitGroup, timeout time.Duration) bool {
	c := make(chan struct{})
	go func() {
		defer close(c)
		wg.Wait()
	}()
	select {
	case <-c:
		return false // completed normally
	case <-time.After(timeout):
		return true // timed out
	}
}

func buildInternalKourierRequest(ip string) (*http.Request, error) {

	port := strconv.Itoa(int(config.HttpPortInternal))
	req, err := http.NewRequest("GET", "http://"+ip+":"+port+config.InternalKourierPath, nil)
	if err != nil {
		return &http.Request{}, err
	}
	req.Host = config.InternalKourierDomain

	return req, nil
}
<commit_msg>Delete unnecessary gateway sync helpers<commit_after><|endoftext|>


<commit_before>package middleware

import (
	"fmt"
	"net/http"
	"strconv"

	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/setting"

	"gopkg.in/macaron.v1"
)

func OrgRedirect() macaron.Handler {
	return func(res http.ResponseWriter, req *http.Request, c *macaron.Context) {
		orgIdValue := req.URL.Query().Get("orgId")
		orgId, err := strconv.ParseInt(orgIdValue, 10, 32)

		if err != nil || orgId == 0 {
			return
		}

		ctx, ok := c.Data["ctx"].(*Context)
		if !ok || !ctx.IsSignedIn {
			return
		}

		if orgId == ctx.OrgId {
			return
		}

		cmd := models.SetUsingOrgCommand{UserId: ctx.UserId, OrgId: orgId}
		if err := bus.Dispatch(&cmd); err != nil {
			if ctx.IsApiRequest() {
				ctx.JsonApiErr(404, "Not found", nil)
			} else {
				ctx.Error(404, "Not found")
			}

			return
		}

		newUrl := setting.ToAbsUrl(fmt.Sprintf("%s?%s", c.Req.URL.Path, c.Req.URL.Query().Encode()))
		c.Redirect(newUrl, 302)
	}
}
<commit_msg>Use strings.TrimPrefix to make sure relative url doesn't start with forward slash<commit_after>package middleware

import (
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"github.com/grafana/grafana/pkg/bus"
	"github.com/grafana/grafana/pkg/models"
	"github.com/grafana/grafana/pkg/setting"

	"gopkg.in/macaron.v1"
)

func OrgRedirect() macaron.Handler {
	return func(res http.ResponseWriter, req *http.Request, c *macaron.Context) {
		orgIdValue := req.URL.Query().Get("orgId")
		orgId, err := strconv.ParseInt(orgIdValue, 10, 32)

		if err != nil || orgId == 0 {
			return
		}

		ctx, ok := c.Data["ctx"].(*Context)
		if !ok || !ctx.IsSignedIn {
			return
		}

		if orgId == ctx.OrgId {
			return
		}

		cmd := models.SetUsingOrgCommand{UserId: ctx.UserId, OrgId: orgId}
		if err := bus.Dispatch(&cmd); err != nil {
			if ctx.IsApiRequest() {
				ctx.JsonApiErr(404, "Not found", nil)
			} else {
				ctx.Error(404, "Not found")
			}

			return
		}

		newURL := setting.ToAbsUrl(fmt.Sprintf("%s?%s", strings.TrimPrefix(c.Req.URL.Path, "/"), c.Req.URL.Query().Encode()))
		c.Redirect(newURL, 302)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package components

import (
	"fmt"

	"github.com/golang/glog"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/loader"
)

// DockerOptionsBuilder adds options for docker to the model
type DockerOptionsBuilder struct {
	*OptionsContext
}

var _ loader.OptionsBuilder = &DockerOptionsBuilder{}

// BuildOptions is responsible for filling in the default setting for docker daemon
func (b *DockerOptionsBuilder) BuildOptions(o interface{}) error {
	clusterSpec := o.(*kops.ClusterSpec)

	sv, err := KubernetesVersion(clusterSpec)
	if err != nil {
		return fmt.Errorf("unable to determine kubernetes version from %q", clusterSpec.KubernetesVersion)
	}

	if clusterSpec.Docker == nil {
		clusterSpec.Docker = &kops.DockerConfig{}
	}

	docker := clusterSpec.Docker

	if fi.StringValue(clusterSpec.Docker.Version) == "" {
		if clusterSpec.KubernetesVersion == "" {
			return fmt.Errorf("KubernetesVersion is required")
		}

		dockerVersion := ""
		if sv.Major == 1 && sv.Minor >= 9 {
			dockerVersion = "17.03.2"
		} else if sv.Major == 1 && sv.Minor >= 8 {
			dockerVersion = "1.13.1"
		} else if sv.Major == 1 && sv.Minor >= 6 {
			dockerVersion = "1.12.6"
		} else if sv.Major == 1 && sv.Minor >= 5 {
			dockerVersion = "1.12.3"
		} else if sv.Major == 1 && sv.Minor <= 4 {
			dockerVersion = "1.11.2"
		}

		if dockerVersion == "" {
			return fmt.Errorf("unknown version of kubernetes %q (cannot infer docker version)", clusterSpec.KubernetesVersion)
		}

		clusterSpec.Docker.Version = &dockerVersion
	}

	if sv.Major == 1 && sv.Minor >= 6 {
		if len(clusterSpec.Docker.LogOpt) == 0 && clusterSpec.Docker.LogDriver == "" {
			// Use built-in docker logging, if not configured otherwise (by the user)
			clusterSpec.Docker.LogDriver = "json-file"
			clusterSpec.Docker.LogOpt = append(clusterSpec.Docker.LogOpt, "max-size=10m")
			clusterSpec.Docker.LogOpt = append(clusterSpec.Docker.LogOpt, "max-file=5")
		}
	}

	docker.LogLevel = fi.String("warn")
	docker.IPTables = fi.Bool(false)
	docker.IPMasq = fi.Bool(false)

	// Note the alternative syntax... with a comma nodeup will try each of the filesystems in turn
	if b.IsKubernetesGTE("1.10") {
		docker.Storage = fi.String("overlay2,overlay,aufs")
	} else {
		docker.Storage = fi.String("overlay,aufs")
	}

	networking := clusterSpec.Networking
	if networking == nil || networking.Classic != nil {
		glog.Warningf("using deprecated (classic) networking")
		docker.Bridge = fi.String("cbr0")
	}

	return nil
}
<commit_msg>Delay overlay2 to 1.11<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package components

import (
	"fmt"

	"github.com/golang/glog"
	"k8s.io/kops/pkg/apis/kops"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/loader"
)

// DockerOptionsBuilder adds options for docker to the model
type DockerOptionsBuilder struct {
	*OptionsContext
}

var _ loader.OptionsBuilder = &DockerOptionsBuilder{}

// BuildOptions is responsible for filling in the default setting for docker daemon
func (b *DockerOptionsBuilder) BuildOptions(o interface{}) error {
	clusterSpec := o.(*kops.ClusterSpec)

	sv, err := KubernetesVersion(clusterSpec)
	if err != nil {
		return fmt.Errorf("unable to determine kubernetes version from %q", clusterSpec.KubernetesVersion)
	}

	if clusterSpec.Docker == nil {
		clusterSpec.Docker = &kops.DockerConfig{}
	}

	docker := clusterSpec.Docker

	if fi.StringValue(clusterSpec.Docker.Version) == "" {
		if clusterSpec.KubernetesVersion == "" {
			return fmt.Errorf("KubernetesVersion is required")
		}

		dockerVersion := ""
		if sv.Major == 1 && sv.Minor >= 9 {
			dockerVersion = "17.03.2"
		} else if sv.Major == 1 && sv.Minor >= 8 {
			dockerVersion = "1.13.1"
		} else if sv.Major == 1 && sv.Minor >= 6 {
			dockerVersion = "1.12.6"
		} else if sv.Major == 1 && sv.Minor >= 5 {
			dockerVersion = "1.12.3"
		} else if sv.Major == 1 && sv.Minor <= 4 {
			dockerVersion = "1.11.2"
		}

		if dockerVersion == "" {
			return fmt.Errorf("unknown version of kubernetes %q (cannot infer docker version)", clusterSpec.KubernetesVersion)
		}

		clusterSpec.Docker.Version = &dockerVersion
	}

	if sv.Major == 1 && sv.Minor >= 6 {
		if len(clusterSpec.Docker.LogOpt) == 0 && clusterSpec.Docker.LogDriver == "" {
			// Use built-in docker logging, if not configured otherwise (by the user)
			clusterSpec.Docker.LogDriver = "json-file"
			clusterSpec.Docker.LogOpt = append(clusterSpec.Docker.LogOpt, "max-size=10m")
			clusterSpec.Docker.LogOpt = append(clusterSpec.Docker.LogOpt, "max-file=5")
		}
	}

	docker.LogLevel = fi.String("warn")
	docker.IPTables = fi.Bool(false)
	docker.IPMasq = fi.Bool(false)

	// Note the alternative syntax... with a comma nodeup will try each of the filesystems in turn
	if b.IsKubernetesGTE("1.11") {
		// TODO(justinsb): figure out whether to use overlay2 on AWS jessie:
		// The ContainerOS image now has docker configured to use overlay2 out-of-the-box
		// and it is an error to specify the flag twice.
		// But Jessie (still our default AWS image) isn't recommended by docker with overlay2
		// (though that may be a kernel issue, and we run a custom kernel on our default image)
		// But we still need to worry about users running generic AMIs (e.g. stock jessie)
		docker.Storage = fi.String("overlay2,overlay,aufs")
	} else {
		docker.Storage = fi.String("overlay,aufs")
	}

	networking := clusterSpec.Networking
	if networking == nil || networking.Classic != nil {
		glog.Warningf("using deprecated (classic) networking")
		docker.Bridge = fi.String("cbr0")
	}

	return nil
}
<|endoftext|>


<commit_before>package opentracing

import (
	"context"
	"io"
	"net/http"

	"github.com/hellofresh/gcloud-opentracing"
	"github.com/hellofresh/janus/pkg/config"
	"github.com/opentracing/opentracing-go"
	log "github.com/sirupsen/logrus"
	jaeger "github.com/uber/jaeger-client-go"
	jaegercfg "github.com/uber/jaeger-client-go/config"
	"github.com/uber/jaeger-client-go/zipkin"
	"github.com/uber/jaeger-lib/metrics"
)

const (
	gcloudTracing = "googleCloud"
	jaegerTracing = "jaeger"
)

// Tracing is the tracing functionality
type Tracing struct {
	config config.Tracing
	tracer opentracing.Tracer
	closer io.Closer
}

type noopCloser struct{}

func (n noopCloser) Close() error { return nil }

// New creates a new instance of Tracing
func New(config config.Tracing) *Tracing {
	return &Tracing{config: config}
}

// Setup a tracer based on the configuration provided
func (t *Tracing) Setup() {
	var err error

	log.Debug("Initializing distributed tracing")
	switch t.config.Provider {
	case gcloudTracing:
		log.Debug("Using google cloud platform (stackdriver trace) as tracing system")
		t.tracer, t.closer, err = t.buildGCloud(t.config.GoogleCloudTracing)
	case jaegerTracing:
		log.Debug("Using Jaeger as tracing system")
		t.tracer, t.closer, err = t.buildJaeger(t.config.ServiceName, t.config.JaegerTracing)
	default:
		log.Debug("No tracer selected")
		t.tracer, t.closer, err = &opentracing.NoopTracer{}, noopCloser{}, nil
	}

	if err != nil {
		log.WithError(err).WithField("provider", t.config.Provider).Warn("Could not initialize tracing")
		return
	}

	opentracing.SetGlobalTracer(t.tracer)
}

// Close tracer
func (t *Tracing) Close() {
	if t.closer != nil {
		t.closer.Close()
	}
}

func (t *Tracing) buildGCloud(config config.GoogleCloudTracing) (opentracing.Tracer, io.Closer, error) {
	tracer, err := gcloudtracer.NewTracer(
		context.Background(),
		gcloudtracer.WithLogger(log.StandardLogger()),
		gcloudtracer.WithProject(config.ProjectID),
		gcloudtracer.WithJWTCredentials(gcloudtracer.JWTCredentials{
			Email:        config.Email,
			PrivateKey:   []byte(config.PrivateKey),
			PrivateKeyID: config.PrivateKeyID,
		}),
	)

	return tracer, noopCloser{}, err
}

func (t *Tracing) buildJaeger(componentName string, c config.JaegerTracing) (opentracing.Tracer, io.Closer, error) {
	cfg := jaegercfg.Configuration{
		Sampler: &jaegercfg.SamplerConfig{
			Type:  c.SamplingType,
			Param: c.SamplingParam,
		},
		Reporter: &jaegercfg.ReporterConfig{
			LogSpans:            c.LogSpans,
			BufferFlushInterval: c.BufferFlushInterval,
			LocalAgentHostPort:  c.SamplingServerURL,
			QueueSize:           c.QueueSize,
		},
	}

	tracerMetrics := jaeger.NewMetrics(metrics.NullFactory, nil)
	tracerLogger := jaegerLoggerAdapter{log.StandardLogger()}
	sampler, err := cfg.Sampler.NewSampler(componentName, tracerMetrics)
	if err != nil {
		return nil, nil, err
	}

	reporter, err := cfg.Reporter.NewReporter(componentName, tracerMetrics, tracerLogger)
	if err != nil {
		return nil, nil, err
	}

	var (
		tracer opentracing.Tracer
		closer io.Closer
	)

	switch c.PropagationFormat {
	case "zipkin":
		log.Debug("Using zipkin b3 http propagation format")
		zipkinPropagator := zipkin.NewZipkinB3HTTPHeaderPropagator()
		tracer, closer = jaeger.NewTracer(componentName, sampler, reporter,
			jaeger.TracerOptions.Metrics(tracerMetrics),
			jaeger.TracerOptions.Logger(tracerLogger),
			jaeger.TracerOptions.Injector(opentracing.HTTPHeaders, zipkinPropagator),
			jaeger.TracerOptions.Extractor(opentracing.HTTPHeaders, zipkinPropagator),
			jaeger.TracerOptions.ZipkinSharedRPCSpan(true),
		)
	default:
		log.Debug("Using jaeger propagation format")
		tracer, closer = jaeger.NewTracer(componentName, sampler, reporter,
			jaeger.TracerOptions.Metrics(tracerMetrics),
			jaeger.TracerOptions.Logger(tracerLogger),
		)
	}

	return tracer, closer, nil
}

// FromContext creates a span from a context that contains a parent span
func FromContext(ctx context.Context, name string) opentracing.Span {
	span, _ := opentracing.StartSpanFromContext(ctx, name)
	return span
}

// ToContext sets a span to a context
func ToContext(r *http.Request, span opentracing.Span) *http.Request {
	return r.WithContext(opentracing.ContextWithSpan(r.Context(), span))
}
<commit_msg>use const if possible<commit_after>package opentracing

import (
	"context"
	"io"
	"net/http"

	"github.com/hellofresh/gcloud-opentracing"
	"github.com/hellofresh/janus/pkg/config"
	"github.com/opentracing/opentracing-go"
	log "github.com/sirupsen/logrus"
	jaeger "github.com/uber/jaeger-client-go"
	jaegercfg "github.com/uber/jaeger-client-go/config"
	"github.com/uber/jaeger-lib/metrics"
)

const (
	gcloudTracing = "googleCloud"
	jaegerTracing = "jaeger"
	zipkin        = "zipkin"
)

// Tracing is the tracing functionality
type Tracing struct {
	config config.Tracing
	tracer opentracing.Tracer
	closer io.Closer
}

type noopCloser struct{}

func (n noopCloser) Close() error { return nil }

// New creates a new instance of Tracing
func New(config config.Tracing) *Tracing {
	return &Tracing{config: config}
}

// Setup a tracer based on the configuration provided
func (t *Tracing) Setup() {
	var err error

	log.Debug("Initializing distributed tracing")
	switch t.config.Provider {
	case gcloudTracing:
		log.Debug("Using google cloud platform (stackdriver trace) as tracing system")
		t.tracer, t.closer, err = t.buildGCloud(t.config.GoogleCloudTracing)
	case jaegerTracing:
		log.Debug("Using Jaeger as tracing system")
		t.tracer, t.closer, err = t.buildJaeger(t.config.ServiceName, t.config.JaegerTracing)
	default:
		log.Debug("No tracer selected")
		t.tracer, t.closer, err = &opentracing.NoopTracer{}, noopCloser{}, nil
	}

	if err != nil {
		log.WithError(err).WithField("provider", t.config.Provider).Warn("Could not initialize tracing")
		return
	}

	opentracing.SetGlobalTracer(t.tracer)
}

// Close tracer
func (t *Tracing) Close() {
	if t.closer != nil {
		t.closer.Close()
	}
}

func (t *Tracing) buildGCloud(config config.GoogleCloudTracing) (opentracing.Tracer, io.Closer, error) {
	tracer, err := gcloudtracer.NewTracer(
		context.Background(),
		gcloudtracer.WithLogger(log.StandardLogger()),
		gcloudtracer.WithProject(config.ProjectID),
		gcloudtracer.WithJWTCredentials(gcloudtracer.JWTCredentials{
			Email:        config.Email,
			PrivateKey:   []byte(config.PrivateKey),
			PrivateKeyID: config.PrivateKeyID,
		}),
	)

	return tracer, noopCloser{}, err
}

func (t *Tracing) buildJaeger(componentName string, c config.JaegerTracing) (opentracing.Tracer, io.Closer, error) {
	cfg := jaegercfg.Configuration{
		Sampler: &jaegercfg.SamplerConfig{
			Type:  c.SamplingType,
			Param: c.SamplingParam,
		},
		Reporter: &jaegercfg.ReporterConfig{
			LogSpans:            c.LogSpans,
			BufferFlushInterval: c.BufferFlushInterval,
			LocalAgentHostPort:  c.SamplingServerURL,
			QueueSize:           c.QueueSize,
		},
	}

	tracerMetrics := jaeger.NewMetrics(metrics.NullFactory, nil)
	tracerLogger := jaegerLoggerAdapter{log.StandardLogger()}
	sampler, err := cfg.Sampler.NewSampler(componentName, tracerMetrics)
	if err != nil {
		return nil, nil, err
	}

	reporter, err := cfg.Reporter.NewReporter(componentName, tracerMetrics, tracerLogger)
	if err != nil {
		return nil, nil, err
	}

	var (
		tracer opentracing.Tracer
		closer io.Closer
	)

	switch c.PropagationFormat {
	case zipkin:
		log.Debug("Using zipkin b3 http propagation format")
		zipkinPropagator := zipkin.NewZipkinB3HTTPHeaderPropagator()
		tracer, closer = jaeger.NewTracer(componentName, sampler, reporter,
			jaeger.TracerOptions.Metrics(tracerMetrics),
			jaeger.TracerOptions.Logger(tracerLogger),
			jaeger.TracerOptions.Injector(opentracing.HTTPHeaders, zipkinPropagator),
			jaeger.TracerOptions.Extractor(opentracing.HTTPHeaders, zipkinPropagator),
			jaeger.TracerOptions.ZipkinSharedRPCSpan(true),
		)
	default:
		log.Debug("Using jaeger propagation format")
		tracer, closer = jaeger.NewTracer(componentName, sampler, reporter,
			jaeger.TracerOptions.Metrics(tracerMetrics),
			jaeger.TracerOptions.Logger(tracerLogger),
		)
	}

	return tracer, closer, nil
}

// FromContext creates a span from a context that contains a parent span
func FromContext(ctx context.Context, name string) opentracing.Span {
	span, _ := opentracing.StartSpanFromContext(ctx, name)
	return span
}

// ToContext sets a span to a context
func ToContext(r *http.Request, span opentracing.Span) *http.Request {
	return r.WithContext(opentracing.ContextWithSpan(r.Context(), span))
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"fmt"
	"net"
	"sync"
	"sync/atomic"
	"testing"
)

func TestPortSet(t *testing.T) {
	pf := &PortSet{}

	// Try to store a port
	pf.Set(9000)

	// Try to load the port
	if alreadySet := pf.LoadOrSet(9000); !alreadySet {
		t.Fatal("didn't load port 9000 correctly")
	}

	if alreadySet := pf.LoadOrSet(4000); alreadySet {
		t.Fatal("didn't store port 4000 correctly")
	}

	if alreadySet := pf.LoadOrSet(4000); !alreadySet {
		t.Fatal("didn't load port 4000 correctly")
	}
}

func TestGetAvailablePort(t *testing.T) {
	var ports PortSet

	N := 100
	var (
		errors int32
		wg     sync.WaitGroup
	)
	wg.Add(N)
	for i := 0; i < N; i++ {
		go func() {
			port := GetAvailablePort("127.0.0.1", 4503, &ports)

			l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", Loopback, port))
			if err != nil {
				atomic.AddInt32(&errors, 1)
			} else {
				l.Close()
			}
			wg.Done()
		}()
	}
	wg.Wait()
	if atomic.LoadInt32(&errors) > 0 {
		t.Fatalf("A port that was available couldn't be used %d times", errors)
	}
}
<commit_msg>Improve error messages for flaky test (#4544)<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import (
	"fmt"
	"net"
	"sync"
	"testing"
)

func TestPortSet(t *testing.T) {
	pf := &PortSet{}

	// Try to store a port
	pf.Set(9000)

	// Try to load the port
	if alreadySet := pf.LoadOrSet(9000); !alreadySet {
		t.Fatal("didn't load port 9000 correctly")
	}

	if alreadySet := pf.LoadOrSet(4000); alreadySet {
		t.Fatal("didn't store port 4000 correctly")
	}

	if alreadySet := pf.LoadOrSet(4000); !alreadySet {
		t.Fatal("didn't load port 4000 correctly")
	}
}

func TestGetAvailablePort(t *testing.T) {
	N := 100

	var (
		ports  PortSet
		lock   sync.Mutex
		wg     sync.WaitGroup
		errors = map[int]error{}
	)

	wg.Add(N)
	for i := 0; i < N; i++ {
		go func() {
			port := GetAvailablePort("127.0.0.1", 4503, &ports)

			l, err := net.Listen("tcp", fmt.Sprintf("%s:%d", Loopback, port))
			if err != nil {
				lock.Lock()
				errors[port] = err
				lock.Unlock()
			} else {
				l.Close()
			}
			wg.Done()
		}()
	}
	wg.Wait()

	for port, err := range errors {
		t.Errorf("available port (%d) couldn't be used: %w", port, err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Ben Darnell

package storage

import (
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/net/context"

	"github.com/cockroachdb/cockroach/pkg/config"
	"github.com/cockroachdb/cockroach/pkg/gossip"
	"github.com/cockroachdb/cockroach/pkg/roachpb"
	"github.com/cockroachdb/cockroach/pkg/util/hlc"
	"github.com/cockroachdb/cockroach/pkg/util/log"
	"github.com/cockroachdb/cockroach/pkg/util/retry"
	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
)

const (
	// replicateQueueMaxSize is the max size of the replicate queue.
	replicateQueueMaxSize = 100

	// replicateQueueTimerDuration is the duration between replication of queued
	// replicas.
	replicateQueueTimerDuration = 0 // zero duration to process replication greedily
)

var (
	// minLeaseTransferInterval controls how frequently leases can be transferred
	// for rebalancing. It does not prevent transferring leases in order to allow
	// a replica to be removed from a range. The value should be some reasonable
	// fraction of the store descriptor gossip interval which currently
	minLeaseTransferInterval = gossip.GossipStoresInterval / 5
)

// replicateQueue manages a queue of replicas which may need to add an
// additional replica to their range.
type replicateQueue struct {
	*baseQueue
	allocator         Allocator
	clock             *hlc.Clock
	updateChan        chan struct{}
	lastLeaseTransfer atomic.Value // read and written by scanner & queue goroutines
}

// newReplicateQueue returns a new instance of replicateQueue.
func newReplicateQueue(
	store *Store, g *gossip.Gossip, allocator Allocator, clock *hlc.Clock, options AllocatorOptions,
) *replicateQueue {
	rq := &replicateQueue{
		allocator:  allocator,
		clock:      clock,
		updateChan: make(chan struct{}, 1),
	}
	rq.baseQueue = newBaseQueue(
		"replicate", rq, store, g,
		queueConfig{
			maxSize:              replicateQueueMaxSize,
			needsLease:           true,
			acceptsUnsplitRanges: store.TestingKnobs().ReplicateQueueAcceptsUnsplit,
			successes:            store.metrics.ReplicateQueueSuccesses,
			failures:             store.metrics.ReplicateQueueFailures,
			pending:              store.metrics.ReplicateQueuePending,
			processingNanos:      store.metrics.ReplicateQueueProcessingNanos,
			purgatory:            store.metrics.ReplicateQueuePurgatory,
		},
	)

	if g != nil { // gossip is nil for some unittests
		// Register a gossip callback to signal queue that replicas in
		// purgatory might be retried due to new store gossip.
		g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(_ string, _ roachpb.Value) {
			select {
			case rq.updateChan <- struct{}{}:
			default:
			}
		})
	}

	return rq
}

func (rq *replicateQueue) shouldQueue(
	ctx context.Context, now hlc.Timestamp, repl *Replica, sysCfg config.SystemConfig,
) (shouldQ bool, priority float64) {
	if !repl.store.splitQueue.Disabled() && repl.needsSplitBySize() {
		// If the range exceeds the split threshold, let that finish first.
		// Ranges must fit in memory on both sender and receiver nodes while
		// being replicated. This supplements the check provided by
		// acceptsUnsplitRanges, which looks at zone config boundaries rather
		// than data size.
		//
		// This check is ignored if the split queue is disabled, since in that
		// case, the split will never come.
		return
	}

	// Find the zone config for this range.
	desc := repl.Desc()
	zone, err := sysCfg.GetZoneConfigForKey(desc.StartKey)
	if err != nil {
		log.Error(ctx, err)
		return
	}

	action, priority := rq.allocator.ComputeAction(zone, desc)
	if action != AllocatorNoop {
		if log.V(2) {
			log.Infof(ctx, "%s repair needed (%s), enqueuing", repl, action)
		}
		return true, priority
	}

	// If we hold the lease, check to see if we should transfer it.
	var leaseStoreID roachpb.StoreID
	if lease, _ := repl.getLease(); lease != nil && lease.Covers(now) {
		leaseStoreID = lease.Replica.StoreID
		if rq.canTransferLease() &&
			rq.allocator.ShouldTransferLease(zone.Constraints, leaseStoreID, desc.RangeID) {
			if log.V(2) {
				log.Infof(ctx, "%s lease transfer needed, enqueuing", repl)
			}
			return true, 0
		}
	}

	// Check for a rebalancing opportunity. Note that leaseStoreID will be 0 if
	// the range doesn't currently have a lease which will allow the current
	// replica to be considered a rebalancing source.
	target, err := rq.allocator.RebalanceTarget(
		zone.Constraints,
		desc.Replicas,
		leaseStoreID,
		desc.RangeID,
	)
	if err != nil {
		log.ErrEventf(ctx, "rebalance target failed: %s", err)
		return false, 0
	}
	if log.V(2) {
		if target != nil {
			log.Infof(ctx, "%s rebalance target found, enqueuing", repl)
		} else {
			log.Infof(ctx, "%s no rebalance target found, not enqueuing", repl)
		}
	}
	return target != nil, 0
}

func (rq *replicateQueue) process(
	ctx context.Context, now hlc.Timestamp, repl *Replica, sysCfg config.SystemConfig,
) error {
	retryOpts := retry.Options{
		InitialBackoff: 50 * time.Millisecond,
		MaxBackoff:     1 * time.Second,
		Multiplier:     2,
		MaxRetries:     5,
	}

	// Use a retry loop in order to backoff in the case of preemptive
	// snapshot errors, usually signalling that a rebalancing
	// reservation could not be made with the selected target.
	for r := retry.StartWithCtx(ctx, retryOpts); r.Next(); {
		if err := rq.processOneChange(ctx, now, repl, sysCfg); err != nil {
			if IsPreemptiveSnapshotError(err) {
				// If ChangeReplicas failed because the preemptive snapshot failed, we
				// log the error but then return success indicating we should retry the
				// operation. The most likely causes of the preemptive snapshot failing are
				// a declined reservation or the remote node being unavailable. In either
				// case we don't want to wait another scanner cycle before reconsidering
				// the range.
				log.Info(ctx, err)
				continue
			}
			return err
		}
		// Enqueue this replica again to see if there are more changes to be made.
		rq.MaybeAdd(repl, rq.clock.Now())
		return nil
	}
	return errors.Errorf("failed to replicate %s after %d retries", repl, retryOpts.MaxRetries)
}

func (rq *replicateQueue) processOneChange(
	ctx context.Context, now hlc.Timestamp, repl *Replica, sysCfg config.SystemConfig,
) error {
	desc := repl.Desc()
	// Find the zone config for this range.
	zone, err := sysCfg.GetZoneConfigForKey(desc.StartKey)
	if err != nil {
		return err
	}
	action, _ := rq.allocator.ComputeAction(zone, desc)

	// Avoid taking action if the range has too many dead replicas to make
	// quorum.
	deadReplicas := rq.allocator.storePool.deadReplicas(desc.RangeID, desc.Replicas)
	quorum := computeQuorum(len(desc.Replicas))
	liveReplicaCount := len(desc.Replicas) - len(deadReplicas)
	if liveReplicaCount < quorum {
		return errors.Errorf("range requires a replication change, but lacks a quorum of live nodes.")
	}

	switch action {
	case AllocatorAdd:
		log.Event(ctx, "adding a new replica")
		newStore, err := rq.allocator.AllocateTarget(
			zone.Constraints,
			desc.Replicas,
			desc.RangeID,
			true,
		)
		if err != nil {
			return err
		}
		newReplica := roachpb.ReplicaDescriptor{
			NodeID:  newStore.Node.NodeID,
			StoreID: newStore.StoreID,
		}

		log.VEventf(ctx, 1, "adding replica to %+v due to under-replication", newReplica)
		if err := rq.addReplica(ctx, repl, newReplica, desc); err != nil {
			return err
		}
	case AllocatorRemove:
		log.Event(ctx, "removing a replica")
		// If the lease holder (our local store) is an overfull store (in terms of
		// leases) allow transferring the lease away.
		leaseHolderStoreID := repl.store.StoreID()
		if rq.allocator.ShouldTransferLease(zone.Constraints, leaseHolderStoreID, desc.RangeID) {
			leaseHolderStoreID = 0
		}
		removeReplica, err := rq.allocator.RemoveTarget(
			zone.Constraints,
			desc.Replicas,
			leaseHolderStoreID,
		)
		if err != nil {
			return err
		}
		if removeReplica.StoreID == repl.store.StoreID() {
			// The local replica was selected as the removal target, but that replica
			// is the leaseholder, so transfer the lease instead. We don't check that
			// the current store has too many leases in this case under the
			// assumption that replica balance is a greater concern. Also note that
			// AllocatorRemove action takes preference over AllocatorNoop
			// (rebalancing) which is where lease transfer would otherwise occur. We
			// need to be able to transfer leases in AllocatorRemove in order to get
			// out of situations where this store is overfull and yet holds all the
			// leases.
			candidates := filterBehindReplicas(repl.RaftStatus(), desc.Replicas)
			target := rq.allocator.TransferLeaseTarget(
				zone.Constraints, candidates, repl.store.StoreID(), desc.RangeID,
				false /* checkTransferLeaseSource */)
			if target != (roachpb.ReplicaDescriptor{}) {
				log.VEventf(ctx, 1, "transferring lease to s%d", target.StoreID)
				if err := repl.AdminTransferLease(target.StoreID); err != nil {
					return errors.Wrapf(err, "%s: unable to transfer lease to s%d", repl, target.StoreID)
				}
				rq.lastLeaseTransfer.Store(timeutil.Now())
				// Do not requeue as we transferred our lease away.
				return nil
			}
		} else {
			log.VEventf(ctx, 1, "removing replica %+v due to over-replication", removeReplica)
			if err := rq.removeReplica(ctx, repl, removeReplica, desc); err != nil {
				return err
			}
		}
	case AllocatorRemoveDead:
		log.Event(ctx, "removing a dead replica")
		if len(deadReplicas) == 0 {
			if log.V(1) {
				log.Warningf(ctx, "Range of replica %s was identified as having dead replicas, but no dead replicas were found.", repl)
			}
			break
		}
		deadReplica := deadReplicas[0]
		log.VEventf(ctx, 1, "removing dead replica %+v from store", deadReplica)
		if err := repl.ChangeReplicas(ctx, roachpb.REMOVE_REPLICA, deadReplica, desc); err != nil {
			return err
		}
	case AllocatorNoop:
		// The Noop case will result if this replica was queued in order to
		// rebalance. Attempt to find a rebalancing target.
		log.Event(ctx, "considering a rebalance")

		if rq.canTransferLease() {
			// We require the lease in order to process replicas, so
			// repl.store.StoreID() corresponds to the lease-holder's store ID.
			candidates := filterBehindReplicas(repl.RaftStatus(), desc.Replicas)
			target := rq.allocator.TransferLeaseTarget(
				zone.Constraints, candidates, repl.store.StoreID(), desc.RangeID,
				true /* checkTransferLeaseSource */)
			if target != (roachpb.ReplicaDescriptor{}) {
				log.VEventf(ctx, 1, "transferring lease to s%d", target.StoreID)
				if err := repl.AdminTransferLease(target.StoreID); err != nil {
					return errors.Wrapf(err, "%s: unable to transfer lease to s%d", repl, target.StoreID)
				}
				rq.lastLeaseTransfer.Store(timeutil.Now())
				// Do not requeue as we transferred our lease away.
				return nil
			}
		}

		rebalanceStore, err := rq.allocator.RebalanceTarget(
			zone.Constraints,
			desc.Replicas,
			repl.store.StoreID(),
			desc.RangeID,
		)
		if err != nil {
			log.ErrEventf(ctx, "rebalance target failed %s", err)
			return nil
		}
		if rebalanceStore == nil {
			log.VEventf(ctx, 1, "no suitable rebalance target")
			// No action was necessary and no rebalance target was found. Return
			// without re-queuing this replica.
			return nil
		}
		rebalanceReplica := roachpb.ReplicaDescriptor{
			NodeID:  rebalanceStore.Node.NodeID,
			StoreID: rebalanceStore.StoreID,
		}
		log.VEventf(ctx, 1, "rebalancing to %+v", rebalanceReplica)
		if err := rq.addReplica(ctx, repl, rebalanceReplica, desc); err != nil {
			return err
		}
	}

	return nil
}

func (rq *replicateQueue) addReplica(
	ctx context.Context,
	repl *Replica,
	repDesc roachpb.ReplicaDescriptor,
	desc *roachpb.RangeDescriptor,
) error {
	return repl.ChangeReplicas(ctx, roachpb.ADD_REPLICA, repDesc, desc)
}

func (rq *replicateQueue) removeReplica(
	ctx context.Context,
	repl *Replica,
	repDesc roachpb.ReplicaDescriptor,
	desc *roachpb.RangeDescriptor,
) error {
	return repl.ChangeReplicas(ctx, roachpb.REMOVE_REPLICA, repDesc, desc)
}

func (rq *replicateQueue) canTransferLease() bool {
	if lastLeaseTransfer := rq.lastLeaseTransfer.Load(); lastLeaseTransfer != nil {
		return timeutil.Since(lastLeaseTransfer.(time.Time)) > minLeaseTransferInterval
	}
	return true
}

func (*replicateQueue) timer() time.Duration {
	return replicateQueueTimerDuration
}

// purgatoryChan returns the replicate queue's store update channel.
func (rq *replicateQueue) purgatoryChan() <-chan struct{} {
	return rq.updateChan
}
<commit_msg>storage: set minLeaseTransferInterval to 1s<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Ben Darnell

package storage

import (
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/net/context"

	"github.com/cockroachdb/cockroach/pkg/config"
	"github.com/cockroachdb/cockroach/pkg/gossip"
	"github.com/cockroachdb/cockroach/pkg/roachpb"
	"github.com/cockroachdb/cockroach/pkg/util/hlc"
	"github.com/cockroachdb/cockroach/pkg/util/log"
	"github.com/cockroachdb/cockroach/pkg/util/retry"
	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
)

const (
	// replicateQueueMaxSize is the max size of the replicate queue.
	replicateQueueMaxSize = 100

	// replicateQueueTimerDuration is the duration between replication of queued
	// replicas.
	replicateQueueTimerDuration = 0 // zero duration to process replication greedily

	// minLeaseTransferInterval controls how frequently leases can be transferred
	// for rebalancing. It does not prevent transferring leases in order to allow
	// a replica to be removed from a range.
	minLeaseTransferInterval = time.Second
)

// replicateQueue manages a queue of replicas which may need to add an
// additional replica to their range.
type replicateQueue struct {
	*baseQueue
	allocator         Allocator
	clock             *hlc.Clock
	updateChan        chan struct{}
	lastLeaseTransfer atomic.Value // read and written by scanner & queue goroutines
}

// newReplicateQueue returns a new instance of replicateQueue.
func newReplicateQueue(
	store *Store, g *gossip.Gossip, allocator Allocator, clock *hlc.Clock, options AllocatorOptions,
) *replicateQueue {
	rq := &replicateQueue{
		allocator:  allocator,
		clock:      clock,
		updateChan: make(chan struct{}, 1),
	}
	rq.baseQueue = newBaseQueue(
		"replicate", rq, store, g,
		queueConfig{
			maxSize:              replicateQueueMaxSize,
			needsLease:           true,
			acceptsUnsplitRanges: store.TestingKnobs().ReplicateQueueAcceptsUnsplit,
			successes:            store.metrics.ReplicateQueueSuccesses,
			failures:             store.metrics.ReplicateQueueFailures,
			pending:              store.metrics.ReplicateQueuePending,
			processingNanos:      store.metrics.ReplicateQueueProcessingNanos,
			purgatory:            store.metrics.ReplicateQueuePurgatory,
		},
	)

	if g != nil { // gossip is nil for some unittests
		// Register a gossip callback to signal queue that replicas in
		// purgatory might be retried due to new store gossip.
		g.RegisterCallback(gossip.MakePrefixPattern(gossip.KeyStorePrefix), func(_ string, _ roachpb.Value) {
			select {
			case rq.updateChan <- struct{}{}:
			default:
			}
		})
	}

	return rq
}

func (rq *replicateQueue) shouldQueue(
	ctx context.Context, now hlc.Timestamp, repl *Replica, sysCfg config.SystemConfig,
) (shouldQ bool, priority float64) {
	if !repl.store.splitQueue.Disabled() && repl.needsSplitBySize() {
		// If the range exceeds the split threshold, let that finish first.
		// Ranges must fit in memory on both sender and receiver nodes while
		// being replicated. This supplements the check provided by
		// acceptsUnsplitRanges, which looks at zone config boundaries rather
		// than data size.
		//
		// This check is ignored if the split queue is disabled, since in that
		// case, the split will never come.
		return
	}

	// Find the zone config for this range.
	desc := repl.Desc()
	zone, err := sysCfg.GetZoneConfigForKey(desc.StartKey)
	if err != nil {
		log.Error(ctx, err)
		return
	}

	action, priority := rq.allocator.ComputeAction(zone, desc)
	if action != AllocatorNoop {
		if log.V(2) {
			log.Infof(ctx, "%s repair needed (%s), enqueuing", repl, action)
		}
		return true, priority
	}

	// If we hold the lease, check to see if we should transfer it.
	var leaseStoreID roachpb.StoreID
	if lease, _ := repl.getLease(); lease != nil && lease.Covers(now) {
		leaseStoreID = lease.Replica.StoreID
		if rq.canTransferLease() &&
			rq.allocator.ShouldTransferLease(zone.Constraints, leaseStoreID, desc.RangeID) {
			if log.V(2) {
				log.Infof(ctx, "%s lease transfer needed, enqueuing", repl)
			}
			return true, 0
		}
	}

	// Check for a rebalancing opportunity. Note that leaseStoreID will be 0 if
	// the range doesn't currently have a lease which will allow the current
	// replica to be considered a rebalancing source.
	target, err := rq.allocator.RebalanceTarget(
		zone.Constraints,
		desc.Replicas,
		leaseStoreID,
		desc.RangeID,
	)
	if err != nil {
		log.ErrEventf(ctx, "rebalance target failed: %s", err)
		return false, 0
	}
	if log.V(2) {
		if target != nil {
			log.Infof(ctx, "%s rebalance target found, enqueuing", repl)
		} else {
			log.Infof(ctx, "%s no rebalance target found, not enqueuing", repl)
		}
	}
	return target != nil, 0
}

func (rq *replicateQueue) process(
	ctx context.Context, now hlc.Timestamp, repl *Replica, sysCfg config.SystemConfig,
) error {
	retryOpts := retry.Options{
		InitialBackoff: 50 * time.Millisecond,
		MaxBackoff:     1 * time.Second,
		Multiplier:     2,
		MaxRetries:     5,
	}

	// Use a retry loop in order to backoff in the case of preemptive
	// snapshot errors, usually signalling that a rebalancing
	// reservation could not be made with the selected target.
	for r := retry.StartWithCtx(ctx, retryOpts); r.Next(); {
		if err := rq.processOneChange(ctx, now, repl, sysCfg); err != nil {
			if IsPreemptiveSnapshotError(err) {
				// If ChangeReplicas failed because the preemptive snapshot failed, we
				// log the error but then return success indicating we should retry the
				// operation. The most likely causes of the preemptive snapshot failing are
				// a declined reservation or the remote node being unavailable. In either
				// case we don't want to wait another scanner cycle before reconsidering
				// the range.
				log.Info(ctx, err)
				continue
			}
			return err
		}
		// Enqueue this replica again to see if there are more changes to be made.
		rq.MaybeAdd(repl, rq.clock.Now())
		return nil
	}
	return errors.Errorf("failed to replicate %s after %d retries", repl, retryOpts.MaxRetries)
}

func (rq *replicateQueue) processOneChange(
	ctx context.Context, now hlc.Timestamp, repl *Replica, sysCfg config.SystemConfig,
) error {
	desc := repl.Desc()
	// Find the zone config for this range.
	zone, err := sysCfg.GetZoneConfigForKey(desc.StartKey)
	if err != nil {
		return err
	}
	action, _ := rq.allocator.ComputeAction(zone, desc)

	// Avoid taking action if the range has too many dead replicas to make
	// quorum.
	deadReplicas := rq.allocator.storePool.deadReplicas(desc.RangeID, desc.Replicas)
	quorum := computeQuorum(len(desc.Replicas))
	liveReplicaCount := len(desc.Replicas) - len(deadReplicas)
	if liveReplicaCount < quorum {
		return errors.Errorf("range requires a replication change, but lacks a quorum of live nodes.")
	}

	switch action {
	case AllocatorAdd:
		log.Event(ctx, "adding a new replica")
		newStore, err := rq.allocator.AllocateTarget(
			zone.Constraints,
			desc.Replicas,
			desc.RangeID,
			true,
		)
		if err != nil {
			return err
		}
		newReplica := roachpb.ReplicaDescriptor{
			NodeID:  newStore.Node.NodeID,
			StoreID: newStore.StoreID,
		}

		log.VEventf(ctx, 1, "adding replica to %+v due to under-replication", newReplica)
		if err := rq.addReplica(ctx, repl, newReplica, desc); err != nil {
			return err
		}
	case AllocatorRemove:
		log.Event(ctx, "removing a replica")
		// If the lease holder (our local store) is an overfull store (in terms of
		// leases) allow transferring the lease away.
		leaseHolderStoreID := repl.store.StoreID()
		if rq.allocator.ShouldTransferLease(zone.Constraints, leaseHolderStoreID, desc.RangeID) {
			leaseHolderStoreID = 0
		}
		removeReplica, err := rq.allocator.RemoveTarget(
			zone.Constraints,
			desc.Replicas,
			leaseHolderStoreID,
		)
		if err != nil {
			return err
		}
		if removeReplica.StoreID == repl.store.StoreID() {
			// The local replica was selected as the removal target, but that replica
			// is the leaseholder, so transfer the lease instead. We don't check that
			// the current store has too many leases in this case under the
			// assumption that replica balance is a greater concern. Also note that
			// AllocatorRemove action takes preference over AllocatorNoop
			// (rebalancing) which is where lease transfer would otherwise occur. We
			// need to be able to transfer leases in AllocatorRemove in order to get
			// out of situations where this store is overfull and yet holds all the
			// leases.
			candidates := filterBehindReplicas(repl.RaftStatus(), desc.Replicas)
			target := rq.allocator.TransferLeaseTarget(
				zone.Constraints, candidates, repl.store.StoreID(), desc.RangeID,
				false /* checkTransferLeaseSource */)
			if target != (roachpb.ReplicaDescriptor{}) {
				log.VEventf(ctx, 1, "transferring lease to s%d", target.StoreID)
				if err := repl.AdminTransferLease(target.StoreID); err != nil {
					return errors.Wrapf(err, "%s: unable to transfer lease to s%d", repl, target.StoreID)
				}
				rq.lastLeaseTransfer.Store(timeutil.Now())
				// Do not requeue as we transferred our lease away.
				return nil
			}
		} else {
			log.VEventf(ctx, 1, "removing replica %+v due to over-replication", removeReplica)
			if err := rq.removeReplica(ctx, repl, removeReplica, desc); err != nil {
				return err
			}
		}
	case AllocatorRemoveDead:
		log.Event(ctx, "removing a dead replica")
		if len(deadReplicas) == 0 {
			if log.V(1) {
				log.Warningf(ctx, "Range of replica %s was identified as having dead replicas, but no dead replicas were found.", repl)
			}
			break
		}
		deadReplica := deadReplicas[0]
		log.VEventf(ctx, 1, "removing dead replica %+v from store", deadReplica)
		if err := repl.ChangeReplicas(ctx, roachpb.REMOVE_REPLICA, deadReplica, desc); err != nil {
			return err
		}
	case AllocatorNoop:
		// The Noop case will result if this replica was queued in order to
		// rebalance. Attempt to find a rebalancing target.
		log.Event(ctx, "considering a rebalance")

		if rq.canTransferLease() {
			// We require the lease in order to process replicas, so
			// repl.store.StoreID() corresponds to the lease-holder's store ID.
			candidates := filterBehindReplicas(repl.RaftStatus(), desc.Replicas)
			target := rq.allocator.TransferLeaseTarget(
				zone.Constraints, candidates, repl.store.StoreID(), desc.RangeID,
				true /* checkTransferLeaseSource */)
			if target != (roachpb.ReplicaDescriptor{}) {
				log.VEventf(ctx, 1, "transferring lease to s%d", target.StoreID)
				if err := repl.AdminTransferLease(target.StoreID); err != nil {
					return errors.Wrapf(err, "%s: unable to transfer lease to s%d", repl, target.StoreID)
				}
				rq.lastLeaseTransfer.Store(timeutil.Now())
				// Do not requeue as we transferred our lease away.
				return nil
			}
		}

		rebalanceStore, err := rq.allocator.RebalanceTarget(
			zone.Constraints,
			desc.Replicas,
			repl.store.StoreID(),
			desc.RangeID,
		)
		if err != nil {
			log.ErrEventf(ctx, "rebalance target failed %s", err)
			return nil
		}
		if rebalanceStore == nil {
			log.VEventf(ctx, 1, "no suitable rebalance target")
			// No action was necessary and no rebalance target was found. Return
			// without re-queuing this replica.
			return nil
		}
		rebalanceReplica := roachpb.ReplicaDescriptor{
			NodeID:  rebalanceStore.Node.NodeID,
			StoreID: rebalanceStore.StoreID,
		}
		log.VEventf(ctx, 1, "rebalancing to %+v", rebalanceReplica)
		if err := rq.addReplica(ctx, repl, rebalanceReplica, desc); err != nil {
			return err
		}
	}

	return nil
}

func (rq *replicateQueue) addReplica(
	ctx context.Context,
	repl *Replica,
	repDesc roachpb.ReplicaDescriptor,
	desc *roachpb.RangeDescriptor,
) error {
	return repl.ChangeReplicas(ctx, roachpb.ADD_REPLICA, repDesc, desc)
}

func (rq *replicateQueue) removeReplica(
	ctx context.Context,
	repl *Replica,
	repDesc roachpb.ReplicaDescriptor,
	desc *roachpb.RangeDescriptor,
) error {
	return repl.ChangeReplicas(ctx, roachpb.REMOVE_REPLICA, repDesc, desc)
}

func (rq *replicateQueue) canTransferLease() bool {
	if lastLeaseTransfer := rq.lastLeaseTransfer.Load(); lastLeaseTransfer != nil {
		return timeutil.Since(lastLeaseTransfer.(time.Time)) > minLeaseTransferInterval
	}
	return true
}

func (*replicateQueue) timer() time.Duration {
	return replicateQueueTimerDuration
}

// purgatoryChan returns the replicate queue's store update channel.
func (rq *replicateQueue) purgatoryChan() <-chan struct{} {
	return rq.updateChan
}
<|endoftext|>