<commit_before>package memory

import (
	"sync/atomic"
	"unsafe"

	"github.com/grafana/metrictank/util"

	"github.com/grafana/metrictank/errors"
	"github.com/grafana/metrictank/expr/tagquery"
	"github.com/grafana/metrictank/schema"
	lru "github.com/hashicorp/golang-lru"
)

// the collision avoidance window defines how many times we try to find a higher
// slot that's free if two record hashes collide
var collisionAvoidanceWindow = uint32(1024)

type recordId uint32

// list of meta records keyed by a unique identifier used as ID
type metaTagRecords struct {
	records  map[recordId]tagquery.MetaTagRecord
	enricher unsafe.Pointer
}

func newMetaTagRecords() *metaTagRecords {
	return &metaTagRecords{
		records: make(map[recordId]tagquery.MetaTagRecord),
	}
}

// upsert inserts or updates a meta tag record according to the given specifications
// it uses the set of tag query expressions as the identity of the record, if a record with the
// same identity is already present then its meta tags get updated to the specified ones.
// If the new record contains no meta tags, then the update is equivalent to a delete.
// Those are the return values:
// 1) The id at which the new record got inserted
// 2) Pointer to the inserted metaTagRecord
// 3) The id of the record that has been replaced if an update was performed
// 4) Pointer to the metaTagRecord that has been replaced if an update was performed, otherwise nil
// 5) Error if an error occurred, otherwise it's nil
func (m metaTagRecords) upsert(record tagquery.MetaTagRecord) (recordId, *tagquery.MetaTagRecord, recordId, *tagquery.MetaTagRecord, error) {
	// after altering meta records we need to reinstantiate the enricher the next time we want to use it
	defer atomic.StorePointer(&m.enricher, nil)

	id := recordId(record.HashExpressions())
	var oldRecord *tagquery.MetaTagRecord
	var oldId recordId

	// loop over existing records, starting from id, trying to find one that has
	// the exact same queries as the one we're upserting
	for i := uint32(0); i < collisionAvoidanceWindow; i++ {
		if existingRecord, ok := m.records[id+recordId(i)]; ok {
			if record.EqualExpressions(&existingRecord) {
				oldRecord = &existingRecord
				oldId = id + recordId(i)
				delete(m.records, oldId)
				break
			}
		}
	}

	if !record.HasMetaTags() {
		return 0, &record, oldId, oldRecord, nil
	}

	// now find the best position to insert the new/updated record, starting from id
	for i := uint32(0); i < collisionAvoidanceWindow; i++ {
		// if we find a free slot, then insert the new record there
		if _, ok := m.records[id]; !ok {
			m.records[id] = record

			return id, &record, oldId, oldRecord, nil
		}

		id++
	}

	return 0, nil, 0, nil, errors.NewInternal("Could not find a free ID to insert record")
}

func (m *metaTagRecords) getEnricher(lookup tagquery.IdTagLookup) *enricher {
	res := (*enricher)(atomic.LoadPointer(&m.enricher))
	if res != nil {
		return res
	}

	// if no enricher is present yet, then we instantiate one and store its reference
	// to reuse it later.
	// there is an unlikely race condition where we receive multiple calls to FindByTag
	// from multiple requests at the same time, which would result in multiple enrichers
	// getting instantiated. in such a case the last one would overwrite the previous
	// ones and there shouldn't be any issues. this case is very unlikely, so it doesn't
	// seem to be worth it to add a lock for that.

	// if provided size is valid, it's not possible for lru.New to return an error
	cache, _ := lru.New(enrichmentCacheSize)
	res = &enricher{
		filters: make([]tagquery.MetricDefinitionFilter, len(m.records)),
		tags:    make([]tagquery.Tags, len(m.records)),
		cache:   cache,
	}

	i := 0
	for _, record := range m.records {
		res.filters[i] = record.GetMetricDefinitionFilter(lookup)
		res.tags[i] = record.MetaTags
		i++
	}

	// to atomically store a pointer, unfortunately we need to use unsafe.Pointer
	atomic.StorePointer(&m.enricher, (unsafe.Pointer)(res))

	return res
}

type enricher struct {
	filters []tagquery.MetricDefinitionFilter
	tags    []tagquery.Tags
	cache   *lru.Cache
}

func (e *enricher) enrich(id schema.MKey, name string, tags []string) tagquery.Tags {
	h := util.NewFnv64aStringWriter()
	h.WriteString(name)
	for i := range tags {
		h.WriteString(";")
		h.WriteString(tags[i])
	}
	sum := h.Sum64()

	cachedRes, ok := e.cache.Get(sum)
	if ok {
		return cachedRes.(tagquery.Tags)
	}

	var res tagquery.Tags
	var matches []int
	for i := range e.filters {
		if e.filters[i](id, name, tags) == tagquery.Pass {
			res = append(res, e.tags[i]...)
			matches = append(matches, i)
		}
	}

	e.cache.Add(sum, res)

	return res
}

// index structure keyed by tag -> value -> list of meta record IDs
type metaTagValue map[string][]recordId
type metaTagIndex map[string]metaTagValue

func (m metaTagIndex) deleteRecord(keyValue tagquery.Tag, id recordId) {
	if values, ok := m[keyValue.Key]; ok {
		if ids, ok := values[keyValue.Value]; ok {
			for i := 0; i < len(ids); i++ {
				if ids[i] == id {
					// no need to keep the order
					ids[i] = ids[len(ids)-1]
					values[keyValue.Value] = ids[:len(ids)-1]

					// no id should ever be present more than once
					return
				}
			}
		}
	}
}

func (m metaTagIndex) insertRecord(keyValue tagquery.Tag, id recordId) {
	var values metaTagValue
	var ok bool

	if values, ok = m[keyValue.Key]; !ok {
		values = make(metaTagValue)
		m[keyValue.Key] = values
	}

	values[keyValue.Value] = append(values[keyValue.Value], id)
}

// getMetaRecordIdsByExpression takes an expression and a bool, it returns all meta record
// ids of the records which match it. the bool indicates whether the result set should be
// inverted. In some cases inverting the result set can minimize its size.
// Minimizing the size of the returned result set will lead to a faster expression evaluation,
// because less meta records will need to be checked against a given MetricDefinition.
// The caller, after receiving the result set, needs to be aware of whether the result set
// is inverted and interpret it accordingly.
func (m metaTagIndex) getMetaRecordIdsByExpression(expr tagquery.Expression, invertSetOfMetaRecords bool) []recordId {
	if expr.OperatesOnTag() {
		return m.getByTag(expr, invertSetOfMetaRecords)
	}
	return m.getByTagValue(expr, invertSetOfMetaRecords)
}

func (m metaTagIndex) getByTag(expr tagquery.Expression, invertSetOfMetaRecords bool) []recordId {
	var res []recordId

	for key := range m {

		if invertSetOfMetaRecords {
			if expr.Matches(key) {
				continue
			}
		} else {
			if !expr.Matches(key) {
				continue
			}
		}

		for _, ids := range m[key] {
			res = append(res, ids...)
		}
	}

	return res
}

func (m metaTagIndex) getByTagValue(expr tagquery.Expression, invertSetOfMetaRecords bool) []recordId {
	if expr.MatchesExactly() {
		return m[expr.GetKey()][expr.GetValue()]
	}

	var res []recordId
	for value, ids := range m[expr.GetKey()] {
		passes := expr.Matches(value)

		if invertSetOfMetaRecords {
			passes = !passes
		}

		if !passes {
			continue
		}

		res = append(res, ids...)
	}

	return res
}
<commit_msg>correctly clear enrichment cache on upsert<commit_after>package memory

import (
	"sync/atomic"
	"unsafe"

	"github.com/grafana/metrictank/util"

	"github.com/grafana/metrictank/errors"
	"github.com/grafana/metrictank/expr/tagquery"
	"github.com/grafana/metrictank/schema"
	lru "github.com/hashicorp/golang-lru"
)

// the collision avoidance window defines how many times we try to find a higher
// slot that's free if two record hashes collide
var collisionAvoidanceWindow = uint32(1024)

type recordId uint32

// list of meta records keyed by a unique identifier used as ID
type metaTagRecords struct {
	records  map[recordId]tagquery.MetaTagRecord
	enricher unsafe.Pointer
}

func newMetaTagRecords() *metaTagRecords {
	return &metaTagRecords{
		records: make(map[recordId]tagquery.MetaTagRecord),
	}
}

// upsert inserts or updates a meta tag record according to the given specifications
// it uses the set of tag query expressions as the identity of the record, if a record with the
// same identity is already present then its meta tags get updated to the specified ones.
// If the new record contains no meta tags, then the update is equivalent to a delete.
// Those are the return values:
// 1) The id at which the new record got inserted
// 2) Pointer to the inserted metaTagRecord
// 3) The id of the record that has been replaced if an update was performed
// 4) Pointer to the metaTagRecord that has been replaced if an update was performed, otherwise nil
// 5) Error if an error occurred, otherwise it's nil
func (m *metaTagRecords) upsert(record tagquery.MetaTagRecord) (recordId, *tagquery.MetaTagRecord, recordId, *tagquery.MetaTagRecord, error) {
	// after altering meta records we need to reinstantiate the enricher the next time we want to use it
	defer atomic.StorePointer(&m.enricher, nil)

	id := recordId(record.HashExpressions())
	var oldRecord *tagquery.MetaTagRecord
	var oldId recordId

	// loop over existing records, starting from id, trying to find one that has
	// the exact same queries as the one we're upserting
	for i := uint32(0); i < collisionAvoidanceWindow; i++ {
		if existingRecord, ok := m.records[id+recordId(i)]; ok {
			if record.EqualExpressions(&existingRecord) {
				oldRecord = &existingRecord
				oldId = id + recordId(i)
				delete(m.records, oldId)
				break
			}
		}
	}

	if !record.HasMetaTags() {
		return 0, &record, oldId, oldRecord, nil
	}

	// now find the best position to insert the new/updated record, starting from id
	for i := uint32(0); i < collisionAvoidanceWindow; i++ {
		// if we find a free slot, then insert the new record there
		if _, ok := m.records[id]; !ok {
			m.records[id] = record

			return id, &record, oldId, oldRecord, nil
		}

		id++
	}

	return 0, nil, 0, nil, errors.NewInternal("Could not find a free ID to insert record")
}

func (m *metaTagRecords) getEnricher(lookup tagquery.IdTagLookup) *enricher {
	res := (*enricher)(atomic.LoadPointer(&m.enricher))
	if res != nil {
		return res
	}

	// if no enricher is present yet, then we instantiate one and store its reference
	// to reuse it later.
	// there is an unlikely race condition where we receive multiple calls to FindByTag
	// from multiple requests at the same time, which would result in multiple enrichers
	// getting instantiated. in such a case the last one would overwrite the previous
	// ones and there shouldn't be any issues. this case is very unlikely, so it doesn't
	// seem to be worth it to add a lock for that.

	// if provided size is valid, it's not possible for lru.New to return an error
	cache, _ := lru.New(enrichmentCacheSize)
	res = &enricher{
		filters: make([]tagquery.MetricDefinitionFilter, len(m.records)),
		tags:    make([]tagquery.Tags, len(m.records)),
		cache:   cache,
	}

	i := 0
	for _, record := range m.records {
		res.filters[i] = record.GetMetricDefinitionFilter(lookup)
		res.tags[i] = record.MetaTags
		i++
	}

	// to atomically store a pointer, unfortunately we need to use unsafe.Pointer
	atomic.StorePointer(&m.enricher, (unsafe.Pointer)(res))

	return res
}

type enricher struct {
	filters []tagquery.MetricDefinitionFilter
	tags    []tagquery.Tags
	cache   *lru.Cache
}

func (e *enricher) enrich(id schema.MKey, name string, tags []string) tagquery.Tags {
	h := util.NewFnv64aStringWriter()
	h.WriteString(name)
	for i := range tags {
		h.WriteString(";")
		h.WriteString(tags[i])
	}
	sum := h.Sum64()

	cachedRes, ok := e.cache.Get(sum)
	if ok {
		return cachedRes.(tagquery.Tags)
	}

	var res tagquery.Tags
	var matches []int
	for i := range e.filters {
		if e.filters[i](id, name, tags) == tagquery.Pass {
			res = append(res, e.tags[i]...)
			matches = append(matches, i)
		}
	}

	e.cache.Add(sum, res)

	return res
}

// index structure keyed by tag -> value -> list of meta record IDs
type metaTagValue map[string][]recordId
type metaTagIndex map[string]metaTagValue

func (m metaTagIndex) deleteRecord(keyValue tagquery.Tag, id recordId) {
	if values, ok := m[keyValue.Key]; ok {
		if ids, ok := values[keyValue.Value]; ok {
			for i := 0; i < len(ids); i++ {
				if ids[i] == id {
					// no need to keep the order
					ids[i] = ids[len(ids)-1]
					values[keyValue.Value] = ids[:len(ids)-1]

					// no id should ever be present more than once
					return
				}
			}
		}
	}
}

func (m metaTagIndex) insertRecord(keyValue tagquery.Tag, id recordId) {
	var values metaTagValue
	var ok bool

	if values, ok = m[keyValue.Key]; !ok {
		values = make(metaTagValue)
		m[keyValue.Key] = values
	}

	values[keyValue.Value] = append(values[keyValue.Value], id)
}

// getMetaRecordIdsByExpression takes an expression and a bool, it returns all meta record
// ids of the records which match it. the bool indicates whether the result set should be
// inverted. In some cases inverting the result set can minimize its size.
// Minimizing the size of the returned result set will lead to a faster expression evaluation,
// because less meta records will need to be checked against a given MetricDefinition.
// The caller, after receiving the result set, needs to be aware of whether the result set
// is inverted and interpret it accordingly.
func (m metaTagIndex) getMetaRecordIdsByExpression(expr tagquery.Expression, invertSetOfMetaRecords bool) []recordId {
	if expr.OperatesOnTag() {
		return m.getByTag(expr, invertSetOfMetaRecords)
	}
	return m.getByTagValue(expr, invertSetOfMetaRecords)
}

func (m metaTagIndex) getByTag(expr tagquery.Expression, invertSetOfMetaRecords bool) []recordId {
	var res []recordId

	for key := range m {

		if invertSetOfMetaRecords {
			if expr.Matches(key) {
				continue
			}
		} else {
			if !expr.Matches(key) {
				continue
			}
		}

		for _, ids := range m[key] {
			res = append(res, ids...)
		}
	}

	return res
}

func (m metaTagIndex) getByTagValue(expr tagquery.Expression, invertSetOfMetaRecords bool) []recordId {
	if expr.MatchesExactly() {
		return m[expr.GetKey()][expr.GetValue()]
	}

	var res []recordId
	for value, ids := range m[expr.GetKey()] {
		passes := expr.Matches(value)

		if invertSetOfMetaRecords {
			passes = !passes
		}

		if !passes {
			continue
		}

		res = append(res, ids...)
	}

	return res
}
<|endoftext|>


<commit_before>package importer

import (
	"testing"

	"github.com/DexterLB/mvm/library"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
)

func testContext(t *testing.T) *Context {
	lib, err := library.New("sqlite3", ":memory:")
	if err != nil {
		t.Fatalf("unable to initialize library: %s\n", err)
	}

	context := NewContext(lib, &Config{
		FileRoot: "./testdata",
	})

	go func() {
		for err := range context.Errors {
			t.Fatalf("context error: %s\n", err)
		}
	}()

	return context
}
<commit_msg>fix DVR option<commit_after>package importer

import (
	"testing"

	"github.com/DexterLB/mvm/library"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
	_ "github.com/orchestrate-io/dvr"
)

func testContext(t *testing.T) *Context {
	lib, err := library.New("sqlite3", ":memory:")
	if err != nil {
		t.Fatalf("unable to initialize library: %s\n", err)
	}

	context := NewContext(lib, &Config{
		FileRoot: "./testdata",
	})

	go func() {
		for err := range context.Errors {
			t.Fatalf("context error: %s\n", err)
		}
	}()

	return context
}
<|endoftext|>


<commit_before>package fortuna

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"testing"
	"time"
)

func newSourceChannel(cs *SourceChannel) error {
	f, err := os.Open("/dev/random")
	if err != nil {
		return err
	}
	buf := new(bytes.Buffer)
	io.CopyN(buf, f, 4096)
	f.Close()
	go func() {
		for {
			if buf.Len() == 0 {
				return
			}
			e := make([]byte, 24)
			_, err := buf.Read(e)
			if err != nil {
				fmt.Fprintf(os.Stderr, "fortuna: error on buffer (%v)\n", err)
				return

			}
			cs.In <- e
			<-time.After(30 * time.Millisecond)
		}
	}()
	return nil
}

func TestSourceChannel(t *testing.T) {
	rng := New(nil)
	cs := NewSourceChannel(rng, 1)
	cs.Start(4)
	err := newSourceChannel(cs)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	}
	err = newSourceChannel(cs)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	}
	err = newSourceChannel(cs)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	}
	err = newSourceChannel(cs)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	}

	<-time.After(1 * time.Second)
	var p = make([]byte, 16384)
	n, err := rng.Read(p)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	} else if n != 16384 {
		fmt.Fprintf(os.Stderr, "fortuna: read %d bytes, expected 16384\n", n)
		t.FailNow()
	}
	//cs.Stop()
}

func TestSourceWriter(t *testing.T) {
	rng := New(nil)
	sw := NewSourceWriter(rng, 2)
	f, err := os.Open("/dev/random")
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	}
	buf := new(bytes.Buffer)
	io.CopyN(buf, f, 4096)
	f.Close()

	n, err := io.Copy(sw, buf)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	} else if n != 4096 {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	}
}

func TestUninitialisedPRNG(t *testing.T) {
	if sc := NewSourceChannel(nil, 3); sc != nil {
		fmt.Fprintln(os.Stderr, "fortuna: new source should fail for uninitialised PRNG")
		t.FailNow()
	}

	if sw := NewSourceWriter(nil, 3); sw != nil {
		fmt.Fprintln(os.Stderr, "fortuna: new source should fail for uninitialised PRNG")
		t.FailNow()
	}

	rng := &Fortuna{}
	if sc := NewSourceChannel(rng, 3); sc != nil {
		fmt.Fprintln(os.Stderr, "fortuna: new source should fail for uninitialised PRNG")
		t.FailNow()
	}

	if sw := NewSourceWriter(rng, 3); sw != nil {
		fmt.Fprintln(os.Stderr, "fortuna: new source should fail for uninitialised PRNG")
		t.FailNow()
	}

}
<commit_msg>Clean up data races.<commit_after>package fortuna

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"testing"
	"time"
)

func newSourceChannel(cs *SourceChannel) error {
	f, err := os.Open("/dev/random")
	if err != nil {
		return err
	}
	buf := new(bytes.Buffer)
	io.CopyN(buf, f, 4096)
	f.Close()
	go func() {
		for {
			if buf.Len() == 0 {
				return
			}
			e := make([]byte, 24)
			_, err := buf.Read(e)
			if err != nil {
				fmt.Fprintf(os.Stderr, "fortuna: error on buffer (%v)\n", err)
				return

			}
			cs.In <- e
			<-time.After(30 * time.Millisecond)
		}
	}()
	return nil
}

func TestSourceChannel(t *testing.T) {
	rng := New(nil)
	cs := NewSourceChannel(rng, 1)
	cs.Start(4)
	err := newSourceChannel(cs)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	}
	err = newSourceChannel(cs)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	}
	err = newSourceChannel(cs)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	}
	err = newSourceChannel(cs)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	}

	<-time.After(1 * time.Second)
	var p = make([]byte, 16384)
	n, err := rng.Read(p)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	} else if n != 16384 {
		fmt.Fprintf(os.Stderr, "fortuna: read %d bytes, expected 16384\n", n)
		t.FailNow()
	}
	cs.Stop()
}

func TestSourceWriter(t *testing.T) {
	rng := New(nil)
	sw := NewSourceWriter(rng, 2)
	f, err := os.Open("/dev/random")
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	}
	buf := new(bytes.Buffer)
	io.CopyN(buf, f, 4096)
	f.Close()

	n, err := io.Copy(sw, buf)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	} else if n != 4096 {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		t.FailNow()
	}
}

func TestUninitialisedPRNG(t *testing.T) {
	if sc := NewSourceChannel(nil, 3); sc != nil {
		fmt.Fprintln(os.Stderr, "fortuna: new source should fail for uninitialised PRNG")
		t.FailNow()
	}

	if sw := NewSourceWriter(nil, 3); sw != nil {
		fmt.Fprintln(os.Stderr, "fortuna: new source should fail for uninitialised PRNG")
		t.FailNow()
	}

	rng := &Fortuna{}
	if sc := NewSourceChannel(rng, 3); sc != nil {
		fmt.Fprintln(os.Stderr, "fortuna: new source should fail for uninitialised PRNG")
		t.FailNow()
	}

	if sw := NewSourceWriter(rng, 3); sw != nil {
		fmt.Fprintln(os.Stderr, "fortuna: new source should fail for uninitialised PRNG")
		t.FailNow()
	}

}
<|endoftext|>


<commit_before>package render

import (
	"strings"

	"github.com/weaveworks/scope/common/mtime"
	"github.com/weaveworks/scope/probe/docker"
	"github.com/weaveworks/scope/probe/endpoint"
	"github.com/weaveworks/scope/probe/kubernetes"
	"github.com/weaveworks/scope/report"
)

// CustomRenderer allow for mapping functions that received the entire topology
// in one call - useful for functions that need to consider the entire graph.
// We should minimise the use of this renderer type, as it is very inflexible.
type CustomRenderer struct {
	RenderFunc func(report.Nodes) report.Nodes
	Renderer
}

// Render implements Renderer
func (c CustomRenderer) Render(rpt report.Report) report.Nodes {
	return c.RenderFunc(c.Renderer.Render(rpt))
}

// ColorConnected colors nodes with the IsConnected key if
// they have edges to or from them.  Edges to/from yourself
// are not counted here (see #656).
func ColorConnected(r Renderer) Renderer {
	return CustomRenderer{
		Renderer: r,
		RenderFunc: func(input report.Nodes) report.Nodes {
			connected := map[string]struct{}{}
			void := struct{}{}

			for id, node := range input {
				if len(node.Adjacency) == 0 {
					continue
				}

				for _, adj := range node.Adjacency {
					if adj != id {
						connected[id] = void
						connected[adj] = void
					}
				}
			}

			output := input.Copy()
			for id := range connected {
				output[id] = output[id].WithLatest(IsConnected, mtime.Now(), "true")
			}
			return output
		},
	}
}

// Filter removes nodes from a view based on a predicate.
type Filter struct {
	Renderer
	FilterFunc func(report.Node) bool
	Silent     bool // true means we don't report stats for how many are filtered
}

// MakeFilter makes a new Filter.
func MakeFilter(f func(report.Node) bool, r Renderer) Renderer {
	return Memoise(&Filter{
		Renderer:   r,
		FilterFunc: f,
	})
}

// MakeSilentFilter makes a new Filter which does not report how many nodes it filters in Stats.
func MakeSilentFilter(f func(report.Node) bool, r Renderer) Renderer {
	return Memoise(&Filter{
		Renderer:   r,
		FilterFunc: f,
		Silent:     true,
	})
}

// Render implements Renderer
func (f *Filter) Render(rpt report.Report) report.Nodes {
	nodes, _ := f.render(rpt)
	return nodes
}

func (f *Filter) render(rpt report.Report) (report.Nodes, int) {
	output := report.Nodes{}
	inDegrees := map[string]int{}
	filtered := 0
	for id, node := range f.Renderer.Render(rpt) {
		if f.FilterFunc(node) {
			output[id] = node
			inDegrees[id] = 0
		} else {
			filtered++
		}
	}

	// Deleted nodes also need to be cut as destinations in adjacency lists.
	for id, node := range output {
		newAdjacency := report.MakeIDList()
		for _, dstID := range node.Adjacency {
			if _, ok := output[dstID]; ok {
				newAdjacency = newAdjacency.Add(dstID)
				inDegrees[dstID]++
			}
		}
		node.Adjacency = newAdjacency
		output[id] = node
	}

	// Remove unconnected pseudo nodes, see #483.
	for id, inDegree := range inDegrees {
		if inDegree > 0 {
			continue
		}
		node := output[id]
		if node.Topology != Pseudo || len(node.Adjacency) > 0 {
			continue
		}
		delete(output, id)
		filtered++
	}
	return output, filtered
}

// Stats implements Renderer
func (f Filter) Stats(rpt report.Report) Stats {
	var upstream = f.Renderer.Stats(rpt)
	if !f.Silent {
		_, filtered := f.render(rpt)
		upstream.FilteredNodes += filtered
	}
	return upstream
}

// IsConnected is the key added to Node.Metadata by ColorConnected
// to indicate a node has an edge pointing to it or from it
const IsConnected = "is_connected"

// Complement takes a FilterFunc f and returns a FilterFunc that has the same
// effects, if any, and returns the opposite truth value.
func Complement(f func(report.Node) bool) func(report.Node) bool {
	return func(node report.Node) bool { return !f(node) }
}

// FilterPseudo produces a renderer that removes pseudo nodes from the given
// renderer
func FilterPseudo(r Renderer) Renderer {
	return MakeFilter(
		func(node report.Node) bool {
			return node.Topology != Pseudo
		},
		r,
	)
}

// FilterUnconnected produces a renderer that filters unconnected nodes
// from the given renderer
func FilterUnconnected(r Renderer) Renderer {
	return MakeFilter(
		func(node report.Node) bool {
			_, ok := node.Latest.Lookup(IsConnected)
			return ok
		},
		ColorConnected(r),
	)
}

// SilentFilterUnconnected produces a renderer that filters unconnected nodes
// from the given renderer; nodes filtered by this are not reported in stats.
func SilentFilterUnconnected(r Renderer) Renderer {
	return MakeSilentFilter(
		func(node report.Node) bool {
			_, ok := node.Latest.Lookup(IsConnected)
			return ok
		},
		ColorConnected(r),
	)
}

// FilterNoop does nothing.
func FilterNoop(in Renderer) Renderer {
	return in
}

// FilterStopped filters out stopped containers.
func FilterStopped(r Renderer) Renderer {
	return MakeFilter(IsRunning, r)
}

// IsRunning checks if the node is a running docker container
func IsRunning(n report.Node) bool {
	state, ok := n.Latest.Lookup(docker.ContainerState)
	return !ok || (state == docker.StateRunning || state == docker.StateRestarting || state == docker.StatePaused)
}

// FilterRunning filters out running containers.
func FilterRunning(r Renderer) Renderer {
	return MakeFilter(Complement(IsRunning), r)
}

// FilterNonProcspied removes endpoints which were not found in procspy.
func FilterNonProcspied(r Renderer) Renderer {
	return MakeFilter(
		func(node report.Node) bool {
			_, ok := node.Latest.Lookup(endpoint.Procspied)
			return ok
		},
		r,
	)
}

// IsSystem checks if the node is a "system" node
func IsSystem(n report.Node) bool {
	containerName, _ := n.Latest.Lookup(docker.ContainerName)
	if _, ok := systemContainerNames[containerName]; ok {
		return false
	}
	imageName, _ := n.Latest.Lookup(docker.ImageName)
	imagePrefix := strings.SplitN(imageName, ":", 2)[0] // :(
	if _, ok := systemImagePrefixes[imagePrefix]; ok {
		return false
	}
	roleLabel, _ := n.Latest.Lookup(docker.LabelPrefix + "works.weave.role")
	if roleLabel == "system" {
		return false
	}
	namespace, _ := n.Latest.Lookup(kubernetes.Namespace)
	if namespace == "kube-system" {
		return false
	}
	podName, _ := n.Latest.Lookup(docker.LabelPrefix + "io.kubernetes.pod.name")
	if strings.HasPrefix(podName, "kube-system/") {
		return false
	}
	return true
}

// FilterSystem is a Renderer which filters out system nodes.
func FilterSystem(r Renderer) Renderer {
	return MakeFilter(IsSystem, r)
}

// FilterApplication is a Renderer which filters out system nodes.
func FilterApplication(r Renderer) Renderer {
	return MakeFilter(Complement(IsSystem), r)
}

var systemContainerNames = map[string]struct{}{
	"weavescope": {},
	"weavedns":   {},
	"weave":      {},
	"weaveproxy": {},
	"weaveexec":  {},
	"ecs-agent":  {},
}

var systemImagePrefixes = map[string]struct{}{
	"weaveworks/scope":                    {},
	"weaveworks/weavedns":                 {},
	"weaveworks/weave":                    {},
	"weaveworks/weaveproxy":               {},
	"weaveworks/weaveexec":                {},
	"amazon/amazon-ecs-agent":             {},
	"beta.gcr.io/google_containers/pause": {},
	"gcr.io/google_containers/pause":      {},
}
<commit_msg>Make the initial procspy filter silent<commit_after>package render

import (
	"strings"

	"github.com/weaveworks/scope/common/mtime"
	"github.com/weaveworks/scope/probe/docker"
	"github.com/weaveworks/scope/probe/endpoint"
	"github.com/weaveworks/scope/probe/kubernetes"
	"github.com/weaveworks/scope/report"
)

// CustomRenderer allow for mapping functions that received the entire topology
// in one call - useful for functions that need to consider the entire graph.
// We should minimise the use of this renderer type, as it is very inflexible.
type CustomRenderer struct {
	RenderFunc func(report.Nodes) report.Nodes
	Renderer
}

// Render implements Renderer
func (c CustomRenderer) Render(rpt report.Report) report.Nodes {
	return c.RenderFunc(c.Renderer.Render(rpt))
}

// ColorConnected colors nodes with the IsConnected key if
// they have edges to or from them.  Edges to/from yourself
// are not counted here (see #656).
func ColorConnected(r Renderer) Renderer {
	return CustomRenderer{
		Renderer: r,
		RenderFunc: func(input report.Nodes) report.Nodes {
			connected := map[string]struct{}{}
			void := struct{}{}

			for id, node := range input {
				if len(node.Adjacency) == 0 {
					continue
				}

				for _, adj := range node.Adjacency {
					if adj != id {
						connected[id] = void
						connected[adj] = void
					}
				}
			}

			output := input.Copy()
			for id := range connected {
				output[id] = output[id].WithLatest(IsConnected, mtime.Now(), "true")
			}
			return output
		},
	}
}

// Filter removes nodes from a view based on a predicate.
type Filter struct {
	Renderer
	FilterFunc func(report.Node) bool
	Silent     bool // true means we don't report stats for how many are filtered
}

// MakeFilter makes a new Filter.
func MakeFilter(f func(report.Node) bool, r Renderer) Renderer {
	return Memoise(&Filter{
		Renderer:   r,
		FilterFunc: f,
	})
}

// MakeSilentFilter makes a new Filter which does not report how many nodes it filters in Stats.
func MakeSilentFilter(f func(report.Node) bool, r Renderer) Renderer {
	return Memoise(&Filter{
		Renderer:   r,
		FilterFunc: f,
		Silent:     true,
	})
}

// Render implements Renderer
func (f *Filter) Render(rpt report.Report) report.Nodes {
	nodes, _ := f.render(rpt)
	return nodes
}

func (f *Filter) render(rpt report.Report) (report.Nodes, int) {
	output := report.Nodes{}
	inDegrees := map[string]int{}
	filtered := 0
	for id, node := range f.Renderer.Render(rpt) {
		if f.FilterFunc(node) {
			output[id] = node
			inDegrees[id] = 0
		} else {
			filtered++
		}
	}

	// Deleted nodes also need to be cut as destinations in adjacency lists.
	for id, node := range output {
		newAdjacency := report.MakeIDList()
		for _, dstID := range node.Adjacency {
			if _, ok := output[dstID]; ok {
				newAdjacency = newAdjacency.Add(dstID)
				inDegrees[dstID]++
			}
		}
		node.Adjacency = newAdjacency
		output[id] = node
	}

	// Remove unconnected pseudo nodes, see #483.
	for id, inDegree := range inDegrees {
		if inDegree > 0 {
			continue
		}
		node := output[id]
		if node.Topology != Pseudo || len(node.Adjacency) > 0 {
			continue
		}
		delete(output, id)
		filtered++
	}
	return output, filtered
}

// Stats implements Renderer
func (f Filter) Stats(rpt report.Report) Stats {
	var upstream = f.Renderer.Stats(rpt)
	if !f.Silent {
		_, filtered := f.render(rpt)
		upstream.FilteredNodes += filtered
	}
	return upstream
}

// IsConnected is the key added to Node.Metadata by ColorConnected
// to indicate a node has an edge pointing to it or from it
const IsConnected = "is_connected"

// Complement takes a FilterFunc f and returns a FilterFunc that has the same
// effects, if any, and returns the opposite truth value.
func Complement(f func(report.Node) bool) func(report.Node) bool {
	return func(node report.Node) bool { return !f(node) }
}

// FilterPseudo produces a renderer that removes pseudo nodes from the given
// renderer
func FilterPseudo(r Renderer) Renderer {
	return MakeFilter(
		func(node report.Node) bool {
			return node.Topology != Pseudo
		},
		r,
	)
}

// FilterUnconnected produces a renderer that filters unconnected nodes
// from the given renderer
func FilterUnconnected(r Renderer) Renderer {
	return MakeFilter(
		func(node report.Node) bool {
			_, ok := node.Latest.Lookup(IsConnected)
			return ok
		},
		ColorConnected(r),
	)
}

// SilentFilterUnconnected produces a renderer that filters unconnected nodes
// from the given renderer; nodes filtered by this are not reported in stats.
func SilentFilterUnconnected(r Renderer) Renderer {
	return MakeSilentFilter(
		func(node report.Node) bool {
			_, ok := node.Latest.Lookup(IsConnected)
			return ok
		},
		ColorConnected(r),
	)
}

// FilterNoop does nothing.
func FilterNoop(in Renderer) Renderer {
	return in
}

// FilterStopped filters out stopped containers.
func FilterStopped(r Renderer) Renderer {
	return MakeFilter(IsRunning, r)
}

// IsRunning checks if the node is a running docker container
func IsRunning(n report.Node) bool {
	state, ok := n.Latest.Lookup(docker.ContainerState)
	return !ok || (state == docker.StateRunning || state == docker.StateRestarting || state == docker.StatePaused)
}

// FilterRunning filters out running containers.
func FilterRunning(r Renderer) Renderer {
	return MakeFilter(Complement(IsRunning), r)
}

// FilterNonProcspied removes endpoints which were not found in procspy.
func FilterNonProcspied(r Renderer) Renderer {
	return MakeSilentFilter(
		func(node report.Node) bool {
			_, ok := node.Latest.Lookup(endpoint.Procspied)
			return ok
		},
		r,
	)
}

// IsSystem checks if the node is a "system" node
func IsSystem(n report.Node) bool {
	containerName, _ := n.Latest.Lookup(docker.ContainerName)
	if _, ok := systemContainerNames[containerName]; ok {
		return false
	}
	imageName, _ := n.Latest.Lookup(docker.ImageName)
	imagePrefix := strings.SplitN(imageName, ":", 2)[0] // :(
	if _, ok := systemImagePrefixes[imagePrefix]; ok {
		return false
	}
	roleLabel, _ := n.Latest.Lookup(docker.LabelPrefix + "works.weave.role")
	if roleLabel == "system" {
		return false
	}
	namespace, _ := n.Latest.Lookup(kubernetes.Namespace)
	if namespace == "kube-system" {
		return false
	}
	podName, _ := n.Latest.Lookup(docker.LabelPrefix + "io.kubernetes.pod.name")
	if strings.HasPrefix(podName, "kube-system/") {
		return false
	}
	return true
}

// FilterSystem is a Renderer which filters out system nodes.
func FilterSystem(r Renderer) Renderer {
	return MakeFilter(IsSystem, r)
}

// FilterApplication is a Renderer which filters out system nodes.
func FilterApplication(r Renderer) Renderer {
	return MakeFilter(Complement(IsSystem), r)
}

var systemContainerNames = map[string]struct{}{
	"weavescope": {},
	"weavedns":   {},
	"weave":      {},
	"weaveproxy": {},
	"weaveexec":  {},
	"ecs-agent":  {},
}

var systemImagePrefixes = map[string]struct{}{
	"weaveworks/scope":                    {},
	"weaveworks/weavedns":                 {},
	"weaveworks/weave":                    {},
	"weaveworks/weaveproxy":               {},
	"weaveworks/weaveexec":                {},
	"amazon/amazon-ecs-agent":             {},
	"beta.gcr.io/google_containers/pause": {},
	"gcr.io/google_containers/pause":      {},
}
<|endoftext|>


<commit_before>package zygo
func init() { GITLASTTAG = "v4.8.1"; GITLASTCOMMIT = "e004037d8173c2839591832264473d8c7e583bc3" }
<commit_msg>v4.8.2<commit_after>package zygo
func init() { GITLASTTAG = "4.8.2"; GITLASTCOMMIT = "4ff8ada2d52f3b1ef3d6af97d6e06ea3957f92a8" }
<|endoftext|>


<commit_before>/*
Copyright 2020-Present Couchbase, Inc.

Use of this software is governed by the Business Source License included in
the file licenses/BSL-Couchbase.txt.  As of the Change Date specified in that
file, in accordance with the Business Source License, use of this software will
be governed by the Apache License, Version 2.0, included in the file
licenses/APL2.txt.
*/

package rest

import (
	"net"
	"net/http"
	"net/url"
	"testing"
	"time"

	"github.com/couchbase/sync_gateway/base"
	"github.com/stretchr/testify/require"
)

// TestX509RoundtripUsingIP is a happy-path roundtrip write test for SG connecting to CBS using valid X.509 certs for authentication.
// The test enforces SG connects using an IP address which is also present in the node cert.
func TestX509RoundtripUsingIP(t *testing.T) {

	if !x509TestsEnabled() {
		t.Skipf("x509 tests not enabled via %s flag", x509TestFlag)
	}

	if base.UnitTestUrlIsWalrus() {
		t.Skip("X509 not supported in Walrus")
	}

	testURL, err := url.Parse(base.UnitTestUrl())
	require.NoError(t, err)
	testIP := net.ParseIP(testURL.Hostname())
	if testIP == nil {
		t.Skipf("Test requires %s to be an IP address, but had: %v", base.TestEnvCouchbaseServerUrl, testURL.Hostname())
	}
	assertHostnameMatch(t, testURL)

	defer base.SetUpTestLogging(base.LevelDebug, base.KeyAll)()

	ca := generateX509CA(t)
	nodePair := generateX509Node(t, ca, []net.IP{testIP}, nil)
	sgPair := generateX509SG(t, ca, base.TestClusterUsername(), time.Now().Add(time.Hour*24))
	teardownFn := saveX509Files(t, ca, nodePair, sgPair)
	defer teardownFn()

	err = loadCertsIntoCouchbaseServer(*testURL, ca, nodePair)
	require.NoError(t, err)

	tb := base.GetTestBucket(t)
	defer tb.Close()

	// force couchbases:// scheme
	tb.BucketSpec.Server = "couchbases://" + testIP.String()
	// use x509 for auth
	tb.BucketSpec.Auth = base.NoPasswordAuthHandler{Handler: tb.BucketSpec.Auth}
	tb.BucketSpec.CACertPath = ca.PEMFilepath
	tb.BucketSpec.Certpath = sgPair.PEMFilepath
	tb.BucketSpec.Keypath = sgPair.KeyFilePath

	rt := NewRestTester(t, &RestTesterConfig{TestBucket: tb})
	defer rt.Close()

	// write a doc to ensure bucket ops work
	tr := rt.SendAdminRequest(http.MethodPut, "/db/"+t.Name(), `{"sgwrite":true}`)
	assertStatus(t, tr, http.StatusCreated)

	// wait for doc to come back over DCP
	err = rt.WaitForDoc(t.Name())
	require.NoError(t, err, "error waiting for doc over DCP")
}

// TestX509RoundtripUsingDomain is a happy-path roundtrip write test for SG connecting to CBS using valid X.509 certs for authentication.
// The test enforces SG connects using a domain name which is also present in the node cert.
func TestX509RoundtripUsingDomain(t *testing.T) {

	if !x509TestsEnabled() {
		t.Skipf("x509 tests not enabled via %s flag", x509TestFlag)
	}

	if base.UnitTestUrlIsWalrus() {
		t.Skip("X509 not supported in Walrus")
	}

	testURL, err := url.Parse(base.UnitTestUrl())
	require.NoError(t, err)
	testIP := net.ParseIP(testURL.Hostname())
	if testIP != nil {
		t.Skipf("Test requires %s to be a domain name, but had an IP: %v", base.TestEnvCouchbaseServerUrl, testURL.Hostname())
	}
	assertHostnameMatch(t, testURL)

	defer base.SetUpTestLogging(base.LevelTrace, base.KeyHTTP, base.KeyCache, base.KeyDCP)()

	ca := generateX509CA(t)
	nodePair := generateX509Node(t, ca, nil, []string{testURL.Hostname()})
	sgPair := generateX509SG(t, ca, base.TestClusterUsername(), time.Now().Add(time.Hour*24))
	teardownFn := saveX509Files(t, ca, nodePair, sgPair)
	defer teardownFn()

	err = loadCertsIntoCouchbaseServer(*testURL, ca, nodePair)
	require.NoError(t, err)

	tb := base.GetTestBucket(t)
	defer tb.Close()

	// force couchbases:// scheme
	tb.BucketSpec.Server = "couchbases://" + testURL.Hostname()
	// use x509 for auth
	tb.BucketSpec.Auth = base.NoPasswordAuthHandler{Handler: tb.BucketSpec.Auth}
	tb.BucketSpec.CACertPath = ca.PEMFilepath
	tb.BucketSpec.Certpath = sgPair.PEMFilepath
	tb.BucketSpec.Keypath = sgPair.KeyFilePath

	rt := NewRestTester(t, &RestTesterConfig{TestBucket: tb})
	defer rt.Close()

	// write a doc to ensure bucket ops work
	tr := rt.SendAdminRequest(http.MethodPut, "/db/"+t.Name(), `{"sgwrite":true}`)
	assertStatus(t, tr, http.StatusCreated)

	// wait for doc to come back over DCP
	err = rt.WaitForDoc(t.Name())
	require.NoError(t, err, "error waiting for doc over DCP")
}
<commit_msg>Split Out X509 Test Setup (#5051)<commit_after>/*
Copyright 2020-Present Couchbase, Inc.

Use of this software is governed by the Business Source License included in
the file licenses/BSL-Couchbase.txt.  As of the Change Date specified in that
file, in accordance with the Business Source License, use of this software will
be governed by the Apache License, Version 2.0, included in the file
licenses/APL2.txt.
*/

package rest

import (
	"net"
	"net/http"
	"net/url"
	"testing"
	"time"

	"github.com/couchbase/sync_gateway/base"
	"github.com/stretchr/testify/require"
)

// TestX509RoundtripUsingIP is a happy-path roundtrip write test for SG connecting to CBS using valid X.509 certs for authentication.
// The test enforces SG connects using an IP address which is also present in the node cert.
func TestX509RoundtripUsingIP(t *testing.T) {
	defer base.SetUpTestLogging(base.LevelDebug, base.KeyAll)()

	tb, teardownFn, _, _, _ := setupX509Tests(t, true)
	defer tb.Close()
	defer teardownFn()

	rt := NewRestTester(t, &RestTesterConfig{TestBucket: tb})
	defer rt.Close()

	// write a doc to ensure bucket ops work
	tr := rt.SendAdminRequest(http.MethodPut, "/db/"+t.Name(), `{"sgwrite":true}`)
	assertStatus(t, tr, http.StatusCreated)

	// wait for doc to come back over DCP
	err := rt.WaitForDoc(t.Name())
	require.NoError(t, err, "error waiting for doc over DCP")
}

// TestX509RoundtripUsingDomain is a happy-path roundtrip write test for SG connecting to CBS using valid X.509 certs for authentication.
// The test enforces SG connects using a domain name which is also present in the node cert.
func TestX509RoundtripUsingDomain(t *testing.T) {
	defer base.SetUpTestLogging(base.LevelDebug, base.KeyAll)()

	tb, teardownFn, _, _, _ := setupX509Tests(t, false)
	defer tb.Close()
	defer teardownFn()

	rt := NewRestTester(t, &RestTesterConfig{TestBucket: tb})
	defer rt.Close()

	// write a doc to ensure bucket ops work
	tr := rt.SendAdminRequest(http.MethodPut, "/db/"+t.Name(), `{"sgwrite":true}`)
	assertStatus(t, tr, http.StatusCreated)

	// wait for doc to come back over DCP
	err := rt.WaitForDoc(t.Name())
	require.NoError(t, err, "error waiting for doc over DCP")
}

func setupX509Tests(t *testing.T, useIPAddress bool) (testBucket *base.TestBucket, teardownFunc func(), caCertPath string, certPath string, keyPath string) {
	if !x509TestsEnabled() {
		t.Skipf("x509 tests not enabled via %s flag", x509TestFlag)
	}

	if base.UnitTestUrlIsWalrus() {
		t.Skip("X509 not supported in Walrus")
	}

	testURL, err := url.Parse(base.UnitTestUrl())
	require.NoError(t, err)
	testIP := net.ParseIP(testURL.Hostname())
	if testIP == nil && useIPAddress {
		t.Skipf("Test requires %s to be an IP address, but had: %v", base.TestEnvCouchbaseServerUrl, testURL.Hostname())
	}

	if testIP != nil && !useIPAddress {
		t.Skipf("Test requires %s to be a domain name, but had an IP: %v", base.TestEnvCouchbaseServerUrl, testURL.Hostname())
	}

	assertHostnameMatch(t, testURL)

	ca := generateX509CA(t)

	var testIPs []net.IP
	var testURls []string

	if useIPAddress {
		testIPs = []net.IP{testIP}
	}

	if !useIPAddress {
		testURls = []string{testURL.Hostname()}
	}

	nodePair := generateX509Node(t, ca, testIPs, testURls)
	sgPair := generateX509SG(t, ca, base.TestClusterUsername(), time.Now().Add(time.Hour*24))
	teardownFn := saveX509Files(t, ca, nodePair, sgPair)

	err = loadCertsIntoCouchbaseServer(*testURL, ca, nodePair)
	require.NoError(t, err)

	tb := base.GetTestBucket(t)

	// force couchbases:// scheme
	if useIPAddress {
		tb.BucketSpec.Server = "couchbases://" + testIP.String()
	} else {
		tb.BucketSpec.Server = "couchbases://" + testURL.Hostname()
	}

	// use x509 for auth
	tb.BucketSpec.Auth = base.NoPasswordAuthHandler{Handler: tb.BucketSpec.Auth}
	tb.BucketSpec.CACertPath = ca.PEMFilepath
	tb.BucketSpec.Certpath = sgPair.PEMFilepath
	tb.BucketSpec.Keypath = sgPair.KeyFilePath

	return tb, teardownFn, caCertPath, certPath, keyPath
}
<|endoftext|>


<commit_before>package robots

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
)

type SlashCommand struct {
	Payload
	Command string `schema:"command"`
}

type Payload struct {
	Token       string  `schema:"token"`
	TeamID      string  `schema:"team_id"`
	TeamDomain  string  `schema:"team_domain,omitempty"`
	ChannelID   string  `schema:"channel_id"`
	ChannelName string  `schema:"channel_name"`
	Timestamp   float64 `schema:"timestamp,omitempty"`
	UserID      string  `schema:"user_id"`
	UserName    string  `schema:"user_name"`
	Text        string  `schema:"text,omitempty"`
	TriggerWord string  `schema:"trigger_word,omitempty"`
	ServiceID   string  `schema:"service_id,omitempty"`
	Robot       string
}

type OutgoingWebHook struct {
	Payload
	TriggerWord string `schema:"trigger_word"`
}

type OutgoingWebHookResponse struct {
	Text      string     `json:"text"`
	Parse     ParseStyle `json:"parse,omitempty"`
	LinkNames bool       `json:"link_names,omitempty"`
	Markdown  bool       `json:"mrkdwn,omitempty"`
}

type ParseStyle string

var (
	ParseStyleFull = ParseStyle("full")
	ParseStyleNone = ParseStyle("none")
)

type IncomingWebhook struct {
	Domain      string       `json:"domain"`
	Channel     string       `json:"channel"`
	Username    string       `json:"username"`
	Text        string       `json:"text"`
	IconEmoji   string       `json:"icon_emoji,omitempty"`
	IconURL     string       `json:"icon_url,omitempty"`
	Attachments []Attachment `json:"attachments,omitempty"`
	UnfurlLinks bool         `json:"unfurl_links,omitempty"`
	Parse       ParseStyle   `json:"parse,omitempty"`
	LinkNames   bool         `json:"link_names,omitempty"`
	Markdown    bool         `json:"mrkdwn,omitempty"`
}

type Attachment struct {
	Fallback   string            `json:"fallback"`
	Pretext    string            `json:"pretext,omitempty"`
	Text       string            `json:"text,omitempty"`
	Color      string            `json:"color,omitempty"`
	Fields     []AttachmentField `json:"fields,omitempty"`
	MarkdownIn []MarkdownField   `json:"mrkdown_in,omitempty"`
}

type MarkdownField string

var (
	MarkdownFieldPretext  = MarkdownField("pretext")
	MarkdownFieldText     = MarkdownField("text")
	MarkdownFieldTitle    = MarkdownField("title")
	MarkdownFieldFields   = MarkdownField("fields")
	MarkdownFieldFallback = MarkdownField("fallback")
)

type AttachmentField struct {
	Title string `json:"title"`
	Value string `json:"value"`
	Short bool   `json:"short,omitempty"`
}

// Send uses the IncomingWebhook API to post a message to a slack channel
func (i *IncomingWebhook) Send() error {
	token := os.Getenv(fmt.Sprintf("%s_TOKEN", strings.ToUpper(i.Domain)))
	if token != "" {
		return fmt.Errorf("Slack token not found for domain %s (expecting %s)", i.Domain, fmt.Sprintf("%s_IN_TOKEN", strings.ToUpper(i.Domain)))
	}
	webhook := url.URL{
		Scheme: "https",
		Host:   i.Domain + ".slack.com",
		Path:   "/services/hooks/incoming-webhook",
	}

	p, err := json.Marshal(i)
	if err != nil {
		return err
	}

	data := url.Values{}
	data.Set("payload", string(p))
	data.Set("token", token)

	webhook.RawQuery = data.Encode()
	resp, err := http.PostForm(webhook.String(), data)
	if resp.StatusCode != 200 {
		message := fmt.Sprintf("ERROR: Non-200 Response from Slack Incoming Webhook API: %s", resp.Status)
		log.Println(message)
	}
	return err
}
<commit_msg>use full url<commit_after>package robots

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"strings"
)

type SlashCommand struct {
	Payload
	Command string `schema:"command"`
}

type Payload struct {
	Token       string  `schema:"token"`
	TeamID      string  `schema:"team_id"`
	TeamDomain  string  `schema:"team_domain,omitempty"`
	ChannelID   string  `schema:"channel_id"`
	ChannelName string  `schema:"channel_name"`
	Timestamp   float64 `schema:"timestamp,omitempty"`
	UserID      string  `schema:"user_id"`
	UserName    string  `schema:"user_name"`
	Text        string  `schema:"text,omitempty"`
	TriggerWord string  `schema:"trigger_word,omitempty"`
	ServiceID   string  `schema:"service_id,omitempty"`
	Robot       string
}

type OutgoingWebHook struct {
	Payload
	TriggerWord string `schema:"trigger_word"`
}

type OutgoingWebHookResponse struct {
	Text      string     `json:"text"`
	Parse     ParseStyle `json:"parse,omitempty"`
	LinkNames bool       `json:"link_names,omitempty"`
	Markdown  bool       `json:"mrkdwn,omitempty"`
}

type ParseStyle string

var (
	ParseStyleFull = ParseStyle("full")
	ParseStyleNone = ParseStyle("none")
)

type IncomingWebhook struct {
	Domain      string       `json:"domain"`
	Channel     string       `json:"channel"`
	Username    string       `json:"username"`
	Text        string       `json:"text"`
	IconEmoji   string       `json:"icon_emoji,omitempty"`
	IconURL     string       `json:"icon_url,omitempty"`
	Attachments []Attachment `json:"attachments,omitempty"`
	UnfurlLinks bool         `json:"unfurl_links,omitempty"`
	Parse       ParseStyle   `json:"parse,omitempty"`
	LinkNames   bool         `json:"link_names,omitempty"`
	Markdown    bool         `json:"mrkdwn,omitempty"`
}

type Attachment struct {
	Fallback   string            `json:"fallback"`
	Pretext    string            `json:"pretext,omitempty"`
	Text       string            `json:"text,omitempty"`
	Color      string            `json:"color,omitempty"`
	Fields     []AttachmentField `json:"fields,omitempty"`
	MarkdownIn []MarkdownField   `json:"mrkdown_in,omitempty"`
}

type MarkdownField string

var (
	MarkdownFieldPretext  = MarkdownField("pretext")
	MarkdownFieldText     = MarkdownField("text")
	MarkdownFieldTitle    = MarkdownField("title")
	MarkdownFieldFields   = MarkdownField("fields")
	MarkdownFieldFallback = MarkdownField("fallback")
)

type AttachmentField struct {
	Title string `json:"title"`
	Value string `json:"value"`
	Short bool   `json:"short,omitempty"`
}

// Send uses the IncomingWebhook API to post a message to a slack channel
func (i *IncomingWebhook) Send() error {
	u := os.Getenv(fmt.Sprintf("%s_IN_URL", strings.ToUpper(i.Domain)))
	if url != "" {
		return fmt.Errorf("Slack incoming webhook url not found for domain %s (check %s)", i.Domain, fmt.Sprintf("%s_IN_URL", strings.ToUpper(i.Domain)))
	}
	webhook, err := url.Parse(u)
	if err != nil {
		return err
	}

	p, err := json.Marshal(i)
	if err != nil {
		return err
	}

	data := url.Values{}
	data.Set("payload", string(p))

	webhook.RawQuery = data.Encode()
	resp, err := http.PostForm(webhook.String(), data)
	if resp.StatusCode != 200 {
		message := fmt.Sprintf("ERROR: Non-200 Response from Slack Incoming Webhook API: %s", resp.Status)
		log.Println(message)
	}
	return err
}
<|endoftext|>


<commit_before>package ecql

import (
	"fmt"
)

type OrderType string

const (
	AscOrder  OrderType = "ASC"
	DescOrder           = "DESC"
)

type OrderBy struct {
	Column string
	OrderType
}

func Asc(col string) OrderBy {
	return OrderBy{col, AscOrder}
}

func Desc(col string) OrderBy {
	return OrderBy{col, DescOrder}
}

type PredicateType int

type Condition struct {
	CQLFragment string
	Values      []interface{}
}

func And(lhs Condition, list ...Condition) Condition {
	cqlfragment := lhs.CQLFragment
	values := lhs.Values
	for _, rhs := range list {
		cqlfragment += " AND " + rhs.CQLFragment
		values = append(values, rhs.Values...)
	}
	return Condition{CQLFragment: cqlfragment, Values: values}
}

func Eq(col string, v interface{}) Condition {
	return Condition{CQLFragment: fmt.Sprintf("%s = ?", col),
		Values: []interface{}{v}}
}

func Gt(col string, v interface{}) Condition {
	return Condition{CQLFragment: fmt.Sprintf("%s > ?", col),
		Values: []interface{}{v}}
}

func Ge(col string, v interface{}) Condition {
	return Condition{CQLFragment: fmt.Sprintf("%s >= ?", col),
		Values: []interface{}{v}}
}

func Lt(col string, v interface{}) Condition {
	return Condition{CQLFragment: fmt.Sprintf("%s < ?", col),
		Values: []interface{}{v}}
}

func Le(col string, v interface{}) Condition {
	return Condition{CQLFragment: fmt.Sprintf("%s <= ?", col),
		Values: []interface{}{v}}
}

func In(col string, v ...interface{}) Condition {
	return Condition{CQLFragment: fmt.Sprintf("%s IN (%s)", qms(len(v)), col),
		Values: v}
}

// EqInt takes is interested in the CQL indexes of the provided struct as a condition
// For convenience, that struct is assumed to follow the same rules as other mappings
func EqInt(i interface{}) Condition {
	values, table := MapTable(i)
	first := true
	condition := True()
	for _, column := range table.KeyColumns {
		keyCondition := Eq(column, values[column])
		if first {
			condition = keyCondition
			first = false
		} else {
			condition = And(condition, keyCondition)
		}

	}
	return condition
}

func True() Condition {
	return Condition{CQLFragment: "true"}
}

// Contains creates the condition 'col CONTAINS value' used to filter elements
// in a collection set, list, or map. Supported on CQL versions >= 3.2.0.
func Contains(col string, v interface{}) Condition {
	return Condition{
		CQLFragment: fmt.Sprintf("%s CONTAINS ?", col),
		Values:      []interface{}{v},
	}
}

// Contains creates the condition 'col CONTAINS KEY value' used to filter elements
// by key in a map. Supported on CQL versions >= 3.2.0.
func ContainsKey(col string, v interface{}) Condition {
	return Condition{
		CQLFragment: fmt.Sprintf("%s CONTAINS KEY ?", col),
		Values:      []interface{}{v},
	}
}
<commit_msg>Fix IN condition typo<commit_after>package ecql

import (
	"fmt"
)

type OrderType string

const (
	AscOrder  OrderType = "ASC"
	DescOrder           = "DESC"
)

type OrderBy struct {
	Column string
	OrderType
}

func Asc(col string) OrderBy {
	return OrderBy{col, AscOrder}
}

func Desc(col string) OrderBy {
	return OrderBy{col, DescOrder}
}

type PredicateType int

type Condition struct {
	CQLFragment string
	Values      []interface{}
}

func And(lhs Condition, list ...Condition) Condition {
	cqlfragment := lhs.CQLFragment
	values := lhs.Values
	for _, rhs := range list {
		cqlfragment += " AND " + rhs.CQLFragment
		values = append(values, rhs.Values...)
	}
	return Condition{CQLFragment: cqlfragment, Values: values}
}

func Eq(col string, v interface{}) Condition {
	return Condition{CQLFragment: fmt.Sprintf("%s = ?", col),
		Values: []interface{}{v}}
}

func Gt(col string, v interface{}) Condition {
	return Condition{CQLFragment: fmt.Sprintf("%s > ?", col),
		Values: []interface{}{v}}
}

func Ge(col string, v interface{}) Condition {
	return Condition{CQLFragment: fmt.Sprintf("%s >= ?", col),
		Values: []interface{}{v}}
}

func Lt(col string, v interface{}) Condition {
	return Condition{CQLFragment: fmt.Sprintf("%s < ?", col),
		Values: []interface{}{v}}
}

func Le(col string, v interface{}) Condition {
	return Condition{CQLFragment: fmt.Sprintf("%s <= ?", col),
		Values: []interface{}{v}}
}

func In(col string, v ...interface{}) Condition {
	return Condition{CQLFragment: fmt.Sprintf("%s IN (%s)", col, qms(len(v))),
		Values: v}
}

// EqInt takes is interested in the CQL indexes of the provided struct as a condition
// For convenience, that struct is assumed to follow the same rules as other mappings
func EqInt(i interface{}) Condition {
	values, table := MapTable(i)
	first := true
	condition := True()
	for _, column := range table.KeyColumns {
		keyCondition := Eq(column, values[column])
		if first {
			condition = keyCondition
			first = false
		} else {
			condition = And(condition, keyCondition)
		}

	}
	return condition
}

func True() Condition {
	return Condition{CQLFragment: "true"}
}

// Contains creates the condition 'col CONTAINS value' used to filter elements
// in a collection set, list, or map. Supported on CQL versions >= 3.2.0.
func Contains(col string, v interface{}) Condition {
	return Condition{
		CQLFragment: fmt.Sprintf("%s CONTAINS ?", col),
		Values:      []interface{}{v},
	}
}

// Contains creates the condition 'col CONTAINS KEY value' used to filter elements
// by key in a map. Supported on CQL versions >= 3.2.0.
func ContainsKey(col string, v interface{}) Condition {
	return Condition{
		CQLFragment: fmt.Sprintf("%s CONTAINS KEY ?", col),
		Values:      []interface{}{v},
	}
}
<|endoftext|>


<commit_before>package graph

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"

	"github.com/docker/docker/image"
	"github.com/docker/docker/pkg/parsers"
	"github.com/docker/docker/utils"
)

const DEFAULTTAG = "latest"

var (
	validTagName = regexp.MustCompile(`^[\w][\w.-]{0,29}$`)
)

type TagStore struct {
	path         string
	graph        *Graph
	mirrors      []string
	Repositories map[string]Repository
	sync.Mutex
	// FIXME: move push/pull-related fields
	// to a helper type
	pullingPool map[string]chan struct{}
	pushingPool map[string]chan struct{}
}

type Repository map[string]string

// update Repository mapping with content of u
func (r Repository) Update(u Repository) {
	for k, v := range u {
		r[k] = v
	}
}

// return true if the contents of u Repository, are wholly contained in r Repository
func (r Repository) Contains(u Repository) bool {
	for k, v := range u {
		// if u's key is not present in r OR u's key is present, but not the same value
		if rv, ok := r[k]; !ok || (ok && rv != v) {
			return false
		}
	}
	return true
}

func NewTagStore(path string, graph *Graph, mirrors []string) (*TagStore, error) {
	abspath, err := filepath.Abs(path)
	if err != nil {
		return nil, err
	}
	store := &TagStore{
		path:         abspath,
		graph:        graph,
		mirrors:      mirrors,
		Repositories: make(map[string]Repository),
		pullingPool:  make(map[string]chan struct{}),
		pushingPool:  make(map[string]chan struct{}),
	}
	// Load the json file if it exists, otherwise create it.
	if err := store.reload(); os.IsNotExist(err) {
		if err := store.save(); err != nil {
			return nil, err
		}
	} else if err != nil {
		return nil, err
	}
	return store, nil
}

func (store *TagStore) save() error {
	// Store the json ball
	jsonData, err := json.Marshal(store)
	if err != nil {
		return err
	}
	if err := ioutil.WriteFile(store.path, jsonData, 0600); err != nil {
		return err
	}
	return nil
}

func (store *TagStore) reload() error {
	jsonData, err := ioutil.ReadFile(store.path)
	if err != nil {
		return err
	}
	if err := json.Unmarshal(jsonData, store); err != nil {
		return err
	}
	return nil
}

func (store *TagStore) LookupImage(name string) (*image.Image, error) {
	// FIXME: standardize on returning nil when the image doesn't exist, and err for everything else
	// (so we can pass all errors here)
	repos, tag := parsers.ParseRepositoryTag(name)
	if tag == "" {
		tag = DEFAULTTAG
	}
	img, err := store.GetImage(repos, tag)
	store.Lock()
	defer store.Unlock()
	if err != nil {
		return nil, err
	} else if img == nil {
		if img, err = store.graph.Get(name); err != nil {
			return nil, err
		}
	}
	return img, nil
}

// Return a reverse-lookup table of all the names which refer to each image
// Eg. {"43b5f19b10584": {"base:latest", "base:v1"}}
func (store *TagStore) ByID() map[string][]string {
	store.Lock()
	defer store.Unlock()
	byID := make(map[string][]string)
	for repoName, repository := range store.Repositories {
		for tag, id := range repository {
			name := repoName + ":" + tag
			if _, exists := byID[id]; !exists {
				byID[id] = []string{name}
			} else {
				byID[id] = append(byID[id], name)
				sort.Strings(byID[id])
			}
		}
	}
	return byID
}

func (store *TagStore) ImageName(id string) string {
	if names, exists := store.ByID()[id]; exists && len(names) > 0 {
		return names[0]
	}
	return utils.TruncateID(id)
}

func (store *TagStore) DeleteAll(id string) error {
	names, exists := store.ByID()[id]
	if !exists || len(names) == 0 {
		return nil
	}
	for _, name := range names {
		if strings.Contains(name, ":") {
			nameParts := strings.Split(name, ":")
			if _, err := store.Delete(nameParts[0], nameParts[1]); err != nil {
				return err
			}
		} else {
			if _, err := store.Delete(name, ""); err != nil {
				return err
			}
		}
	}
	return nil
}

func (store *TagStore) Delete(repoName, tag string) (bool, error) {
	store.Lock()
	defer store.Unlock()
	deleted := false
	if err := store.reload(); err != nil {
		return false, err
	}
	if r, exists := store.Repositories[repoName]; exists {
		if tag != "" {
			if _, exists2 := r[tag]; exists2 {
				delete(r, tag)
				if len(r) == 0 {
					delete(store.Repositories, repoName)
				}
				deleted = true
			} else {
				return false, fmt.Errorf("No such tag: %s:%s", repoName, tag)
			}
		} else {
			delete(store.Repositories, repoName)
			deleted = true
		}
	} else {
		return false, fmt.Errorf("No such repository: %s", repoName)
	}
	return deleted, store.save()
}

func (store *TagStore) Set(repoName, tag, imageName string, force bool) error {
	img, err := store.LookupImage(imageName)
	store.Lock()
	defer store.Unlock()
	if err != nil {
		return err
	}
	if tag == "" {
		tag = DEFAULTTAG
	}
	if err := validateRepoName(repoName); err != nil {
		return err
	}
	if err := ValidateTagName(tag); err != nil {
		return err
	}
	if err := store.reload(); err != nil {
		return err
	}
	var repo Repository
	if r, exists := store.Repositories[repoName]; exists {
		repo = r
	} else {
		repo = make(map[string]string)
		if old, exists := store.Repositories[repoName]; exists && !force {
			return fmt.Errorf("Conflict: Tag %s:%s is already set to %s", repoName, tag, old)
		}
		store.Repositories[repoName] = repo
	}
	repo[tag] = img.ID
	return store.save()
}

func (store *TagStore) Get(repoName string) (Repository, error) {
	store.Lock()
	defer store.Unlock()
	if err := store.reload(); err != nil {
		return nil, err
	}
	if r, exists := store.Repositories[repoName]; exists {
		return r, nil
	}
	return nil, nil
}

func (store *TagStore) GetImage(repoName, tagOrID string) (*image.Image, error) {
	repo, err := store.Get(repoName)
	store.Lock()
	defer store.Unlock()
	if err != nil {
		return nil, err
	} else if repo == nil {
		return nil, nil
	}
	if revision, exists := repo[tagOrID]; exists {
		return store.graph.Get(revision)
	}
	// If no matching tag is found, search through images for a matching image id
	for _, revision := range repo {
		if strings.HasPrefix(revision, tagOrID) {
			return store.graph.Get(revision)
		}
	}
	return nil, nil
}

func (store *TagStore) GetRepoRefs() map[string][]string {
	store.Lock()
	reporefs := make(map[string][]string)

	for name, repository := range store.Repositories {
		for tag, id := range repository {
			shortID := utils.TruncateID(id)
			reporefs[shortID] = append(reporefs[shortID], fmt.Sprintf("%s:%s", name, tag))
		}
	}
	store.Unlock()
	return reporefs
}

// isOfficialName returns whether a repo name is considered an official
// repository.  Official repositories are repos with names within
// the library namespace or which default to the library namespace
// by not providing one.
func isOfficialName(name string) bool {
	if strings.HasPrefix(name, "library/") {
		return true
	}
	if strings.IndexRune(name, '/') == -1 {
		return true
	}
	return false
}

// Validate the name of a repository
func validateRepoName(name string) error {
	if name == "" {
		return fmt.Errorf("Repository name can't be empty")
	}
	return nil
}

// Validate the name of a tag
func ValidateTagName(name string) error {
	if name == "" {
		return fmt.Errorf("Tag name can't be empty")
	}
	if !validTagName.MatchString(name) {
		return fmt.Errorf("Illegal tag name (%s): only [A-Za-z0-9_.-] are allowed, minimum 2, maximum 30 in length", name)
	}
	return nil
}

func (store *TagStore) poolAdd(kind, key string) (chan struct{}, error) {
	store.Lock()
	defer store.Unlock()

	if c, exists := store.pullingPool[key]; exists {
		return c, fmt.Errorf("pull %s is already in progress", key)
	}
	if c, exists := store.pushingPool[key]; exists {
		return c, fmt.Errorf("push %s is already in progress", key)
	}

	c := make(chan struct{})
	switch kind {
	case "pull":
		store.pullingPool[key] = c
	case "push":
		store.pushingPool[key] = c
	default:
		return nil, fmt.Errorf("Unknown pool type")
	}
	return c, nil
}

func (store *TagStore) poolRemove(kind, key string) error {
	store.Lock()
	defer store.Unlock()
	switch kind {
	case "pull":
		if c, exists := store.pullingPool[key]; exists {
			close(c)
			delete(store.pullingPool, key)
		}
	case "push":
		if c, exists := store.pushingPool[key]; exists {
			close(c)
			delete(store.pushingPool, key)
		}
	default:
		return fmt.Errorf("Unknown pool type")
	}
	return nil
}
<commit_msg>Increase the tag length limit from 30 to 128<commit_after>package graph

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"sync"

	"github.com/docker/docker/image"
	"github.com/docker/docker/pkg/parsers"
	"github.com/docker/docker/utils"
)

const DEFAULTTAG = "latest"

var (
	validTagName = regexp.MustCompile(`^[\w][\w.-]{0,127}$`)
)

type TagStore struct {
	path         string
	graph        *Graph
	mirrors      []string
	Repositories map[string]Repository
	sync.Mutex
	// FIXME: move push/pull-related fields
	// to a helper type
	pullingPool map[string]chan struct{}
	pushingPool map[string]chan struct{}
}

type Repository map[string]string

// update Repository mapping with content of u
func (r Repository) Update(u Repository) {
	for k, v := range u {
		r[k] = v
	}
}

// return true if the contents of u Repository, are wholly contained in r Repository
func (r Repository) Contains(u Repository) bool {
	for k, v := range u {
		// if u's key is not present in r OR u's key is present, but not the same value
		if rv, ok := r[k]; !ok || (ok && rv != v) {
			return false
		}
	}
	return true
}

func NewTagStore(path string, graph *Graph, mirrors []string) (*TagStore, error) {
	abspath, err := filepath.Abs(path)
	if err != nil {
		return nil, err
	}
	store := &TagStore{
		path:         abspath,
		graph:        graph,
		mirrors:      mirrors,
		Repositories: make(map[string]Repository),
		pullingPool:  make(map[string]chan struct{}),
		pushingPool:  make(map[string]chan struct{}),
	}
	// Load the json file if it exists, otherwise create it.
	if err := store.reload(); os.IsNotExist(err) {
		if err := store.save(); err != nil {
			return nil, err
		}
	} else if err != nil {
		return nil, err
	}
	return store, nil
}

func (store *TagStore) save() error {
	// Store the json ball
	jsonData, err := json.Marshal(store)
	if err != nil {
		return err
	}
	if err := ioutil.WriteFile(store.path, jsonData, 0600); err != nil {
		return err
	}
	return nil
}

func (store *TagStore) reload() error {
	jsonData, err := ioutil.ReadFile(store.path)
	if err != nil {
		return err
	}
	if err := json.Unmarshal(jsonData, store); err != nil {
		return err
	}
	return nil
}

func (store *TagStore) LookupImage(name string) (*image.Image, error) {
	// FIXME: standardize on returning nil when the image doesn't exist, and err for everything else
	// (so we can pass all errors here)
	repos, tag := parsers.ParseRepositoryTag(name)
	if tag == "" {
		tag = DEFAULTTAG
	}
	img, err := store.GetImage(repos, tag)
	store.Lock()
	defer store.Unlock()
	if err != nil {
		return nil, err
	} else if img == nil {
		if img, err = store.graph.Get(name); err != nil {
			return nil, err
		}
	}
	return img, nil
}

// Return a reverse-lookup table of all the names which refer to each image
// Eg. {"43b5f19b10584": {"base:latest", "base:v1"}}
func (store *TagStore) ByID() map[string][]string {
	store.Lock()
	defer store.Unlock()
	byID := make(map[string][]string)
	for repoName, repository := range store.Repositories {
		for tag, id := range repository {
			name := repoName + ":" + tag
			if _, exists := byID[id]; !exists {
				byID[id] = []string{name}
			} else {
				byID[id] = append(byID[id], name)
				sort.Strings(byID[id])
			}
		}
	}
	return byID
}

func (store *TagStore) ImageName(id string) string {
	if names, exists := store.ByID()[id]; exists && len(names) > 0 {
		return names[0]
	}
	return utils.TruncateID(id)
}

func (store *TagStore) DeleteAll(id string) error {
	names, exists := store.ByID()[id]
	if !exists || len(names) == 0 {
		return nil
	}
	for _, name := range names {
		if strings.Contains(name, ":") {
			nameParts := strings.Split(name, ":")
			if _, err := store.Delete(nameParts[0], nameParts[1]); err != nil {
				return err
			}
		} else {
			if _, err := store.Delete(name, ""); err != nil {
				return err
			}
		}
	}
	return nil
}

func (store *TagStore) Delete(repoName, tag string) (bool, error) {
	store.Lock()
	defer store.Unlock()
	deleted := false
	if err := store.reload(); err != nil {
		return false, err
	}
	if r, exists := store.Repositories[repoName]; exists {
		if tag != "" {
			if _, exists2 := r[tag]; exists2 {
				delete(r, tag)
				if len(r) == 0 {
					delete(store.Repositories, repoName)
				}
				deleted = true
			} else {
				return false, fmt.Errorf("No such tag: %s:%s", repoName, tag)
			}
		} else {
			delete(store.Repositories, repoName)
			deleted = true
		}
	} else {
		return false, fmt.Errorf("No such repository: %s", repoName)
	}
	return deleted, store.save()
}

func (store *TagStore) Set(repoName, tag, imageName string, force bool) error {
	img, err := store.LookupImage(imageName)
	store.Lock()
	defer store.Unlock()
	if err != nil {
		return err
	}
	if tag == "" {
		tag = DEFAULTTAG
	}
	if err := validateRepoName(repoName); err != nil {
		return err
	}
	if err := ValidateTagName(tag); err != nil {
		return err
	}
	if err := store.reload(); err != nil {
		return err
	}
	var repo Repository
	if r, exists := store.Repositories[repoName]; exists {
		repo = r
	} else {
		repo = make(map[string]string)
		if old, exists := store.Repositories[repoName]; exists && !force {
			return fmt.Errorf("Conflict: Tag %s:%s is already set to %s", repoName, tag, old)
		}
		store.Repositories[repoName] = repo
	}
	repo[tag] = img.ID
	return store.save()
}

func (store *TagStore) Get(repoName string) (Repository, error) {
	store.Lock()
	defer store.Unlock()
	if err := store.reload(); err != nil {
		return nil, err
	}
	if r, exists := store.Repositories[repoName]; exists {
		return r, nil
	}
	return nil, nil
}

func (store *TagStore) GetImage(repoName, tagOrID string) (*image.Image, error) {
	repo, err := store.Get(repoName)
	store.Lock()
	defer store.Unlock()
	if err != nil {
		return nil, err
	} else if repo == nil {
		return nil, nil
	}
	if revision, exists := repo[tagOrID]; exists {
		return store.graph.Get(revision)
	}
	// If no matching tag is found, search through images for a matching image id
	for _, revision := range repo {
		if strings.HasPrefix(revision, tagOrID) {
			return store.graph.Get(revision)
		}
	}
	return nil, nil
}

func (store *TagStore) GetRepoRefs() map[string][]string {
	store.Lock()
	reporefs := make(map[string][]string)

	for name, repository := range store.Repositories {
		for tag, id := range repository {
			shortID := utils.TruncateID(id)
			reporefs[shortID] = append(reporefs[shortID], fmt.Sprintf("%s:%s", name, tag))
		}
	}
	store.Unlock()
	return reporefs
}

// isOfficialName returns whether a repo name is considered an official
// repository.  Official repositories are repos with names within
// the library namespace or which default to the library namespace
// by not providing one.
func isOfficialName(name string) bool {
	if strings.HasPrefix(name, "library/") {
		return true
	}
	if strings.IndexRune(name, '/') == -1 {
		return true
	}
	return false
}

// Validate the name of a repository
func validateRepoName(name string) error {
	if name == "" {
		return fmt.Errorf("Repository name can't be empty")
	}
	return nil
}

// Validate the name of a tag
func ValidateTagName(name string) error {
	if name == "" {
		return fmt.Errorf("Tag name can't be empty")
	}
	if !validTagName.MatchString(name) {
		return fmt.Errorf("Illegal tag name (%s): only [A-Za-z0-9_.-] are allowed, minimum 2, maximum 30 in length", name)
	}
	return nil
}

func (store *TagStore) poolAdd(kind, key string) (chan struct{}, error) {
	store.Lock()
	defer store.Unlock()

	if c, exists := store.pullingPool[key]; exists {
		return c, fmt.Errorf("pull %s is already in progress", key)
	}
	if c, exists := store.pushingPool[key]; exists {
		return c, fmt.Errorf("push %s is already in progress", key)
	}

	c := make(chan struct{})
	switch kind {
	case "pull":
		store.pullingPool[key] = c
	case "push":
		store.pushingPool[key] = c
	default:
		return nil, fmt.Errorf("Unknown pool type")
	}
	return c, nil
}

func (store *TagStore) poolRemove(kind, key string) error {
	store.Lock()
	defer store.Unlock()
	switch kind {
	case "pull":
		if c, exists := store.pullingPool[key]; exists {
			close(c)
			delete(store.pullingPool, key)
		}
	case "push":
		if c, exists := store.pushingPool[key]; exists {
			close(c)
			delete(store.pushingPool, key)
		}
	default:
		return fmt.Errorf("Unknown pool type")
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package admission

import (
	"bytes"
	"context"
	"fmt"
	"net/http"
	"os/exec"
	"strings"

	"github.com/onsi/ginkgo"
	"github.com/stretchr/testify/assert"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"k8s.io/ingress-nginx/test/e2e/framework"
)

var _ = framework.IngressNginxDescribe("[Serial] admission controller", func() {
	f := framework.NewDefaultFramework("admission")

	ginkgo.BeforeEach(func() {
		f.NewEchoDeployment()
		f.NewSlowEchoDeployment()
	})

	ginkgo.AfterEach(func() {
		err := uninstallChart(f)
		assert.Nil(ginkgo.GinkgoT(), err, "uninstalling helm chart")
	})

	ginkgo.It("should not allow overlaps of host and paths without canary annotations", func() {
		host := "admission-test"

		firstIngress := framework.NewSingleIngress("first-ingress", "/", host, f.Namespace, framework.EchoService, 80, nil)
		_, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), firstIngress, metav1.CreateOptions{})
		assert.Nil(ginkgo.GinkgoT(), err, "creating ingress")

		f.WaitForNginxServer(host,
			func(server string) bool {
				return strings.Contains(server, fmt.Sprintf("server_name %v", host))
			})

		secondIngress := framework.NewSingleIngress("second-ingress", "/", host, f.Namespace, framework.EchoService, 80, nil)
		_, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), secondIngress, metav1.CreateOptions{})
		assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress with the same host and path should return an error")
	})

	ginkgo.It("should allow overlaps of host and paths with canary annotation", func() {
		host := "admission-test"

		firstIngress := framework.NewSingleIngress("first-ingress", "/", host, f.Namespace, framework.EchoService, 80, nil)
		_, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), firstIngress, metav1.CreateOptions{})
		assert.Nil(ginkgo.GinkgoT(), err, "creating ingress")

		f.WaitForNginxServer(host,
			func(server string) bool {
				return strings.Contains(server, fmt.Sprintf("server_name %v", host))
			})

		canaryAnnotations := map[string]string{
			"nginx.ingress.kubernetes.io/canary":           "true",
			"nginx.ingress.kubernetes.io/canary-by-header": "CanaryByHeader",
		}
		secondIngress := framework.NewSingleIngress("second-ingress", "/", host, f.Namespace, framework.SlowEchoService, 80, canaryAnnotations)
		_, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), secondIngress, metav1.CreateOptions{})
		assert.Nil(ginkgo.GinkgoT(), err, "creating an ingress with the same host and path should not return an error using a canary annotation")
	})

	ginkgo.It("should return an error if there is an error validating the ingress definition", func() {
		host := "admission-test"

		annotations := map[string]string{
			"nginx.ingress.kubernetes.io/configuration-snippet": "something invalid",
		}
		firstIngress := framework.NewSingleIngress("first-ingress", "/", host, f.Namespace, framework.EchoService, 80, annotations)
		_, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), firstIngress, metav1.CreateOptions{})
		assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress with invalid configuration should return an error")
	})

	ginkgo.It("should not return an error the ingress definition uses the deprecated extensions package", func() {
		err := createIngress(f.Namespace, validIngress)
		assert.Nil(ginkgo.GinkgoT(), err, "creating an ingress using kubectl")

		f.WaitForNginxConfiguration(func(cfg string) bool {
			return strings.Contains(cfg, "extensions")
		})

		f.HTTPTestClient().
			GET("/").
			WithHeader("Host", "extensions").
			Expect().
			Status(http.StatusOK)
	})

	ginkgo.It("should return an error if the ingress definition uses the deprecated extensions package and invalid annotations", func() {
		err := createIngress(f.Namespace, invalidIngress)
		assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress using kubectl")

		_, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), "extensions", metav1.GetOptions{})
		if !apierrors.IsNotFound(err) {
			assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress with invalid configuration should return an error")
		}
	})

	ginkgo.It("should not return an error if the Ingress V1 definition is valid", func() {
		if !f.IsIngressV1Ready {
			ginkgo.Skip("Test requires Kubernetes v1.19 or higher")
		}

		err := createIngress(f.Namespace, validV1Ingress)
		assert.Nil(ginkgo.GinkgoT(), err, "creating an ingress using kubectl")

		f.WaitForNginxConfiguration(func(cfg string) bool {
			return strings.Contains(cfg, "extensions")
		})

		f.HTTPTestClient().
			GET("/").
			WithHeader("Host", "extensions").
			Expect().
			Status(http.StatusOK)
	})

	ginkgo.It("should return an error if the Ingress V1 definition contains invalid annotations", func() {
		if !f.IsIngressV1Ready {
			ginkgo.Skip("Test requires Kubernetes v1.19 or higher")
		}

		err := createIngress(f.Namespace, invalidV1Ingress)
		assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress using kubectl")

		_, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), "extensions", metav1.GetOptions{})
		if !apierrors.IsNotFound(err) {
			assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress with invalid configuration should return an error")
		}
	})
})

func uninstallChart(f *framework.Framework) error {
	cmd := exec.Command("helm", "uninstall", "--namespace", f.Namespace, "nginx-ingress")
	_, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("unexpected error uninstalling ingress-nginx release: %v", err)
	}

	return nil
}

const (
	validIngress = `
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: extensions
spec:
  rules:
  - host: extensions
    http:
      paths:
      - path: /
        backend:
          serviceName: echo
          servicePort: 80
---
`

	invalidIngress = `
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: extensions
  annotations:
    nginx.ingress.kubernetes.io/configuration-snippet: |
      invalid directive
spec:
  rules:
  - host: extensions
    http:
      paths:
      - path: /
        backend:
          serviceName: echo
          servicePort: 80
---
`

	validV1Ingress = `
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: extensions
spec:
  rules:
  - host: extensions
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: echo
            port:
              number: 80

---
`

	invalidV1Ingress = `
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: extensions
  annotations:
    nginx.ingress.kubernetes.io/configuration-snippet: |
      invalid directive
spec:
  rules:
  - host: extensions
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: echo
            port:
              number: 80
---
`
)

func createIngress(namespace, ingressDefinition string) error {
	var (
		execErr bytes.Buffer
	)

	cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%v --warnings-as-errors=false apply --namespace %s -f -", framework.KubectlPath, namespace))
	cmd.Stdin = strings.NewReader(ingressDefinition)
	cmd.Stderr = &execErr

	err := cmd.Run()
	if err != nil {
		stderr := strings.TrimSpace(execErr.String())
		return fmt.Errorf("Kubectl error: %v\n%v", err, stderr)
	}

	return nil
}
<commit_msg>e2e test for requiring memcached setting to configure global rate limit<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package admission

import (
	"bytes"
	"context"
	"fmt"
	"net/http"
	"os/exec"
	"strings"

	"github.com/onsi/ginkgo"
	"github.com/stretchr/testify/assert"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"k8s.io/ingress-nginx/test/e2e/framework"
)

var _ = framework.IngressNginxDescribe("[Serial] admission controller", func() {
	f := framework.NewDefaultFramework("admission")

	ginkgo.BeforeEach(func() {
		f.NewEchoDeployment()
		f.NewSlowEchoDeployment()
	})

	ginkgo.AfterEach(func() {
		err := uninstallChart(f)
		assert.Nil(ginkgo.GinkgoT(), err, "uninstalling helm chart")
	})

	ginkgo.It("reject ingress with global-rate-limit annotations when memcached is not configured", func() {
		host := "admission-test"

		annotations := map[string]string{
			"nginx.ingress.kubernetes.io/global-rate-limit":        "100",
			"nginx.ingress.kubernetes.io/global-rate-limit-window": "1m",
		}
		ing := framework.NewSingleIngress("first-ingress", "/", host, f.Namespace, framework.EchoService, 80, annotations)

		ginkgo.By("rejects ingress when memcached is not configured")

		_, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), ing, metav1.CreateOptions{})
		assert.NotNil(ginkgo.GinkgoT(), err, "creating ingress with global throttle annotations when memcached is not configured")

		ginkgo.By("accepts ingress when memcached is not configured")

		f.UpdateNginxConfigMapData("global-rate-limit-memcached-host", "memc.default.svc.cluster.local")

		_, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), ing, metav1.CreateOptions{})
		assert.Nil(ginkgo.GinkgoT(), err, "creating ingress with global throttle annotations when memcached is configured")

		f.WaitForNginxServer(host,
			func(server string) bool {
				return strings.Contains(server, fmt.Sprintf("server_name %v", host))
			})
	})

	ginkgo.It("should not allow overlaps of host and paths without canary annotations", func() {
		host := "admission-test"

		firstIngress := framework.NewSingleIngress("first-ingress", "/", host, f.Namespace, framework.EchoService, 80, nil)
		_, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), firstIngress, metav1.CreateOptions{})
		assert.Nil(ginkgo.GinkgoT(), err, "creating ingress")

		f.WaitForNginxServer(host,
			func(server string) bool {
				return strings.Contains(server, fmt.Sprintf("server_name %v", host))
			})

		secondIngress := framework.NewSingleIngress("second-ingress", "/", host, f.Namespace, framework.EchoService, 80, nil)
		_, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), secondIngress, metav1.CreateOptions{})
		assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress with the same host and path should return an error")
	})

	ginkgo.It("should allow overlaps of host and paths with canary annotation", func() {
		host := "admission-test"

		firstIngress := framework.NewSingleIngress("first-ingress", "/", host, f.Namespace, framework.EchoService, 80, nil)
		_, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), firstIngress, metav1.CreateOptions{})
		assert.Nil(ginkgo.GinkgoT(), err, "creating ingress")

		f.WaitForNginxServer(host,
			func(server string) bool {
				return strings.Contains(server, fmt.Sprintf("server_name %v", host))
			})

		canaryAnnotations := map[string]string{
			"nginx.ingress.kubernetes.io/canary":           "true",
			"nginx.ingress.kubernetes.io/canary-by-header": "CanaryByHeader",
		}
		secondIngress := framework.NewSingleIngress("second-ingress", "/", host, f.Namespace, framework.SlowEchoService, 80, canaryAnnotations)
		_, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), secondIngress, metav1.CreateOptions{})
		assert.Nil(ginkgo.GinkgoT(), err, "creating an ingress with the same host and path should not return an error using a canary annotation")
	})

	ginkgo.It("should return an error if there is an error validating the ingress definition", func() {
		host := "admission-test"

		annotations := map[string]string{
			"nginx.ingress.kubernetes.io/configuration-snippet": "something invalid",
		}
		firstIngress := framework.NewSingleIngress("first-ingress", "/", host, f.Namespace, framework.EchoService, 80, annotations)
		_, err := f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Create(context.TODO(), firstIngress, metav1.CreateOptions{})
		assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress with invalid configuration should return an error")
	})

	ginkgo.It("should not return an error the ingress definition uses the deprecated extensions package", func() {
		err := createIngress(f.Namespace, validIngress)
		assert.Nil(ginkgo.GinkgoT(), err, "creating an ingress using kubectl")

		f.WaitForNginxConfiguration(func(cfg string) bool {
			return strings.Contains(cfg, "extensions")
		})

		f.HTTPTestClient().
			GET("/").
			WithHeader("Host", "extensions").
			Expect().
			Status(http.StatusOK)
	})

	ginkgo.It("should return an error if the ingress definition uses the deprecated extensions package and invalid annotations", func() {
		err := createIngress(f.Namespace, invalidIngress)
		assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress using kubectl")

		_, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), "extensions", metav1.GetOptions{})
		if !apierrors.IsNotFound(err) {
			assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress with invalid configuration should return an error")
		}
	})

	ginkgo.It("should not return an error if the Ingress V1 definition is valid", func() {
		if !f.IsIngressV1Ready {
			ginkgo.Skip("Test requires Kubernetes v1.19 or higher")
		}

		err := createIngress(f.Namespace, validV1Ingress)
		assert.Nil(ginkgo.GinkgoT(), err, "creating an ingress using kubectl")

		f.WaitForNginxConfiguration(func(cfg string) bool {
			return strings.Contains(cfg, "extensions")
		})

		f.HTTPTestClient().
			GET("/").
			WithHeader("Host", "extensions").
			Expect().
			Status(http.StatusOK)
	})

	ginkgo.It("should return an error if the Ingress V1 definition contains invalid annotations", func() {
		if !f.IsIngressV1Ready {
			ginkgo.Skip("Test requires Kubernetes v1.19 or higher")
		}

		err := createIngress(f.Namespace, invalidV1Ingress)
		assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress using kubectl")

		_, err = f.KubeClientSet.NetworkingV1beta1().Ingresses(f.Namespace).Get(context.TODO(), "extensions", metav1.GetOptions{})
		if !apierrors.IsNotFound(err) {
			assert.NotNil(ginkgo.GinkgoT(), err, "creating an ingress with invalid configuration should return an error")
		}
	})
})

func uninstallChart(f *framework.Framework) error {
	cmd := exec.Command("helm", "uninstall", "--namespace", f.Namespace, "nginx-ingress")
	_, err := cmd.CombinedOutput()
	if err != nil {
		return fmt.Errorf("unexpected error uninstalling ingress-nginx release: %v", err)
	}

	return nil
}

const (
	validIngress = `
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: extensions
spec:
  rules:
  - host: extensions
    http:
      paths:
      - path: /
        backend:
          serviceName: echo
          servicePort: 80
---
`

	invalidIngress = `
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: extensions
  annotations:
    nginx.ingress.kubernetes.io/configuration-snippet: |
      invalid directive
spec:
  rules:
  - host: extensions
    http:
      paths:
      - path: /
        backend:
          serviceName: echo
          servicePort: 80
---
`

	validV1Ingress = `
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: extensions
spec:
  rules:
  - host: extensions
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: echo
            port:
              number: 80

---
`

	invalidV1Ingress = `
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
  name: extensions
  annotations:
    nginx.ingress.kubernetes.io/configuration-snippet: |
      invalid directive
spec:
  rules:
  - host: extensions
    http:
      paths:
      - path: /
        pathType: Prefix
        backend:
          service:
            name: echo
            port:
              number: 80
---
`
)

func createIngress(namespace, ingressDefinition string) error {
	var (
		execErr bytes.Buffer
	)

	cmd := exec.Command("/bin/bash", "-c", fmt.Sprintf("%v --warnings-as-errors=false apply --namespace %s -f -", framework.KubectlPath, namespace))
	cmd.Stdin = strings.NewReader(ingressDefinition)
	cmd.Stderr = &execErr

	err := cmd.Run()
	if err != nil {
		stderr := strings.TrimSpace(execErr.String())
		return fmt.Errorf("Kubectl error: %v\n%v", err, stderr)
	}

	return nil
}
<|endoftext|>