<commit_before>package parser

import (
	"bytes"
	"regexp"
	"strings"

	"github.com/yuin/goldmark/ast"
	"github.com/yuin/goldmark/text"
	"github.com/yuin/goldmark/util"
)

var allowedBlockTags = map[string]bool{
	"address":    true,
	"article":    true,
	"aside":      true,
	"base":       true,
	"basefont":   true,
	"blockquote": true,
	"body":       true,
	"caption":    true,
	"center":     true,
	"col":        true,
	"colgroup":   true,
	"dd":         true,
	"details":    true,
	"dialog":     true,
	"dir":        true,
	"div":        true,
	"dl":         true,
	"dt":         true,
	"fieldset":   true,
	"figcaption": true,
	"figure":     true,
	"footer":     true,
	"form":       true,
	"frame":      true,
	"frameset":   true,
	"h1":         true,
	"h2":         true,
	"h3":         true,
	"h4":         true,
	"h5":         true,
	"h6":         true,
	"head":       true,
	"header":     true,
	"hr":         true,
	"html":       true,
	"iframe":     true,
	"legend":     true,
	"li":         true,
	"link":       true,
	"main":       true,
	"menu":       true,
	"menuitem":   true,
	"meta":       true,
	"nav":        true,
	"noframes":   true,
	"ol":         true,
	"optgroup":   true,
	"option":     true,
	"p":          true,
	"param":      true,
	"section":    true,
	"source":     true,
	"summary":    true,
	"table":      true,
	"tbody":      true,
	"td":         true,
	"tfoot":      true,
	"th":         true,
	"thead":      true,
	"title":      true,
	"tr":         true,
	"track":      true,
	"ul":         true,
}

var htmlBlockType1OpenRegexp = regexp.MustCompile(`(?i)^[ ]{0,3}<(script|pre|style)(?:\s.*|>.*|/>.*|)\n?$`)
var htmlBlockType1CloseRegexp = regexp.MustCompile(`(?i)^[ ]{0,3}(?:[^ ].*|)</(?:script|pre|style)>.*`)

var htmlBlockType2OpenRegexp = regexp.MustCompile(`^[ ]{0,3}<!\-\-`)
var htmlBlockType2Close = []byte{'-', '-', '>'}

var htmlBlockType3OpenRegexp = regexp.MustCompile(`^[ ]{0,3}<\?`)
var htmlBlockType3Close = []byte{'?', '>'}

var htmlBlockType4OpenRegexp = regexp.MustCompile(`^[ ]{0,3}<![A-Z]+.*\n?$`)
var htmlBlockType4Close = []byte{'>'}

var htmlBlockType5OpenRegexp = regexp.MustCompile(`^[ ]{0,3}<\!\[CDATA\[`)
var htmlBlockType5Close = []byte{']', ']', '>'}

var htmlBlockType6Regexp = regexp.MustCompile(`^[ ]{0,3}</?([a-zA-Z0-9]+)(?:\s.*|>.*|/>.*|)\n?$`)

var htmlBlockType7Regexp = regexp.MustCompile(`^[ ]{0,3}<(/)?([a-zA-Z0-9]+)(` + attributePattern + `*)(:?>|/>)\s*\n?$`)

type htmlBlockParser struct {
}

var defaultHTMLBlockParser = &htmlBlockParser{}

// NewHTMLBlockParser return a new BlockParser that can parse html
// blocks.
func NewHTMLBlockParser() BlockParser {
	return defaultHTMLBlockParser
}

func (b *htmlBlockParser) Trigger() []byte {
	return []byte{'<'}
}

func (b *htmlBlockParser) Open(parent ast.Node, reader text.Reader, pc Context) (ast.Node, State) {
	var node *ast.HTMLBlock
	line, segment := reader.PeekLine()
	last := pc.LastOpenedBlock().Node
	if pos := pc.BlockOffset(); pos < 0 || line[pos] != '<' {
		return nil, NoChildren
	}

	if m := htmlBlockType1OpenRegexp.FindSubmatchIndex(line); m != nil {
		node = ast.NewHTMLBlock(ast.HTMLBlockType1)
	} else if htmlBlockType2OpenRegexp.Match(line) {
		node = ast.NewHTMLBlock(ast.HTMLBlockType2)
	} else if htmlBlockType3OpenRegexp.Match(line) {
		node = ast.NewHTMLBlock(ast.HTMLBlockType3)
	} else if htmlBlockType4OpenRegexp.Match(line) {
		node = ast.NewHTMLBlock(ast.HTMLBlockType4)
	} else if htmlBlockType5OpenRegexp.Match(line) {
		node = ast.NewHTMLBlock(ast.HTMLBlockType5)
	} else if match := htmlBlockType7Regexp.FindSubmatchIndex(line); match != nil {
		isCloseTag := match[2] > -1 && bytes.Equal(line[match[2]:match[3]], []byte("/"))
		hasAttr := match[6] != match[7]
		tagName := strings.ToLower(string(line[match[4]:match[5]]))
		_, ok := allowedBlockTags[tagName]
		if ok {
			node = ast.NewHTMLBlock(ast.HTMLBlockType6)
		} else if tagName != "script" && tagName != "style" && tagName != "pre" && !ast.IsParagraph(last) && !(isCloseTag && hasAttr) { // type 7 can not interrupt paragraph
			node = ast.NewHTMLBlock(ast.HTMLBlockType7)
		}
	}
	if node == nil {
		if match := htmlBlockType6Regexp.FindSubmatchIndex(line); match != nil {
			tagName := string(line[match[2]:match[3]])
			_, ok := allowedBlockTags[strings.ToLower(tagName)]
			if ok {
				node = ast.NewHTMLBlock(ast.HTMLBlockType6)
			}
		}
	}
	if node != nil {
		reader.Advance(segment.Len() - 1)
		node.Lines().Append(segment)
		return node, NoChildren
	}
	return nil, NoChildren
}

func (b *htmlBlockParser) Continue(node ast.Node, reader text.Reader, pc Context) State {
	htmlBlock := node.(*ast.HTMLBlock)
	lines := htmlBlock.Lines()
	line, segment := reader.PeekLine()
	var closurePattern []byte

	switch htmlBlock.HTMLBlockType {
	case ast.HTMLBlockType1:
		if lines.Len() == 1 {
			firstLine := lines.At(0)
			if htmlBlockType1CloseRegexp.Match(firstLine.Value(reader.Source())) {
				return Close
			}
		}
		if htmlBlockType1CloseRegexp.Match(line) {
			htmlBlock.ClosureLine = segment
			reader.Advance(segment.Len() - 1)
			return Close
		}
	case ast.HTMLBlockType2:
		closurePattern = htmlBlockType2Close
		fallthrough
	case ast.HTMLBlockType3:
		if closurePattern == nil {
			closurePattern = htmlBlockType3Close
		}
		fallthrough
	case ast.HTMLBlockType4:
		if closurePattern == nil {
			closurePattern = htmlBlockType4Close
		}
		fallthrough
	case ast.HTMLBlockType5:
		if closurePattern == nil {
			closurePattern = htmlBlockType5Close
		}

		if lines.Len() == 1 {
			firstLine := lines.At(0)
			if bytes.Contains(firstLine.Value(reader.Source()), closurePattern) {
				return Close
			}
		}
		if bytes.Contains(line, closurePattern) {
			htmlBlock.ClosureLine = segment
			reader.Advance(segment.Len() - 1)
			return Close
		}

	case ast.HTMLBlockType6, ast.HTMLBlockType7:
		if util.IsBlank(line) {
			return Close
		}
	}
	node.Lines().Append(segment)
	reader.Advance(segment.Len() - 1)
	return Continue | NoChildren
}

func (b *htmlBlockParser) Close(node ast.Node, reader text.Reader, pc Context) {
	// nothing to do
}

func (b *htmlBlockParser) CanInterruptParagraph() bool {
	return true
}

func (b *htmlBlockParser) CanAcceptIndentedLine() bool {
	return false
}
<commit_msg>Fixes #81<commit_after>package parser

import (
	"bytes"
	"regexp"
	"strings"

	"github.com/yuin/goldmark/ast"
	"github.com/yuin/goldmark/text"
	"github.com/yuin/goldmark/util"
)

var allowedBlockTags = map[string]bool{
	"address":    true,
	"article":    true,
	"aside":      true,
	"base":       true,
	"basefont":   true,
	"blockquote": true,
	"body":       true,
	"caption":    true,
	"center":     true,
	"col":        true,
	"colgroup":   true,
	"dd":         true,
	"details":    true,
	"dialog":     true,
	"dir":        true,
	"div":        true,
	"dl":         true,
	"dt":         true,
	"fieldset":   true,
	"figcaption": true,
	"figure":     true,
	"footer":     true,
	"form":       true,
	"frame":      true,
	"frameset":   true,
	"h1":         true,
	"h2":         true,
	"h3":         true,
	"h4":         true,
	"h5":         true,
	"h6":         true,
	"head":       true,
	"header":     true,
	"hr":         true,
	"html":       true,
	"iframe":     true,
	"legend":     true,
	"li":         true,
	"link":       true,
	"main":       true,
	"menu":       true,
	"menuitem":   true,
	"meta":       true,
	"nav":        true,
	"noframes":   true,
	"ol":         true,
	"optgroup":   true,
	"option":     true,
	"p":          true,
	"param":      true,
	"section":    true,
	"source":     true,
	"summary":    true,
	"table":      true,
	"tbody":      true,
	"td":         true,
	"tfoot":      true,
	"th":         true,
	"thead":      true,
	"title":      true,
	"tr":         true,
	"track":      true,
	"ul":         true,
}

var htmlBlockType1OpenRegexp = regexp.MustCompile(`(?i)^[ ]{0,3}<(script|pre|style)(?:\s.*|>.*|/>.*|)\n?$`)
var htmlBlockType1CloseRegexp = regexp.MustCompile(`(?i)^.*</(?:script|pre|style)>.*`)

var htmlBlockType2OpenRegexp = regexp.MustCompile(`^[ ]{0,3}<!\-\-`)
var htmlBlockType2Close = []byte{'-', '-', '>'}

var htmlBlockType3OpenRegexp = regexp.MustCompile(`^[ ]{0,3}<\?`)
var htmlBlockType3Close = []byte{'?', '>'}

var htmlBlockType4OpenRegexp = regexp.MustCompile(`^[ ]{0,3}<![A-Z]+.*\n?$`)
var htmlBlockType4Close = []byte{'>'}

var htmlBlockType5OpenRegexp = regexp.MustCompile(`^[ ]{0,3}<\!\[CDATA\[`)
var htmlBlockType5Close = []byte{']', ']', '>'}

var htmlBlockType6Regexp = regexp.MustCompile(`^[ ]{0,3}</?([a-zA-Z0-9]+)(?:\s.*|>.*|/>.*|)\n?$`)

var htmlBlockType7Regexp = regexp.MustCompile(`^[ ]{0,3}<(/)?([a-zA-Z0-9]+)(` + attributePattern + `*)(:?>|/>)\s*\n?$`)

type htmlBlockParser struct {
}

var defaultHTMLBlockParser = &htmlBlockParser{}

// NewHTMLBlockParser return a new BlockParser that can parse html
// blocks.
func NewHTMLBlockParser() BlockParser {
	return defaultHTMLBlockParser
}

func (b *htmlBlockParser) Trigger() []byte {
	return []byte{'<'}
}

func (b *htmlBlockParser) Open(parent ast.Node, reader text.Reader, pc Context) (ast.Node, State) {
	var node *ast.HTMLBlock
	line, segment := reader.PeekLine()
	last := pc.LastOpenedBlock().Node
	if pos := pc.BlockOffset(); pos < 0 || line[pos] != '<' {
		return nil, NoChildren
	}

	if m := htmlBlockType1OpenRegexp.FindSubmatchIndex(line); m != nil {
		node = ast.NewHTMLBlock(ast.HTMLBlockType1)
	} else if htmlBlockType2OpenRegexp.Match(line) {
		node = ast.NewHTMLBlock(ast.HTMLBlockType2)
	} else if htmlBlockType3OpenRegexp.Match(line) {
		node = ast.NewHTMLBlock(ast.HTMLBlockType3)
	} else if htmlBlockType4OpenRegexp.Match(line) {
		node = ast.NewHTMLBlock(ast.HTMLBlockType4)
	} else if htmlBlockType5OpenRegexp.Match(line) {
		node = ast.NewHTMLBlock(ast.HTMLBlockType5)
	} else if match := htmlBlockType7Regexp.FindSubmatchIndex(line); match != nil {
		isCloseTag := match[2] > -1 && bytes.Equal(line[match[2]:match[3]], []byte("/"))
		hasAttr := match[6] != match[7]
		tagName := strings.ToLower(string(line[match[4]:match[5]]))
		_, ok := allowedBlockTags[tagName]
		if ok {
			node = ast.NewHTMLBlock(ast.HTMLBlockType6)
		} else if tagName != "script" && tagName != "style" && tagName != "pre" && !ast.IsParagraph(last) && !(isCloseTag && hasAttr) { // type 7 can not interrupt paragraph
			node = ast.NewHTMLBlock(ast.HTMLBlockType7)
		}
	}
	if node == nil {
		if match := htmlBlockType6Regexp.FindSubmatchIndex(line); match != nil {
			tagName := string(line[match[2]:match[3]])
			_, ok := allowedBlockTags[strings.ToLower(tagName)]
			if ok {
				node = ast.NewHTMLBlock(ast.HTMLBlockType6)
			}
		}
	}
	if node != nil {
		reader.Advance(segment.Len() - 1)
		node.Lines().Append(segment)
		return node, NoChildren
	}
	return nil, NoChildren
}

func (b *htmlBlockParser) Continue(node ast.Node, reader text.Reader, pc Context) State {
	htmlBlock := node.(*ast.HTMLBlock)
	lines := htmlBlock.Lines()
	line, segment := reader.PeekLine()
	var closurePattern []byte

	switch htmlBlock.HTMLBlockType {
	case ast.HTMLBlockType1:
		if lines.Len() == 1 {
			firstLine := lines.At(0)
			if htmlBlockType1CloseRegexp.Match(firstLine.Value(reader.Source())) {
				return Close
			}
		}
		if htmlBlockType1CloseRegexp.Match(line) {
			htmlBlock.ClosureLine = segment
			reader.Advance(segment.Len() - 1)
			return Close
		}
	case ast.HTMLBlockType2:
		closurePattern = htmlBlockType2Close
		fallthrough
	case ast.HTMLBlockType3:
		if closurePattern == nil {
			closurePattern = htmlBlockType3Close
		}
		fallthrough
	case ast.HTMLBlockType4:
		if closurePattern == nil {
			closurePattern = htmlBlockType4Close
		}
		fallthrough
	case ast.HTMLBlockType5:
		if closurePattern == nil {
			closurePattern = htmlBlockType5Close
		}

		if lines.Len() == 1 {
			firstLine := lines.At(0)
			if bytes.Contains(firstLine.Value(reader.Source()), closurePattern) {
				return Close
			}
		}
		if bytes.Contains(line, closurePattern) {
			htmlBlock.ClosureLine = segment
			reader.Advance(segment.Len() - 1)
			return Close
		}

	case ast.HTMLBlockType6, ast.HTMLBlockType7:
		if util.IsBlank(line) {
			return Close
		}
	}
	node.Lines().Append(segment)
	reader.Advance(segment.Len() - 1)
	return Continue | NoChildren
}

func (b *htmlBlockParser) Close(node ast.Node, reader text.Reader, pc Context) {
	// nothing to do
}

func (b *htmlBlockParser) CanInterruptParagraph() bool {
	return true
}

func (b *htmlBlockParser) CanAcceptIndentedLine() bool {
	return false
}
<|endoftext|>


<commit_before>// Copyright 2019, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"context"
	"fmt"
	"net/http"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	"go.opentelemetry.io/otel/api/core"
	"go.opentelemetry.io/otel/api/global"
	export "go.opentelemetry.io/otel/sdk/export/metric"
	"go.opentelemetry.io/otel/sdk/export/metric/aggregator"
	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
	"go.opentelemetry.io/otel/sdk/metric/batcher/defaultkeys"
	"go.opentelemetry.io/otel/sdk/metric/controller/push"
	"go.opentelemetry.io/otel/sdk/metric/selector/simple"
)

// Exporter is an implementation of metric.Exporter that sends metrics to
// Prometheus.
type Exporter struct {
	handler http.Handler

	registerer prometheus.Registerer
	gatherer   prometheus.Gatherer

	snapshot export.CheckpointSet
	onError  func(error)

	defaultSummaryQuantiles []float64
}

var _ export.Exporter = &Exporter{}
var _ http.Handler = &Exporter{}

// Config is a set of configs for the tally reporter.
type Config struct {
	// Registry is the prometheus registry that will be used as the default Registerer and
	// Gatherer if these are not specified.
	//
	// If not set a new empty Registry is created.
	Registry *prometheus.Registry

	// Registerer is the prometheus registerer to register
	// metrics with.
	//
	// If not specified the Registry will be used as default.
	Registerer prometheus.Registerer

	// Gatherer is the prometheus gatherer to gather
	// metrics with.
	//
	// If not specified the Registry will be used as default.
	Gatherer prometheus.Gatherer

	// DefaultSummaryQuantiles is the default summary quantiles
	// to use. Use nil to specify the system-default summary quantiles.
	DefaultSummaryQuantiles []float64

	// OnError is a function that handle errors that may occur while exporting metrics.
	// TODO: This should be refactored or even removed once we have a better error handling mechanism.
	OnError func(error)
}

// NewRawExporter returns a new prometheus exporter for prometheus metrics
// for use in a pipeline.
func NewRawExporter(config Config) (*Exporter, error) {
	if config.Registry == nil {
		config.Registry = prometheus.NewRegistry()
	}

	if config.Registerer == nil {
		config.Registerer = config.Registry
	}

	if config.Gatherer == nil {
		config.Gatherer = config.Registry
	}

	if config.OnError == nil {
		config.OnError = func(err error) {
			fmt.Println(err.Error())
		}
	}

	e := &Exporter{
		handler:                 promhttp.HandlerFor(config.Gatherer, promhttp.HandlerOpts{}),
		registerer:              config.Registerer,
		gatherer:                config.Gatherer,
		defaultSummaryQuantiles: config.DefaultSummaryQuantiles,
	}

	c := newCollector(e)
	if err := config.Registerer.Register(c); err != nil {
		config.OnError(fmt.Errorf("cannot register the collector: %w", err))
	}

	return e, nil
}

// InstallNewPipeline instantiates a NewExportPipeline and registers it globally.
// Typically called as:
// pipeline, hf, err := prometheus.InstallNewPipeline(prometheus.Config{...})
// if err != nil {
// 	...
// }
// http.HandleFunc("/metrics", hf)
// defer pipeline.Stop()
// ... Done
func InstallNewPipeline(config Config) (*push.Controller, http.HandlerFunc, error) {
	controller, hf, err := NewExportPipeline(config)
	if err != nil {
		return controller, hf, err
	}
	global.SetMeterProvider(controller)
	return controller, hf, err
}

// NewExportPipeline sets up a complete export pipeline with the recommended setup,
// chaining a NewRawExporter into the recommended selectors and batchers.
func NewExportPipeline(config Config) (*push.Controller, http.HandlerFunc, error) {
	selector := simple.NewWithExactMeasure()
	exporter, err := NewRawExporter(config)
	if err != nil {
		return nil, nil, err
	}

	// Prometheus needs to use a stateful batcher since counters (and histogram since they are a collection of Counters)
	// are cumulative (i.e., monotonically increasing values) and should not be resetted after each export.
	//
	// Prometheus uses this approach to be resilient to scrape failures.
	// If a Prometheus server tries to scrape metrics from a host and fails for some reason,
	// it could try again on the next scrape and no data would be lost, only resolution.
	//
	// Gauges (or LastValues) and Summaries are an exception to this and have different behaviors.
	batcher := defaultkeys.New(selector, sdkmetric.NewDefaultLabelEncoder(), false)
	pusher := push.New(batcher, exporter, time.Second)
	pusher.Start()

	return pusher, exporter.ServeHTTP, nil
}

// Export exports the provide metric record to prometheus.
func (e *Exporter) Export(_ context.Context, checkpointSet export.CheckpointSet) error {
	e.snapshot = checkpointSet
	return nil
}

// collector implements prometheus.Collector interface.
type collector struct {
	exp *Exporter
}

var _ prometheus.Collector = (*collector)(nil)

func newCollector(exporter *Exporter) *collector {
	return &collector{
		exp: exporter,
	}
}

func (c *collector) Describe(ch chan<- *prometheus.Desc) {
	if c.exp.snapshot == nil {
		return
	}

	c.exp.snapshot.ForEach(func(record export.Record) {
		ch <- c.toDesc(&record)
	})
}

// Collect exports the last calculated CheckpointSet.
//
// Collect is invoked whenever prometheus.Gatherer is also invoked.
// For example, when the HTTP endpoint is invoked by Prometheus.
func (c *collector) Collect(ch chan<- prometheus.Metric) {
	if c.exp.snapshot == nil {
		return
	}

	c.exp.snapshot.ForEach(func(record export.Record) {
		agg := record.Aggregator()
		numberKind := record.Descriptor().NumberKind()
		labels := labelValues(record.Labels())
		desc := c.toDesc(&record)

		// TODO: implement histogram export when the histogram aggregation is done.
		//  https://github.com/open-telemetry/opentelemetry-go/issues/317

		if dist, ok := agg.(aggregator.Distribution); ok {
			// TODO: summaries values are never being resetted.
			//  As measures are recorded, new records starts to have less impact on these summaries.
			//  We should implement an solution that is similar to the Prometheus Clients
			//  using a rolling window for summaries could be a solution.
			//
			//  References:
			// 	https://www.robustperception.io/how-does-a-prometheus-summary-work
			//  https://github.com/prometheus/client_golang/blob/fa4aa9000d2863904891d193dea354d23f3d712a/prometheus/summary.go#L135
			c.exportSummary(ch, dist, numberKind, desc, labels)
		} else if sum, ok := agg.(aggregator.Sum); ok {
			c.exportCounter(ch, sum, numberKind, desc, labels)
		} else if gauge, ok := agg.(aggregator.LastValue); ok {
			c.exportGauge(ch, gauge, numberKind, desc, labels)
		}
	})
}

func (c *collector) exportGauge(ch chan<- prometheus.Metric, gauge aggregator.LastValue, kind core.NumberKind, desc *prometheus.Desc, labels []string) {
	lastValue, _, err := gauge.LastValue()
	if err != nil {
		c.exp.onError(err)
		return
	}

	m, err := prometheus.NewConstMetric(desc, prometheus.GaugeValue, lastValue.CoerceToFloat64(kind), labels...)
	if err != nil {
		c.exp.onError(err)
		return
	}

	ch <- m
}

func (c *collector) exportCounter(ch chan<- prometheus.Metric, sum aggregator.Sum, kind core.NumberKind, desc *prometheus.Desc, labels []string) {
	v, err := sum.Sum()
	if err != nil {
		c.exp.onError(err)
		return
	}

	m, err := prometheus.NewConstMetric(desc, prometheus.CounterValue, v.CoerceToFloat64(kind), labels...)
	if err != nil {
		c.exp.onError(err)
		return
	}

	ch <- m
}

func (c *collector) exportSummary(ch chan<- prometheus.Metric, dist aggregator.Distribution, kind core.NumberKind, desc *prometheus.Desc, labels []string) {
	count, err := dist.Count()
	if err != nil {
		c.exp.onError(err)
		return
	}

	var sum core.Number
	sum, err = dist.Sum()
	if err != nil {
		c.exp.onError(err)
		return
	}

	quantiles := make(map[float64]float64)
	for _, quantile := range c.exp.defaultSummaryQuantiles {
		q, _ := dist.Quantile(quantile)
		quantiles[quantile] = q.CoerceToFloat64(kind)
	}

	m, err := prometheus.NewConstSummary(desc, uint64(count), sum.CoerceToFloat64(kind), quantiles, labels...)
	if err != nil {
		c.exp.onError(err)
		return
	}

	ch <- m
}

func (c *collector) toDesc(metric *export.Record) *prometheus.Desc {
	desc := metric.Descriptor()
	labels := labelsKeys(metric.Labels())
	return prometheus.NewDesc(sanitize(desc.Name()), desc.Description(), labels, nil)
}

func (e *Exporter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	e.handler.ServeHTTP(w, r)
}

func labelsKeys(labels export.Labels) []string {
	keys := make([]string, 0, labels.Len())
	for _, kv := range labels.Ordered() {
		keys = append(keys, sanitize(string(kv.Key)))
	}
	return keys
}

func labelValues(labels export.Labels) []string {
	// TODO(paivagustavo): parse the labels.Encoded() instead of calling `Emit()` directly
	//  this would avoid unnecessary allocations.
	values := make([]string, 0, labels.Len())
	for _, label := range labels.Ordered() {
		values = append(values, label.Value.Emit())
	}
	return values
}
<commit_msg>use stateful batcher on prometheus exporter (#428)<commit_after>// Copyright 2019, OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"context"
	"fmt"
	"net/http"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	"go.opentelemetry.io/otel/api/core"
	"go.opentelemetry.io/otel/api/global"
	export "go.opentelemetry.io/otel/sdk/export/metric"
	"go.opentelemetry.io/otel/sdk/export/metric/aggregator"
	sdkmetric "go.opentelemetry.io/otel/sdk/metric"
	"go.opentelemetry.io/otel/sdk/metric/batcher/defaultkeys"
	"go.opentelemetry.io/otel/sdk/metric/controller/push"
	"go.opentelemetry.io/otel/sdk/metric/selector/simple"
)

// Exporter is an implementation of metric.Exporter that sends metrics to
// Prometheus.
type Exporter struct {
	handler http.Handler

	registerer prometheus.Registerer
	gatherer   prometheus.Gatherer

	snapshot export.CheckpointSet
	onError  func(error)

	defaultSummaryQuantiles []float64
}

var _ export.Exporter = &Exporter{}
var _ http.Handler = &Exporter{}

// Config is a set of configs for the tally reporter.
type Config struct {
	// Registry is the prometheus registry that will be used as the default Registerer and
	// Gatherer if these are not specified.
	//
	// If not set a new empty Registry is created.
	Registry *prometheus.Registry

	// Registerer is the prometheus registerer to register
	// metrics with.
	//
	// If not specified the Registry will be used as default.
	Registerer prometheus.Registerer

	// Gatherer is the prometheus gatherer to gather
	// metrics with.
	//
	// If not specified the Registry will be used as default.
	Gatherer prometheus.Gatherer

	// DefaultSummaryQuantiles is the default summary quantiles
	// to use. Use nil to specify the system-default summary quantiles.
	DefaultSummaryQuantiles []float64

	// OnError is a function that handle errors that may occur while exporting metrics.
	// TODO: This should be refactored or even removed once we have a better error handling mechanism.
	OnError func(error)
}

// NewRawExporter returns a new prometheus exporter for prometheus metrics
// for use in a pipeline.
func NewRawExporter(config Config) (*Exporter, error) {
	if config.Registry == nil {
		config.Registry = prometheus.NewRegistry()
	}

	if config.Registerer == nil {
		config.Registerer = config.Registry
	}

	if config.Gatherer == nil {
		config.Gatherer = config.Registry
	}

	if config.OnError == nil {
		config.OnError = func(err error) {
			fmt.Println(err.Error())
		}
	}

	e := &Exporter{
		handler:                 promhttp.HandlerFor(config.Gatherer, promhttp.HandlerOpts{}),
		registerer:              config.Registerer,
		gatherer:                config.Gatherer,
		defaultSummaryQuantiles: config.DefaultSummaryQuantiles,
	}

	c := newCollector(e)
	if err := config.Registerer.Register(c); err != nil {
		config.OnError(fmt.Errorf("cannot register the collector: %w", err))
	}

	return e, nil
}

// InstallNewPipeline instantiates a NewExportPipeline and registers it globally.
// Typically called as:
// pipeline, hf, err := prometheus.InstallNewPipeline(prometheus.Config{...})
// if err != nil {
// 	...
// }
// http.HandleFunc("/metrics", hf)
// defer pipeline.Stop()
// ... Done
func InstallNewPipeline(config Config) (*push.Controller, http.HandlerFunc, error) {
	controller, hf, err := NewExportPipeline(config)
	if err != nil {
		return controller, hf, err
	}
	global.SetMeterProvider(controller)
	return controller, hf, err
}

// NewExportPipeline sets up a complete export pipeline with the recommended setup,
// chaining a NewRawExporter into the recommended selectors and batchers.
func NewExportPipeline(config Config) (*push.Controller, http.HandlerFunc, error) {
	selector := simple.NewWithExactMeasure()
	exporter, err := NewRawExporter(config)
	if err != nil {
		return nil, nil, err
	}

	// Prometheus needs to use a stateful batcher since counters (and histogram since they are a collection of Counters)
	// are cumulative (i.e., monotonically increasing values) and should not be resetted after each export.
	//
	// Prometheus uses this approach to be resilient to scrape failures.
	// If a Prometheus server tries to scrape metrics from a host and fails for some reason,
	// it could try again on the next scrape and no data would be lost, only resolution.
	//
	// Gauges (or LastValues) and Summaries are an exception to this and have different behaviors.
	batcher := defaultkeys.New(selector, sdkmetric.NewDefaultLabelEncoder(), true)
	pusher := push.New(batcher, exporter, time.Second)
	pusher.Start()

	return pusher, exporter.ServeHTTP, nil
}

// Export exports the provide metric record to prometheus.
func (e *Exporter) Export(_ context.Context, checkpointSet export.CheckpointSet) error {
	e.snapshot = checkpointSet
	return nil
}

// collector implements prometheus.Collector interface.
type collector struct {
	exp *Exporter
}

var _ prometheus.Collector = (*collector)(nil)

func newCollector(exporter *Exporter) *collector {
	return &collector{
		exp: exporter,
	}
}

func (c *collector) Describe(ch chan<- *prometheus.Desc) {
	if c.exp.snapshot == nil {
		return
	}

	c.exp.snapshot.ForEach(func(record export.Record) {
		ch <- c.toDesc(&record)
	})
}

// Collect exports the last calculated CheckpointSet.
//
// Collect is invoked whenever prometheus.Gatherer is also invoked.
// For example, when the HTTP endpoint is invoked by Prometheus.
func (c *collector) Collect(ch chan<- prometheus.Metric) {
	if c.exp.snapshot == nil {
		return
	}

	c.exp.snapshot.ForEach(func(record export.Record) {
		agg := record.Aggregator()
		numberKind := record.Descriptor().NumberKind()
		labels := labelValues(record.Labels())
		desc := c.toDesc(&record)

		// TODO: implement histogram export when the histogram aggregation is done.
		//  https://github.com/open-telemetry/opentelemetry-go/issues/317

		if dist, ok := agg.(aggregator.Distribution); ok {
			// TODO: summaries values are never being resetted.
			//  As measures are recorded, new records starts to have less impact on these summaries.
			//  We should implement an solution that is similar to the Prometheus Clients
			//  using a rolling window for summaries could be a solution.
			//
			//  References:
			// 	https://www.robustperception.io/how-does-a-prometheus-summary-work
			//  https://github.com/prometheus/client_golang/blob/fa4aa9000d2863904891d193dea354d23f3d712a/prometheus/summary.go#L135
			c.exportSummary(ch, dist, numberKind, desc, labels)
		} else if sum, ok := agg.(aggregator.Sum); ok {
			c.exportCounter(ch, sum, numberKind, desc, labels)
		} else if gauge, ok := agg.(aggregator.LastValue); ok {
			c.exportGauge(ch, gauge, numberKind, desc, labels)
		}
	})
}

func (c *collector) exportGauge(ch chan<- prometheus.Metric, gauge aggregator.LastValue, kind core.NumberKind, desc *prometheus.Desc, labels []string) {
	lastValue, _, err := gauge.LastValue()
	if err != nil {
		c.exp.onError(err)
		return
	}

	m, err := prometheus.NewConstMetric(desc, prometheus.GaugeValue, lastValue.CoerceToFloat64(kind), labels...)
	if err != nil {
		c.exp.onError(err)
		return
	}

	ch <- m
}

func (c *collector) exportCounter(ch chan<- prometheus.Metric, sum aggregator.Sum, kind core.NumberKind, desc *prometheus.Desc, labels []string) {
	v, err := sum.Sum()
	if err != nil {
		c.exp.onError(err)
		return
	}

	m, err := prometheus.NewConstMetric(desc, prometheus.CounterValue, v.CoerceToFloat64(kind), labels...)
	if err != nil {
		c.exp.onError(err)
		return
	}

	ch <- m
}

func (c *collector) exportSummary(ch chan<- prometheus.Metric, dist aggregator.Distribution, kind core.NumberKind, desc *prometheus.Desc, labels []string) {
	count, err := dist.Count()
	if err != nil {
		c.exp.onError(err)
		return
	}

	var sum core.Number
	sum, err = dist.Sum()
	if err != nil {
		c.exp.onError(err)
		return
	}

	quantiles := make(map[float64]float64)
	for _, quantile := range c.exp.defaultSummaryQuantiles {
		q, _ := dist.Quantile(quantile)
		quantiles[quantile] = q.CoerceToFloat64(kind)
	}

	m, err := prometheus.NewConstSummary(desc, uint64(count), sum.CoerceToFloat64(kind), quantiles, labels...)
	if err != nil {
		c.exp.onError(err)
		return
	}

	ch <- m
}

func (c *collector) toDesc(metric *export.Record) *prometheus.Desc {
	desc := metric.Descriptor()
	labels := labelsKeys(metric.Labels())
	return prometheus.NewDesc(sanitize(desc.Name()), desc.Description(), labels, nil)
}

func (e *Exporter) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	e.handler.ServeHTTP(w, r)
}

func labelsKeys(labels export.Labels) []string {
	keys := make([]string, 0, labels.Len())
	for _, kv := range labels.Ordered() {
		keys = append(keys, sanitize(string(kv.Key)))
	}
	return keys
}

func labelValues(labels export.Labels) []string {
	// TODO(paivagustavo): parse the labels.Encoded() instead of calling `Emit()` directly
	//  this would avoid unnecessary allocations.
	values := make([]string, 0, labels.Len())
	for _, label := range labels.Ordered() {
		values = append(values, label.Value.Emit())
	}
	return values
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"sync"

	"time"

	"github.com/minio/minio-go"
)

func usage() {
	fmt.Println("perf <object-size-in-MB> <parallel-upload-count>")
	os.Exit(0)
}

func performanceTest(client *minio.Core, bucket, objectPrefix string, objSize int64, threadCount int) (bandwidth float64, objsPerSec float64, delta float64) {
	var wg = &sync.WaitGroup{}
	t1 := time.Now()
	for i := 0; i < threadCount; i++ {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			// Start all the goroutines at the same time
			o, _, err := client.GetObject(bucket, fmt.Sprintf("%s.%d", objectPrefix, i), minio.RequestHeaders{})
			if err != nil {
				fmt.Println(err)
			}
			io.Copy(ioutil.Discard, o)
		}(i)
	}
	wg.Wait() // Wait till all go routines finish
	delta = time.Since(t1).Seconds()
	bandwidth = float64(objSize*int64(threadCount)) / delta / 1024 / 1024 // in MBps
	objsPerSec = float64(threadCount) / delta
	return bandwidth, objsPerSec, delta
}

func main() {
	bucket := "testbucket"
	objectPrefix := "testobject"
	if len(os.Args) != 3 {
		usage()
	}

	objSize, err := strconv.Atoi(os.Args[1])
	if err != nil {
		log.Print(err)
		usage()
	}

	threadCount, err := strconv.Atoi(os.Args[2])
	if err != nil {
		log.Print(err)
		usage()
	}
	client, err := minio.NewCore(os.Getenv("MINIO_ENDPOINT"), os.Getenv("MINIO_ACCESS_KEY"), os.Getenv("MINIO_SECRET_KEY"), false)
	if err != nil {
		log.Fatal(err)
	}

	client.MakeBucket(bucket, "") // Ignore "bucket-exists" error

	bandwidth, objsPerSec, delta := performanceTest(client, bucket, objectPrefix, int64(objSize), threadCount)
	t := struct {
		ObjSize     int64
		ThreadCount int
		Delta       float64
		Bandwidth   float64
		ObjsPerSec  float64
	}{
		int64(objSize), threadCount, delta, bandwidth, objsPerSec,
	}
	b, err := json.Marshal(t)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(string(b))
}
<commit_msg>log error on io.Copy<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"sync"

	"time"

	"github.com/minio/minio-go"
)

func usage() {
	fmt.Println("perf <object-size-in-MB> <parallel-upload-count>")
	os.Exit(0)
}

func performanceTest(client *minio.Core, bucket, objectPrefix string, objSize int64, threadCount int) (bandwidth float64, objsPerSec float64, delta float64) {
	var wg = &sync.WaitGroup{}
	t1 := time.Now()
	for i := 0; i < threadCount; i++ {
		wg.Add(1)
		go func(i int) {
			defer wg.Done()
			// Start all the goroutines at the same time
			o, _, err := client.GetObject(bucket, fmt.Sprintf("%s.%d", objectPrefix, i), minio.RequestHeaders{})
			if err != nil {
				fmt.Println(err)
			}
			_, err = io.Copy(ioutil.Discard, o)
			if err != nil {
				fmt.Println(err)
			}
		}(i)
	}
	wg.Wait() // Wait till all go routines finish
	delta = time.Since(t1).Seconds()
	bandwidth = float64(objSize*int64(threadCount)) / delta / 1024 / 1024 // in MBps
	objsPerSec = float64(threadCount) / delta
	return bandwidth, objsPerSec, delta
}

func main() {
	bucket := "testbucket"
	objectPrefix := "testobject"
	if len(os.Args) != 3 {
		usage()
	}

	objSize, err := strconv.Atoi(os.Args[1])
	if err != nil {
		log.Print(err)
		usage()
	}
	objSize = objSize * 1024 * 1024
	threadCount, err := strconv.Atoi(os.Args[2])
	if err != nil {
		log.Print(err)
		usage()
	}
	client, err := minio.NewCore(os.Getenv("MINIO_ENDPOINT"), os.Getenv("MINIO_ACCESS_KEY"), os.Getenv("MINIO_SECRET_KEY"), false)
	if err != nil {
		log.Fatal(err)
	}

	client.MakeBucket(bucket, "") // Ignore "bucket-exists" error

	bandwidth, objsPerSec, delta := performanceTest(client, bucket, objectPrefix, int64(objSize), threadCount)
	t := struct {
		ObjSize     int64
		ThreadCount int
		Delta       float64
		Bandwidth   float64
		ObjsPerSec  float64
	}{
		int64(objSize), threadCount, delta, bandwidth, objsPerSec,
	}
	b, err := json.Marshal(t)
	if err != nil {
		log.Fatal(err)
	}
	fmt.Println(string(b))
}
<|endoftext|>


<commit_before>package shell

import (
	"context"
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/cluster"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"io"

	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
)

func init() {
	Commands = append(Commands, &commandClusterCheck{})
}

type commandClusterCheck struct {
}

func (c *commandClusterCheck) Name() string {
	return "cluster.check"
}

func (c *commandClusterCheck) Help() string {
	return `check current cluster network connectivity

	cluster.check

`
}

func (c *commandClusterCheck) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	clusterPsCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	if err = clusterPsCommand.Parse(args); err != nil {
		return nil
	}

	// collect filers
	var filers []pb.ServerAddress
	err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
		resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
			ClientType: cluster.FilerType,
		})

		for _, node := range resp.ClusterNodes {
			filers = append(filers, pb.ServerAddress(node.Address))
		}
		return err
	})
	if err != nil {
		return
	}
	fmt.Fprintf(writer, "the cluster has %d filers: %+v\n", len(filers), filers)

	// collect volume servers
	var volumeServers []pb.ServerAddress
	t, _, err := collectTopologyInfo(commandEnv, 0)
	if err != nil {
		return err
	}
	for _, dc := range t.DataCenterInfos {
		for _, r := range dc.RackInfos {
			for _, dn := range r.DataNodeInfos {
				volumeServers = append(volumeServers, pb.NewServerAddressFromDataNode(dn))
			}
		}
	}
	fmt.Fprintf(writer, "the cluster has %d volume servers: %+v\n", len(volumeServers), volumeServers)

	// collect all masters
	var masters []pb.ServerAddress
	for _, master := range commandEnv.MasterClient.GetMasters() {
		masters = append(masters, master)
	}

	// check from master to volume servers
	for _, master := range masters {
		for _, volumeServer := range volumeServers {
			fmt.Fprintf(writer, "checking master %s to volume server %s ... ", string(master), string(volumeServer))
			err := pb.WithMasterClient(false, master, commandEnv.option.GrpcDialOption, func(client master_pb.SeaweedClient) error {
				_, err := client.Ping(context.Background(), &master_pb.PingRequest{
					Target:     string(volumeServer),
					TargetType: cluster.VolumeServerType,
				})
				return err
			})
			if err == nil {
				fmt.Fprintf(writer, "ok\n")
			} else {
				fmt.Fprintf(writer, "%v\n", err)
			}
		}
	}

	// check between masters
	for _, sourceMaster := range masters {
		for _, targetMaster := range masters {
			if sourceMaster == targetMaster {
				continue
			}
			fmt.Fprintf(writer, "checking master %s to %s ... ", string(sourceMaster), string(targetMaster))
			err := pb.WithMasterClient(false, sourceMaster, commandEnv.option.GrpcDialOption, func(client master_pb.SeaweedClient) error {
				_, err := client.Ping(context.Background(), &master_pb.PingRequest{
					Target:     string(targetMaster),
					TargetType: cluster.MasterType,
				})
				return err
			})
			if err == nil {
				fmt.Fprintf(writer, "ok\n")
			} else {
				fmt.Fprintf(writer, "%v\n", err)
			}
		}
	}

	// check from volume servers to masters
	for _, volumeServer := range volumeServers {
		for _, master := range masters {
			fmt.Fprintf(writer, "checking volume server %s to master %s ... ", string(volumeServer), string(master))
			err := pb.WithVolumeServerClient(false, volumeServer, commandEnv.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
				_, err := client.Ping(context.Background(), &volume_server_pb.PingRequest{
					Target:     string(master),
					TargetType: cluster.MasterType,
				})
				return err
			})
			if err == nil {
				fmt.Fprintf(writer, "ok\n")
			} else {
				fmt.Fprintf(writer, "%v\n", err)
			}
		}
	}

	// check from filers to masters
	for _, filer := range filers {
		for _, master := range masters {
			fmt.Fprintf(writer, "checking filer %s to master %s ... ", string(filer), string(master))
			err := pb.WithFilerClient(false, filer, commandEnv.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
				_, err := client.Ping(context.Background(), &filer_pb.PingRequest{
					Target:     string(master),
					TargetType: cluster.MasterType,
				})
				return err
			})
			if err == nil {
				fmt.Fprintf(writer, "ok\n")
			} else {
				fmt.Fprintf(writer, "%v\n", err)
			}
		}
	}

	// check from filers to volume servers
	for _, filer := range filers {
		for _, volumeServer := range volumeServers {
			fmt.Fprintf(writer, "checking filer %s to volume server %s ... ", string(filer), string(volumeServer))
			err := pb.WithFilerClient(false, filer, commandEnv.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
				_, err := client.Ping(context.Background(), &filer_pb.PingRequest{
					Target:     string(volumeServer),
					TargetType: cluster.VolumeServerType,
				})
				return err
			})
			if err == nil {
				fmt.Fprintf(writer, "ok\n")
			} else {
				fmt.Fprintf(writer, "%v\n", err)
			}
		}
	}

	// check between volume servers
	for _, sourceVolumeServer := range volumeServers {
		for _, targetVolumeServer := range volumeServers {
			if sourceVolumeServer == targetVolumeServer {
				continue
			}
			fmt.Fprintf(writer, "checking volume server %s to %s ... ", string(sourceVolumeServer), string(targetVolumeServer))
			err := pb.WithVolumeServerClient(false, sourceVolumeServer, commandEnv.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
				_, err := client.Ping(context.Background(), &volume_server_pb.PingRequest{
					Target:     string(targetVolumeServer),
					TargetType: cluster.VolumeServerType,
				})
				return err
			})
			if err == nil {
				fmt.Fprintf(writer, "ok\n")
			} else {
				fmt.Fprintf(writer, "%v\n", err)
			}
		}
	}

	// check between filers, and need to connect to itself
	for _, sourceFiler := range filers {
		for _, targetFiler := range filers {
			fmt.Fprintf(writer, "checking filer %s to %s ... ", string(sourceFiler), string(targetFiler))
			err := pb.WithFilerClient(false, sourceFiler, commandEnv.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
				_, err := client.Ping(context.Background(), &filer_pb.PingRequest{
					Target:     string(targetFiler),
					TargetType: cluster.FilerType,
				})
				return err
			})
			if err == nil {
				fmt.Fprintf(writer, "ok\n")
			} else {
				fmt.Fprintf(writer, "%v\n", err)
			}
		}
	}

	return nil
}
<commit_msg>check missing hdd disk type<commit_after>package shell

import (
	"context"
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/cluster"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"io"

	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
)

func init() {
	Commands = append(Commands, &commandClusterCheck{})
}

type commandClusterCheck struct {
}

func (c *commandClusterCheck) Name() string {
	return "cluster.check"
}

func (c *commandClusterCheck) Help() string {
	return `check current cluster network connectivity

	cluster.check

`
}

func (c *commandClusterCheck) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {

	clusterPsCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	if err = clusterPsCommand.Parse(args); err != nil {
		return nil
	}

	// collect topology information
	topologyInfo, volumeSizeLimitMb, err := collectTopologyInfo(commandEnv, 0)
	if err != nil {
		return err
	}
	fmt.Fprintf(writer, "Topology volumeSizeLimit:%d MB%s\n", volumeSizeLimitMb, diskInfosToString(topologyInfo.DiskInfos))

	emptyDiskTypeDiskInfo, emptyDiskTypeFound := topologyInfo.DiskInfos[""]
	hddDiskTypeDiskInfo, hddDiskTypeFound := topologyInfo.DiskInfos["hdd"]
	if !emptyDiskTypeFound && !hddDiskTypeFound || emptyDiskTypeDiskInfo.VolumeCount == 0 && hddDiskTypeDiskInfo.VolumeCount == 0 {
		return fmt.Errorf("Need to a hdd disk type!")
	}

	// collect filers
	var filers []pb.ServerAddress
	err = commandEnv.MasterClient.WithClient(false, func(client master_pb.SeaweedClient) error {
		resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
			ClientType: cluster.FilerType,
		})

		for _, node := range resp.ClusterNodes {
			filers = append(filers, pb.ServerAddress(node.Address))
		}
		return err
	})
	if err != nil {
		return
	}
	fmt.Fprintf(writer, "the cluster has %d filers: %+v\n", len(filers), filers)

	// collect volume servers
	var volumeServers []pb.ServerAddress
	t, _, err := collectTopologyInfo(commandEnv, 0)
	if err != nil {
		return err
	}
	for _, dc := range t.DataCenterInfos {
		for _, r := range dc.RackInfos {
			for _, dn := range r.DataNodeInfos {
				volumeServers = append(volumeServers, pb.NewServerAddressFromDataNode(dn))
			}
		}
	}
	fmt.Fprintf(writer, "the cluster has %d volume servers: %+v\n", len(volumeServers), volumeServers)

	// collect all masters
	var masters []pb.ServerAddress
	for _, master := range commandEnv.MasterClient.GetMasters() {
		masters = append(masters, master)
	}

	// check from master to volume servers
	for _, master := range masters {
		for _, volumeServer := range volumeServers {
			fmt.Fprintf(writer, "checking master %s to volume server %s ... ", string(master), string(volumeServer))
			err := pb.WithMasterClient(false, master, commandEnv.option.GrpcDialOption, func(client master_pb.SeaweedClient) error {
				_, err := client.Ping(context.Background(), &master_pb.PingRequest{
					Target:     string(volumeServer),
					TargetType: cluster.VolumeServerType,
				})
				return err
			})
			if err == nil {
				fmt.Fprintf(writer, "ok\n")
			} else {
				fmt.Fprintf(writer, "%v\n", err)
			}
		}
	}

	// check between masters
	for _, sourceMaster := range masters {
		for _, targetMaster := range masters {
			if sourceMaster == targetMaster {
				continue
			}
			fmt.Fprintf(writer, "checking master %s to %s ... ", string(sourceMaster), string(targetMaster))
			err := pb.WithMasterClient(false, sourceMaster, commandEnv.option.GrpcDialOption, func(client master_pb.SeaweedClient) error {
				_, err := client.Ping(context.Background(), &master_pb.PingRequest{
					Target:     string(targetMaster),
					TargetType: cluster.MasterType,
				})
				return err
			})
			if err == nil {
				fmt.Fprintf(writer, "ok\n")
			} else {
				fmt.Fprintf(writer, "%v\n", err)
			}
		}
	}

	// check from volume servers to masters
	for _, volumeServer := range volumeServers {
		for _, master := range masters {
			fmt.Fprintf(writer, "checking volume server %s to master %s ... ", string(volumeServer), string(master))
			err := pb.WithVolumeServerClient(false, volumeServer, commandEnv.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
				_, err := client.Ping(context.Background(), &volume_server_pb.PingRequest{
					Target:     string(master),
					TargetType: cluster.MasterType,
				})
				return err
			})
			if err == nil {
				fmt.Fprintf(writer, "ok\n")
			} else {
				fmt.Fprintf(writer, "%v\n", err)
			}
		}
	}

	// check from filers to masters
	for _, filer := range filers {
		for _, master := range masters {
			fmt.Fprintf(writer, "checking filer %s to master %s ... ", string(filer), string(master))
			err := pb.WithFilerClient(false, filer, commandEnv.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
				_, err := client.Ping(context.Background(), &filer_pb.PingRequest{
					Target:     string(master),
					TargetType: cluster.MasterType,
				})
				return err
			})
			if err == nil {
				fmt.Fprintf(writer, "ok\n")
			} else {
				fmt.Fprintf(writer, "%v\n", err)
			}
		}
	}

	// check from filers to volume servers
	for _, filer := range filers {
		for _, volumeServer := range volumeServers {
			fmt.Fprintf(writer, "checking filer %s to volume server %s ... ", string(filer), string(volumeServer))
			err := pb.WithFilerClient(false, filer, commandEnv.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
				_, err := client.Ping(context.Background(), &filer_pb.PingRequest{
					Target:     string(volumeServer),
					TargetType: cluster.VolumeServerType,
				})
				return err
			})
			if err == nil {
				fmt.Fprintf(writer, "ok\n")
			} else {
				fmt.Fprintf(writer, "%v\n", err)
			}
		}
	}

	// check between volume servers
	for _, sourceVolumeServer := range volumeServers {
		for _, targetVolumeServer := range volumeServers {
			if sourceVolumeServer == targetVolumeServer {
				continue
			}
			fmt.Fprintf(writer, "checking volume server %s to %s ... ", string(sourceVolumeServer), string(targetVolumeServer))
			err := pb.WithVolumeServerClient(false, sourceVolumeServer, commandEnv.option.GrpcDialOption, func(client volume_server_pb.VolumeServerClient) error {
				_, err := client.Ping(context.Background(), &volume_server_pb.PingRequest{
					Target:     string(targetVolumeServer),
					TargetType: cluster.VolumeServerType,
				})
				return err
			})
			if err == nil {
				fmt.Fprintf(writer, "ok\n")
			} else {
				fmt.Fprintf(writer, "%v\n", err)
			}
		}
	}

	// check between filers, and need to connect to itself
	for _, sourceFiler := range filers {
		for _, targetFiler := range filers {
			fmt.Fprintf(writer, "checking filer %s to %s ... ", string(sourceFiler), string(targetFiler))
			err := pb.WithFilerClient(false, sourceFiler, commandEnv.option.GrpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
				_, err := client.Ping(context.Background(), &filer_pb.PingRequest{
					Target:     string(targetFiler),
					TargetType: cluster.FilerType,
				})
				return err
			})
			if err == nil {
				fmt.Fprintf(writer, "ok\n")
			} else {
				fmt.Fprintf(writer, "%v\n", err)
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cluster

import (
	"context"
	"fmt"
	"regexp"
	"strings"

	v1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"

	analyzer_util "istio.io/istio/galley/pkg/config/analysis/analyzers/util"
	"istio.io/istio/pkg/config/resource"
	"istio.io/istio/tools/bug-report/pkg/common"
	"istio.io/istio/tools/bug-report/pkg/util/path"
	"istio.io/pkg/log"
)

type ResourceType int

const (
	Namespace ResourceType = iota
	Deployment
	Pod
	Label
	Annotation
	Container
)

var versionRegex = regexp.MustCompile(`.*(\d\.\d\.\d).*`)

// ParsePath parses path into its components. Input must have the form namespace/deployment/pod/container.
func ParsePath(path string) (namespace string, deployment, pod string, container string, err error) {
	pv := strings.Split(path, "/")
	if len(pv) != 4 {
		return "", "", "", "", fmt.Errorf("bad path %s, must be namespace/deployment/pod/container", path)
	}
	return pv[0], pv[1], pv[2], pv[3], nil
}

// GetClusterResources returns cluster resources for the given REST config and k8s Clientset.
func GetClusterResources(ctx context.Context, clientset *kubernetes.Clientset) (*Resources, error) {
	var errs []string
	out := &Resources{
		Labels:      make(map[string]map[string]string),
		Annotations: make(map[string]map[string]string),
		Pod:         make(map[string]*corev1.Pod),
	}
	namespaces, err := clientset.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
	if err != nil {
		return nil, err
	}
	for _, ns := range namespaces.Items {
		// skip system namesapces
		if analyzer_util.IsSystemNamespace(resource.Namespace(ns.Name)) {
			continue
		}
		pods, err := clientset.CoreV1().Pods(ns.Name).List(ctx, metav1.ListOptions{})
		if err != nil {
			return nil, err
		}
		replicasets, err := clientset.AppsV1().ReplicaSets(ns.Name).List(ctx, metav1.ListOptions{})
		if err != nil {
			return nil, err
		}
		for i, p := range pods.Items {
			deployment := getOwnerDeployment(&p, replicasets.Items)
			for _, c := range p.Spec.Containers {
				out.insertContainer(ns.Name, deployment, p.Name, c.Name)
			}
			out.Labels[PodKey(p.Namespace, p.Name)] = p.Labels
			out.Annotations[PodKey(p.Namespace, p.Name)] = p.Annotations
			out.Pod[PodKey(p.Namespace, p.Name)] = &pods.Items[i]
		}
	}
	if len(errs) != 0 {
		log.Warn(strings.Join(errs, "\n"))
	}
	return out, nil
}

// Resources defines a tree of cluster resource names.
type Resources struct {
	// Root is the first level in the cluster resource hierarchy.
	// Each level in the hierarchy is a map[string]interface{} to the next level.
	// The levels are: namespaces/deployments/pods/containers.
	Root map[string]interface{}
	// Labels maps a pod name to a map of labels key-values.
	Labels map[string]map[string]string
	// Annotations maps a pod name to a map of annotation key-values.
	Annotations map[string]map[string]string
	// Pod maps a pod name to its Pod info. The key is namespace/pod-name.
	Pod map[string]*corev1.Pod
}

func (r *Resources) insertContainer(namespace, deployment, pod, container string) {
	if r.Root == nil {
		r.Root = make(map[string]interface{})
	}
	if r.Root[namespace] == nil {
		r.Root[namespace] = make(map[string]interface{})
	}
	d := r.Root[namespace].(map[string]interface{})
	if d[deployment] == nil {
		d[deployment] = make(map[string]interface{})
	}
	p := d[deployment].(map[string]interface{})
	if p[pod] == nil {
		p[pod] = make(map[string]interface{})
	}
	c := p[pod].(map[string]interface{})
	c[container] = nil
}

// ContainerRestarts returns the number of container restarts for the given container.
func (r *Resources) ContainerRestarts(namespace, pod, container string) int {
	for _, cs := range r.Pod[PodKey(namespace, pod)].Status.ContainerStatuses {
		if cs.Name == container {
			return int(cs.RestartCount)
		}
	}
	return 0
}

// IsDiscoveryContainer reports whether the given container is the Istio discovery container.
func (r *Resources) IsDiscoveryContainer(clusterVersion, namespace, pod, container string) bool {
	return common.IsDiscoveryContainer(clusterVersion, container, r.Labels[PodKey(namespace, pod)])
}

// PodIstioVersion returns the Istio version for the given pod, if either the proxy or discovery are one of its
// containers and the tag is in a parseable format.
func (r *Resources) PodIstioVersion(namespace, pod string) string {
	p := r.Pod[PodKey(namespace, pod)]
	if p == nil {
		return ""
	}

	for _, c := range p.Spec.Containers {
		if c.Name == common.ProxyContainerName || c.Name == common.DiscoveryContainerName {
			return imageToVersion(c.Image)
		}
	}
	return ""
}

// String implements the Stringer interface.
func (r *Resources) String() string {
	return resourcesStringImpl(r.Root, "")
}

func resourcesStringImpl(node interface{}, prefix string) string {
	out := ""
	if node == nil {
		return ""
	}
	nv := node.(map[string]interface{})
	for k, n := range nv {
		out += prefix + k + "\n"
		out += resourcesStringImpl(n, prefix+"  ")
	}

	return out
}

// PodKey returns a unique key based on the namespace and pod name.
func PodKey(namespace, pod string) string {
	return path.Path{namespace, pod}.String()
}

func getOwnerDeployment(pod *corev1.Pod, replicasets []v1.ReplicaSet) string {
	for _, o := range pod.OwnerReferences {
		if o.Kind == "ReplicaSet" {
			for _, rs := range replicasets {
				if rs.Name == o.Name {
					for _, oo := range rs.OwnerReferences {
						if oo.Kind == "Deployment" {
							return oo.Name
						}
					}
				}
			}
		}
	}
	return ""
}

func imageToVersion(imageStr string) string {
	vs := versionRegex.FindStringSubmatch(imageStr)
	if len(vs) != 2 {
		return ""
	}
	return vs[0]
}
<commit_msg>Remove unuse errs (#34602)<commit_after>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cluster

import (
	"context"
	"fmt"
	"regexp"
	"strings"

	v1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"

	analyzer_util "istio.io/istio/galley/pkg/config/analysis/analyzers/util"
	"istio.io/istio/pkg/config/resource"
	"istio.io/istio/tools/bug-report/pkg/common"
	"istio.io/istio/tools/bug-report/pkg/util/path"
)

type ResourceType int

const (
	Namespace ResourceType = iota
	Deployment
	Pod
	Label
	Annotation
	Container
)

var versionRegex = regexp.MustCompile(`.*(\d\.\d\.\d).*`)

// ParsePath parses path into its components. Input must have the form namespace/deployment/pod/container.
func ParsePath(path string) (namespace string, deployment, pod string, container string, err error) {
	pv := strings.Split(path, "/")
	if len(pv) != 4 {
		return "", "", "", "", fmt.Errorf("bad path %s, must be namespace/deployment/pod/container", path)
	}
	return pv[0], pv[1], pv[2], pv[3], nil
}

// GetClusterResources returns cluster resources for the given REST config and k8s Clientset.
func GetClusterResources(ctx context.Context, clientset *kubernetes.Clientset) (*Resources, error) {
	out := &Resources{
		Labels:      make(map[string]map[string]string),
		Annotations: make(map[string]map[string]string),
		Pod:         make(map[string]*corev1.Pod),
	}
	namespaces, err := clientset.CoreV1().Namespaces().List(ctx, metav1.ListOptions{})
	if err != nil {
		return nil, err
	}
	for _, ns := range namespaces.Items {
		// skip system namesapces
		if analyzer_util.IsSystemNamespace(resource.Namespace(ns.Name)) {
			continue
		}
		pods, err := clientset.CoreV1().Pods(ns.Name).List(ctx, metav1.ListOptions{})
		if err != nil {
			return nil, err
		}
		replicasets, err := clientset.AppsV1().ReplicaSets(ns.Name).List(ctx, metav1.ListOptions{})
		if err != nil {
			return nil, err
		}
		for i, p := range pods.Items {
			deployment := getOwnerDeployment(&p, replicasets.Items)
			for _, c := range p.Spec.Containers {
				out.insertContainer(ns.Name, deployment, p.Name, c.Name)
			}
			out.Labels[PodKey(p.Namespace, p.Name)] = p.Labels
			out.Annotations[PodKey(p.Namespace, p.Name)] = p.Annotations
			out.Pod[PodKey(p.Namespace, p.Name)] = &pods.Items[i]
		}
	}
	return out, nil
}

// Resources defines a tree of cluster resource names.
type Resources struct {
	// Root is the first level in the cluster resource hierarchy.
	// Each level in the hierarchy is a map[string]interface{} to the next level.
	// The levels are: namespaces/deployments/pods/containers.
	Root map[string]interface{}
	// Labels maps a pod name to a map of labels key-values.
	Labels map[string]map[string]string
	// Annotations maps a pod name to a map of annotation key-values.
	Annotations map[string]map[string]string
	// Pod maps a pod name to its Pod info. The key is namespace/pod-name.
	Pod map[string]*corev1.Pod
}

func (r *Resources) insertContainer(namespace, deployment, pod, container string) {
	if r.Root == nil {
		r.Root = make(map[string]interface{})
	}
	if r.Root[namespace] == nil {
		r.Root[namespace] = make(map[string]interface{})
	}
	d := r.Root[namespace].(map[string]interface{})
	if d[deployment] == nil {
		d[deployment] = make(map[string]interface{})
	}
	p := d[deployment].(map[string]interface{})
	if p[pod] == nil {
		p[pod] = make(map[string]interface{})
	}
	c := p[pod].(map[string]interface{})
	c[container] = nil
}

// ContainerRestarts returns the number of container restarts for the given container.
func (r *Resources) ContainerRestarts(namespace, pod, container string) int {
	for _, cs := range r.Pod[PodKey(namespace, pod)].Status.ContainerStatuses {
		if cs.Name == container {
			return int(cs.RestartCount)
		}
	}
	return 0
}

// IsDiscoveryContainer reports whether the given container is the Istio discovery container.
func (r *Resources) IsDiscoveryContainer(clusterVersion, namespace, pod, container string) bool {
	return common.IsDiscoveryContainer(clusterVersion, container, r.Labels[PodKey(namespace, pod)])
}

// PodIstioVersion returns the Istio version for the given pod, if either the proxy or discovery are one of its
// containers and the tag is in a parseable format.
func (r *Resources) PodIstioVersion(namespace, pod string) string {
	p := r.Pod[PodKey(namespace, pod)]
	if p == nil {
		return ""
	}

	for _, c := range p.Spec.Containers {
		if c.Name == common.ProxyContainerName || c.Name == common.DiscoveryContainerName {
			return imageToVersion(c.Image)
		}
	}
	return ""
}

// String implements the Stringer interface.
func (r *Resources) String() string {
	return resourcesStringImpl(r.Root, "")
}

func resourcesStringImpl(node interface{}, prefix string) string {
	out := ""
	if node == nil {
		return ""
	}
	nv := node.(map[string]interface{})
	for k, n := range nv {
		out += prefix + k + "\n"
		out += resourcesStringImpl(n, prefix+"  ")
	}

	return out
}

// PodKey returns a unique key based on the namespace and pod name.
func PodKey(namespace, pod string) string {
	return path.Path{namespace, pod}.String()
}

func getOwnerDeployment(pod *corev1.Pod, replicasets []v1.ReplicaSet) string {
	for _, o := range pod.OwnerReferences {
		if o.Kind == "ReplicaSet" {
			for _, rs := range replicasets {
				if rs.Name == o.Name {
					for _, oo := range rs.OwnerReferences {
						if oo.Kind == "Deployment" {
							return oo.Name
						}
					}
				}
			}
		}
	}
	return ""
}

func imageToVersion(imageStr string) string {
	vs := versionRegex.FindStringSubmatch(imageStr)
	if len(vs) != 2 {
		return ""
	}
	return vs[0]
}
<|endoftext|>


<commit_before>package cache

import "github.com/phzfi/RIC/server/ops"
import "github.com/phzfi/RIC/server/images"

type Operator struct {
	cache  *Cache
	tokens chan bool
}

func MakeOperator(mm uint64) Operator {
	o := Operator{NewLRU(mm), make(chan bool, 4)}
	for i := 0; i < 4; i++ {
		o.tokens <- true
	}
	return o
}

func (o Operator) GetBlob(operations ...ops.Operation) (blob images.ImageBlob, err error) {

	blob, found := o.cache.GetBlob(operations)
	if found {
		return blob, nil
	}

	t := <-o.tokens
	img := images.NewImage()
	defer img.Destroy()

	for _, op := range operations {
		err = op.Apply(img)
		if err != nil {
			return
		}
	}
	o.tokens <- t

	blob = img.Blob()
	o.cache.AddBlob(operations, blob)
	return

}
<commit_msg>Limit concurrent operations to 2<commit_after>package cache

import "github.com/phzfi/RIC/server/ops"
import "github.com/phzfi/RIC/server/images"

type Operator struct {
	cache  *Cache
	tokens chan bool
}

func MakeOperator(mm uint64) Operator {
	o := Operator{NewLRU(mm), make(chan bool, 4)}
	for i := 0; i < 2; i++ {
		o.tokens <- true
	}
	return o
}

func (o Operator) GetBlob(operations ...ops.Operation) (blob images.ImageBlob, err error) {

	blob, found := o.cache.GetBlob(operations)
	if found {
		return blob, nil
	}

	t := <-o.tokens
	img := images.NewImage()
	defer img.Destroy()

	for _, op := range operations {
		err = op.Apply(img)
		if err != nil {
			return
		}
	}
	o.tokens <- t

	blob = img.Blob()
	o.cache.AddBlob(operations, blob)
	return

}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"

	c "github.com/couchbase/indexing/secondary/common"
	protobuf "github.com/couchbase/indexing/secondary/protobuf/query"
	queryclient "github.com/couchbase/indexing/secondary/queryport/client"
)

var (
	server   string
	scanType string

	indexName string
	bucket    string

	low   string
	high  string
	equal string
	incl  uint

	limit    int64
	pageSize int64
)

func parseArgs() {
	flag.StringVar(&server, "server", "localhost:7000", "query server address")
	flag.StringVar(&scanType, "type", "scanAll", "Scan command")
	flag.StringVar(&indexName, "index", "", "Index name")
	flag.StringVar(&bucket, "bucket", "default", "Bucket name")
	flag.StringVar(&low, "low", "", "Range: [low]")
	flag.StringVar(&high, "high", "", "Range: [high]")
	flag.StringVar(&equal, "equal", "", "Range: [key]")
	flag.UintVar(&incl, "incl", 0, "Range: 0|1|2|3")
	flag.Int64Var(&limit, "limit", 10, "Row limit")
	flag.Int64Var(&pageSize, "buffersz", 0, "Rows buffer size per internal message")

	flag.Parse()
}

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s -type scanAll -index idx1 -bucket default\n", os.Args[0])
	flag.PrintDefaults()
}

func main() {
	var err error
	var statsResp *protobuf.IndexStatistics
	var keys [][]byte

	parseArgs()

	if indexName == "" {
		usage()
		os.Exit(1)
	}

	config := c.SystemConfig.SectionConfig("queryport.client.", true)
	client := queryclient.NewClient(server, config)
	if equal != "" {
		keys = append(keys, []byte(equal))
	}

	switch scanType {
	case "scan":
		err = client.Scan(indexName, bucket, []byte(low), []byte(high), keys, uint32(incl), pageSize, false, limit, scanCallback)
	case "scanAll":
		err = client.ScanAll(indexName, bucket, pageSize, limit, scanCallback)
	case "stats":
		statsResp, err = client.Statistics(indexName, bucket, []byte(low), []byte(high), keys, uint32(incl))
		if err == nil {
			fmt.Println("Stats: ", statsResp)
		}
	}

	if err != nil {
		fmt.Fprintf(os.Stderr, "Error occured %v\n", err)
	}

	client.Close()
}

func scanCallback(res interface{}) bool {
	switch r := res.(type) {
	case *protobuf.ResponseStream:
		fmt.Println("StreamResponse: ", res.(*protobuf.ResponseStream).String())
	case error:
		fmt.Println("Error: ", r)
	}
	return true
}
<commit_msg>querycmd: Add support for create index, drop index and list index<commit_after>package main

import (
	"flag"
	"fmt"

	"os"
	"strings"

	c "github.com/couchbase/indexing/secondary/common"
	protobuf "github.com/couchbase/indexing/secondary/protobuf/query"
	queryclient "github.com/couchbase/indexing/secondary/queryport/client"
	"github.com/couchbaselabs/query/expression"
	"github.com/couchbaselabs/query/parser/n1ql"
)

var (
	server string
	opType string

	indexName string
	bucket    string

	low   string
	high  string
	equal string
	incl  uint

	limit    int64
	pageSize int64

	fields     string
	isPrimary  bool
	instanceId string
)

const (
	using    = "lsm"
	exprType = "N1QL"
	partnExp = ""
	where    = ""
)

func parseArgs() {
	flag.StringVar(&server, "server", "localhost:7000", "index server or scan server address")
	flag.StringVar(&opType, "type", "scanAll", "Index command (scan|stats|scanAll|create|drop|list)")
	flag.StringVar(&indexName, "index", "", "Index name")
	flag.StringVar(&bucket, "bucket", "default", "Bucket name")
	flag.StringVar(&low, "low", "", "Range: [low]")
	flag.StringVar(&high, "high", "", "Range: [high]")
	flag.StringVar(&equal, "equal", "", "Range: [key]")
	flag.UintVar(&incl, "incl", 0, "Range: 0|1|2|3")
	flag.Int64Var(&limit, "limit", 10, "Row limit")
	flag.Int64Var(&pageSize, "buffersz", 0, "Rows buffer size per internal message")
	flag.StringVar(&fields, "fields", "", "Comma separated on-index fields")
	flag.BoolVar(&isPrimary, "primary", false, "Is primary index")
	flag.StringVar(&instanceId, "instanceid", "", "Index instanceId")

	flag.Parse()
}

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s -type scanAll -index idx1 -bucket default\n", os.Args[0])
	flag.PrintDefaults()
}

func main() {
	var err error
	var statsResp *protobuf.IndexStatistics
	var keys [][]byte

	parseArgs()

	switch opType {

	case "create":

		if !isPrimary && (fields == "" || indexName == "") {
			fmt.Println("Invalid fields or index name")
			usage()
			os.Exit(1)
		}

		client := queryclient.NewClusterClient(server)
		var secExprs []string
		fields := strings.Split(fields, ",")
		for _, field := range fields {
			expr, err := n1ql.ParseExpression(field)
			if err != nil {
				fmt.Printf("Error occured: Invalid field (%v) %v ", field, err)
			}

			secExprs = append(secExprs, expression.NewStringer().Visit(expr))
		}

		info, err := client.CreateIndex(indexName, bucket, using, exprType, partnExp, where, secExprs, isPrimary)
		if err == nil {
			fmt.Println("Index created")
			printIndexInfo(*info)
		} else {
			fmt.Println("Error occured:", err)
		}

	case "drop":
		if instanceId == "" {
			fmt.Println("Invalid instanceId")
			usage()
			os.Exit(1)
		}

		client := queryclient.NewClusterClient(server)
		err := client.DropIndex(instanceId)
		if err == nil {
			fmt.Println("Index dropped")
		} else {
			fmt.Println("Error occured:", err)
		}
	case "list":
		client := queryclient.NewClusterClient(server)
		infos, err := client.List()
		if err != nil {
			fmt.Println("Error occured:", err)
		}

		fmt.Println("Indexes:")
		for _, info := range infos {
			printIndexInfo(info)
		}

	default:
		if indexName == "" {
			usage()
			os.Exit(1)
		}
		config := c.SystemConfig.SectionConfig("queryport.client.", true)
		client := queryclient.NewClient(server, config)
		if equal != "" {
			keys = append(keys, []byte(equal))
		}

		switch opType {
		case "scan":
			err = client.Scan(indexName, bucket, []byte(low), []byte(high), keys, uint32(incl), pageSize, false, limit, scanCallback)
		case "scanAll":
			err = client.ScanAll(indexName, bucket, pageSize, limit, scanCallback)
		case "stats":
			statsResp, err = client.Statistics(indexName, bucket, []byte(low), []byte(high), keys, uint32(incl))
			if err == nil {
				fmt.Println("Stats: ", statsResp)
			}
		}

		if err != nil {
			fmt.Fprintf(os.Stderr, "Error occured %v\n", err)
		}

		client.Close()
	}
}

func scanCallback(res interface{}) bool {
	switch r := res.(type) {
	case *protobuf.ResponseStream:
		fmt.Println("StreamResponse: ", res.(*protobuf.ResponseStream).String())
	case error:
		fmt.Println("Error: ", r)
	}
	return true
}

func printIndexInfo(info queryclient.IndexInfo) {
	fmt.Printf("Index:%s/%s, Id:%s, Using:%s, Exprs:%v, isPrimary:%v\n",
		info.Name, info.Bucket, info.DefnID, info.Using, info.SecExprs, info.IsPrimary)
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package featuretests

import (
	"bytes"
	"fmt"

	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"
	"gopkg.in/juju/charm.v6-unstable"
	"gopkg.in/juju/charmrepo.v2-unstable"

	"github.com/juju/juju/cmd/juju/crossmodel"
	jujutesting "github.com/juju/juju/juju/testing"
	"github.com/juju/juju/permission"
	"github.com/juju/juju/state"
	"github.com/juju/juju/testcharms"
	"github.com/juju/juju/testing"
	"github.com/juju/juju/testing/factory"
)

type crossmodelSuite struct {
	jujutesting.JujuConnSuite
}

func (s *crossmodelSuite) TestListEndpoints(c *gc.C) {
	ch := s.AddTestingCharm(c, "riak")
	s.AddTestingService(c, "riakservice", ch)
	ch = s.AddTestingCharm(c, "varnish")
	s.AddTestingService(c, "varnishservice", ch)

	_, err := testing.RunCommand(c, crossmodel.NewOfferCommand(),
		"riakservice:endpoint", "local:/u/me/riak")
	c.Assert(err, jc.ErrorIsNil)
	_, err = testing.RunCommand(c, crossmodel.NewOfferCommand(),
		"varnishservice:webcache", "local:/u/me/varnish")
	c.Assert(err, jc.ErrorIsNil)

	// TODO(wallyworld) - list with filters when supported
	ctx, err := testing.RunCommand(c, crossmodel.NewListEndpointsCommand(),
		"--format", "yaml")
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(ctx.Stdout.(*bytes.Buffer).String(), gc.Equals, `
local:
  riak:
    store: local
    url: /u/me/riak
    endpoints:
      endpoint:
        interface: http
        role: provider
  varnish:
    store: local
    url: /u/me/varnish
    endpoints:
      webcache:
        interface: varnish
        role: provider
`[1:])
}

func (s *crossmodelSuite) TestShow(c *gc.C) {
	ch := s.AddTestingCharm(c, "riak")
	s.AddTestingService(c, "riakservice", ch)
	ch = s.AddTestingCharm(c, "varnish")
	s.AddTestingService(c, "varnishservice", ch)

	_, err := testing.RunCommand(c, crossmodel.NewOfferCommand(),
		"riakservice:endpoint", "local:/u/me/riak")
	c.Assert(err, jc.ErrorIsNil)
	_, err = testing.RunCommand(c, crossmodel.NewOfferCommand(),
		"varnishservice:webcache", "local:/u/me/varnish")
	c.Assert(err, jc.ErrorIsNil)

	// TODO(wallyworld) - list with filters when supported
	ctx, err := testing.RunCommand(c, crossmodel.NewShowOfferedEndpointCommand(),
		"local:/u/me/varnish", "--format", "yaml")
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(ctx.Stdout.(*bytes.Buffer).String(), gc.Equals, `
varnishservice:
  endpoints:
    webcache:
      interface: varnish
      role: provider
  description: Another popular database
`[1:])
}

func (s *crossmodelSuite) TestFind(c *gc.C) {
	ch := s.AddTestingCharm(c, "riak")
	s.AddTestingService(c, "riakservice", ch)
	ch = s.AddTestingCharm(c, "varnish")
	s.AddTestingService(c, "varnishservice", ch)

	_, err := testing.RunCommand(c, crossmodel.NewOfferCommand(),
		"riakservice:endpoint", "local:/u/you/riak")
	c.Assert(err, jc.ErrorIsNil)
	_, err = testing.RunCommand(c, crossmodel.NewOfferCommand(),
		"varnishservice:webcache", "local:/u/me/varnish")
	c.Assert(err, jc.ErrorIsNil)

	// TODO(wallyworld) - find with interface and endpoint name filters when supported
	ctx, err := testing.RunCommand(c, crossmodel.NewFindEndpointsCommand(),
		"local:/u/me", "--format", "yaml")
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(ctx.Stdout.(*bytes.Buffer).String(), gc.Equals, `
local:/u/me/varnish:
  endpoints:
    webcache:
      interface: varnish
      role: provider
`[1:])
}

func (s *crossmodelSuite) TestAddRelationFromURL(c *gc.C) {
	c.Skip("add relation from URL not currently supported")

	ch := s.AddTestingCharm(c, "wordpress")
	s.AddTestingService(c, "wordpress", ch)
	ch = s.AddTestingCharm(c, "mysql")
	s.AddTestingService(c, "mysql", ch)

	_, err := testing.RunCommand(c, crossmodel.NewOfferCommand(),
		"mysql:server", "local:/u/me/hosted-mysql")
	c.Assert(err, jc.ErrorIsNil)
	_, err = runJujuCommand(c, "add-relation", "wordpress", "local:/u/me/hosted-mysql")
	c.Assert(err, jc.ErrorIsNil)
	svc, err := s.State.RemoteApplication("hosted-mysql")
	c.Assert(err, jc.ErrorIsNil)
	rel, err := svc.Relations()
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(rel, gc.HasLen, 1)
	c.Assert(rel[0].Endpoints(), jc.SameContents, []state.Endpoint{
		{
			ApplicationName: "wordpress",
			Relation: charm.Relation{
				Name:      "db",
				Role:      "requirer",
				Interface: "mysql",
				Limit:     1,
				Scope:     "global",
			},
		}, {
			ApplicationName: "hosted-mysql",
			Relation: charm.Relation{Name: "server",
				Role:      "provider",
				Interface: "mysql",
				Scope:     "global"},
		},
	})
}

func (s *crossmodelSuite) assertAddRelationSameControllerSuccess(c *gc.C, otherModeluser string) {
	_, err := runJujuCommand(c, "add-relation", "wordpress", otherModeluser+"/othermodel.mysql")
	c.Assert(err, jc.ErrorIsNil)
	svc, err := s.State.RemoteApplication("mysql")
	c.Assert(err, jc.ErrorIsNil)
	rel, err := svc.Relations()
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(rel, gc.HasLen, 1)
	c.Assert(rel[0].Endpoints(), jc.SameContents, []state.Endpoint{
		{
			ApplicationName: "wordpress",
			Relation: charm.Relation{
				Name:      "db",
				Role:      "requirer",
				Interface: "mysql",
				Limit:     1,
				Scope:     "global",
			},
		}, {
			ApplicationName: "mysql",
			Relation: charm.Relation{Name: "server",
				Role:      "provider",
				Interface: "mysql",
				Scope:     "global"},
		},
	})
}

func (s *crossmodelSuite) TestAddRelationSameControllerSameOwner(c *gc.C) {
	ch := s.AddTestingCharm(c, "wordpress")
	s.AddTestingService(c, "wordpress", ch)

	otherModel := s.Factory.MakeModel(c, &factory.ModelParams{Name: "othermodel"})
	s.AddCleanup(func(*gc.C) { otherModel.Close() })

	mysql := testcharms.Repo.CharmDir("mysql")
	ident := fmt.Sprintf("%s-%d", mysql.Meta().Name, mysql.Revision())
	curl := charm.MustParseURL("local:quantal/" + ident)
	repo, err := charmrepo.InferRepository(
		curl,
		charmrepo.NewCharmStoreParams{},
		testcharms.Repo.Path())
	c.Assert(err, jc.ErrorIsNil)
	ch, err = jujutesting.PutCharm(otherModel, curl, repo, false)
	c.Assert(err, jc.ErrorIsNil)
	_, err = otherModel.AddApplication(state.AddApplicationArgs{
		Name:  "mysql",
		Charm: ch,
	})
	s.assertAddRelationSameControllerSuccess(c, "admin")
}

func (s *crossmodelSuite) TestAddRelationSameControllerPermissionDenied(c *gc.C) {
	ch := s.AddTestingCharm(c, "wordpress")
	s.AddTestingService(c, "wordpress", ch)

	otherOwner := s.Factory.MakeUser(c, &factory.UserParams{Name: "otheruser"})
	otherModel := s.Factory.MakeModel(c, &factory.ModelParams{Name: "othermodel", Owner: otherOwner.Tag()})
	s.AddCleanup(func(*gc.C) { otherModel.Close() })

	mysql := testcharms.Repo.CharmDir("mysql")
	ident := fmt.Sprintf("%s-%d", mysql.Meta().Name, mysql.Revision())
	curl := charm.MustParseURL("local:quantal/" + ident)
	repo, err := charmrepo.InferRepository(
		curl,
		charmrepo.NewCharmStoreParams{},
		testcharms.Repo.Path())
	c.Assert(err, jc.ErrorIsNil)
	ch, err = jujutesting.PutCharm(otherModel, curl, repo, false)
	c.Assert(err, jc.ErrorIsNil)
	_, err = otherModel.AddApplication(state.AddApplicationArgs{
		Name:  "mysql",
		Charm: ch,
	})

	context, err := runJujuCommand(c, "add-relation", "wordpress", "otheruser/othermodel.mysql")
	c.Assert(err, gc.NotNil)
	c.Assert(testing.Stderr(context), jc.Contains, "You do not have permission to add a relation")
}

func (s *crossmodelSuite) TestAddRelationSameControllerPermissionAllowed(c *gc.C) {
	ch := s.AddTestingCharm(c, "wordpress")
	s.AddTestingService(c, "wordpress", ch)

	otherOwner := s.Factory.MakeUser(c, &factory.UserParams{Name: "otheruser"})
	otherModel := s.Factory.MakeModel(c, &factory.ModelParams{Name: "othermodel", Owner: otherOwner.Tag()})
	s.AddCleanup(func(*gc.C) { otherModel.Close() })

	// Users with write permission to the model can add relations.
	otherFactory := factory.NewFactory(otherModel)
	otherFactory.MakeModelUser(c, &factory.ModelUserParams{User: "admin", Access: permission.WriteAccess})

	mysql := testcharms.Repo.CharmDir("mysql")
	ident := fmt.Sprintf("%s-%d", mysql.Meta().Name, mysql.Revision())
	curl := charm.MustParseURL("local:quantal/" + ident)
	repo, err := charmrepo.InferRepository(
		curl,
		charmrepo.NewCharmStoreParams{},
		testcharms.Repo.Path())
	c.Assert(err, jc.ErrorIsNil)
	ch, err = jujutesting.PutCharm(otherModel, curl, repo, false)
	c.Assert(err, jc.ErrorIsNil)
	_, err = otherModel.AddApplication(state.AddApplicationArgs{
		Name:  "mysql",
		Charm: ch,
	})
	s.assertAddRelationSameControllerSuccess(c, "otheruser")
}
<commit_msg>Fix tests<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package featuretests

import (
	"bytes"
	"fmt"

	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"
	"gopkg.in/juju/charm.v6-unstable"
	"gopkg.in/juju/charmrepo.v2-unstable"

	"github.com/juju/juju/cmd/juju/crossmodel"
	jujucrossmodel "github.com/juju/juju/core/crossmodel"
	jujutesting "github.com/juju/juju/juju/testing"
	"github.com/juju/juju/permission"
	"github.com/juju/juju/state"
	"github.com/juju/juju/testcharms"
	"github.com/juju/juju/testing"
	"github.com/juju/juju/testing/factory"
)

type crossmodelSuite struct {
	jujutesting.JujuConnSuite
}

func (s *crossmodelSuite) TestListEndpoints(c *gc.C) {
	ch := s.AddTestingCharm(c, "riak")
	s.AddTestingService(c, "riakservice", ch)
	ch = s.AddTestingCharm(c, "varnish")
	s.AddTestingService(c, "varnishservice", ch)

	_, err := testing.RunCommand(c, crossmodel.NewOfferCommand(),
		"riakservice:endpoint", "local:/u/me/riak")
	c.Assert(err, jc.ErrorIsNil)
	_, err = testing.RunCommand(c, crossmodel.NewOfferCommand(),
		"varnishservice:webcache", "local:/u/me/varnish")
	c.Assert(err, jc.ErrorIsNil)

	// TODO(wallyworld) - list with filters when supported
	ctx, err := testing.RunCommand(c, crossmodel.NewListEndpointsCommand(),
		"--format", "yaml")
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(ctx.Stdout.(*bytes.Buffer).String(), gc.Equals, `
local:
  riak:
    store: local
    url: /u/me/riak
    endpoints:
      endpoint:
        interface: http
        role: provider
  varnish:
    store: local
    url: /u/me/varnish
    endpoints:
      webcache:
        interface: varnish
        role: provider
`[1:])
}

func (s *crossmodelSuite) TestShow(c *gc.C) {
	ch := s.AddTestingCharm(c, "riak")
	s.AddTestingService(c, "riakservice", ch)
	ch = s.AddTestingCharm(c, "varnish")
	s.AddTestingService(c, "varnishservice", ch)

	_, err := testing.RunCommand(c, crossmodel.NewOfferCommand(),
		"riakservice:endpoint", "local:/u/me/riak")
	c.Assert(err, jc.ErrorIsNil)
	_, err = testing.RunCommand(c, crossmodel.NewOfferCommand(),
		"varnishservice:webcache", "local:/u/me/varnish")
	c.Assert(err, jc.ErrorIsNil)

	// TODO(wallyworld) - list with filters when supported
	ctx, err := testing.RunCommand(c, crossmodel.NewShowOfferedEndpointCommand(),
		"local:/u/me/varnish", "--format", "yaml")
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(ctx.Stdout.(*bytes.Buffer).String(), gc.Equals, `
varnishservice:
  endpoints:
    webcache:
      interface: varnish
      role: provider
  description: Another popular database
`[1:])
}

func (s *crossmodelSuite) TestFind(c *gc.C) {
	ch := s.AddTestingCharm(c, "riak")
	s.AddTestingService(c, "riakservice", ch)
	ch = s.AddTestingCharm(c, "varnish")
	s.AddTestingService(c, "varnishservice", ch)

	_, err := testing.RunCommand(c, crossmodel.NewOfferCommand(),
		"riakservice:endpoint", "local:/u/you/riak")
	c.Assert(err, jc.ErrorIsNil)
	_, err = testing.RunCommand(c, crossmodel.NewOfferCommand(),
		"varnishservice:webcache", "local:/u/me/varnish")
	c.Assert(err, jc.ErrorIsNil)

	// TODO(wallyworld) - find with interface and endpoint name filters when supported
	ctx, err := testing.RunCommand(c, crossmodel.NewFindEndpointsCommand(),
		"local:/u/me", "--format", "yaml")
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(ctx.Stdout.(*bytes.Buffer).String(), gc.Equals, `
local:/u/me/varnish:
  endpoints:
    webcache:
      interface: varnish
      role: provider
`[1:])
}

func (s *crossmodelSuite) TestAddRelationFromURL(c *gc.C) {
	c.Skip("add relation from URL not currently supported")

	ch := s.AddTestingCharm(c, "wordpress")
	s.AddTestingService(c, "wordpress", ch)
	ch = s.AddTestingCharm(c, "mysql")
	s.AddTestingService(c, "mysql", ch)

	_, err := testing.RunCommand(c, crossmodel.NewOfferCommand(),
		"mysql:server", "local:/u/me/hosted-mysql")
	c.Assert(err, jc.ErrorIsNil)
	_, err = runJujuCommand(c, "add-relation", "wordpress", "local:/u/me/hosted-mysql")
	c.Assert(err, jc.ErrorIsNil)
	svc, err := s.State.RemoteApplication("hosted-mysql")
	c.Assert(err, jc.ErrorIsNil)
	rel, err := svc.Relations()
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(rel, gc.HasLen, 1)
	c.Assert(rel[0].Endpoints(), jc.SameContents, []state.Endpoint{
		{
			ApplicationName: "wordpress",
			Relation: charm.Relation{
				Name:      "db",
				Role:      "requirer",
				Interface: "mysql",
				Limit:     1,
				Scope:     "global",
			},
		}, {
			ApplicationName: "hosted-mysql",
			Relation: charm.Relation{Name: "server",
				Role:      "provider",
				Interface: "mysql",
				Scope:     "global"},
		},
	})
}

func (s *crossmodelSuite) assertAddRelationSameControllerSuccess(c *gc.C, otherModeluser string) {
	_, err := runJujuCommand(c, "add-relation", "wordpress", otherModeluser+"/othermodel.hosted-mysql")
	c.Assert(err, jc.ErrorIsNil)
	app, err := s.State.RemoteApplication("hosted-mysql")
	c.Assert(err, jc.ErrorIsNil)
	rel, err := app.Relations()
	c.Assert(err, jc.ErrorIsNil)
	c.Assert(rel, gc.HasLen, 1)
	c.Assert(rel[0].Endpoints(), jc.SameContents, []state.Endpoint{
		{
			ApplicationName: "wordpress",
			Relation: charm.Relation{
				Name:      "db",
				Role:      "requirer",
				Interface: "mysql",
				Limit:     1,
				Scope:     "global",
			},
		}, {
			ApplicationName: "hosted-mysql",
			Relation: charm.Relation{Name: "server",
				Role:      "provider",
				Interface: "mysql",
				Scope:     "global"},
		},
	})
}

func (s *crossmodelSuite) TestAddRelationSameControllerSameOwner(c *gc.C) {
	ch := s.AddTestingCharm(c, "wordpress")
	s.AddTestingService(c, "wordpress", ch)

	otherModel := s.Factory.MakeModel(c, &factory.ModelParams{Name: "othermodel"})
	s.AddCleanup(func(*gc.C) { otherModel.Close() })

	mysql := testcharms.Repo.CharmDir("mysql")
	ident := fmt.Sprintf("%s-%d", mysql.Meta().Name, mysql.Revision())
	curl := charm.MustParseURL("local:quantal/" + ident)
	repo, err := charmrepo.InferRepository(
		curl,
		charmrepo.NewCharmStoreParams{},
		testcharms.Repo.Path())
	c.Assert(err, jc.ErrorIsNil)
	ch, err = jujutesting.PutCharm(otherModel, curl, repo, false)
	c.Assert(err, jc.ErrorIsNil)
	_, err = otherModel.AddApplication(state.AddApplicationArgs{
		Name:  "mysql",
		Charm: ch,
	})
	c.Assert(err, jc.ErrorIsNil)
	offersAPi := state.NewApplicationOffers(otherModel)
	_, err = offersAPi.AddOffer(jujucrossmodel.AddApplicationOfferArgs{
		ApplicationURL:  "local:/u/me/hosted-mysql",
		ApplicationName: "mysql",
		Endpoints:       map[string]string{"database": "server"},
	})
	c.Assert(err, jc.ErrorIsNil)
	s.assertAddRelationSameControllerSuccess(c, "admin")
}

func (s *crossmodelSuite) TestAddRelationSameControllerPermissionDenied(c *gc.C) {
	ch := s.AddTestingCharm(c, "wordpress")
	s.AddTestingService(c, "wordpress", ch)

	otherOwner := s.Factory.MakeUser(c, &factory.UserParams{Name: "otheruser"})
	otherModel := s.Factory.MakeModel(c, &factory.ModelParams{Name: "othermodel", Owner: otherOwner.Tag()})
	s.AddCleanup(func(*gc.C) { otherModel.Close() })

	mysql := testcharms.Repo.CharmDir("mysql")
	ident := fmt.Sprintf("%s-%d", mysql.Meta().Name, mysql.Revision())
	curl := charm.MustParseURL("local:quantal/" + ident)
	repo, err := charmrepo.InferRepository(
		curl,
		charmrepo.NewCharmStoreParams{},
		testcharms.Repo.Path())
	c.Assert(err, jc.ErrorIsNil)
	ch, err = jujutesting.PutCharm(otherModel, curl, repo, false)
	c.Assert(err, jc.ErrorIsNil)
	_, err = otherModel.AddApplication(state.AddApplicationArgs{
		Name:  "mysql",
		Charm: ch,
	})

	context, err := runJujuCommand(c, "add-relation", "wordpress", "otheruser/othermodel.mysql")
	c.Assert(err, gc.NotNil)
	c.Assert(testing.Stderr(context), jc.Contains, "You do not have permission to add a relation")
}

func (s *crossmodelSuite) TestAddRelationSameControllerPermissionAllowed(c *gc.C) {
	ch := s.AddTestingCharm(c, "wordpress")
	s.AddTestingService(c, "wordpress", ch)

	otherOwner := s.Factory.MakeUser(c, &factory.UserParams{Name: "otheruser"})
	otherModel := s.Factory.MakeModel(c, &factory.ModelParams{Name: "othermodel", Owner: otherOwner.Tag()})
	s.AddCleanup(func(*gc.C) { otherModel.Close() })

	// Users with write permission to the model can add relations.
	otherFactory := factory.NewFactory(otherModel)
	otherFactory.MakeModelUser(c, &factory.ModelUserParams{User: "admin", Access: permission.WriteAccess})

	mysql := testcharms.Repo.CharmDir("mysql")
	ident := fmt.Sprintf("%s-%d", mysql.Meta().Name, mysql.Revision())
	curl := charm.MustParseURL("local:quantal/" + ident)
	repo, err := charmrepo.InferRepository(
		curl,
		charmrepo.NewCharmStoreParams{},
		testcharms.Repo.Path())
	c.Assert(err, jc.ErrorIsNil)
	ch, err = jujutesting.PutCharm(otherModel, curl, repo, false)
	c.Assert(err, jc.ErrorIsNil)
	_, err = otherModel.AddApplication(state.AddApplicationArgs{
		Name:  "mysql",
		Charm: ch,
	})
	c.Assert(err, jc.ErrorIsNil)
	offersAPi := state.NewApplicationOffers(otherModel)
	_, err = offersAPi.AddOffer(jujucrossmodel.AddApplicationOfferArgs{
		ApplicationURL:  "local:/u/me/hosted-mysql",
		ApplicationName: "mysql",
		Endpoints:       map[string]string{"database": "server"},
	})
	c.Assert(err, jc.ErrorIsNil)
	s.assertAddRelationSameControllerSuccess(c, "otheruser")
}
<|endoftext|>


<commit_before>package models

import (
	"database/sql"
	"errors"
	sq "github.com/Masterminds/squirrel"
	log "github.com/Sirupsen/logrus"
	"net/http"
	"time"
)

// PostgreSQL Statement Builder instance
var psql = sq.StatementBuilder.PlaceholderFormat(sq.Dollar)

// Default and maximum number of datum returned by bulk API queries
// Used when obtaining and displaying many datum of a given structure
const defaultNumResults uint64 = 30

type emptyQuery struct{}

func (_ emptyQuery) ToSql() (string, []interface{}, error) {
	return "", nil, nil
}

// getExecResultCode is a standard way to extract an HTTP error out of an SQL result
func getExecResultCode(result sql.Result, err error) (int, error) {
	if err != nil {
		return http.StatusInternalServerError, err
	}
	numRows, err := result.RowsAffected()
	if err != nil {
		return http.StatusInternalServerError, err
	}
	if numRows == 0 {
		return http.StatusNotFound, sql.ErrNoRows
	}
	if numRows != 1 {
		return http.StatusInternalServerError, errors.New("multiple rows affected")
	}
	return http.StatusNoContent, nil
}

// getExecDoNothingResultCode is like getExecResultCode,
// but should be used when the SQL operation affecting no rows is not an error
func getExecDoNothingResultCode(result sql.Result, err error) (int, error) {
	code, err := getExecResultCode
	if code == http.StatusNotFound {
		return http.StatusNoContent, nil
	}
	return code, err
}

// https://coderwall.com/p/cp5fya/measuring-execution-time-in-go
func logTime(start time.Time, name string) {
	elapsed := time.Since(start)
	log.WithField("took", elapsed).
		Infof("completed execution of %s", name)
}

// https://goo.gl/BPVkA6
func stringUnique(s []string) []string {
	seen := make(map[string]struct{}, len(s))
	j := 0
	for _, v := range s {
		if _, ok := seen[v]; ok {
			continue
		}
		seen[v] = struct{}{}
		s[j] = v
		j++
	}
	return s[:j]
}
<commit_msg>im dumb<commit_after>package models

import (
	"database/sql"
	"errors"
	sq "github.com/Masterminds/squirrel"
	log "github.com/Sirupsen/logrus"
	"net/http"
	"time"
)

// PostgreSQL Statement Builder instance
var psql = sq.StatementBuilder.PlaceholderFormat(sq.Dollar)

// Default and maximum number of datum returned by bulk API queries
// Used when obtaining and displaying many datum of a given structure
const defaultNumResults uint64 = 30

type emptyQuery struct{}

func (_ emptyQuery) ToSql() (string, []interface{}, error) {
	return "", nil, nil
}

// getExecResultCode is a standard way to extract an HTTP error out of an SQL result
func getExecResultCode(result sql.Result, err error) (int, error) {
	if err != nil {
		return http.StatusInternalServerError, err
	}
	numRows, err := result.RowsAffected()
	if err != nil {
		return http.StatusInternalServerError, err
	}
	if numRows == 0 {
		return http.StatusNotFound, sql.ErrNoRows
	}
	if numRows != 1 {
		return http.StatusInternalServerError, errors.New("multiple rows affected")
	}
	return http.StatusNoContent, nil
}

// getExecDoNothingResultCode is like getExecResultCode,
// but should be used when the SQL operation affecting no rows is not an error
func getExecDoNothingResultCode(result sql.Result, err error) (int, error) {
	code, err := getExecResultCode(result, err)
	if code == http.StatusNotFound {
		return http.StatusNoContent, nil
	}
	return code, err
}

// https://coderwall.com/p/cp5fya/measuring-execution-time-in-go
func logTime(start time.Time, name string) {
	elapsed := time.Since(start)
	log.WithField("took", elapsed).
		Infof("completed execution of %s", name)
}

// https://goo.gl/BPVkA6
func stringUnique(s []string) []string {
	seen := make(map[string]struct{}, len(s))
	j := 0
	for _, v := range s {
		if _, ok := seen[v]; ok {
			continue
		}
		seen[v] = struct{}{}
		s[j] = v
		j++
	}
	return s[:j]
}
<|endoftext|>