<commit_before>package client

import (
	"fmt"
	"text/tabwriter"

	"golang.org/x/net/context"

	"github.com/docker/engine-api/types"
	"github.com/docker/engine-api/types/filters"
	Cli "github.com/hyperhq/hypercli/cli"
	"github.com/hyperhq/hypercli/opts"
	flag "github.com/hyperhq/hypercli/pkg/mflag"
)

// CmdSnapshot is the parent subcommand for all snapshot commands
//
// Usage: docker snapshot <COMMAND> <OPTS>
func (cli *DockerCli) CmdSnapshot(args ...string) error {
	description := Cli.DockerCommands["snaphot"].Description + "\n\nSnapshots:\n"
	commands := [][]string{
		{"create", "Create a snaphot"},
		{"inspect", "Return low-level information on a snaphot"},
		{"ls", "List snaphots"},
		{"rm", "Remove a snaphot"},
	}

	for _, cmd := range commands {
		description += fmt.Sprintf("  %-25.25s%s\n", cmd[0], cmd[1])
	}

	description += "\nRun 'hyper snaphot COMMAND --help' for more information on a command"
	cmd := Cli.Subcmd("snaphot", []string{"[COMMAND]"}, description, false)

	cmd.Require(flag.Exact, 0)
	err := cmd.ParseFlags(args, true)
	cmd.Usage()
	return err
}

// CmdSnapshotLs outputs a list of Docker snapshots.
//
// Usage: docker snapshot ls [OPTIONS]
func (cli *DockerCli) CmdSnapshotLs(args ...string) error {
	cmd := Cli.Subcmd("snapshot ls", nil, "List snapshots", true)

	quiet := cmd.Bool([]string{"q", "-quiet"}, false, "Only display snapshot names")
	flFilter := opts.NewListOpts(nil)
	cmd.Var(&flFilter, []string{"f", "-filter"}, "Provide filter values (i.e. 'dangling=true')")

	cmd.Require(flag.Exact, 0)
	cmd.ParseFlags(args, true)

	volFilterArgs := filters.NewArgs()
	for _, f := range flFilter.GetAll() {
		var err error
		volFilterArgs, err = filters.ParseFlag(f, volFilterArgs)
		if err != nil {
			return err
		}
	}

	snapshots, err := cli.client.SnapshotList(context.Background(), volFilterArgs)
	if err != nil {
		return err
	}

	w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
	if !*quiet {
		for _, warn := range snapshots.Warnings {
			fmt.Fprintln(cli.err, warn)
		}
		fmt.Fprintf(w, "Snapshot Name \tVolume\tSize")
		fmt.Fprintf(w, "\n")
	}

	for _, vol := range snapshots.Snapshots {
		if *quiet {
			fmt.Fprintln(w, vol.Name)
			continue
		}
		fmt.Fprintf(w, "%s\t%s\t%d\n", vol.Name, vol.Volume, vol.Size)
	}
	w.Flush()
	return nil
}

// CmdSnapshotInspect displays low-level information on one or more snapshots.
//
// Usage: docker snapshot inspect [OPTIONS] snapshot [snapshot...]
func (cli *DockerCli) CmdSnapshotInspect(args ...string) error {
	cmd := Cli.Subcmd("snapshot inspect", []string{"snapshot [snapshot...]"}, "Return low-level information on a snapshot", true)
	tmplStr := cmd.String([]string{"f", "-format"}, "", "Format the output using the given go template")

	cmd.Require(flag.Min, 1)
	cmd.ParseFlags(args, true)

	if err := cmd.Parse(args); err != nil {
		return nil
	}

	inspectSearcher := func(name string) (interface{}, []byte, error) {
		i, err := cli.client.SnapshotInspect(context.Background(), name)
		return i, nil, err
	}

	return cli.inspectElements(*tmplStr, cmd.Args(), inspectSearcher)
}

// CmdSnapshotCreate creates a new snapshot.
//
// Usage: docker snapshot create [OPTIONS]
func (cli *DockerCli) CmdSnapshotCreate(args ...string) error {
	cmd := Cli.Subcmd("snapshot create", []string{"-v volume"}, "Create a snapshot", true)
	flForce := cmd.Bool([]string{"f", "-force"}, false, "Force to create snapshot, needed if volume is in use")
	flVolume := cmd.String([]string{"v", "-volume"}, "", "Specify volume to create snapshot")
	flName := cmd.String([]string{"-name"}, "", "Specify snapshot name")

	cmd.Require(flag.Exact, 0)
	cmd.ParseFlags(args, true)

	volReq := types.SnapshotCreateRequest{
		Name:   *flName,
		Volume: *flVolume,
		Force:  *flForce,
	}

	vol, err := cli.client.SnapshotCreate(context.Background(), volReq)
	if err != nil {
		return err
	}

	fmt.Fprintf(cli.out, "%s\n", vol.Name)
	return nil
}

// CmdSnapshotRm removes one or more snapshots.
//
// Usage: docker snapshot rm snapshot [snapshot...]
func (cli *DockerCli) CmdSnapshotRm(args ...string) error {
	cmd := Cli.Subcmd("snapshot rm", []string{"snapshot [snapshot...]"}, "Remove a snapshot", true)
	cmd.Require(flag.Min, 1)
	cmd.ParseFlags(args, true)

	var status = 0

	for _, name := range cmd.Args() {
		if err := cli.client.SnapshotRemove(context.Background(), name); err != nil {
			fmt.Fprintf(cli.err, "%s\n", err)
			status = 1
			continue
		}
		fmt.Fprintf(cli.out, "%s\n", name)
	}

	if status != 0 {
		return Cli.StatusError{StatusCode: status}
	}
	return nil
}
<commit_msg>more typos<commit_after>package client

import (
	"fmt"
	"text/tabwriter"

	"golang.org/x/net/context"

	"github.com/docker/engine-api/types"
	"github.com/docker/engine-api/types/filters"
	Cli "github.com/hyperhq/hypercli/cli"
	"github.com/hyperhq/hypercli/opts"
	flag "github.com/hyperhq/hypercli/pkg/mflag"
)

// CmdSnapshot is the parent subcommand for all snapshot commands
//
// Usage: docker snapshot <COMMAND> <OPTS>
func (cli *DockerCli) CmdSnapshot(args ...string) error {
	description := Cli.DockerCommands["snapshot"].Description + "\n\nSnapshots:\n"
	commands := [][]string{
		{"create", "Create a snapshot"},
		{"inspect", "Return low-level information on a snapshot"},
		{"ls", "List snapshots"},
		{"rm", "Remove a snapshot"},
	}

	for _, cmd := range commands {
		description += fmt.Sprintf("  %-25.25s%s\n", cmd[0], cmd[1])
	}

	description += "\nRun 'hyper snapshot COMMAND --help' for more information on a command"
	cmd := Cli.Subcmd("snapshot", []string{"[COMMAND]"}, description, false)

	cmd.Require(flag.Exact, 0)
	err := cmd.ParseFlags(args, true)
	cmd.Usage()
	return err
}

// CmdSnapshotLs outputs a list of Docker snapshots.
//
// Usage: docker snapshot ls [OPTIONS]
func (cli *DockerCli) CmdSnapshotLs(args ...string) error {
	cmd := Cli.Subcmd("snapshot ls", nil, "List snapshots", true)

	quiet := cmd.Bool([]string{"q", "-quiet"}, false, "Only display snapshot names")
	flFilter := opts.NewListOpts(nil)
	cmd.Var(&flFilter, []string{"f", "-filter"}, "Provide filter values (i.e. 'dangling=true')")

	cmd.Require(flag.Exact, 0)
	cmd.ParseFlags(args, true)

	volFilterArgs := filters.NewArgs()
	for _, f := range flFilter.GetAll() {
		var err error
		volFilterArgs, err = filters.ParseFlag(f, volFilterArgs)
		if err != nil {
			return err
		}
	}

	snapshots, err := cli.client.SnapshotList(context.Background(), volFilterArgs)
	if err != nil {
		return err
	}

	w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0)
	if !*quiet {
		for _, warn := range snapshots.Warnings {
			fmt.Fprintln(cli.err, warn)
		}
		fmt.Fprintf(w, "Snapshot Name \tVolume\tSize")
		fmt.Fprintf(w, "\n")
	}

	for _, vol := range snapshots.Snapshots {
		if *quiet {
			fmt.Fprintln(w, vol.Name)
			continue
		}
		fmt.Fprintf(w, "%s\t%s\t%d\n", vol.Name, vol.Volume, vol.Size)
	}
	w.Flush()
	return nil
}

// CmdSnapshotInspect displays low-level information on one or more snapshots.
//
// Usage: docker snapshot inspect [OPTIONS] snapshot [snapshot...]
func (cli *DockerCli) CmdSnapshotInspect(args ...string) error {
	cmd := Cli.Subcmd("snapshot inspect", []string{"snapshot [snapshot...]"}, "Return low-level information on a snapshot", true)
	tmplStr := cmd.String([]string{"f", "-format"}, "", "Format the output using the given go template")

	cmd.Require(flag.Min, 1)
	cmd.ParseFlags(args, true)

	if err := cmd.Parse(args); err != nil {
		return nil
	}

	inspectSearcher := func(name string) (interface{}, []byte, error) {
		i, err := cli.client.SnapshotInspect(context.Background(), name)
		return i, nil, err
	}

	return cli.inspectElements(*tmplStr, cmd.Args(), inspectSearcher)
}

// CmdSnapshotCreate creates a new snapshot.
//
// Usage: docker snapshot create [OPTIONS]
func (cli *DockerCli) CmdSnapshotCreate(args ...string) error {
	cmd := Cli.Subcmd("snapshot create", []string{"-v volume"}, "Create a snapshot", true)
	flForce := cmd.Bool([]string{"f", "-force"}, false, "Force to create snapshot, needed if volume is in use")
	flVolume := cmd.String([]string{"v", "-volume"}, "", "Specify volume to create snapshot")
	flName := cmd.String([]string{"-name"}, "", "Specify snapshot name")

	cmd.Require(flag.Exact, 0)
	cmd.ParseFlags(args, true)

	volReq := types.SnapshotCreateRequest{
		Name:   *flName,
		Volume: *flVolume,
		Force:  *flForce,
	}

	vol, err := cli.client.SnapshotCreate(context.Background(), volReq)
	if err != nil {
		return err
	}

	fmt.Fprintf(cli.out, "%s\n", vol.Name)
	return nil
}

// CmdSnapshotRm removes one or more snapshots.
//
// Usage: docker snapshot rm snapshot [snapshot...]
func (cli *DockerCli) CmdSnapshotRm(args ...string) error {
	cmd := Cli.Subcmd("snapshot rm", []string{"snapshot [snapshot...]"}, "Remove a snapshot", true)
	cmd.Require(flag.Min, 1)
	cmd.ParseFlags(args, true)

	var status = 0

	for _, name := range cmd.Args() {
		if err := cli.client.SnapshotRemove(context.Background(), name); err != nil {
			fmt.Fprintf(cli.err, "%s\n", err)
			status = 1
			continue
		}
		fmt.Fprintf(cli.out, "%s\n", name)
	}

	if status != 0 {
		return Cli.StatusError{StatusCode: status}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package batchprocessor // import "go.opentelemetry.io/collector/processor/batchprocessor"

import (
	"context"
	"runtime"
	"sync"
	"time"

	"go.opencensus.io/stats"
	"go.opencensus.io/tag"
	"go.uber.org/zap"

	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/config/configtelemetry"
	"go.opentelemetry.io/collector/consumer"
	"go.opentelemetry.io/collector/pdata/plog"
	"go.opentelemetry.io/collector/pdata/pmetric"
	"go.opentelemetry.io/collector/pdata/ptrace"
)

// batch_processor is a component that accepts spans and metrics, places them
// into batches and sends downstream.
//
// batch_processor implements consumer.Traces and consumer.Metrics
//
// Batches are sent out with any of the following conditions:
// - batch size reaches cfg.SendBatchSize
// - cfg.Timeout is elapsed since the timestamp when the previous batch was sent out.
type batchProcessor struct {
	logger           *zap.Logger
	exportCtx        context.Context
	timer            *time.Timer
	timeout          time.Duration
	sendBatchSize    int
	sendBatchMaxSize int

	newItem chan interface{}
	batch   batch

	shutdownC  chan struct{}
	goroutines sync.WaitGroup

	telemetryLevel configtelemetry.Level
}

type batch interface {
	// export the current batch
	export(ctx context.Context, sendBatchMaxSize int, returnBytes bool) (sentBatchSize int, sentBatchBytes int, err error)

	// itemCount returns the size of the current batch
	itemCount() int

	// size returns the size in bytes of the current batch
	size() int

	// add item to the current batch
	add(item interface{})
}

var _ consumer.Traces = (*batchProcessor)(nil)
var _ consumer.Metrics = (*batchProcessor)(nil)
var _ consumer.Logs = (*batchProcessor)(nil)

func newBatchProcessor(set component.ProcessorCreateSettings, cfg *Config, batch batch, telemetryLevel configtelemetry.Level) (*batchProcessor, error) {
	exportCtx, err := tag.New(context.Background(), tag.Insert(processorTagKey, cfg.ID().String()))
	if err != nil {
		return nil, err
	}
	return &batchProcessor{
		logger:         set.Logger,
		exportCtx:      exportCtx,
		telemetryLevel: telemetryLevel,

		sendBatchSize:    int(cfg.SendBatchSize),
		sendBatchMaxSize: int(cfg.SendBatchMaxSize),
		timeout:          cfg.Timeout,
		newItem:          make(chan interface{}, runtime.NumCPU()),
		batch:            batch,
		shutdownC:        make(chan struct{}, 1),
	}, nil
}

func (bp *batchProcessor) Capabilities() consumer.Capabilities {
	return consumer.Capabilities{MutatesData: true}
}

// Start is invoked during service startup.
func (bp *batchProcessor) Start(context.Context, component.Host) error {
	bp.goroutines.Add(1)
	go bp.startProcessingCycle()
	return nil
}

// Shutdown is invoked during service shutdown.
func (bp *batchProcessor) Shutdown(context.Context) error {
	close(bp.shutdownC)

	// Wait until all goroutines are done.
	bp.goroutines.Wait()
	return nil
}

func (bp *batchProcessor) startProcessingCycle() {
	defer bp.goroutines.Done()
	bp.timer = time.NewTimer(bp.timeout)
	for {
		select {
		case <-bp.shutdownC:
		DONE:
			for {
				select {
				case item := <-bp.newItem:
					bp.processItem(item)
				default:
					break DONE
				}
			}
			// This is the close of the channel
			if bp.batch.itemCount() > 0 {
				// TODO: Set a timeout on sendTraces or
				// make it cancellable using the context that Shutdown gets as a parameter
				bp.sendItems(statTimeoutTriggerSend)
			}
			return
		case item := <-bp.newItem:
			if item == nil {
				continue
			}
			bp.processItem(item)
		case <-bp.timer.C:
			if bp.batch.itemCount() > 0 {
				bp.sendItems(statTimeoutTriggerSend)
			}
			bp.resetTimer()
		}
	}
}

func (bp *batchProcessor) processItem(item interface{}) {
	bp.batch.add(item)
	sent := false
	for bp.batch.itemCount() >= bp.sendBatchSize {
		sent = true
		bp.sendItems(statBatchSizeTriggerSend)
	}

	if sent {
		bp.stopTimer()
		bp.resetTimer()
	}
}

func (bp *batchProcessor) stopTimer() {
	if !bp.timer.Stop() {
		<-bp.timer.C
	}
}

func (bp *batchProcessor) resetTimer() {
	bp.timer.Reset(bp.timeout)
}

func (bp *batchProcessor) sendItems(triggerMeasure *stats.Int64Measure) {
	detailed := bp.telemetryLevel == configtelemetry.LevelDetailed
	sent, bytes, err := bp.batch.export(bp.exportCtx, bp.sendBatchMaxSize, detailed)
	if err != nil {
		bp.logger.Warn("Sender failed", zap.Error(err))
	} else {
		// Add that it came form the trace pipeline?
		stats.Record(bp.exportCtx, triggerMeasure.M(1), statBatchSendSize.M(int64(sent)))
		if detailed {
			stats.Record(bp.exportCtx, statBatchSendSizeBytes.M(int64(bytes)))
		}
	}
}

// ConsumeTraces implements TracesProcessor
func (bp *batchProcessor) ConsumeTraces(_ context.Context, td ptrace.Traces) error {
	bp.newItem <- td
	return nil
}

// ConsumeMetrics implements MetricsProcessor
func (bp *batchProcessor) ConsumeMetrics(_ context.Context, md pmetric.Metrics) error {
	// First thing is convert into a different internal format
	bp.newItem <- md
	return nil
}

// ConsumeLogs implements LogsProcessor
func (bp *batchProcessor) ConsumeLogs(_ context.Context, ld plog.Logs) error {
	bp.newItem <- ld
	return nil
}

// newBatchTracesProcessor creates a new batch processor that batches traces by size or with timeout
func newBatchTracesProcessor(set component.ProcessorCreateSettings, next consumer.Traces, cfg *Config, telemetryLevel configtelemetry.Level) (*batchProcessor, error) {
	return newBatchProcessor(set, cfg, newBatchTraces(next), telemetryLevel)
}

// newBatchMetricsProcessor creates a new batch processor that batches metrics by size or with timeout
func newBatchMetricsProcessor(set component.ProcessorCreateSettings, next consumer.Metrics, cfg *Config, telemetryLevel configtelemetry.Level) (*batchProcessor, error) {
	return newBatchProcessor(set, cfg, newBatchMetrics(next), telemetryLevel)
}

// newBatchLogsProcessor creates a new batch processor that batches logs by size or with timeout
func newBatchLogsProcessor(set component.ProcessorCreateSettings, next consumer.Logs, cfg *Config, telemetryLevel configtelemetry.Level) (*batchProcessor, error) {
	return newBatchProcessor(set, cfg, newBatchLogs(next), telemetryLevel)
}

type batchTraces struct {
	nextConsumer consumer.Traces
	traceData    ptrace.Traces
	spanCount    int
	sizer        ptrace.Sizer
}

func newBatchTraces(nextConsumer consumer.Traces) *batchTraces {
	return &batchTraces{nextConsumer: nextConsumer, traceData: ptrace.NewTraces(), sizer: ptrace.NewProtoMarshaler().(ptrace.Sizer)}
}

// add updates current batchTraces by adding new TraceData object
func (bt *batchTraces) add(item interface{}) {
	td := item.(ptrace.Traces)
	newSpanCount := td.SpanCount()
	if newSpanCount == 0 {
		return
	}

	bt.spanCount += newSpanCount
	td.ResourceSpans().MoveAndAppendTo(bt.traceData.ResourceSpans())
}

func (bt *batchTraces) export(ctx context.Context, sendBatchMaxSize int, returnBytes bool) (int, int, error) {
	var req ptrace.Traces
	var sent int
	var bytes int
	if sendBatchMaxSize > 0 && bt.itemCount() > sendBatchMaxSize {
		req = splitTraces(sendBatchMaxSize, bt.traceData)
		bt.spanCount -= sendBatchMaxSize
		sent = sendBatchMaxSize
	} else {
		req = bt.traceData
		sent = bt.spanCount
		bt.traceData = ptrace.NewTraces()
		bt.spanCount = 0
	}
	if returnBytes {
		bytes = bt.sizer.TracesSize(req)
	}
	return sent, bytes, bt.nextConsumer.ConsumeTraces(ctx, req)
}

func (bt *batchTraces) itemCount() int {
	return bt.spanCount
}

func (bt *batchTraces) size() int {
	return bt.sizer.TracesSize(bt.traceData)
}

type batchMetrics struct {
	nextConsumer   consumer.Metrics
	metricData     pmetric.Metrics
	dataPointCount int
	sizer          pmetric.Sizer
}

func newBatchMetrics(nextConsumer consumer.Metrics) *batchMetrics {
	return &batchMetrics{nextConsumer: nextConsumer, metricData: pmetric.NewMetrics(), sizer: pmetric.NewProtoMarshaler().(pmetric.Sizer)}
}

func (bm *batchMetrics) export(ctx context.Context, sendBatchMaxSize int, returnBytes bool) (int, int, error) {
	var req pmetric.Metrics
	var sent int
	var bytes int
	if sendBatchMaxSize > 0 && bm.dataPointCount > sendBatchMaxSize {
		req = splitMetrics(sendBatchMaxSize, bm.metricData)
		bm.dataPointCount -= sendBatchMaxSize
		sent = sendBatchMaxSize
	} else {
		req = bm.metricData
		sent = bm.dataPointCount
		bm.metricData = pmetric.NewMetrics()
		bm.dataPointCount = 0
	}
	if returnBytes {
		bytes = bm.sizer.MetricsSize(req)
	}
	return sent, bytes, bm.nextConsumer.ConsumeMetrics(ctx, req)
}

func (bm *batchMetrics) itemCount() int {
	return bm.dataPointCount
}

func (bm *batchMetrics) size() int {
	return bm.sizer.MetricsSize(bm.metricData)
}

func (bm *batchMetrics) add(item interface{}) {
	md := item.(pmetric.Metrics)

	newDataPointCount := md.DataPointCount()
	if newDataPointCount == 0 {
		return
	}
	bm.dataPointCount += newDataPointCount
	md.ResourceMetrics().MoveAndAppendTo(bm.metricData.ResourceMetrics())
}

type batchLogs struct {
	nextConsumer consumer.Logs
	logData      plog.Logs
	logCount     int
	sizer        plog.Sizer
}

func newBatchLogs(nextConsumer consumer.Logs) *batchLogs {
	return &batchLogs{nextConsumer: nextConsumer, logData: plog.NewLogs(), sizer: plog.NewProtoMarshaler().(plog.Sizer)}
}

func (bl *batchLogs) export(ctx context.Context, sendBatchMaxSize int, returnBytes bool) (int, int, error) {
	var req plog.Logs
	var sent int
	var bytes int
	if sendBatchMaxSize > 0 && bl.logCount > sendBatchMaxSize {
		req = splitLogs(sendBatchMaxSize, bl.logData)
		bl.logCount -= sendBatchMaxSize
		sent = sendBatchMaxSize
	} else {
		req = bl.logData
		sent = bl.logCount
		bl.logData = plog.NewLogs()
		bl.logCount = 0
	}
	if returnBytes {
		bytes = bl.sizer.LogsSize(req)
	}
	return sent, bytes, bl.nextConsumer.ConsumeLogs(ctx, req)
}

func (bl *batchLogs) itemCount() int {
	return bl.logCount
}

func (bl *batchLogs) size() int {
	return bl.sizer.LogsSize(bl.logData)
}

func (bl *batchLogs) add(item interface{}) {
	ld := item.(plog.Logs)

	newLogsCount := ld.LogRecordCount()
	if newLogsCount == 0 {
		return
	}
	bl.logCount += newLogsCount
	ld.ResourceLogs().MoveAndAppendTo(bl.logData.ResourceLogs())
}
<commit_msg>Remove unused func from internal interface batch (#5462)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package batchprocessor // import "go.opentelemetry.io/collector/processor/batchprocessor"

import (
	"context"
	"runtime"
	"sync"
	"time"

	"go.opencensus.io/stats"
	"go.opencensus.io/tag"
	"go.uber.org/zap"

	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/config/configtelemetry"
	"go.opentelemetry.io/collector/consumer"
	"go.opentelemetry.io/collector/pdata/plog"
	"go.opentelemetry.io/collector/pdata/pmetric"
	"go.opentelemetry.io/collector/pdata/ptrace"
)

// batch_processor is a component that accepts spans and metrics, places them
// into batches and sends downstream.
//
// batch_processor implements consumer.Traces and consumer.Metrics
//
// Batches are sent out with any of the following conditions:
// - batch size reaches cfg.SendBatchSize
// - cfg.Timeout is elapsed since the timestamp when the previous batch was sent out.
type batchProcessor struct {
	logger           *zap.Logger
	exportCtx        context.Context
	timer            *time.Timer
	timeout          time.Duration
	sendBatchSize    int
	sendBatchMaxSize int

	newItem chan interface{}
	batch   batch

	shutdownC  chan struct{}
	goroutines sync.WaitGroup

	telemetryLevel configtelemetry.Level
}

type batch interface {
	// export the current batch
	export(ctx context.Context, sendBatchMaxSize int, returnBytes bool) (sentBatchSize int, sentBatchBytes int, err error)

	// itemCount returns the size of the current batch
	itemCount() int

	// add item to the current batch
	add(item interface{})
}

var _ consumer.Traces = (*batchProcessor)(nil)
var _ consumer.Metrics = (*batchProcessor)(nil)
var _ consumer.Logs = (*batchProcessor)(nil)

func newBatchProcessor(set component.ProcessorCreateSettings, cfg *Config, batch batch, telemetryLevel configtelemetry.Level) (*batchProcessor, error) {
	exportCtx, err := tag.New(context.Background(), tag.Insert(processorTagKey, cfg.ID().String()))
	if err != nil {
		return nil, err
	}
	return &batchProcessor{
		logger:         set.Logger,
		exportCtx:      exportCtx,
		telemetryLevel: telemetryLevel,

		sendBatchSize:    int(cfg.SendBatchSize),
		sendBatchMaxSize: int(cfg.SendBatchMaxSize),
		timeout:          cfg.Timeout,
		newItem:          make(chan interface{}, runtime.NumCPU()),
		batch:            batch,
		shutdownC:        make(chan struct{}, 1),
	}, nil
}

func (bp *batchProcessor) Capabilities() consumer.Capabilities {
	return consumer.Capabilities{MutatesData: true}
}

// Start is invoked during service startup.
func (bp *batchProcessor) Start(context.Context, component.Host) error {
	bp.goroutines.Add(1)
	go bp.startProcessingCycle()
	return nil
}

// Shutdown is invoked during service shutdown.
func (bp *batchProcessor) Shutdown(context.Context) error {
	close(bp.shutdownC)

	// Wait until all goroutines are done.
	bp.goroutines.Wait()
	return nil
}

func (bp *batchProcessor) startProcessingCycle() {
	defer bp.goroutines.Done()
	bp.timer = time.NewTimer(bp.timeout)
	for {
		select {
		case <-bp.shutdownC:
		DONE:
			for {
				select {
				case item := <-bp.newItem:
					bp.processItem(item)
				default:
					break DONE
				}
			}
			// This is the close of the channel
			if bp.batch.itemCount() > 0 {
				// TODO: Set a timeout on sendTraces or
				// make it cancellable using the context that Shutdown gets as a parameter
				bp.sendItems(statTimeoutTriggerSend)
			}
			return
		case item := <-bp.newItem:
			if item == nil {
				continue
			}
			bp.processItem(item)
		case <-bp.timer.C:
			if bp.batch.itemCount() > 0 {
				bp.sendItems(statTimeoutTriggerSend)
			}
			bp.resetTimer()
		}
	}
}

func (bp *batchProcessor) processItem(item interface{}) {
	bp.batch.add(item)
	sent := false
	for bp.batch.itemCount() >= bp.sendBatchSize {
		sent = true
		bp.sendItems(statBatchSizeTriggerSend)
	}

	if sent {
		bp.stopTimer()
		bp.resetTimer()
	}
}

func (bp *batchProcessor) stopTimer() {
	if !bp.timer.Stop() {
		<-bp.timer.C
	}
}

func (bp *batchProcessor) resetTimer() {
	bp.timer.Reset(bp.timeout)
}

func (bp *batchProcessor) sendItems(triggerMeasure *stats.Int64Measure) {
	detailed := bp.telemetryLevel == configtelemetry.LevelDetailed
	sent, bytes, err := bp.batch.export(bp.exportCtx, bp.sendBatchMaxSize, detailed)
	if err != nil {
		bp.logger.Warn("Sender failed", zap.Error(err))
	} else {
		// Add that it came form the trace pipeline?
		stats.Record(bp.exportCtx, triggerMeasure.M(1), statBatchSendSize.M(int64(sent)))
		if detailed {
			stats.Record(bp.exportCtx, statBatchSendSizeBytes.M(int64(bytes)))
		}
	}
}

// ConsumeTraces implements TracesProcessor
func (bp *batchProcessor) ConsumeTraces(_ context.Context, td ptrace.Traces) error {
	bp.newItem <- td
	return nil
}

// ConsumeMetrics implements MetricsProcessor
func (bp *batchProcessor) ConsumeMetrics(_ context.Context, md pmetric.Metrics) error {
	// First thing is convert into a different internal format
	bp.newItem <- md
	return nil
}

// ConsumeLogs implements LogsProcessor
func (bp *batchProcessor) ConsumeLogs(_ context.Context, ld plog.Logs) error {
	bp.newItem <- ld
	return nil
}

// newBatchTracesProcessor creates a new batch processor that batches traces by size or with timeout
func newBatchTracesProcessor(set component.ProcessorCreateSettings, next consumer.Traces, cfg *Config, telemetryLevel configtelemetry.Level) (*batchProcessor, error) {
	return newBatchProcessor(set, cfg, newBatchTraces(next), telemetryLevel)
}

// newBatchMetricsProcessor creates a new batch processor that batches metrics by size or with timeout
func newBatchMetricsProcessor(set component.ProcessorCreateSettings, next consumer.Metrics, cfg *Config, telemetryLevel configtelemetry.Level) (*batchProcessor, error) {
	return newBatchProcessor(set, cfg, newBatchMetrics(next), telemetryLevel)
}

// newBatchLogsProcessor creates a new batch processor that batches logs by size or with timeout
func newBatchLogsProcessor(set component.ProcessorCreateSettings, next consumer.Logs, cfg *Config, telemetryLevel configtelemetry.Level) (*batchProcessor, error) {
	return newBatchProcessor(set, cfg, newBatchLogs(next), telemetryLevel)
}

type batchTraces struct {
	nextConsumer consumer.Traces
	traceData    ptrace.Traces
	spanCount    int
	sizer        ptrace.Sizer
}

func newBatchTraces(nextConsumer consumer.Traces) *batchTraces {
	return &batchTraces{nextConsumer: nextConsumer, traceData: ptrace.NewTraces(), sizer: ptrace.NewProtoMarshaler().(ptrace.Sizer)}
}

// add updates current batchTraces by adding new TraceData object
func (bt *batchTraces) add(item interface{}) {
	td := item.(ptrace.Traces)
	newSpanCount := td.SpanCount()
	if newSpanCount == 0 {
		return
	}

	bt.spanCount += newSpanCount
	td.ResourceSpans().MoveAndAppendTo(bt.traceData.ResourceSpans())
}

func (bt *batchTraces) export(ctx context.Context, sendBatchMaxSize int, returnBytes bool) (int, int, error) {
	var req ptrace.Traces
	var sent int
	var bytes int
	if sendBatchMaxSize > 0 && bt.itemCount() > sendBatchMaxSize {
		req = splitTraces(sendBatchMaxSize, bt.traceData)
		bt.spanCount -= sendBatchMaxSize
		sent = sendBatchMaxSize
	} else {
		req = bt.traceData
		sent = bt.spanCount
		bt.traceData = ptrace.NewTraces()
		bt.spanCount = 0
	}
	if returnBytes {
		bytes = bt.sizer.TracesSize(req)
	}
	return sent, bytes, bt.nextConsumer.ConsumeTraces(ctx, req)
}

func (bt *batchTraces) itemCount() int {
	return bt.spanCount
}

type batchMetrics struct {
	nextConsumer   consumer.Metrics
	metricData     pmetric.Metrics
	dataPointCount int
	sizer          pmetric.Sizer
}

func newBatchMetrics(nextConsumer consumer.Metrics) *batchMetrics {
	return &batchMetrics{nextConsumer: nextConsumer, metricData: pmetric.NewMetrics(), sizer: pmetric.NewProtoMarshaler().(pmetric.Sizer)}
}

func (bm *batchMetrics) export(ctx context.Context, sendBatchMaxSize int, returnBytes bool) (int, int, error) {
	var req pmetric.Metrics
	var sent int
	var bytes int
	if sendBatchMaxSize > 0 && bm.dataPointCount > sendBatchMaxSize {
		req = splitMetrics(sendBatchMaxSize, bm.metricData)
		bm.dataPointCount -= sendBatchMaxSize
		sent = sendBatchMaxSize
	} else {
		req = bm.metricData
		sent = bm.dataPointCount
		bm.metricData = pmetric.NewMetrics()
		bm.dataPointCount = 0
	}
	if returnBytes {
		bytes = bm.sizer.MetricsSize(req)
	}
	return sent, bytes, bm.nextConsumer.ConsumeMetrics(ctx, req)
}

func (bm *batchMetrics) itemCount() int {
	return bm.dataPointCount
}

func (bm *batchMetrics) add(item interface{}) {
	md := item.(pmetric.Metrics)

	newDataPointCount := md.DataPointCount()
	if newDataPointCount == 0 {
		return
	}
	bm.dataPointCount += newDataPointCount
	md.ResourceMetrics().MoveAndAppendTo(bm.metricData.ResourceMetrics())
}

type batchLogs struct {
	nextConsumer consumer.Logs
	logData      plog.Logs
	logCount     int
	sizer        plog.Sizer
}

func newBatchLogs(nextConsumer consumer.Logs) *batchLogs {
	return &batchLogs{nextConsumer: nextConsumer, logData: plog.NewLogs(), sizer: plog.NewProtoMarshaler().(plog.Sizer)}
}

func (bl *batchLogs) export(ctx context.Context, sendBatchMaxSize int, returnBytes bool) (int, int, error) {
	var req plog.Logs
	var sent int
	var bytes int
	if sendBatchMaxSize > 0 && bl.logCount > sendBatchMaxSize {
		req = splitLogs(sendBatchMaxSize, bl.logData)
		bl.logCount -= sendBatchMaxSize
		sent = sendBatchMaxSize
	} else {
		req = bl.logData
		sent = bl.logCount
		bl.logData = plog.NewLogs()
		bl.logCount = 0
	}
	if returnBytes {
		bytes = bl.sizer.LogsSize(req)
	}
	return sent, bytes, bl.nextConsumer.ConsumeLogs(ctx, req)
}

func (bl *batchLogs) itemCount() int {
	return bl.logCount
}

func (bl *batchLogs) add(item interface{}) {
	ld := item.(plog.Logs)

	newLogsCount := ld.LogRecordCount()
	if newLogsCount == 0 {
		return
	}
	bl.logCount += newLogsCount
	ld.ResourceLogs().MoveAndAppendTo(bl.logData.ResourceLogs())
}
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.21"
<commit_msg>: 0.3.22 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.22"
<|endoftext|>


<commit_before>package herd

import (
	"fmt"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/filter"
	subproto "github.com/Symantec/Dominator/proto/sub"
	"os"
	"path"
)

type state struct {
}

func (sub *Sub) buildUpdateRequest(request *subproto.UpdateRequest) {
	fmt.Println("buildUpdateRequest()") // TODO(rgooch): Delete debugging.
	subFS := sub.fileSystem
	requiredImage := sub.herd.getImage(sub.requiredImage)
	requiredFS := requiredImage.FileSystem
	filter := requiredImage.Filter
	var state state
	compareDirectories(request, &state, &subFS.Directory, &requiredFS.Directory,
		"", filter)
	// TODO(rgooch): Implement this.
}

func compareDirectories(request *subproto.UpdateRequest, state *state,
	subDirectory, requiredDirectory *filesystem.Directory,
	parentName string, filter *filter.Filter) {
	requiredPathName := path.Join(parentName, requiredDirectory.Name)
	// First look for entries that should be deleted.
	makeSubDirectory := false
	if subDirectory == nil {
		makeSubDirectory = true
	} else {
		subPathName := path.Join(parentName, subDirectory.Name)
		for name, subEntry := range subDirectory.EntriesByName {
			pathname := path.Join(subPathName, entryName(subEntry))
			if filter.Match(pathname) {
				continue
			}
			if _, ok := requiredDirectory.EntriesByName[name]; !ok {
				request.PathsToDelete = append(request.PathsToDelete, pathname)
				fmt.Printf("Delete: %s\n", pathname) // HACK
			}
		}
		if !filesystem.CompareDirectoriesMetadata(subDirectory,
			requiredDirectory, os.Stdout) {
			fmt.Printf("Different directory: %s...\n", requiredPathName) // HACK
			makeSubDirectory = true
			// TODO(rgooch): Update metadata.
		}
	}
	if makeSubDirectory {
		var newdir subproto.Directory
		newdir.Name = requiredPathName
		newdir.Mode = requiredDirectory.Mode
		newdir.Uid = requiredDirectory.Uid
		newdir.Gid = requiredDirectory.Gid
		request.DirectoriesToMake = append(request.DirectoriesToMake, newdir)
	}
	for name, requiredEntry := range requiredDirectory.EntriesByName {
		pathname := path.Join(requiredPathName, entryName(requiredEntry))
		if filter.Match(pathname) {
			continue
		}
		if subDirectory == nil {
			compareEntries(request, state, nil, requiredEntry, requiredPathName,
				filter)
		} else {
			if subEntry, ok := subDirectory.EntriesByName[name]; ok {
				compareEntries(request, state, subEntry, requiredEntry,
					requiredPathName, filter)
			} else {
				compareEntries(request, state, nil, requiredEntry,
					requiredPathName, filter)
			}
		}
	}
}

func entryName(entry interface{}) string {
	switch e := entry.(type) {
	case *filesystem.RegularFile:
		return e.Name
	case *filesystem.Symlink:
		return e.Name
	case *filesystem.File:
		return e.Name
	case *filesystem.Directory:
		return e.Name
	}
	panic("Unsupported entry type")
}

func compareEntries(request *subproto.UpdateRequest, state *state,
	subEntry, requiredEntry interface{},
	parentName string, filter *filter.Filter) {
	switch re := requiredEntry.(type) {
	case *filesystem.RegularFile:
		compareRegularFile(request, state, subEntry, re, parentName)
		return
	case *filesystem.Symlink:
		compareSymlink(request, state, subEntry, re, parentName)
		return
	case *filesystem.File:
		compareFile(request, state, subEntry, re, parentName)
		return
	case *filesystem.Directory:
		compareDirectory(request, state, subEntry, re, parentName, filter)
		return
	}
	panic("Unsupported entry type")
}

func compareRegularFile(request *subproto.UpdateRequest, state *state,
	subEntry interface{}, requiredRegularFile *filesystem.RegularFile,
	parentName string) {
	if subRegularFile, ok := subEntry.(*filesystem.RegularFile); ok {
		sameMetadata := filesystem.CompareRegularInodesMetadata(
			subRegularFile.Inode(), requiredRegularFile.Inode(),
			os.Stdout)
		sameData := filesystem.CompareRegularInodesData(subRegularFile.Inode(),
			requiredRegularFile.Inode(), os.Stdout)
		if sameMetadata && sameData {
			return
		}
		fmt.Printf("Different rfile: %s...\n", requiredRegularFile.Name) // HACK
	} else {
		fmt.Printf("Add rfile: %s...\n", requiredRegularFile.Name) // HACK
	}
	// TODO(rgooch): Delete entry and replace.
}

func compareSymlink(request *subproto.UpdateRequest, state *state,
	subEntry interface{}, requiredSymlink *filesystem.Symlink,
	parentName string) {
	if subSymlink, ok := subEntry.(*filesystem.Symlink); ok {
		if filesystem.CompareSymlinkInodes(subSymlink.Inode(),
			requiredSymlink.Inode(), os.Stdout) {
			return
		}
		fmt.Printf("Different symlink: %s...\n", requiredSymlink.Name) // HACK
	} else {
		fmt.Printf("Add symlink: %s...\n", requiredSymlink.Name) // HACK
	}
	// TODO(rgooch): Delete entry and replace.
}

func compareFile(request *subproto.UpdateRequest, state *state,
	subEntry interface{}, requiredFile *filesystem.File,
	parentName string) {
	if subFile, ok := subEntry.(*filesystem.File); ok {
		if filesystem.CompareInodes(subFile.Inode(), requiredFile.Inode(),
			os.Stdout) {
			return
		}
		fmt.Printf("Different file: %s...\n", requiredFile.Name) // HACK
	} else {
		fmt.Printf("Add file: %s...\n", requiredFile.Name) // HACK
	}
	// TODO(rgooch): Delete entry and replace.
}

func compareDirectory(request *subproto.UpdateRequest, state *state,
	subEntry interface{}, requiredDirectory *filesystem.Directory,
	parentName string, filter *filter.Filter) {
	if subDirectory, ok := subEntry.(*filesystem.Directory); ok {
		compareDirectories(request, state, subDirectory, requiredDirectory,
			parentName, filter)
	} else {
		compareDirectories(request, state, nil, requiredDirectory, parentName,
			filter)
	}
}
<commit_msg>Incremental work on buildUpdateRequest().<commit_after>package herd

import (
	"fmt"
	"github.com/Symantec/Dominator/lib/filesystem"
	"github.com/Symantec/Dominator/lib/filter"
	subproto "github.com/Symantec/Dominator/proto/sub"
	"os"
	"path"
)

type state struct {
	subInodeToRequiredInode map[uint64]uint64
}

func (sub *Sub) buildUpdateRequest(request *subproto.UpdateRequest) {
	fmt.Println("buildUpdateRequest()") // TODO(rgooch): Delete debugging.
	subFS := sub.fileSystem
	requiredImage := sub.herd.getImage(sub.requiredImage)
	requiredFS := requiredImage.FileSystem
	filter := requiredImage.Filter
	var state state
	state.subInodeToRequiredInode = make(map[uint64]uint64)
	compareDirectories(request, &state, &subFS.Directory, &requiredFS.Directory,
		"", filter)
	// TODO(rgooch): Implement this.
}

func compareDirectories(request *subproto.UpdateRequest, state *state,
	subDirectory, requiredDirectory *filesystem.Directory,
	parentName string, filter *filter.Filter) {
	requiredPathName := path.Join(parentName, requiredDirectory.Name)
	// First look for entries that should be deleted.
	makeSubDirectory := false
	if subDirectory == nil {
		makeSubDirectory = true
	} else {
		subPathName := path.Join(parentName, subDirectory.Name)
		for name, subEntry := range subDirectory.EntriesByName {
			pathname := path.Join(subPathName, entryName(subEntry))
			if filter.Match(pathname) {
				continue
			}
			if _, ok := requiredDirectory.EntriesByName[name]; !ok {
				request.PathsToDelete = append(request.PathsToDelete, pathname)
				fmt.Printf("Delete: %s\n", pathname) // HACK
			}
		}
		if !filesystem.CompareDirectoriesMetadata(subDirectory,
			requiredDirectory, os.Stdout) {
			fmt.Printf("Different directory: %s...\n", requiredPathName) // HACK
			makeSubDirectory = true
			// TODO(rgooch): Update metadata.
		}
	}
	if makeSubDirectory {
		var newdir subproto.Directory
		newdir.Name = requiredPathName
		newdir.Mode = requiredDirectory.Mode
		newdir.Uid = requiredDirectory.Uid
		newdir.Gid = requiredDirectory.Gid
		request.DirectoriesToMake = append(request.DirectoriesToMake, newdir)
	}
	for name, requiredEntry := range requiredDirectory.EntriesByName {
		pathname := path.Join(requiredPathName, entryName(requiredEntry))
		if filter.Match(pathname) {
			continue
		}
		if subDirectory == nil {
			compareEntries(request, state, nil, requiredEntry, requiredPathName,
				filter)
		} else {
			if subEntry, ok := subDirectory.EntriesByName[name]; ok {
				compareEntries(request, state, subEntry, requiredEntry,
					requiredPathName, filter)
			} else {
				compareEntries(request, state, nil, requiredEntry,
					requiredPathName, filter)
			}
		}
	}
}

func entryName(entry interface{}) string {
	switch e := entry.(type) {
	case *filesystem.RegularFile:
		return e.Name
	case *filesystem.Symlink:
		return e.Name
	case *filesystem.File:
		return e.Name
	case *filesystem.Directory:
		return e.Name
	}
	panic("Unsupported entry type")
}

func compareEntries(request *subproto.UpdateRequest, state *state,
	subEntry, requiredEntry interface{},
	parentName string, filter *filter.Filter) {
	switch re := requiredEntry.(type) {
	case *filesystem.RegularFile:
		compareRegularFile(request, state, subEntry, re, parentName)
		return
	case *filesystem.Symlink:
		compareSymlink(request, state, subEntry, re, parentName)
		return
	case *filesystem.File:
		compareFile(request, state, subEntry, re, parentName)
		return
	case *filesystem.Directory:
		compareDirectory(request, state, subEntry, re, parentName, filter)
		return
	}
	panic("Unsupported entry type")
}

func compareRegularFile(request *subproto.UpdateRequest, state *state,
	subEntry interface{}, requiredRegularFile *filesystem.RegularFile,
	parentName string) {
	debugFilename := path.Join(parentName, requiredRegularFile.Name)
	if subRegularFile, ok := subEntry.(*filesystem.RegularFile); ok {
		if requiredInode, ok :=
			state.subInodeToRequiredInode[subRegularFile.InodeNumber]; ok {
			if requiredInode == requiredRegularFile.InodeNumber {
				//
				fmt.Printf("Different links: %s...\n", debugFilename) // HACK
			}
		} else {
			state.subInodeToRequiredInode[subRegularFile.InodeNumber] =
				requiredRegularFile.InodeNumber
		}
		sameMetadata := filesystem.CompareRegularInodesMetadata(
			subRegularFile.Inode(), requiredRegularFile.Inode(),
			os.Stdout)
		sameData := filesystem.CompareRegularInodesData(subRegularFile.Inode(),
			requiredRegularFile.Inode(), os.Stdout)
		if sameMetadata && sameData {
			return
		}
		fmt.Printf("Different rfile: %s...\n", debugFilename) // HACK
	} else {
		fmt.Printf("Add rfile: %s...\n", debugFilename) // HACK
	}
	// TODO(rgooch): Delete entry and replace.
}

func compareSymlink(request *subproto.UpdateRequest, state *state,
	subEntry interface{}, requiredSymlink *filesystem.Symlink,
	parentName string) {
	debugFilename := path.Join(parentName, requiredSymlink.Name)
	if subSymlink, ok := subEntry.(*filesystem.Symlink); ok {
		if requiredInode, ok :=
			state.subInodeToRequiredInode[subSymlink.InodeNumber]; ok {
			if requiredInode != requiredSymlink.InodeNumber {
				fmt.Printf("Different links: %s...\n", debugFilename) // HACK
			}
		} else {
			state.subInodeToRequiredInode[subSymlink.InodeNumber] =
				requiredSymlink.InodeNumber
		}
		if filesystem.CompareSymlinkInodes(subSymlink.Inode(),
			requiredSymlink.Inode(), os.Stdout) {
			return
		}
		fmt.Printf("Different symlink: %s...\n", debugFilename) // HACK
	} else {
		fmt.Printf("Add symlink: %s...\n", debugFilename) // HACK
	}
	// TODO(rgooch): Delete entry and replace.
}

func compareFile(request *subproto.UpdateRequest, state *state,
	subEntry interface{}, requiredFile *filesystem.File,
	parentName string) {
	debugFilename := path.Join(parentName, requiredFile.Name)
	if subFile, ok := subEntry.(*filesystem.File); ok {
		if requiredInode, ok :=
			state.subInodeToRequiredInode[subFile.InodeNumber]; ok {
			if requiredInode != requiredFile.InodeNumber {
				fmt.Printf("Different links: %s...\n", debugFilename) // HACK
			}
		} else {
			state.subInodeToRequiredInode[subFile.InodeNumber] =
				requiredFile.InodeNumber
		}
		if filesystem.CompareInodes(subFile.Inode(), requiredFile.Inode(),
			os.Stdout) {
			return
		}
		fmt.Printf("Different file: %s...\n", debugFilename) // HACK
	} else {
		fmt.Printf("Add file: %s...\n", debugFilename) // HACK
	}
	// TODO(rgooch): Delete entry and replace.
}

func compareDirectory(request *subproto.UpdateRequest, state *state,
	subEntry interface{}, requiredDirectory *filesystem.Directory,
	parentName string, filter *filter.Filter) {
	if subDirectory, ok := subEntry.(*filesystem.Directory); ok {
		compareDirectories(request, state, subDirectory, requiredDirectory,
			parentName, filter)
	} else {
		compareDirectories(request, state, nil, requiredDirectory, parentName,
			filter)
	}
}
<|endoftext|>


<commit_before>package buffer

import (
	"crypto/rand"
	"fmt"
	"io"
	"testing"
	"unsafe"
)

func randBytes(n int) (b []byte, err error) {
	b = make([]byte, n)
	_, err = io.ReadFull(rand.Reader, b)
	return
}

func TestMemclr(t *testing.T) {
	// All sizes up to 32 bytes.
	var sizes []int
	for i := 0; i <= 32; i++ {
		sizes = append(sizes, i)
	}

	// And a few hand-chosen sizes.
	sizes = append(sizes, []int{
		39, 41, 64, 127, 128, 129,
		1<<20 - 1,
		1 << 20,
		1<<20 + 1,
	}...)

	// For each size, fill a buffer with random bytes and then zero it.
	for _, size := range sizes {
		size := size
		t.Run(fmt.Sprintf("size=%d", size), func(t *testing.T) {
			// Generate
			b, err := randBytes(size)
			if err != nil {
				t.Fatalf("randBytes: %v", err)
			}

			// Clear
			var p unsafe.Pointer
			if len(b) != 0 {
				p = unsafe.Pointer(&b[0])
			}

			memclr(p, uintptr(len(b)))

			// Check
			for i, x := range b {
				if x != 0 {
					t.Fatalf("non-zero byte %d at offset %d", x, i)
				}
			}
		})
	}
}

func BenchmarkOutMessageReset(b *testing.B) {
	// A single buffer, which should fit in some level of CPU cache.
	b.Run("Single buffer", func(b *testing.B) {
		var om OutMessage
		for i := 0; i < b.N; i++ {
			om.Reset()
		}

		b.SetBytes(int64(om.offset))
	})

	// Many megabytes worth of buffers, which should defeat the CPU cache.
	b.Run("Many buffers", func(b *testing.B) {
		// The number of messages; intentionally a power of two.
		const numMessages = 128

		var oms [numMessages]OutMessage
		if s := unsafe.Sizeof(oms); s < 128<<20 {
			panic(fmt.Sprintf("Array is too small; total size: %d", s))
		}

		for i := 0; i < b.N; i++ {
			oms[i%numMessages].Reset()
		}

		b.SetBytes(int64(oms[0].offset))
	})
}

func BenchmarkOutMessageGrowShrink(b *testing.B) {
	// A single buffer, which should fit in some level of CPU cache.
	b.Run("Single buffer", func(b *testing.B) {
		var om OutMessage
		for i := 0; i < b.N; i++ {
			om.Grow(MaxReadSize)
			om.ShrinkTo(OutMessageInitialSize)
		}

		b.SetBytes(int64(MaxReadSize))
	})

	// Many megabytes worth of buffers, which should defeat the CPU cache.
	b.Run("Many buffers", func(b *testing.B) {
		// The number of messages; intentionally a power of two.
		const numMessages = 128

		var oms [numMessages]OutMessage
		if s := unsafe.Sizeof(oms); s < 128<<20 {
			panic(fmt.Sprintf("Array is too small; total size: %d", s))
		}

		for i := 0; i < b.N; i++ {
			oms[i%numMessages].Grow(MaxReadSize)
			oms[i%numMessages].ShrinkTo(OutMessageInitialSize)
		}

		b.SetBytes(int64(MaxReadSize))
	})
}
<commit_msg>buffer: add more tests.<commit_after>package buffer

import (
	"crypto/rand"
	"fmt"
	"io"
	"reflect"
	"testing"
	"unsafe"
)

func toByteSlice(p unsafe.Pointer, n int) []byte {
	sh := reflect.SliceHeader{
		Data: uintptr(p),
		Len:  n,
		Cap:  n,
	}

	return *(*[]byte)(unsafe.Pointer(&sh))
}

// fillWithGarbage writes random data to [p, p+n).
func fillWithGarbage(p unsafe.Pointer, n int) (err error) {
	b := toByteSlice(p, n)
	_, err = io.ReadFull(rand.Reader, b)
	return
}

func randBytes(n int) (b []byte, err error) {
	b = make([]byte, n)
	_, err = io.ReadFull(rand.Reader, b)
	return
}

// findNonZero finds the offset of the first non-zero byte in [p, p+n). If
// none, it returns n.
func findNonZero(p unsafe.Pointer, n int) int {
	b := toByteSlice(p, n)
	for i, x := range b {
		if x != 0 {
			return i
		}
	}

	return n
}

func TestMemclr(t *testing.T) {
	// All sizes up to 32 bytes.
	var sizes []int
	for i := 0; i <= 32; i++ {
		sizes = append(sizes, i)
	}

	// And a few hand-chosen sizes.
	sizes = append(sizes, []int{
		39, 41, 64, 127, 128, 129,
		1<<20 - 1,
		1 << 20,
		1<<20 + 1,
	}...)

	// For each size, fill a buffer with random bytes and then zero it.
	for _, size := range sizes {
		size := size
		t.Run(fmt.Sprintf("size=%d", size), func(t *testing.T) {
			// Generate
			b, err := randBytes(size)
			if err != nil {
				t.Fatalf("randBytes: %v", err)
			}

			// Clear
			var p unsafe.Pointer
			if len(b) != 0 {
				p = unsafe.Pointer(&b[0])
			}

			memclr(p, uintptr(len(b)))

			// Check
			if i := findNonZero(p, len(b)); i != len(b) {
				t.Fatalf("non-zero byte at offset %d", i)
			}
		})
	}
}

func TestOutMessageReset(t *testing.T) {
	var om OutMessage
	h := om.OutHeader()

	const trials = 100
	for i := 0; i < trials; i++ {
		err := fillWithGarbage(unsafe.Pointer(h), int(unsafe.Sizeof(*h)))
		if err != nil {
			t.Fatalf("fillWithGarbage: %v", err)
		}

		om.Reset()
		if h.Len != 0 {
			t.Fatalf("non-zero Len %v", h.Len)
		}

		if h.Error != 0 {
			t.Fatalf("non-zero Error %v", h.Error)
		}

		if h.Unique != 0 {
			t.Fatalf("non-zero Unique %v", h.Unique)
		}
	}
}

func TestOutMessageGrow(t *testing.T) {
	var om OutMessage

	// Overwrite with garbage.
	err := fillWithGarbage(unsafe.Pointer(&om), int(unsafe.Sizeof(om)))
	if err != nil {
		t.Fatalf("fillWithGarbage: %v", err)
	}

	// Zero the header.
	om.Reset()

	// Grow to the max size. This should zero the message.
	if p := om.Grow(MaxReadSize); p == nil {
		t.Fatal("Grow returned nil")
	}

	// Check that everything has been zeroed.
	b := om.Bytes()
	for i, x := range b {
		if x != 0 {
			t.Fatalf("non-zero byte 0x%02x at offset %d", x, i)
		}
	}
}

func BenchmarkOutMessageReset(b *testing.B) {
	// A single buffer, which should fit in some level of CPU cache.
	b.Run("Single buffer", func(b *testing.B) {
		var om OutMessage
		for i := 0; i < b.N; i++ {
			om.Reset()
		}

		b.SetBytes(int64(om.offset))
	})

	// Many megabytes worth of buffers, which should defeat the CPU cache.
	b.Run("Many buffers", func(b *testing.B) {
		// The number of messages; intentionally a power of two.
		const numMessages = 128

		var oms [numMessages]OutMessage
		if s := unsafe.Sizeof(oms); s < 128<<20 {
			panic(fmt.Sprintf("Array is too small; total size: %d", s))
		}

		for i := 0; i < b.N; i++ {
			oms[i%numMessages].Reset()
		}

		b.SetBytes(int64(oms[0].offset))
	})
}

func BenchmarkOutMessageGrowShrink(b *testing.B) {
	// A single buffer, which should fit in some level of CPU cache.
	b.Run("Single buffer", func(b *testing.B) {
		var om OutMessage
		for i := 0; i < b.N; i++ {
			om.Grow(MaxReadSize)
			om.ShrinkTo(OutMessageInitialSize)
		}

		b.SetBytes(int64(MaxReadSize))
	})

	// Many megabytes worth of buffers, which should defeat the CPU cache.
	b.Run("Many buffers", func(b *testing.B) {
		// The number of messages; intentionally a power of two.
		const numMessages = 128

		var oms [numMessages]OutMessage
		if s := unsafe.Sizeof(oms); s < 128<<20 {
			panic(fmt.Sprintf("Array is too small; total size: %d", s))
		}

		for i := 0; i < b.N; i++ {
			oms[i%numMessages].Grow(MaxReadSize)
			oms[i%numMessages].ShrinkTo(OutMessageInitialSize)
		}

		b.SetBytes(int64(MaxReadSize))
	})
}
<|endoftext|>


<commit_before>package version

// Version of Functions
var Version = "0.3.426"
<commit_msg>fnserver: 0.3.427 release [skip ci]<commit_after>package version

// Version of Functions
var Version = "0.3.427"
<|endoftext|>


<commit_before>package models

import (
	"bytes"
	"sort"
	"strconv"

	"github.com/go-macaron/binding"
	"github.com/grafana/metrictank/idx"
	pickle "github.com/kisielk/og-rek"
	"gopkg.in/macaron.v1"
)

type FromTo struct {
	From  string `json:"from" form:"from"`
	Until string `json:"until" form:"until"`
	To    string `json:"to" form:"to"` // graphite uses 'until' but we allow to alternatively cause it's shorter
	Tz    string `json:"tz" form:"tz"`
}

type GraphiteRender struct {
	FromTo
	MaxDataPoints uint32   `json:"maxDataPoints" form:"maxDataPoints" binding:"Default(800)"`
	Targets       []string `json:"target" form:"target"`
	TargetsRails  []string `form:"target[]"` // # Rails/PHP/jQuery common practice format: ?target[]=path.1&target[]=path.2 -> like graphite, we allow this.
	Format        string   `json:"format" form:"format" binding:"In(,json,msgp,pickle)"`
	NoProxy       bool     `json:"local" form:"local"` //this is set to true by graphite-web when it passes request to cluster servers
	Process       string   `json:"process" form:"process" binding:"In(,none,stable,any);Default(stable)"`
}

func (gr GraphiteRender) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {
	if len(gr.Targets) == 0 {
		if len(gr.TargetsRails) == 0 {
			errs = append(errs, binding.Error{
				FieldNames:     []string{"target"},
				Classification: "RequiredError",
				Message:        "Required",
			})
			return errs
		}
		gr.Targets = gr.TargetsRails
	}
	for _, val := range gr.Targets {
		if val == "" {
			errs = append(errs, binding.Error{
				FieldNames:     []string{"target"},
				Classification: "RequiredError",
				Message:        "Required",
			})
		}
	}
	return errs
}

type GraphiteTags struct {
	Filter string `json:"filter"`
	From   int64  `json:"from"`
}

type GraphiteTagsResp []GraphiteTagResp

type GraphiteAutoCompleteTags struct {
	Expr      []string `json:"expr" form:"expr"`
	TagPrefix string   `json:"tagPrefix" form:"tagPrefix"`
	From      int64    `json:"from" form:"from"`
	Limit     uint16   `json:"limit" form:"limit"`
}

type GraphiteAutoCompleteTagValues struct {
	Expr        []string `json:"expr" form:"expr"`
	Tag         string   `json:"tag", form:"tag"`
	ValuePrefix string   `json:"valuePrefix" form:"valuePrefix"`
	From        int64    `json:"from" form:"from"`
	Limit       uint16   `json:"limit" form:"limit"`
}

type GraphiteTagResp struct {
	Tag string `json:"tag"`
}

type GraphiteTagDetails struct {
	Tag    string `json:"tag"`
	Filter string `json:"filter"`
	From   int64  `json:"from"`
}

type GraphiteTagDetailsResp struct {
	Tag    string                        `json:"tag"`
	Values []GraphiteTagDetailsValueResp `json:"values"`
}

type GraphiteTagDetailsValueResp struct {
	Count uint64 `json:"count"`
	Value string `json:"value"`
}

type GraphiteTagFindSeries struct {
	Expr []string `json:"expr"`
	From int64    `json:"from"`
}

type GraphiteTagFindSeriesResp struct {
	Series []string `json:"series"`
}

type GraphiteFind struct {
	FromTo
	Query  string `json:"query" form:"query" binding:"Required"`
	Format string `json:"format" form:"format" binding:"In(,completer,json,treejson,pickle)"`
	Jsonp  string `json:"jsonp" form:"jsonp"`
}

type MetricsDelete struct {
	Query string `json:"query" form:"query" binding:"Required"`
}

type MetricNames []idx.Archive

func (defs MetricNames) MarshalJSONFast(b []byte) ([]byte, error) {
	seen := make(map[string]struct{})

	names := make([]string, 0, len(defs))

	for i := 0; i < len(defs); i++ {
		_, ok := seen[defs[i].Name]
		if !ok {
			names = append(names, defs[i].Name)
			seen[defs[i].Name] = struct{}{}
		}
	}
	sort.Strings(names)
	b = append(b, '[')
	for _, name := range names {
		b = strconv.AppendQuoteToASCII(b, name)
		b = append(b, ',')
	}
	if len(defs) != 0 {
		b = b[:len(b)-1] // cut last comma
	}
	b = append(b, ']')
	return b, nil
}

func (defs MetricNames) MarshalJSON() ([]byte, error) {
	return defs.MarshalJSONFast(nil)
}

type SeriesCompleter map[string][]SeriesCompleterItem

func NewSeriesCompleter() SeriesCompleter {
	return SeriesCompleter(map[string][]SeriesCompleterItem{"metrics": make([]SeriesCompleterItem, 0)})
}

func (c SeriesCompleter) Add(e SeriesCompleterItem) {
	c["metrics"] = append(c["metrics"], e)
}

type SeriesCompleterItem struct {
	Path   string `json:"path"`
	Name   string `json:"name"`
	IsLeaf string `json:"is_leaf"`
}

type SeriesPickle []SeriesPickleItem

func (s SeriesPickle) Pickle(buf []byte) ([]byte, error) {
	buffer := bytes.NewBuffer(buf)
	encoder := pickle.NewEncoder(buffer)
	err := encoder.Encode(s)
	return buffer.Bytes(), err
}

type SeriesPickleItem struct {
	Path      string    `pickle:"path"`
	IsLeaf    bool      `pickle:"isLeaf"`
	Intervals [][]int64 `pickle:"intervals"` // list of (start,end) tuples
}

func NewSeriesPickleItem(path string, isLeaf bool, intervals [][]int64) SeriesPickleItem {
	return SeriesPickleItem{
		Path:      path,
		IsLeaf:    isLeaf,
		Intervals: intervals,
	}
}

type SeriesTree []SeriesTreeItem

func NewSeriesTree() *SeriesTree {
	return new(SeriesTree)
}

func (s *SeriesTree) Add(i *SeriesTreeItem) {
	*s = append(*s, *i)
}

type SeriesTreeItem struct {
	AllowChildren int            `json:"allowChildren"`
	Expandable    int            `json:"expandable"`
	Leaf          int            `json:"leaf"`
	ID            string         `json:"id"`
	Text          string         `json:"text"`
	Context       map[string]int `json:"context"` // unused
}
<commit_msg>accept form values<commit_after>package models

import (
	"bytes"
	"sort"
	"strconv"

	"github.com/go-macaron/binding"
	"github.com/grafana/metrictank/idx"
	pickle "github.com/kisielk/og-rek"
	"gopkg.in/macaron.v1"
)

type FromTo struct {
	From  string `json:"from" form:"from"`
	Until string `json:"until" form:"until"`
	To    string `json:"to" form:"to"` // graphite uses 'until' but we allow to alternatively cause it's shorter
	Tz    string `json:"tz" form:"tz"`
}

type GraphiteRender struct {
	FromTo
	MaxDataPoints uint32   `json:"maxDataPoints" form:"maxDataPoints" binding:"Default(800)"`
	Targets       []string `json:"target" form:"target"`
	TargetsRails  []string `form:"target[]"` // # Rails/PHP/jQuery common practice format: ?target[]=path.1&target[]=path.2 -> like graphite, we allow this.
	Format        string   `json:"format" form:"format" binding:"In(,json,msgp,pickle)"`
	NoProxy       bool     `json:"local" form:"local"` //this is set to true by graphite-web when it passes request to cluster servers
	Process       string   `json:"process" form:"process" binding:"In(,none,stable,any);Default(stable)"`
}

func (gr GraphiteRender) Validate(ctx *macaron.Context, errs binding.Errors) binding.Errors {
	if len(gr.Targets) == 0 {
		if len(gr.TargetsRails) == 0 {
			errs = append(errs, binding.Error{
				FieldNames:     []string{"target"},
				Classification: "RequiredError",
				Message:        "Required",
			})
			return errs
		}
		gr.Targets = gr.TargetsRails
	}
	for _, val := range gr.Targets {
		if val == "" {
			errs = append(errs, binding.Error{
				FieldNames:     []string{"target"},
				Classification: "RequiredError",
				Message:        "Required",
			})
		}
	}
	return errs
}

type GraphiteTags struct {
	Filter string `json:"filter" form:"filter"`
	From   int64  `json:"from" form:"from"`
}

type GraphiteTagsResp []GraphiteTagResp

type GraphiteAutoCompleteTags struct {
	Expr      []string `json:"expr" form:"expr"`
	TagPrefix string   `json:"tagPrefix" form:"tagPrefix"`
	From      int64    `json:"from" form:"from"`
	Limit     uint16   `json:"limit" form:"limit"`
}

type GraphiteAutoCompleteTagValues struct {
	Expr        []string `json:"expr" form:"expr"`
	Tag         string   `json:"tag", form:"tag"`
	ValuePrefix string   `json:"valuePrefix" form:"valuePrefix"`
	From        int64    `json:"from" form:"from"`
	Limit       uint16   `json:"limit" form:"limit"`
}

type GraphiteTagResp struct {
	Tag string `json:"tag"`
}

type GraphiteTagDetails struct {
	Tag    string `json:"tag" form:"tag"`
	Filter string `json:"filter" form:"filter"`
	From   int64  `json:"from" form:"from"`
}

type GraphiteTagDetailsResp struct {
	Tag    string                        `json:"tag"`
	Values []GraphiteTagDetailsValueResp `json:"values"`
}

type GraphiteTagDetailsValueResp struct {
	Count uint64 `json:"count"`
	Value string `json:"value"`
}

type GraphiteTagFindSeries struct {
	Expr []string `json:"expr" form:"expr"`
	From int64    `json:"from" form:"from"`
}

type GraphiteTagFindSeriesResp struct {
	Series []string `json:"series"`
}

type GraphiteFind struct {
	FromTo
	Query  string `json:"query" form:"query" binding:"Required"`
	Format string `json:"format" form:"format" binding:"In(,completer,json,treejson,pickle)"`
	Jsonp  string `json:"jsonp" form:"jsonp"`
}

type MetricsDelete struct {
	Query string `json:"query" form:"query" binding:"Required"`
}

type MetricNames []idx.Archive

func (defs MetricNames) MarshalJSONFast(b []byte) ([]byte, error) {
	seen := make(map[string]struct{})

	names := make([]string, 0, len(defs))

	for i := 0; i < len(defs); i++ {
		_, ok := seen[defs[i].Name]
		if !ok {
			names = append(names, defs[i].Name)
			seen[defs[i].Name] = struct{}{}
		}
	}
	sort.Strings(names)
	b = append(b, '[')
	for _, name := range names {
		b = strconv.AppendQuoteToASCII(b, name)
		b = append(b, ',')
	}
	if len(defs) != 0 {
		b = b[:len(b)-1] // cut last comma
	}
	b = append(b, ']')
	return b, nil
}

func (defs MetricNames) MarshalJSON() ([]byte, error) {
	return defs.MarshalJSONFast(nil)
}

type SeriesCompleter map[string][]SeriesCompleterItem

func NewSeriesCompleter() SeriesCompleter {
	return SeriesCompleter(map[string][]SeriesCompleterItem{"metrics": make([]SeriesCompleterItem, 0)})
}

func (c SeriesCompleter) Add(e SeriesCompleterItem) {
	c["metrics"] = append(c["metrics"], e)
}

type SeriesCompleterItem struct {
	Path   string `json:"path"`
	Name   string `json:"name"`
	IsLeaf string `json:"is_leaf"`
}

type SeriesPickle []SeriesPickleItem

func (s SeriesPickle) Pickle(buf []byte) ([]byte, error) {
	buffer := bytes.NewBuffer(buf)
	encoder := pickle.NewEncoder(buffer)
	err := encoder.Encode(s)
	return buffer.Bytes(), err
}

type SeriesPickleItem struct {
	Path      string    `pickle:"path"`
	IsLeaf    bool      `pickle:"isLeaf"`
	Intervals [][]int64 `pickle:"intervals"` // list of (start,end) tuples
}

func NewSeriesPickleItem(path string, isLeaf bool, intervals [][]int64) SeriesPickleItem {
	return SeriesPickleItem{
		Path:      path,
		IsLeaf:    isLeaf,
		Intervals: intervals,
	}
}

type SeriesTree []SeriesTreeItem

func NewSeriesTree() *SeriesTree {
	return new(SeriesTree)
}

func (s *SeriesTree) Add(i *SeriesTreeItem) {
	*s = append(*s, *i)
}

type SeriesTreeItem struct {
	AllowChildren int            `json:"allowChildren"`
	Expandable    int            `json:"expandable"`
	Leaf          int            `json:"leaf"`
	ID            string         `json:"id"`
	Text          string         `json:"text"`
	Context       map[string]int `json:"context"` // unused
}
<|endoftext|>


<commit_before>package stack

import (
	"context"
	"fmt"
	"os"

	"github.com/cozy/checkup"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/scheduler"
	"github.com/cozy/cozy-stack/pkg/utils"
	"github.com/google/gops/agent"
)

var log = logger.WithNamespace("stack")

type gopAgent struct{}

func (g gopAgent) Shutdown(ctx context.Context) error {
	fmt.Print("  shutting down gops...")
	agent.Close()
	fmt.Println("ok.")
	return nil
}

// Start is used to initialize all the
func Start() (broker jobs.Broker, schder scheduler.Scheduler, processes utils.Shutdowner, err error) {
	if config.IsDevRelease() {
		fmt.Println(`                           !! DEVELOPMENT RELEASE !!
You are running a development release which may deactivate some very important
security features. Please do not use this binary as your production server.
`)
	}

	err = agent.Listen(agent.Options{})
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error on gops agent: %s\n", err)
	}

	// Check that we can properly reach CouchDB.
	db, err := checkup.HTTPChecker{
		URL:         config.CouchURL().String(),
		MustContain: `"version":"2`,
	}.Check()
	if err != nil {
		err = fmt.Errorf("Could not reach Couchdb 2.0 database: %s", err.Error())
		return
	}
	if db.Status() == checkup.Down {
		err = fmt.Errorf("Could not reach Couchdb 2.0 database:\n%s", db.String())
		return
	}
	if db.Status() != checkup.Healthy {
		log.Warnf("CouchDB does not seem to be in a healthy state, "+
			"the cozy-stack will be starting anyway:\n%s", db.String())
	}

	// Init the main global connection to the swift server
	fsURL := config.FsURL()
	if fsURL.Scheme == config.SchemeSwift {
		if err = config.InitSwiftConnection(fsURL); err != nil {
			return
		}
	}

	// Start update cron for auto-updates
	cronUpdates, err := instance.StartUpdateCron()
	if err != nil {
		return
	}

	jobsConfig := config.GetConfig().Jobs
	nbWorkers := jobsConfig.Workers
	if cli := jobsConfig.Redis.Client(); cli != nil {
		broker = jobs.NewRedisBroker(nbWorkers, cli)
		schder = scheduler.NewRedisScheduler(cli)
	} else {
		broker = jobs.NewMemBroker(nbWorkers)
		schder = scheduler.NewMemScheduler()
	}
	if err = broker.Start(jobs.GetWorkersList()); err != nil {
		return
	}
	if err = schder.Start(broker); err != nil {
		return
	}

	// Global shutdowner that composes all the running processes of the stack
	processes = utils.NewGroupShutdown(
		broker,
		schder,
		cronUpdates,
		gopAgent{},
	)
	return
}
<commit_msg>Increase attempts at connecting to couchdb at startup<commit_after>package stack

import (
	"context"
	"fmt"
	"os"
	"time"

	"github.com/cozy/checkup"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/cozy/cozy-stack/pkg/logger"
	"github.com/cozy/cozy-stack/pkg/scheduler"
	"github.com/cozy/cozy-stack/pkg/utils"
	"github.com/google/gops/agent"
)

var log = logger.WithNamespace("stack")

type gopAgent struct{}

func (g gopAgent) Shutdown(ctx context.Context) error {
	fmt.Print("  shutting down gops...")
	agent.Close()
	fmt.Println("ok.")
	return nil
}

// Start is used to initialize all the
func Start() (broker jobs.Broker, schder scheduler.Scheduler, processes utils.Shutdowner, err error) {
	if config.IsDevRelease() {
		fmt.Println(`                           !! DEVELOPMENT RELEASE !!
You are running a development release which may deactivate some very important
security features. Please do not use this binary as your production server.
`)
	}

	err = agent.Listen(agent.Options{})
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error on gops agent: %s\n", err)
	}

	// Check that we can properly reach CouchDB.
	db, err := checkup.HTTPChecker{
		URL:            config.CouchURL().String(),
		MustContain:    `"version":"2`,
		Attempts:       5,
		AttemptSpacing: 1 * time.Second,
	}.Check()
	if err != nil {
		err = fmt.Errorf("Could not reach Couchdb 2.0 database: %s", err.Error())
		return
	}
	if db.Status() == checkup.Down {
		err = fmt.Errorf("Could not reach Couchdb 2.0 database:\n%s", db.String())
		return
	}
	if db.Status() != checkup.Healthy {
		log.Warnf("CouchDB does not seem to be in a healthy state, "+
			"the cozy-stack will be starting anyway:\n%s", db.String())
	}

	// Init the main global connection to the swift server
	fsURL := config.FsURL()
	if fsURL.Scheme == config.SchemeSwift {
		if err = config.InitSwiftConnection(fsURL); err != nil {
			return
		}
	}

	// Start update cron for auto-updates
	cronUpdates, err := instance.StartUpdateCron()
	if err != nil {
		return
	}

	jobsConfig := config.GetConfig().Jobs
	nbWorkers := jobsConfig.Workers
	if cli := jobsConfig.Redis.Client(); cli != nil {
		broker = jobs.NewRedisBroker(nbWorkers, cli)
		schder = scheduler.NewRedisScheduler(cli)
	} else {
		broker = jobs.NewMemBroker(nbWorkers)
		schder = scheduler.NewMemScheduler()
	}
	if err = broker.Start(jobs.GetWorkersList()); err != nil {
		return
	}
	if err = schder.Start(broker); err != nil {
		return
	}

	// Global shutdowner that composes all the running processes of the stack
	processes = utils.NewGroupShutdown(
		broker,
		schder,
		cronUpdates,
		gopAgent{},
	)
	return
}
<|endoftext|>


<commit_before>// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package tikv

import (
	"bytes"
	"context"
	"math"
	"time"

	"github.com/pingcap/errors"
	"github.com/pingcap/failpoint"
	"github.com/pingcap/kvproto/pkg/kvrpcpb"
	"github.com/pingcap/kvproto/pkg/metapb"
	"github.com/pingcap/kvproto/pkg/pdpb"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/store/tikv/tikvrpc"
	"github.com/pingcap/tidb/util"
	"github.com/pingcap/tidb/util/logutil"
	"github.com/pingcap/tidb/util/stringutil"
	"go.uber.org/zap"
)

func equalRegionStartKey(key, regionStartKey []byte) bool {
	if bytes.Equal(key, regionStartKey) {
		return true
	}
	return false
}

func (s *tikvStore) splitBatchRegionsReq(bo *Backoffer, keys [][]byte, scatter bool) (*tikvrpc.Response, error) {
	// equalRegionStartKey is used to filter split keys.
	// If the split key is equal to the start key of the region, then the key has been split, we need to skip the split key.
	groups, _, err := s.regionCache.GroupKeysByRegion(bo, keys, equalRegionStartKey)
	if err != nil {
		return nil, errors.Trace(err)
	}

	var batches []batch
	for regionID, groupKeys := range groups {
		batches = appendKeyBatches(batches, regionID, groupKeys, rawBatchPutSize)
	}

	if len(batches) == 0 {
		return nil, nil
	}
	// The first time it enters this function.
	if bo.totalSleep == 0 {
		logutil.BgLogger().Info("split batch regions request",
			zap.Int("split key count", len(keys)),
			zap.Int("batch count", len(batches)),
			zap.Uint64("first batch, region ID", batches[0].regionID.id),
			zap.Stringer("first split key", kv.Key(batches[0].keys[0])))
	}
	if len(batches) == 1 {
		resp := s.batchSendSingleRegion(bo, batches[0], scatter)
		return resp.resp, errors.Trace(resp.err)
	}
	ch := make(chan singleBatchResp, len(batches))
	for _, batch1 := range batches {
		go func(b batch) {
			backoffer, cancel := bo.Fork()
			defer cancel()

			util.WithRecovery(func() {
				select {
				case ch <- s.batchSendSingleRegion(backoffer, b, scatter):
				case <-bo.ctx.Done():
					ch <- singleBatchResp{err: bo.ctx.Err()}
				}
			}, func(r interface{}) {
				if r != nil {
					ch <- singleBatchResp{err: errors.Errorf("%v", r)}
				}
			})
		}(batch1)
	}

	srResp := &kvrpcpb.SplitRegionResponse{Regions: make([]*metapb.Region, 0, len(keys)*2)}
	for i := 0; i < len(batches); i++ {
		batchResp := <-ch
		if batchResp.err != nil {
			logutil.BgLogger().Info("batch split regions failed", zap.Error(batchResp.err))
			if err == nil {
				err = batchResp.err
			}
		}

		// If the split succeeds and the scatter fails, we also need to add the region IDs.
		if batchResp.resp != nil {
			spResp := batchResp.resp.Resp.(*kvrpcpb.SplitRegionResponse)
			regions := spResp.GetRegions()
			srResp.Regions = append(srResp.Regions, regions...)
		}
	}
	return &tikvrpc.Response{Resp: srResp}, errors.Trace(err)
}

func (s *tikvStore) batchSendSingleRegion(bo *Backoffer, batch batch, scatter bool) singleBatchResp {
	failpoint.Inject("MockSplitRegionTimeout", func(val failpoint.Value) {
		if val.(bool) {
			time.Sleep(time.Second*1 + time.Millisecond*10)
		}
	})

	req := tikvrpc.NewRequest(tikvrpc.CmdSplitRegion, &kvrpcpb.SplitRegionRequest{
		SplitKeys: batch.keys,
	}, kvrpcpb.Context{
		Priority: kvrpcpb.CommandPri_Normal,
	})

	sender := NewRegionRequestSender(s.regionCache, s.client)
	resp, err := sender.SendReq(bo, req, batch.regionID, readTimeoutShort)

	batchResp := singleBatchResp{resp: resp}
	if err != nil {
		batchResp.err = errors.Trace(err)
		return batchResp
	}
	regionErr, err := resp.GetRegionError()
	if err != nil {
		batchResp.err = errors.Trace(err)
		return batchResp
	}
	if regionErr != nil {
		err := bo.Backoff(BoRegionMiss, errors.New(regionErr.String()))
		if err != nil {
			batchResp.err = errors.Trace(err)
			return batchResp
		}
		resp, err = s.splitBatchRegionsReq(bo, batch.keys, scatter)
		batchResp.resp = resp
		batchResp.err = err
		return batchResp
	}

	spResp := resp.Resp.(*kvrpcpb.SplitRegionResponse)
	regions := spResp.GetRegions()
	if len(regions) > 0 {
		// Divide a region into n, one of them may not need to be scattered,
		// so n-1 needs to be scattered to other stores.
		spResp.Regions = regions[:len(regions)-1]
	}
	logutil.BgLogger().Info("batch split regions complete",
		zap.Uint64("batch region ID", batch.regionID.id),
		zap.Stringer("first at", kv.Key(batch.keys[0])),
		zap.Stringer("first new region left", stringutil.MemoizeStr(func() string {
			if len(spResp.Regions) == 0 {
				return ""
			}
			return logutil.Hex(spResp.Regions[0]).String()
		})),
		zap.Int("new region count", len(spResp.Regions)))

	if !scatter {
		if len(spResp.Regions) == 0 {
			return batchResp
		}
		return batchResp
	}

	for i, r := range spResp.Regions {
		if err = s.scatterRegion(r.Id); err == nil {
			logutil.BgLogger().Info("batch split regions, scatter region complete",
				zap.Uint64("batch region ID", batch.regionID.id),
				zap.Stringer("at", kv.Key(batch.keys[i])),
				zap.Stringer("new region left", logutil.Hex(r)))
			continue
		}

		logutil.BgLogger().Info("batch split regions, scatter region failed",
			zap.Uint64("batch region ID", batch.regionID.id),
			zap.Stringer("at", kv.Key(batch.keys[i])),
			zap.Stringer("new region left", logutil.Hex(r)),
			zap.Error(err))
		if batchResp.err == nil {
			batchResp.err = err
		}
		if ErrPDServerTimeout.Equal(err) {
			break
		}
	}
	return batchResp
}

// SplitRegions splits regions by splitKeys.
func (s *tikvStore) SplitRegions(ctx context.Context, splitKeys [][]byte, scatter bool) (regionIDs []uint64, err error) {
	bo := NewBackoffer(ctx, int(math.Min(float64(len(splitKeys))*splitRegionBackoff, maxSplitRegionsBackoff)))
	resp, err := s.splitBatchRegionsReq(bo, splitKeys, scatter)
	regionIDs = make([]uint64, 0, len(splitKeys))
	if resp != nil && resp.Resp != nil {
		spResp := resp.Resp.(*kvrpcpb.SplitRegionResponse)
		for _, r := range spResp.Regions {
			regionIDs = append(regionIDs, r.Id)
		}
		logutil.BgLogger().Info("split regions complete", zap.Int("region count", len(regionIDs)), zap.Uint64s("region IDs", regionIDs))
	}
	return regionIDs, errors.Trace(err)
}

func (s *tikvStore) scatterRegion(regionID uint64) error {
	failpoint.Inject("MockScatterRegionTimeout", func(val failpoint.Value) {
		if val.(bool) {
			failpoint.Return(ErrPDServerTimeout)
		}
	})

	logutil.BgLogger().Info("start scatter region",
		zap.Uint64("regionID", regionID))
	bo := NewBackoffer(context.Background(), scatterRegionBackoff)
	for {
		err := s.pdClient.ScatterRegion(context.Background(), regionID)
		if err == nil {
			break
		}
		err = bo.Backoff(BoPDRPC, errors.New(err.Error()))
		if err != nil {
			return errors.Trace(err)
		}
	}
	logutil.BgLogger().Debug("scatter region complete",
		zap.Uint64("regionID", regionID))
	return nil
}

// WaitScatterRegionFinish implements SplitableStore interface.
// backOff is the back off time of the wait scatter region.(Milliseconds)
// if backOff <= 0, the default wait scatter back off time will be used.
func (s *tikvStore) WaitScatterRegionFinish(regionID uint64, backOff int) error {
	if backOff <= 0 {
		backOff = waitScatterRegionFinishBackoff
	}
	logutil.BgLogger().Info("wait scatter region",
		zap.Uint64("regionID", regionID), zap.Int("backoff(ms)", backOff))

	bo := NewBackoffer(context.Background(), backOff)
	logFreq := 0
	for {
		resp, err := s.pdClient.GetOperator(context.Background(), regionID)
		if err == nil && resp != nil {
			if !bytes.Equal(resp.Desc, []byte("scatter-region")) || resp.Status != pdpb.OperatorStatus_RUNNING {
				logutil.BgLogger().Info("wait scatter region finished",
					zap.Uint64("regionID", regionID))
				return nil
			}
			if logFreq%10 == 0 {
				logutil.BgLogger().Info("wait scatter region",
					zap.Uint64("regionID", regionID),
					zap.String("reverse", string(resp.Desc)),
					zap.String("status", pdpb.OperatorStatus_name[int32(resp.Status)]))
			}
			logFreq++
		}
		if err != nil {
			err = bo.Backoff(BoRegionMiss, errors.New(err.Error()))
		} else {
			err = bo.Backoff(BoRegionMiss, errors.New("wait scatter region timeout"))
		}
		if err != nil {
			return errors.Trace(err)
		}
	}
}

// CheckRegionInScattering uses to check whether scatter region finished.
func (s *tikvStore) CheckRegionInScattering(regionID uint64) (bool, error) {
	bo := NewBackoffer(context.Background(), locateRegionMaxBackoff)
	for {
		resp, err := s.pdClient.GetOperator(context.Background(), regionID)
		if err == nil && resp != nil {
			if !bytes.Equal(resp.Desc, []byte("scatter-region")) || resp.Status != pdpb.OperatorStatus_RUNNING {
				return false, nil
			}
		}
		if err != nil {
			err = bo.Backoff(BoRegionMiss, errors.New(err.Error()))
		} else {
			return true, nil
		}
		if err != nil {
			return true, errors.Trace(err)
		}
	}
}
<commit_msg>executor: make TestSplitRegionTimeout stable (#13152)<commit_after>// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package tikv

import (
	"bytes"
	"context"
	"math"

	"github.com/pingcap/errors"
	"github.com/pingcap/failpoint"
	"github.com/pingcap/kvproto/pkg/kvrpcpb"
	"github.com/pingcap/kvproto/pkg/metapb"
	"github.com/pingcap/kvproto/pkg/pdpb"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/store/tikv/tikvrpc"
	"github.com/pingcap/tidb/util"
	"github.com/pingcap/tidb/util/logutil"
	"github.com/pingcap/tidb/util/stringutil"
	"go.uber.org/zap"
)

func equalRegionStartKey(key, regionStartKey []byte) bool {
	if bytes.Equal(key, regionStartKey) {
		return true
	}
	return false
}

func (s *tikvStore) splitBatchRegionsReq(bo *Backoffer, keys [][]byte, scatter bool) (*tikvrpc.Response, error) {
	// equalRegionStartKey is used to filter split keys.
	// If the split key is equal to the start key of the region, then the key has been split, we need to skip the split key.
	groups, _, err := s.regionCache.GroupKeysByRegion(bo, keys, equalRegionStartKey)
	if err != nil {
		return nil, errors.Trace(err)
	}

	var batches []batch
	for regionID, groupKeys := range groups {
		batches = appendKeyBatches(batches, regionID, groupKeys, rawBatchPutSize)
	}

	if len(batches) == 0 {
		return nil, nil
	}
	// The first time it enters this function.
	if bo.totalSleep == 0 {
		logutil.BgLogger().Info("split batch regions request",
			zap.Int("split key count", len(keys)),
			zap.Int("batch count", len(batches)),
			zap.Uint64("first batch, region ID", batches[0].regionID.id),
			zap.Stringer("first split key", kv.Key(batches[0].keys[0])))
	}
	if len(batches) == 1 {
		resp := s.batchSendSingleRegion(bo, batches[0], scatter)
		return resp.resp, errors.Trace(resp.err)
	}
	ch := make(chan singleBatchResp, len(batches))
	for _, batch1 := range batches {
		go func(b batch) {
			backoffer, cancel := bo.Fork()
			defer cancel()

			util.WithRecovery(func() {
				select {
				case ch <- s.batchSendSingleRegion(backoffer, b, scatter):
				case <-bo.ctx.Done():
					ch <- singleBatchResp{err: bo.ctx.Err()}
				}
			}, func(r interface{}) {
				if r != nil {
					ch <- singleBatchResp{err: errors.Errorf("%v", r)}
				}
			})
		}(batch1)
	}

	srResp := &kvrpcpb.SplitRegionResponse{Regions: make([]*metapb.Region, 0, len(keys)*2)}
	for i := 0; i < len(batches); i++ {
		batchResp := <-ch
		if batchResp.err != nil {
			logutil.BgLogger().Info("batch split regions failed", zap.Error(batchResp.err))
			if err == nil {
				err = batchResp.err
			}
		}

		// If the split succeeds and the scatter fails, we also need to add the region IDs.
		if batchResp.resp != nil {
			spResp := batchResp.resp.Resp.(*kvrpcpb.SplitRegionResponse)
			regions := spResp.GetRegions()
			srResp.Regions = append(srResp.Regions, regions...)
		}
	}
	return &tikvrpc.Response{Resp: srResp}, errors.Trace(err)
}

func (s *tikvStore) batchSendSingleRegion(bo *Backoffer, batch batch, scatter bool) singleBatchResp {
	failpoint.Inject("MockSplitRegionTimeout", func(val failpoint.Value) {
		if val.(bool) {
			if _, ok := bo.ctx.Deadline(); ok {
				<-bo.ctx.Done()
			}
		}
	})

	req := tikvrpc.NewRequest(tikvrpc.CmdSplitRegion, &kvrpcpb.SplitRegionRequest{
		SplitKeys: batch.keys,
	}, kvrpcpb.Context{
		Priority: kvrpcpb.CommandPri_Normal,
	})

	sender := NewRegionRequestSender(s.regionCache, s.client)
	resp, err := sender.SendReq(bo, req, batch.regionID, readTimeoutShort)

	batchResp := singleBatchResp{resp: resp}
	if err != nil {
		batchResp.err = errors.Trace(err)
		return batchResp
	}
	regionErr, err := resp.GetRegionError()
	if err != nil {
		batchResp.err = errors.Trace(err)
		return batchResp
	}
	if regionErr != nil {
		err := bo.Backoff(BoRegionMiss, errors.New(regionErr.String()))
		if err != nil {
			batchResp.err = errors.Trace(err)
			return batchResp
		}
		resp, err = s.splitBatchRegionsReq(bo, batch.keys, scatter)
		batchResp.resp = resp
		batchResp.err = err
		return batchResp
	}

	spResp := resp.Resp.(*kvrpcpb.SplitRegionResponse)
	regions := spResp.GetRegions()
	if len(regions) > 0 {
		// Divide a region into n, one of them may not need to be scattered,
		// so n-1 needs to be scattered to other stores.
		spResp.Regions = regions[:len(regions)-1]
	}
	logutil.BgLogger().Info("batch split regions complete",
		zap.Uint64("batch region ID", batch.regionID.id),
		zap.Stringer("first at", kv.Key(batch.keys[0])),
		zap.Stringer("first new region left", stringutil.MemoizeStr(func() string {
			if len(spResp.Regions) == 0 {
				return ""
			}
			return logutil.Hex(spResp.Regions[0]).String()
		})),
		zap.Int("new region count", len(spResp.Regions)))

	if !scatter {
		if len(spResp.Regions) == 0 {
			return batchResp
		}
		return batchResp
	}

	for i, r := range spResp.Regions {
		if err = s.scatterRegion(r.Id); err == nil {
			logutil.BgLogger().Info("batch split regions, scatter region complete",
				zap.Uint64("batch region ID", batch.regionID.id),
				zap.Stringer("at", kv.Key(batch.keys[i])),
				zap.Stringer("new region left", logutil.Hex(r)))
			continue
		}

		logutil.BgLogger().Info("batch split regions, scatter region failed",
			zap.Uint64("batch region ID", batch.regionID.id),
			zap.Stringer("at", kv.Key(batch.keys[i])),
			zap.Stringer("new region left", logutil.Hex(r)),
			zap.Error(err))
		if batchResp.err == nil {
			batchResp.err = err
		}
		if ErrPDServerTimeout.Equal(err) {
			break
		}
	}
	return batchResp
}

// SplitRegions splits regions by splitKeys.
func (s *tikvStore) SplitRegions(ctx context.Context, splitKeys [][]byte, scatter bool) (regionIDs []uint64, err error) {
	bo := NewBackoffer(ctx, int(math.Min(float64(len(splitKeys))*splitRegionBackoff, maxSplitRegionsBackoff)))
	resp, err := s.splitBatchRegionsReq(bo, splitKeys, scatter)
	regionIDs = make([]uint64, 0, len(splitKeys))
	if resp != nil && resp.Resp != nil {
		spResp := resp.Resp.(*kvrpcpb.SplitRegionResponse)
		for _, r := range spResp.Regions {
			regionIDs = append(regionIDs, r.Id)
		}
		logutil.BgLogger().Info("split regions complete", zap.Int("region count", len(regionIDs)), zap.Uint64s("region IDs", regionIDs))
	}
	return regionIDs, errors.Trace(err)
}

func (s *tikvStore) scatterRegion(regionID uint64) error {
	failpoint.Inject("MockScatterRegionTimeout", func(val failpoint.Value) {
		if val.(bool) {
			failpoint.Return(ErrPDServerTimeout)
		}
	})

	logutil.BgLogger().Info("start scatter region",
		zap.Uint64("regionID", regionID))
	bo := NewBackoffer(context.Background(), scatterRegionBackoff)
	for {
		err := s.pdClient.ScatterRegion(context.Background(), regionID)
		if err == nil {
			break
		}
		err = bo.Backoff(BoPDRPC, errors.New(err.Error()))
		if err != nil {
			return errors.Trace(err)
		}
	}
	logutil.BgLogger().Debug("scatter region complete",
		zap.Uint64("regionID", regionID))
	return nil
}

// WaitScatterRegionFinish implements SplitableStore interface.
// backOff is the back off time of the wait scatter region.(Milliseconds)
// if backOff <= 0, the default wait scatter back off time will be used.
func (s *tikvStore) WaitScatterRegionFinish(regionID uint64, backOff int) error {
	if backOff <= 0 {
		backOff = waitScatterRegionFinishBackoff
	}
	logutil.BgLogger().Info("wait scatter region",
		zap.Uint64("regionID", regionID), zap.Int("backoff(ms)", backOff))

	bo := NewBackoffer(context.Background(), backOff)
	logFreq := 0
	for {
		resp, err := s.pdClient.GetOperator(context.Background(), regionID)
		if err == nil && resp != nil {
			if !bytes.Equal(resp.Desc, []byte("scatter-region")) || resp.Status != pdpb.OperatorStatus_RUNNING {
				logutil.BgLogger().Info("wait scatter region finished",
					zap.Uint64("regionID", regionID))
				return nil
			}
			if logFreq%10 == 0 {
				logutil.BgLogger().Info("wait scatter region",
					zap.Uint64("regionID", regionID),
					zap.String("reverse", string(resp.Desc)),
					zap.String("status", pdpb.OperatorStatus_name[int32(resp.Status)]))
			}
			logFreq++
		}
		if err != nil {
			err = bo.Backoff(BoRegionMiss, errors.New(err.Error()))
		} else {
			err = bo.Backoff(BoRegionMiss, errors.New("wait scatter region timeout"))
		}
		if err != nil {
			return errors.Trace(err)
		}
	}
}

// CheckRegionInScattering uses to check whether scatter region finished.
func (s *tikvStore) CheckRegionInScattering(regionID uint64) (bool, error) {
	bo := NewBackoffer(context.Background(), locateRegionMaxBackoff)
	for {
		resp, err := s.pdClient.GetOperator(context.Background(), regionID)
		if err == nil && resp != nil {
			if !bytes.Equal(resp.Desc, []byte("scatter-region")) || resp.Status != pdpb.OperatorStatus_RUNNING {
				return false, nil
			}
		}
		if err != nil {
			err = bo.Backoff(BoRegionMiss, errors.New(err.Error()))
		} else {
			return true, nil
		}
		if err != nil {
			return true, errors.Trace(err)
		}
	}
}
<|endoftext|>


<commit_before>package types

import (
	"encoding/json"
)

type Template struct {
	Id                      string           `json:"id,omitempty" header:"ID"`
	Name                    string           `json:"name,omitempty" header:"NAME"`
	GenericImgId            string           `json:"generic_image_id,omitempty" header:"GENERIC_IMAGE_ID"`
	ServiceList             []string         `json:"service_list,omitempty" header:"SERVICE_LIST"`
	ConfigurationAttributes *json.RawMessage `json:"configuration_attributes,omitempty" header:"CONFIGURATION_ATTRIBUTES"`
}

type TemplateScript struct {
	Id               string          `json:"id" header:"ID"`
	Type             string          `json:"type" header:"TYPE"`
	Template_Id      string          `json:"template_id" header:"TEMPLATE_ID"`
	Script_Id        string          `json:"script_id" header:"SCRIPT_ID"`
	Parameter_Values json.RawMessage `json:"parameter_values" header:"PARAMETER_VALUES"`
	Execution_Order  int             `json:"execution_order" header:"EXECUTION_ORDER"`
}

type TemplateServer struct {
	Id             string `json:"id"  header:"ID"`
	Name           string `json:"name" header:"NAME"`
	Fqdn           string `json:"fqdn" header:"FQDN"`
	State          string `json:"state" header:"STATE"`
	Public_ip      string `json:"public_ip" header:"PUBLIC_IP"`
	Workspace_id   string `json:"workspace_id" header:"WORKSPACE_ID"`
	Template_id    string `json:"template_id" header:"TEMPLATE_ID"`
	Server_plan_id string `json:"server_plan_id" header:"SERVER_PLAN_ID"`
	Ssh_profile_id string `json:"ssh_profile_id" header:"SSH_PROFILE_ID"`
}

type TemplateScriptCredentials interface{}
<commit_msg>change columns order<commit_after>package types

import (
	"encoding/json"
)

type Template struct {
	Id                      string           `json:"id,omitempty" header:"ID"`
	Name                    string           `json:"name,omitempty" header:"NAME"`
	GenericImgId            string           `json:"generic_image_id,omitempty" header:"GENERIC_IMAGE_ID"`
	ServiceList             []string         `json:"service_list,omitempty" header:"SERVICE_LIST"`
	ConfigurationAttributes *json.RawMessage `json:"configuration_attributes,omitempty" header:"CONFIGURATION_ATTRIBUTES"`
}

type TemplateScript struct {
	Id               string          `json:"id" header:"ID"`
	Type             string          `json:"type" header:"TYPE"`
	Execution_Order  int             `json:"execution_order" header:"EXECUTION_ORDER"`
	Template_Id      string          `json:"template_id" header:"TEMPLATE_ID"`
	Script_Id        string          `json:"script_id" header:"SCRIPT_ID"`
	Parameter_Values json.RawMessage `json:"parameter_values" header:"PARAMETER_VALUES"`
}

type TemplateServer struct {
	Id             string `json:"id"  header:"ID"`
	Name           string `json:"name" header:"NAME"`
	Fqdn           string `json:"fqdn" header:"FQDN"`
	State          string `json:"state" header:"STATE"`
	Public_ip      string `json:"public_ip" header:"PUBLIC_IP"`
	Workspace_id   string `json:"workspace_id" header:"WORKSPACE_ID"`
	Template_id    string `json:"template_id" header:"TEMPLATE_ID"`
	Server_plan_id string `json:"server_plan_id" header:"SERVER_PLAN_ID"`
	Ssh_profile_id string `json:"ssh_profile_id" header:"SSH_PROFILE_ID"`
}

type TemplateScriptCredentials interface{}
<|endoftext|>


<commit_before>package driver

import (
	"flag"
	"fmt"
	"log"
	"reflect"
	"strings"
	"sync"

	"github.com/chrislusf/glow/driver/plan"
	"github.com/chrislusf/glow/flow"
	"github.com/chrislusf/glow/netchan"
)

type TaskOption struct {
	ContextId    int
	TaskGroupId  int
	FistTaskName string
	Inputs       string
}

var taskOption TaskOption

func init() {
	flag.IntVar(&taskOption.ContextId, "glow.flow.id", -1, "flow id")
	flag.IntVar(&taskOption.TaskGroupId, "glow.taskGroup.id", -1, "task group id")
	flag.StringVar(&taskOption.FistTaskName, "glow.task.name", "", "name of first task in the task group")
	flag.StringVar(&taskOption.Inputs, "glow.taskGroup.inputs", "", "comma and @ seperated input locations")

	flow.RegisterTaskRunner(NewTaskRunner(&taskOption))
}

type TaskRunner struct {
	option *TaskOption
	Tasks  []*flow.Task
}

func NewTaskRunner(option *TaskOption) *TaskRunner {
	return &TaskRunner{option: option}
}

func (tr *TaskRunner) IsTaskMode() bool {
	return tr.option.TaskGroupId >= 0 && tr.option.ContextId >= 0
}

// if this should not run, return false
func (tr *TaskRunner) Run(fc *flow.FlowContext) {

	taskGroups := plan.GroupTasks(fc)

	tr.Tasks = taskGroups[tr.option.TaskGroupId].Tasks

	if len(tr.Tasks) == 0 {
		log.Println("How can the task group has no tasks!")
		return
	}

	// println("taskGroup", tr.Tasks[0].Name(), "starts")
	// 4. setup task input and output channels
	var wg sync.WaitGroup
	tr.connectInputsAndOutputs(&wg)
	// 6. starts to run the task locally
	for _, task := range tr.Tasks {
		// println("run task", task.Name())
		wg.Add(1)
		go func(task *flow.Task) {
			defer wg.Done()
			task.RunTask()
		}(task)
	}
	// 7. need to close connected output channels
	wg.Wait()
	// println("taskGroup", tr.Tasks[0].Name(), "finishes")
}

func (tr *TaskRunner) connectInputsAndOutputs(wg *sync.WaitGroup) {
	name2Location := make(map[string]string)
	if tr.option.Inputs != "" {
		for _, nameLocation := range strings.Split(tr.option.Inputs, ",") {
			// println("input:", nameLocation)
			nl := strings.Split(nameLocation, "@")
			name2Location[nl[0]] = nl[1]
		}
	}
	tr.connectExternalInputChannels(wg)
	tr.connectExternalInputs(wg, name2Location)
	tr.connectInternalInputsAndOutputs(wg)
	tr.connectExternalOutputs(wg)
}

func (tr *TaskRunner) connectInternalInputsAndOutputs(wg *sync.WaitGroup) {
	for i, _ := range tr.Tasks {
		if i == len(tr.Tasks)-1 {
			continue
		}
		currentShard, nextShard := tr.Tasks[i].Outputs[0], tr.Tasks[i+1].Inputs[0]

		currentShard.SetupReadingChans()

		wg.Add(1)
		go func(currentShard, nextShard *flow.DatasetShard, i int) {
			defer wg.Done()
			for {
				if t, ok := currentShard.WriteChan.Recv(); ok {
					nextShard.SendForRead(t)
				} else {
					nextShard.CloseRead()
					break
				}
			}
		}(currentShard, nextShard, i)
	}
}

func (tr *TaskRunner) connectExternalInputs(wg *sync.WaitGroup, name2Location map[string]string) {
	task := tr.Tasks[0]
	for i, shard := range task.Inputs {
		d := shard.Parent
		readChanName := shard.Name()
		// println("taskGroup", tr.option.TaskGroupId, "task", task.Name(), "trying to read from:", readChanName, len(task.InputChans))
		rawChan, err := netchan.GetDirectReadChannel(readChanName, name2Location[readChanName])
		if err != nil {
			log.Panic(err)
		}
		netchan.ConnectRawReadChannelToTyped(rawChan, task.InputChans[i], d.Type, wg)
	}
}

func (tr *TaskRunner) connectExternalInputChannels(wg *sync.WaitGroup) {
	// this is only for Channel dataset
	firstTask := tr.Tasks[0]
	if firstTask.Inputs != nil {
		return
	}
	ds := firstTask.Outputs[0].Parent
	for i, _ := range ds.ExternalInputChans {
		inputChanName := fmt.Sprintf("ct-%d-input-%d-p-%d", tr.option.ContextId, ds.Id, i)
		rawChan, err := netchan.GetLocalReadChannel(inputChanName)
		if err != nil {
			log.Panic(err)
		}
		typedInputChan := make(chan reflect.Value)
		netchan.ConnectRawReadChannelToTyped(rawChan, typedInputChan, ds.Type, wg)
		firstTask.InputChans = append(firstTask.InputChans, typedInputChan)
	}
}

func (tr *TaskRunner) connectExternalOutputs(wg *sync.WaitGroup) {
	task := tr.Tasks[len(tr.Tasks)-1]
	for _, shard := range task.Outputs {
		writeChanName := shard.Name()
		// println("taskGroup", tr.option.TaskGroupId, "step", task.Step.Id, "task", task.Id, "writing to:", writeChanName)
		rawChan, err := netchan.GetLocalSendChannel(writeChanName, wg)
		if err != nil {
			log.Panic(err)
		}
		netchan.ConnectTypedWriteChannelToRaw(shard.WriteChan, rawChan, wg)
	}
}
<commit_msg>ensure run the correct flow context<commit_after>package driver

import (
	"flag"
	"fmt"
	"log"
	"reflect"
	"strings"
	"sync"

	"github.com/chrislusf/glow/driver/plan"
	"github.com/chrislusf/glow/flow"
	"github.com/chrislusf/glow/netchan"
)

type TaskOption struct {
	ContextId    int
	TaskGroupId  int
	FistTaskName string
	Inputs       string
}

var taskOption TaskOption

func init() {
	flag.IntVar(&taskOption.ContextId, "glow.flow.id", -1, "flow id")
	flag.IntVar(&taskOption.TaskGroupId, "glow.taskGroup.id", -1, "task group id")
	flag.StringVar(&taskOption.FistTaskName, "glow.task.name", "", "name of first task in the task group")
	flag.StringVar(&taskOption.Inputs, "glow.taskGroup.inputs", "", "comma and @ seperated input locations")

	flow.RegisterTaskRunner(NewTaskRunner(&taskOption))
}

type TaskRunner struct {
	option *TaskOption
	Tasks  []*flow.Task
}

func NewTaskRunner(option *TaskOption) *TaskRunner {
	return &TaskRunner{option: option}
}

func (tr *TaskRunner) IsTaskMode() bool {
	return tr.option.TaskGroupId >= 0 && tr.option.ContextId >= 0
}

// if this should not run, return false
func (tr *TaskRunner) Run(fc *flow.FlowContext) {
	if fc.Id != tr.option.ContextId {
		return
	}

	taskGroups := plan.GroupTasks(fc)

	tr.Tasks = taskGroups[tr.option.TaskGroupId].Tasks

	if len(tr.Tasks) == 0 {
		log.Println("How can the task group has no tasks!")
		return
	}

	// println("taskGroup", tr.Tasks[0].Name(), "starts")
	// 4. setup task input and output channels
	var wg sync.WaitGroup
	tr.connectInputsAndOutputs(&wg)
	// 6. starts to run the task locally
	for _, task := range tr.Tasks {
		// println("run task", task.Name())
		wg.Add(1)
		go func(task *flow.Task) {
			defer wg.Done()
			task.RunTask()
		}(task)
	}
	// 7. need to close connected output channels
	wg.Wait()
	// println("taskGroup", tr.Tasks[0].Name(), "finishes")
}

func (tr *TaskRunner) connectInputsAndOutputs(wg *sync.WaitGroup) {
	name2Location := make(map[string]string)
	if tr.option.Inputs != "" {
		for _, nameLocation := range strings.Split(tr.option.Inputs, ",") {
			// println("input:", nameLocation)
			nl := strings.Split(nameLocation, "@")
			name2Location[nl[0]] = nl[1]
		}
	}
	tr.connectExternalInputChannels(wg)
	tr.connectExternalInputs(wg, name2Location)
	tr.connectInternalInputsAndOutputs(wg)
	tr.connectExternalOutputs(wg)
}

func (tr *TaskRunner) connectInternalInputsAndOutputs(wg *sync.WaitGroup) {
	for i, _ := range tr.Tasks {
		if i == len(tr.Tasks)-1 {
			continue
		}
		currentShard, nextShard := tr.Tasks[i].Outputs[0], tr.Tasks[i+1].Inputs[0]

		currentShard.SetupReadingChans()

		wg.Add(1)
		go func(currentShard, nextShard *flow.DatasetShard, i int) {
			defer wg.Done()
			for {
				if t, ok := currentShard.WriteChan.Recv(); ok {
					nextShard.SendForRead(t)
				} else {
					nextShard.CloseRead()
					break
				}
			}
		}(currentShard, nextShard, i)
	}
}

func (tr *TaskRunner) connectExternalInputs(wg *sync.WaitGroup, name2Location map[string]string) {
	task := tr.Tasks[0]
	for i, shard := range task.Inputs {
		d := shard.Parent
		readChanName := shard.Name()
		// println("taskGroup", tr.option.TaskGroupId, "task", task.Name(), "trying to read from:", readChanName, len(task.InputChans))
		rawChan, err := netchan.GetDirectReadChannel(readChanName, name2Location[readChanName])
		if err != nil {
			log.Panic(err)
		}
		netchan.ConnectRawReadChannelToTyped(rawChan, task.InputChans[i], d.Type, wg)
	}
}

func (tr *TaskRunner) connectExternalInputChannels(wg *sync.WaitGroup) {
	// this is only for Channel dataset
	firstTask := tr.Tasks[0]
	if firstTask.Inputs != nil {
		return
	}
	ds := firstTask.Outputs[0].Parent
	for i, _ := range ds.ExternalInputChans {
		inputChanName := fmt.Sprintf("ct-%d-input-%d-p-%d", tr.option.ContextId, ds.Id, i)
		rawChan, err := netchan.GetLocalReadChannel(inputChanName)
		if err != nil {
			log.Panic(err)
		}
		typedInputChan := make(chan reflect.Value)
		netchan.ConnectRawReadChannelToTyped(rawChan, typedInputChan, ds.Type, wg)
		firstTask.InputChans = append(firstTask.InputChans, typedInputChan)
	}
}

func (tr *TaskRunner) connectExternalOutputs(wg *sync.WaitGroup) {
	task := tr.Tasks[len(tr.Tasks)-1]
	for _, shard := range task.Outputs {
		writeChanName := shard.Name()
		// println("taskGroup", tr.option.TaskGroupId, "step", task.Step.Id, "task", task.Id, "writing to:", writeChanName)
		rawChan, err := netchan.GetLocalSendChannel(writeChanName, wg)
		if err != nil {
			log.Panic(err)
		}
		netchan.ConnectTypedWriteChannelToRaw(shard.WriteChan, rawChan, wg)
	}
}
<|endoftext|>