<commit_before>package controllers

import (
	"fmt"

	"bitbucket.org/ikeikeikeike/antenna/lib/mailer"
	"github.com/astaxie/beego/context"
)

type ErrorController struct {
	BaseController
}

func (c *ErrorController) Error400() {
	mailer.SendStackTrace(traceMessage(c.Ctx))
	c.TplNames = "errors/400.tpl"
}

func (c *ErrorController) Error401() {
	mailer.SendStackTrace(traceMessage(c.Ctx))
	c.TplNames = "errors/401.tpl"
}

func (c *ErrorController) Error403() {
	mailer.SendStackTrace(traceMessage(c.Ctx))
	c.TplNames = "errors/403.tpl"
}

func (c *ErrorController) Error404() {
	c.TplNames = "errors/404.tpl"
}

func (c *ErrorController) Error500() {
	mailer.SendStackTrace(traceMessage(c.Ctx))
	c.TplNames = "errors/500.tpl"
}

func (c *ErrorController) Error503() {
	mailer.SendStackTrace(traceMessage(c.Ctx))
	c.TplNames = "errors/503.tpl"
}

func (c *ErrorController) ErrorDb() {
	mailer.SendStackTrace(traceMessage(c.Ctx))
	c.TplNames = "errors/dberror.tpl"
}

func traceMessage(ctx *context.Context) []string {
	return []string{
		fmt.Sprintf("Url: %s", ctx.Input.Url()),
		fmt.Sprintf("Uri: %s", ctx.Input.Uri()),
		fmt.Sprintf("Method: %s", ctx.Input.Method()),
		fmt.Sprintf("Params: %v", ctx.Input.Params),
		fmt.Sprintf("UserAgent: %s", ctx.Input.UserAgent()),
		"",
	}
}
<commit_msg>More section refer in error message template<commit_after>package controllers

import (
	"fmt"

	"bitbucket.org/ikeikeikeike/antenna/lib/mailer"
	"github.com/astaxie/beego/context"
)

type ErrorController struct {
	BaseController
}

func (c *ErrorController) Error400() {
	mailer.SendStackTrace(traceMessage(c.Ctx))
	c.TplNames = "errors/400.tpl"
}

func (c *ErrorController) Error401() {
	mailer.SendStackTrace(traceMessage(c.Ctx))
	c.TplNames = "errors/401.tpl"
}

func (c *ErrorController) Error403() {
	mailer.SendStackTrace(traceMessage(c.Ctx))
	c.TplNames = "errors/403.tpl"
}

func (c *ErrorController) Error404() {
	c.TplNames = "errors/404.tpl"
}

func (c *ErrorController) Error500() {
	mailer.SendStackTrace(traceMessage(c.Ctx))
	c.TplNames = "errors/500.tpl"
}

func (c *ErrorController) Error503() {
	mailer.SendStackTrace(traceMessage(c.Ctx))
	c.TplNames = "errors/503.tpl"
}

func (c *ErrorController) ErrorDb() {
	mailer.SendStackTrace(traceMessage(c.Ctx))
	c.TplNames = "errors/dberror.tpl"
}

func traceMessage(ctx *context.Context) []string {
	return []string{
		fmt.Sprintf("Url: %s", ctx.Input.Url()),
		fmt.Sprintf("Uri: %s", ctx.Input.Uri()),
		fmt.Sprintf("Method: %s", ctx.Input.Method()),
		fmt.Sprintf("Params: %v", ctx.Input.Params),
		fmt.Sprintf("Referer: %s", ctx.Input.Referer()),
		fmt.Sprintf("UserAgent: %s", ctx.Input.UserAgent()),
		"",
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package securecookie

import (
	"crypto/aes"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/base64"
	"errors"
	"fmt"
	"strings"
	"testing"
)

var testCookies = []interface{}{
	map[string]string{"foo": "bar"},
	map[string]string{"baz": "ding"},
}

var testStrings = []string{"foo", "bar", "baz"}

func TestSecureCookie(t *testing.T) {
	// TODO test too old / too new timestamps
	compareMaps := func(m1, m2 map[string]interface{}) error {
		if len(m1) != len(m2) {
			return errors.New("different maps")
		}
		for k, v := range m1 {
			if m2[k] != v {
				return fmt.Errorf("Different value for key %v: expected %v, got %v", k, m2[k], v)
			}
		}
		return nil
	}

	s1 := New([]byte("12345"), []byte("1234567890123456"))
	s2 := New([]byte("54321"), []byte("6543210987654321"))
	value := map[string]interface{}{
		"foo": "bar",
		"baz": 128,
	}

	for i := 0; i < 50; i++ {
		// Running this multiple times to check if any special character
		// breaks encoding/decoding.
		encoded, err1 := s1.Encode("sid", value)
		if err1 != nil {
			t.Error(err1)
			continue
		}
		dst := make(map[string]interface{})
		err2 := s1.Decode("sid", encoded, &dst)
		if err2 != nil {
			t.Fatalf("%v: %v", err2, encoded)
		}
		if err := compareMaps(dst, value); err != nil {
			t.Fatalf("Expected %v, got %v.", value, dst)
		}
		dst2 := make(map[string]interface{})
		err3 := s2.Decode("sid", encoded, &dst2)
		if err3 == nil {
			t.Fatalf("Expected failure decoding.")
		}
	}
}

func TestDecodeInvalid(t *testing.T) {
	// List of invalid cookies, which must not be accepted, base64-decoded
	// (they will be encoded before passing to Decode).
	invalidCookies := []string{
		"",
		" ",
		"\n",
		"||",
		"|||",
		"cookie",
	}
	s := New([]byte("12345"), nil)
	var dst string
	for i, v := range invalidCookies {
		err := s.Decode("name", base64.StdEncoding.EncodeToString([]byte(v)), &dst)
		if err == nil {
			t.Fatalf("%d: expected failure decoding", i)
		}
	}
}

func TestAuthentication(t *testing.T) {
	hash := hmac.New(sha256.New, []byte("secret-key"))
	for _, value := range testStrings {
		hash.Reset()
		signed := createMac(hash, []byte(value))
		hash.Reset()
		err := verifyMac(hash, []byte(value), signed)
		if err != nil {
			t.Error(err)
		}
	}
}

func TestEncription(t *testing.T) {
	block, err := aes.NewCipher([]byte("1234567890123456"))
	if err != nil {
		t.Fatalf("Block could not be created")
	}
	var encrypted, decrypted []byte
	for _, value := range testStrings {
		if encrypted, err = encrypt(block, []byte(value)); err != nil {
			t.Error(err)
		} else {
			if decrypted, err = decrypt(block, encrypted); err != nil {
				t.Error(err)
			}
			if string(decrypted) != value {
				t.Errorf("Expected %v, got %v.", value, string(decrypted))
			}
		}
	}
}

func TestSerialization(t *testing.T) {
	var (
		serialized   []byte
		deserialized map[string]string
		err          error
	)
	for _, value := range testCookies {
		if serialized, err = serialize(value); err != nil {
			t.Error(err)
		} else {
			deserialized = make(map[string]string)
			if err = deserialize(serialized, &deserialized); err != nil {
				t.Error(err)
			}
			if fmt.Sprintf("%v", deserialized) != fmt.Sprintf("%v", value) {
				t.Errorf("Expected %v, got %v.", value, deserialized)
			}
		}
	}
}

func TestEncoding(t *testing.T) {
	for _, value := range testStrings {
		encoded := encode([]byte(value))
		decoded, err := decode(encoded)
		if err != nil {
			t.Error(err)
		} else if string(decoded) != value {
			t.Errorf("Expected %v, got %s.", value, string(decoded))
		}
	}
}

func TestMultiError(t *testing.T) {
	s1, s2 := New(nil, nil), New(nil, nil)
	_, err := EncodeMulti("sid", "value", s1, s2)
	if len(err.(MultiError)) != 2 {
		t.Errorf("Expected 2 errors, got %s.", err)
	} else {
		if strings.Index(err.Error(), "hash key is not set") == -1 {
			t.Errorf("Expected missing hash key error, got %s.", err.Error())
		}
	}
}

func TestMultiNoCodecs(t *testing.T) {
	_, err := EncodeMulti("foo", "bar")
	if err != errNoCodecs {
		t.Errorf("EncodeMulti: bad value for error, got: %v", err)
	}

	var dst []byte
	err = DecodeMulti("foo", "bar", &dst)
	if err != errNoCodecs {
		t.Errorf("DecodeMulti: bad value for error, got: %v", err)
	}
}

func TestMissingKey(t *testing.T) {
	s1 := New(nil, nil)

	var dst []byte
	err := s1.Decode("sid", "value", &dst)
	if err != errHashKeyNotSet {
		t.Fatalf("Expected %#v, got %#v", errHashKeyNotSet, err)
	}
}

// ----------------------------------------------------------------------------

type FooBar struct {
	Foo int
	Bar string
}

func TestCustomType(t *testing.T) {
	s1 := New([]byte("12345"), []byte("1234567890123456"))
	// Type is not registered in gob. (!!!)
	src := &FooBar{42, "bar"}
	encoded, _ := s1.Encode("sid", src)

	dst := &FooBar{}
	_ = s1.Decode("sid", encoded, dst)
	if dst.Foo != 42 || dst.Bar != "bar" {
		t.Fatalf("Expected %#v, got %#v", src, dst)
	}
}
<commit_msg>Use reflect.DeepEqual instead<commit_after>// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package securecookie

import (
	"crypto/aes"
	"crypto/hmac"
	"crypto/sha256"
	"encoding/base64"
	"fmt"
	"reflect"
	"strings"
	"testing"
)

var testCookies = []interface{}{
	map[string]string{"foo": "bar"},
	map[string]string{"baz": "ding"},
}

var testStrings = []string{"foo", "bar", "baz"}

func TestSecureCookie(t *testing.T) {
	// TODO test too old / too new timestamps
	s1 := New([]byte("12345"), []byte("1234567890123456"))
	s2 := New([]byte("54321"), []byte("6543210987654321"))
	value := map[string]interface{}{
		"foo": "bar",
		"baz": 128,
	}

	for i := 0; i < 50; i++ {
		// Running this multiple times to check if any special character
		// breaks encoding/decoding.
		encoded, err1 := s1.Encode("sid", value)
		if err1 != nil {
			t.Error(err1)
			continue
		}
		dst := make(map[string]interface{})
		err2 := s1.Decode("sid", encoded, &dst)
		if err2 != nil {
			t.Fatalf("%v: %v", err2, encoded)
		}
		if !reflect.DeepEqual(dst, value) {
			t.Fatalf("Expected %v, got %v.", value, dst)
		}
		dst2 := make(map[string]interface{})
		err3 := s2.Decode("sid", encoded, &dst2)
		if err3 == nil {
			t.Fatalf("Expected failure decoding.")
		}
	}
}

func TestDecodeInvalid(t *testing.T) {
	// List of invalid cookies, which must not be accepted, base64-decoded
	// (they will be encoded before passing to Decode).
	invalidCookies := []string{
		"",
		" ",
		"\n",
		"||",
		"|||",
		"cookie",
	}
	s := New([]byte("12345"), nil)
	var dst string
	for i, v := range invalidCookies {
		err := s.Decode("name", base64.StdEncoding.EncodeToString([]byte(v)), &dst)
		if err == nil {
			t.Fatalf("%d: expected failure decoding", i)
		}
	}
}

func TestAuthentication(t *testing.T) {
	hash := hmac.New(sha256.New, []byte("secret-key"))
	for _, value := range testStrings {
		hash.Reset()
		signed := createMac(hash, []byte(value))
		hash.Reset()
		err := verifyMac(hash, []byte(value), signed)
		if err != nil {
			t.Error(err)
		}
	}
}

func TestEncription(t *testing.T) {
	block, err := aes.NewCipher([]byte("1234567890123456"))
	if err != nil {
		t.Fatalf("Block could not be created")
	}
	var encrypted, decrypted []byte
	for _, value := range testStrings {
		if encrypted, err = encrypt(block, []byte(value)); err != nil {
			t.Error(err)
		} else {
			if decrypted, err = decrypt(block, encrypted); err != nil {
				t.Error(err)
			}
			if string(decrypted) != value {
				t.Errorf("Expected %v, got %v.", value, string(decrypted))
			}
		}
	}
}

func TestSerialization(t *testing.T) {
	var (
		serialized   []byte
		deserialized map[string]string
		err          error
	)
	for _, value := range testCookies {
		if serialized, err = serialize(value); err != nil {
			t.Error(err)
		} else {
			deserialized = make(map[string]string)
			if err = deserialize(serialized, &deserialized); err != nil {
				t.Error(err)
			}
			if fmt.Sprintf("%v", deserialized) != fmt.Sprintf("%v", value) {
				t.Errorf("Expected %v, got %v.", value, deserialized)
			}
		}
	}
}

func TestEncoding(t *testing.T) {
	for _, value := range testStrings {
		encoded := encode([]byte(value))
		decoded, err := decode(encoded)
		if err != nil {
			t.Error(err)
		} else if string(decoded) != value {
			t.Errorf("Expected %v, got %s.", value, string(decoded))
		}
	}
}

func TestMultiError(t *testing.T) {
	s1, s2 := New(nil, nil), New(nil, nil)
	_, err := EncodeMulti("sid", "value", s1, s2)
	if len(err.(MultiError)) != 2 {
		t.Errorf("Expected 2 errors, got %s.", err)
	} else {
		if strings.Index(err.Error(), "hash key is not set") == -1 {
			t.Errorf("Expected missing hash key error, got %s.", err.Error())
		}
	}
}

func TestMultiNoCodecs(t *testing.T) {
	_, err := EncodeMulti("foo", "bar")
	if err != errNoCodecs {
		t.Errorf("EncodeMulti: bad value for error, got: %v", err)
	}

	var dst []byte
	err = DecodeMulti("foo", "bar", &dst)
	if err != errNoCodecs {
		t.Errorf("DecodeMulti: bad value for error, got: %v", err)
	}
}

func TestMissingKey(t *testing.T) {
	s1 := New(nil, nil)

	var dst []byte
	err := s1.Decode("sid", "value", &dst)
	if err != errHashKeyNotSet {
		t.Fatalf("Expected %#v, got %#v", errHashKeyNotSet, err)
	}
}

// ----------------------------------------------------------------------------

type FooBar struct {
	Foo int
	Bar string
}

func TestCustomType(t *testing.T) {
	s1 := New([]byte("12345"), []byte("1234567890123456"))
	// Type is not registered in gob. (!!!)
	src := &FooBar{42, "bar"}
	encoded, _ := s1.Encode("sid", src)

	dst := &FooBar{}
	_ = s1.Decode("sid", encoded, dst)
	if dst.Foo != 42 || dst.Bar != "bar" {
		t.Fatalf("Expected %#v, got %#v", src, dst)
	}
}
<|endoftext|>


<commit_before>package exec

import (
	"archive/tar"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"strings"

	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/worker"
)

const taskProcessPropertyName = "concourse:task-process"
const taskExitStatusPropertyName = "concourse:exit-status"

// MissingInputsError is returned when any of the task's required inputs are
// missing.
type MissingInputsError struct {
	Inputs []string
}

// Error prints a human-friendly message listing the inputs that were missing.
func (err MissingInputsError) Error() string {
	return fmt.Sprintf("missing inputs: %s", strings.Join(err.Inputs, ", "))
}

type MissingTaskImageSourceError struct {
	SourceName string
}

func (err MissingTaskImageSourceError) Error() string {
	return fmt.Sprintf(`missing image artifact source: %s
make sure there's a corresponding 'get' step, or a task that produces it as an output`, err.SourceName)
}

// TaskStep executes a TaskConfig, whose inputs will be fetched from the
// worker.ArtifactRepository and outputs will be added to the worker.ArtifactRepository.
type TaskStep struct {
	logger            lager.Logger
	containerID       worker.Identifier
	metadata          worker.Metadata
	tags              atc.Tags
	teamID            int
	delegate          TaskDelegate
	privileged        Privileged
	configSource      TaskConfigSource
	workerPool        worker.Client
	artifactsRoot     string
	resourceTypes     atc.VersionedResourceTypes
	inputMapping      map[string]string
	outputMapping     map[string]string
	imageArtifactName string
	clock             clock.Clock
	repo              *worker.ArtifactRepository

	process garden.Process

	exitStatus int
}

func newTaskStep(
	logger lager.Logger,
	containerID worker.Identifier,
	metadata worker.Metadata,
	tags atc.Tags,
	teamID int,
	delegate TaskDelegate,
	privileged Privileged,
	configSource TaskConfigSource,
	workerPool worker.Client,
	artifactsRoot string,
	resourceTypes atc.VersionedResourceTypes,
	inputMapping map[string]string,
	outputMapping map[string]string,
	imageArtifactName string,
	clock clock.Clock,
) TaskStep {
	return TaskStep{
		logger:            logger,
		containerID:       containerID,
		metadata:          metadata,
		tags:              tags,
		teamID:            teamID,
		delegate:          delegate,
		privileged:        privileged,
		configSource:      configSource,
		workerPool:        workerPool,
		artifactsRoot:     artifactsRoot,
		resourceTypes:     resourceTypes,
		inputMapping:      inputMapping,
		outputMapping:     outputMapping,
		imageArtifactName: imageArtifactName,
		clock:             clock,
	}
}

// Using finishes construction of the TaskStep and returns a *TaskStep. If the
// *TaskStep errors, its error is reported to the delegate.
func (step TaskStep) Using(prev Step, repo *worker.ArtifactRepository) Step {
	step.repo = repo

	return errorReporter{
		Step:          &step,
		ReportFailure: step.delegate.Failed,
	}
}

// Run will first load the TaskConfig. A worker will be selected based on the
// TaskConfig's platform, the TaskStep's tags, and prioritized by availability
// of volumes for the TaskConfig's inputs. Inputs that did not have volumes
// available on the worker will be streamed in to the container.
//
// If any inputs are not available in the worker.ArtifactRepository, MissingInputsError
// is returned.
//
// Once all the inputs are satisfies, the task's script will be executed, and
// the RunStep indicates that it's ready, and any signals will be forwarded to
// the script.
//
// If the script exits successfully, the outputs specified in the TaskConfig
// are registered with the worker.ArtifactRepository. If no outputs are specified, the
// task's entire working directory is registered as an ArtifactSource under the
// name of the task.
func (step *TaskStep) Run(signals <-chan os.Signal, ready chan<- struct{}) error {
	var err error
	var found bool

	processIO := garden.ProcessIO{
		Stdout: step.delegate.Stdout(),
		Stderr: step.delegate.Stderr(),
	}

	deprecationConfigSource := DeprecationConfigSource{
		Delegate: step.configSource,
		Stderr:   step.delegate.Stderr(),
	}

	config, err := deprecationConfigSource.FetchConfig(step.repo)
	if err != nil {
		return err
	}

	step.metadata.EnvironmentVariables = step.envForParams(config.Params)

	runContainerID := step.containerID
	runContainerID.Stage = db.ContainerStageRun

	container, found, err := step.workerPool.FindContainerForIdentifier(
		step.logger.Session("found-container"),
		runContainerID,
	)

	if err == nil && found {
		exitStatusProp, err := container.Property(taskExitStatusPropertyName)
		if err == nil {
			step.logger.Info("already-exited", lager.Data{"status": exitStatusProp})

			// process already completed; recover result

			_, err = fmt.Sscanf(exitStatusProp, "%d", &step.exitStatus)
			if err != nil {
				return err
			}

			step.registerSource(config, container)
			return nil
		}

		processID, err := container.Property(taskProcessPropertyName)
		if err != nil {
			// rogue container? perhaps did not shut down cleanly.
			return err
		}

		step.logger.Info("already-running", lager.Data{"process-id": processID})

		// process still running; re-attach
		step.process, err = container.Attach(processID, processIO)
		if err != nil {
			return err
		}

		step.logger.Info("attached")
	} else {
		// container does not exist; new session

		step.delegate.Initializing(config)

		workerSpec := worker.WorkerSpec{
			Platform: config.Platform,
			Tags:     step.tags,
			TeamID:   step.teamID,
		}

		if config.ImageResource != nil {
			workerSpec.ResourceType = config.ImageResource.Type
		}

		container, err = step.createContainer(config, signals)
		if err != nil {
			return err
		}

		step.delegate.Started()

		step.process, err = container.Run(garden.ProcessSpec{
			Path: config.Run.Path,
			Args: config.Run.Args,
			Env:  step.envForParams(config.Params),

			Dir: path.Join(step.artifactsRoot, config.Run.Dir),
			TTY: &garden.TTYSpec{},
		}, processIO)
		if err != nil {
			return err
		}

		err = container.SetProperty(taskProcessPropertyName, step.process.ID())
		if err != nil {
			return err
		}
	}

	close(ready)

	exited := make(chan struct{})
	var processStatus int
	var processErr error

	go func() {
		processStatus, processErr = step.process.Wait()
		close(exited)
	}()

	select {
	case <-signals:
		step.registerSource(config, container)

		err = container.Stop(false)
		if err != nil {
			step.logger.Error("stopping-container", err)
		}

		<-exited

		return ErrInterrupted

	case <-exited:
		if processErr != nil {
			return processErr
		}

		step.registerSource(config, container)

		step.exitStatus = processStatus

		err := container.SetProperty(taskExitStatusPropertyName, fmt.Sprintf("%d", processStatus))
		if err != nil {
			return err
		}

		step.delegate.Finished(ExitStatus(processStatus))

		return nil
	}
}

func (step *TaskStep) createContainer(config atc.TaskConfig, signals <-chan os.Signal) (worker.Container, error) {
	imageSpec := worker.ImageSpec{
		Privileged: bool(step.privileged),
	}
	if step.imageArtifactName != "" {
		source, found := step.repo.SourceFor(worker.ArtifactName(step.imageArtifactName))
		if !found {
			return nil, MissingTaskImageSourceError{step.imageArtifactName}
		}

		imageSpec.ImageArtifactSource = source
		imageSpec.ImageArtifactName = worker.ArtifactName(step.imageArtifactName)
	} else {
		imageSpec.ImageURL = config.Image
		imageSpec.ImageResource = config.ImageResource
	}

	runContainerID := step.containerID
	runContainerID.Stage = db.ContainerStageRun

	containerSpec := worker.ContainerSpec{
		Platform:  config.Platform,
		Tags:      step.tags,
		TeamID:    step.teamID,
		ImageSpec: imageSpec,
		User:      config.Run.User,
		Dir:       step.artifactsRoot,

		Inputs:  []worker.InputSource{},
		Outputs: worker.OutputPaths{},
	}

	var missingInputs []string
	for _, input := range config.Inputs {
		inputName := input.Name
		if sourceName, ok := step.inputMapping[inputName]; ok {
			inputName = sourceName
		}

		source, found := step.repo.SourceFor(worker.ArtifactName(inputName))
		if !found {
			missingInputs = append(missingInputs, inputName)
			continue
		}

		containerSpec.Inputs = append(containerSpec.Inputs, &taskInputSource{
			name:          worker.ArtifactName(inputName),
			config:        input,
			source:        source,
			artifactsRoot: step.artifactsRoot,
		})
	}

	if len(missingInputs) > 0 {
		return nil, MissingInputsError{missingInputs}
	}

	for _, output := range config.Outputs {
		path := artifactsPath(output, step.artifactsRoot)
		containerSpec.Outputs[output.Name] = path
	}

	container, err := step.workerPool.FindOrCreateBuildContainer(
		step.logger,
		signals,
		step.delegate,
		runContainerID,
		step.metadata,
		containerSpec,
		step.resourceTypes,
	)
	if err != nil {
		return nil, err
	}

	return container, nil
}

func (step *TaskStep) registerSource(config atc.TaskConfig, container worker.Container) {
	volumeMounts := container.VolumeMounts()

	step.logger.Debug("registering-outputs", lager.Data{"config": config})

	for _, output := range config.Outputs {
		outputName := output.Name
		if destinationName, ok := step.outputMapping[output.Name]; ok {
			outputName = destinationName
		}

		outputPath := artifactsPath(output, step.artifactsRoot)

		for _, mount := range volumeMounts {
			if mount.MountPath == outputPath {
				source := newVolumeSource(step.logger, mount.Volume)
				step.repo.RegisterSource(worker.ArtifactName(outputName), source)
			}
		}
	}
}

// Result indicates Success as true if the script's exit status was 0.
//
// It also indicates ExitStatus as the exit status of the script.
//
// All other types are ignored.
func (step *TaskStep) Result(x interface{}) bool {
	switch v := x.(type) {
	case *Success:
		*v = step.exitStatus == 0
		return true

	case *ExitStatus:
		*v = ExitStatus(step.exitStatus)
		return true

	default:
		return false
	}
}

func (TaskStep) envForParams(params map[string]string) []string {
	env := make([]string, 0, len(params))

	for k, v := range params {
		env = append(env, k+"="+v)
	}

	return env
}

type volumeSource struct {
	logger lager.Logger
	volume worker.Volume
}

func newVolumeSource(
	logger lager.Logger,
	volume worker.Volume,
) *volumeSource {
	return &volumeSource{
		logger: logger,
		volume: volume,
	}
}

func (src *volumeSource) StreamTo(destination worker.ArtifactDestination) error {
	out, err := src.volume.StreamOut(".")
	if err != nil {
		return err
	}

	defer out.Close()

	return destination.StreamIn(".", out)
}

func (src *volumeSource) StreamFile(filename string) (io.ReadCloser, error) {
	out, err := src.volume.StreamOut(filename)
	if err != nil {
		return nil, err
	}

	tarReader := tar.NewReader(out)

	_, err = tarReader.Next()
	if err != nil {
		return nil, FileNotFoundError{Path: filename}
	}

	return fileReadCloser{
		Reader: tarReader,
		Closer: out,
	}, nil
}

func (src *volumeSource) VolumeOn(w worker.Worker) (worker.Volume, bool, error) {
	return w.LookupVolume(src.logger, src.volume.Handle())
}

type taskInputSource struct {
	name          worker.ArtifactName
	config        atc.TaskInputConfig
	source        worker.ArtifactSource
	artifactsRoot string
}

func (s *taskInputSource) Name() worker.ArtifactName     { return s.name }
func (s *taskInputSource) Source() worker.ArtifactSource { return s.source }

func (s *taskInputSource) DestinationPath() string {
	subdir := s.config.Path
	if s.config.Path == "" {
		subdir = s.config.Name
	}

	return filepath.Join(s.artifactsRoot, subdir)
}

func artifactsPath(outputConfig atc.TaskOutputConfig, artifactsRoot string) string {
	outputSrc := outputConfig.Path
	if len(outputSrc) == 0 {
		outputSrc = outputConfig.Name
	}

	return path.Join(artifactsRoot, outputSrc) + "/"
}
<commit_msg>remove unused workerSpec var<commit_after>package exec

import (
	"archive/tar"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"strings"

	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/garden"
	"code.cloudfoundry.org/lager"
	"github.com/concourse/atc"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/worker"
)

const taskProcessPropertyName = "concourse:task-process"
const taskExitStatusPropertyName = "concourse:exit-status"

// MissingInputsError is returned when any of the task's required inputs are
// missing.
type MissingInputsError struct {
	Inputs []string
}

// Error prints a human-friendly message listing the inputs that were missing.
func (err MissingInputsError) Error() string {
	return fmt.Sprintf("missing inputs: %s", strings.Join(err.Inputs, ", "))
}

type MissingTaskImageSourceError struct {
	SourceName string
}

func (err MissingTaskImageSourceError) Error() string {
	return fmt.Sprintf(`missing image artifact source: %s
make sure there's a corresponding 'get' step, or a task that produces it as an output`, err.SourceName)
}

// TaskStep executes a TaskConfig, whose inputs will be fetched from the
// worker.ArtifactRepository and outputs will be added to the worker.ArtifactRepository.
type TaskStep struct {
	logger            lager.Logger
	containerID       worker.Identifier
	metadata          worker.Metadata
	tags              atc.Tags
	teamID            int
	delegate          TaskDelegate
	privileged        Privileged
	configSource      TaskConfigSource
	workerPool        worker.Client
	artifactsRoot     string
	resourceTypes     atc.VersionedResourceTypes
	inputMapping      map[string]string
	outputMapping     map[string]string
	imageArtifactName string
	clock             clock.Clock
	repo              *worker.ArtifactRepository

	process garden.Process

	exitStatus int
}

func newTaskStep(
	logger lager.Logger,
	containerID worker.Identifier,
	metadata worker.Metadata,
	tags atc.Tags,
	teamID int,
	delegate TaskDelegate,
	privileged Privileged,
	configSource TaskConfigSource,
	workerPool worker.Client,
	artifactsRoot string,
	resourceTypes atc.VersionedResourceTypes,
	inputMapping map[string]string,
	outputMapping map[string]string,
	imageArtifactName string,
	clock clock.Clock,
) TaskStep {
	return TaskStep{
		logger:            logger,
		containerID:       containerID,
		metadata:          metadata,
		tags:              tags,
		teamID:            teamID,
		delegate:          delegate,
		privileged:        privileged,
		configSource:      configSource,
		workerPool:        workerPool,
		artifactsRoot:     artifactsRoot,
		resourceTypes:     resourceTypes,
		inputMapping:      inputMapping,
		outputMapping:     outputMapping,
		imageArtifactName: imageArtifactName,
		clock:             clock,
	}
}

// Using finishes construction of the TaskStep and returns a *TaskStep. If the
// *TaskStep errors, its error is reported to the delegate.
func (step TaskStep) Using(prev Step, repo *worker.ArtifactRepository) Step {
	step.repo = repo

	return errorReporter{
		Step:          &step,
		ReportFailure: step.delegate.Failed,
	}
}

// Run will first load the TaskConfig. A worker will be selected based on the
// TaskConfig's platform, the TaskStep's tags, and prioritized by availability
// of volumes for the TaskConfig's inputs. Inputs that did not have volumes
// available on the worker will be streamed in to the container.
//
// If any inputs are not available in the worker.ArtifactRepository, MissingInputsError
// is returned.
//
// Once all the inputs are satisfies, the task's script will be executed, and
// the RunStep indicates that it's ready, and any signals will be forwarded to
// the script.
//
// If the script exits successfully, the outputs specified in the TaskConfig
// are registered with the worker.ArtifactRepository. If no outputs are specified, the
// task's entire working directory is registered as an ArtifactSource under the
// name of the task.
func (step *TaskStep) Run(signals <-chan os.Signal, ready chan<- struct{}) error {
	var err error
	var found bool

	processIO := garden.ProcessIO{
		Stdout: step.delegate.Stdout(),
		Stderr: step.delegate.Stderr(),
	}

	deprecationConfigSource := DeprecationConfigSource{
		Delegate: step.configSource,
		Stderr:   step.delegate.Stderr(),
	}

	config, err := deprecationConfigSource.FetchConfig(step.repo)
	if err != nil {
		return err
	}

	step.metadata.EnvironmentVariables = step.envForParams(config.Params)

	runContainerID := step.containerID
	runContainerID.Stage = db.ContainerStageRun

	container, found, err := step.workerPool.FindContainerForIdentifier(
		step.logger.Session("found-container"),
		runContainerID,
	)

	if err == nil && found {
		exitStatusProp, err := container.Property(taskExitStatusPropertyName)
		if err == nil {
			step.logger.Info("already-exited", lager.Data{"status": exitStatusProp})

			// process already completed; recover result

			_, err = fmt.Sscanf(exitStatusProp, "%d", &step.exitStatus)
			if err != nil {
				return err
			}

			step.registerSource(config, container)
			return nil
		}

		processID, err := container.Property(taskProcessPropertyName)
		if err != nil {
			// rogue container? perhaps did not shut down cleanly.
			return err
		}

		step.logger.Info("already-running", lager.Data{"process-id": processID})

		// process still running; re-attach
		step.process, err = container.Attach(processID, processIO)
		if err != nil {
			return err
		}

		step.logger.Info("attached")
	} else {
		// container does not exist; new session

		step.delegate.Initializing(config)

		container, err = step.createContainer(config, signals)
		if err != nil {
			return err
		}

		step.delegate.Started()

		step.process, err = container.Run(garden.ProcessSpec{
			Path: config.Run.Path,
			Args: config.Run.Args,
			Env:  step.envForParams(config.Params),

			Dir: path.Join(step.artifactsRoot, config.Run.Dir),
			TTY: &garden.TTYSpec{},
		}, processIO)
		if err != nil {
			return err
		}

		err = container.SetProperty(taskProcessPropertyName, step.process.ID())
		if err != nil {
			return err
		}
	}

	close(ready)

	exited := make(chan struct{})
	var processStatus int
	var processErr error

	go func() {
		processStatus, processErr = step.process.Wait()
		close(exited)
	}()

	select {
	case <-signals:
		step.registerSource(config, container)

		err = container.Stop(false)
		if err != nil {
			step.logger.Error("stopping-container", err)
		}

		<-exited

		return ErrInterrupted

	case <-exited:
		if processErr != nil {
			return processErr
		}

		step.registerSource(config, container)

		step.exitStatus = processStatus

		err := container.SetProperty(taskExitStatusPropertyName, fmt.Sprintf("%d", processStatus))
		if err != nil {
			return err
		}

		step.delegate.Finished(ExitStatus(processStatus))

		return nil
	}
}

func (step *TaskStep) createContainer(config atc.TaskConfig, signals <-chan os.Signal) (worker.Container, error) {
	imageSpec := worker.ImageSpec{
		Privileged: bool(step.privileged),
	}
	if step.imageArtifactName != "" {
		source, found := step.repo.SourceFor(worker.ArtifactName(step.imageArtifactName))
		if !found {
			return nil, MissingTaskImageSourceError{step.imageArtifactName}
		}

		imageSpec.ImageArtifactSource = source
		imageSpec.ImageArtifactName = worker.ArtifactName(step.imageArtifactName)
	} else {
		imageSpec.ImageURL = config.Image
		imageSpec.ImageResource = config.ImageResource
	}

	runContainerID := step.containerID
	runContainerID.Stage = db.ContainerStageRun

	containerSpec := worker.ContainerSpec{
		Platform:  config.Platform,
		Tags:      step.tags,
		TeamID:    step.teamID,
		ImageSpec: imageSpec,
		User:      config.Run.User,
		Dir:       step.artifactsRoot,

		Inputs:  []worker.InputSource{},
		Outputs: worker.OutputPaths{},
	}

	var missingInputs []string
	for _, input := range config.Inputs {
		inputName := input.Name
		if sourceName, ok := step.inputMapping[inputName]; ok {
			inputName = sourceName
		}

		source, found := step.repo.SourceFor(worker.ArtifactName(inputName))
		if !found {
			missingInputs = append(missingInputs, inputName)
			continue
		}

		containerSpec.Inputs = append(containerSpec.Inputs, &taskInputSource{
			name:          worker.ArtifactName(inputName),
			config:        input,
			source:        source,
			artifactsRoot: step.artifactsRoot,
		})
	}

	if len(missingInputs) > 0 {
		return nil, MissingInputsError{missingInputs}
	}

	for _, output := range config.Outputs {
		path := artifactsPath(output, step.artifactsRoot)
		containerSpec.Outputs[output.Name] = path
	}

	container, err := step.workerPool.FindOrCreateBuildContainer(
		step.logger,
		signals,
		step.delegate,
		runContainerID,
		step.metadata,
		containerSpec,
		step.resourceTypes,
	)
	if err != nil {
		return nil, err
	}

	return container, nil
}

func (step *TaskStep) registerSource(config atc.TaskConfig, container worker.Container) {
	volumeMounts := container.VolumeMounts()

	step.logger.Debug("registering-outputs", lager.Data{"config": config})

	for _, output := range config.Outputs {
		outputName := output.Name
		if destinationName, ok := step.outputMapping[output.Name]; ok {
			outputName = destinationName
		}

		outputPath := artifactsPath(output, step.artifactsRoot)

		for _, mount := range volumeMounts {
			if mount.MountPath == outputPath {
				source := newVolumeSource(step.logger, mount.Volume)
				step.repo.RegisterSource(worker.ArtifactName(outputName), source)
			}
		}
	}
}

// Result indicates Success as true if the script's exit status was 0.
//
// It also indicates ExitStatus as the exit status of the script.
//
// All other types are ignored.
func (step *TaskStep) Result(x interface{}) bool {
	switch v := x.(type) {
	case *Success:
		*v = step.exitStatus == 0
		return true

	case *ExitStatus:
		*v = ExitStatus(step.exitStatus)
		return true

	default:
		return false
	}
}

func (TaskStep) envForParams(params map[string]string) []string {
	env := make([]string, 0, len(params))

	for k, v := range params {
		env = append(env, k+"="+v)
	}

	return env
}

type volumeSource struct {
	logger lager.Logger
	volume worker.Volume
}

func newVolumeSource(
	logger lager.Logger,
	volume worker.Volume,
) *volumeSource {
	return &volumeSource{
		logger: logger,
		volume: volume,
	}
}

func (src *volumeSource) StreamTo(destination worker.ArtifactDestination) error {
	out, err := src.volume.StreamOut(".")
	if err != nil {
		return err
	}

	defer out.Close()

	return destination.StreamIn(".", out)
}

func (src *volumeSource) StreamFile(filename string) (io.ReadCloser, error) {
	out, err := src.volume.StreamOut(filename)
	if err != nil {
		return nil, err
	}

	tarReader := tar.NewReader(out)

	_, err = tarReader.Next()
	if err != nil {
		return nil, FileNotFoundError{Path: filename}
	}

	return fileReadCloser{
		Reader: tarReader,
		Closer: out,
	}, nil
}

func (src *volumeSource) VolumeOn(w worker.Worker) (worker.Volume, bool, error) {
	return w.LookupVolume(src.logger, src.volume.Handle())
}

type taskInputSource struct {
	name          worker.ArtifactName
	config        atc.TaskInputConfig
	source        worker.ArtifactSource
	artifactsRoot string
}

func (s *taskInputSource) Name() worker.ArtifactName     { return s.name }
func (s *taskInputSource) Source() worker.ArtifactSource { return s.source }

func (s *taskInputSource) DestinationPath() string {
	subdir := s.config.Path
	if s.config.Path == "" {
		subdir = s.config.Name
	}

	return filepath.Join(s.artifactsRoot, subdir)
}

func artifactsPath(outputConfig atc.TaskOutputConfig, artifactsRoot string) string {
	outputSrc := outputConfig.Path
	if len(outputSrc) == 0 {
		outputSrc = outputConfig.Name
	}

	return path.Join(artifactsRoot, outputSrc) + "/"
}
<|endoftext|>


<commit_before>package core

import (
	"net/url"
	"os"
	"path"
	"time"

	"github.com/secsy/goftp"
	"github.com/snickers/snickers/db"
	"github.com/snickers/snickers/types"
)

// FTPUpload uploades the file using FTP. Job Destination should be
// in format: ftp://login:password@host/path
func FTPUpload(jobID string) error {
	dbInstance, err := db.GetDatabase()
	job, _ := dbInstance.RetrieveJob(jobID)

	job.Status = types.jobUploading
	dbInstance.UpdateJob(job.ID, job)

	u, err := url.Parse(job.Destination)
	if err != nil {
		return err
	}

	pw, isSet := u.User.Password()
	if !isSet {
		pw = ""
	}

	config := goftp.Config{
		User:               u.User.Username(),
		Password:           pw,
		ConnectionsPerHost: 10,
		Timeout:            10 * time.Second,
		Logger:             os.Stderr,
	}

	client, err := goftp.DialConfig(config, u.Host+":21")
	if err != nil {
		return err
	}

	localFile, err := os.Open(job.LocalDestination)
	if err != nil {
		return err
	}

	err = client.Store(u.Path, localFile)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>fix tests<commit_after>package core

import (
	"net/url"
	"os"
	"time"

	"github.com/secsy/goftp"
	"github.com/snickers/snickers/db"
	"github.com/snickers/snickers/types"
)

// FTPUpload uploades the file using FTP. Job Destination should be
// in format: ftp://login:password@host/path
func FTPUpload(jobID string) error {
	dbInstance, err := db.GetDatabase()
	job, _ := dbInstance.RetrieveJob(jobID)

	job.Status = types.JobUploading
	dbInstance.UpdateJob(job.ID, job)

	u, err := url.Parse(job.Destination)
	if err != nil {
		return err
	}

	pw, isSet := u.User.Password()
	if !isSet {
		pw = ""
	}

	config := goftp.Config{
		User:               u.User.Username(),
		Password:           pw,
		ConnectionsPerHost: 10,
		Timeout:            10 * time.Second,
		Logger:             os.Stderr,
	}

	client, err := goftp.DialConfig(config, u.Host+":21")
	if err != nil {
		return err
	}

	localFile, err := os.Open(job.LocalDestination)
	if err != nil {
		return err
	}

	err = client.Store(u.Path, localFile)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package message

type Message struct {
	From      string
	Previous  string
	Seq       string
	Timestamp string

	Content   string

	Hash      string
	Signature string
}
<commit_msg>[message] Update placeholder<commit_after>package message

import (
	"time"
)

type Message struct {
	From      string
	Previous  string
	Seq       int
	Timestamp time.Time

	Content string

	Hash      string
	Signature string
}

func (m *Message) IsValid() bool {
	return true
}
<|endoftext|>


<commit_before>package core

import (
	"encoding/json"
	"strconv"
	"sync"
	"time"

	"github.com/af83/edwig/logger"
	"github.com/af83/edwig/model"
)

type SubscriptionId string

type Subscription struct {
	model.ClockConsumer
	manager Subscriptions

	id         SubscriptionId
	kind       string
	externalId string

	resourcesByObjectID map[string]*SubscribedResource
	subscriptionOptions map[string]string
}

type SubscribedResource struct {
	Reference        model.Reference
	RetryCount       int
	SubscribedAt     time.Time
	SubscribedUntil  time.Time
	LastStates       map[string]lastState `json:",omitempty"`
	resourcesOptions map[string]string
}

func NewResource(ref model.Reference) SubscribedResource {
	ressource := SubscribedResource{
		Reference:        ref,
		LastStates:       make(map[string]lastState),
		resourcesOptions: make(map[string]string),
	}

	return ressource
}

func (sr *SubscribedResource) ResourcesOptions() map[string]string {
	return sr.resourcesOptions
}

type APISubscription struct {
	Kind       string
	References []model.Reference
}

func (subscription *Subscription) SetDefinition(apisub *APISubscription) {
	subscription.kind = apisub.Kind
	for _, ref := range apisub.References {
		if ref.ObjectId != nil {
			subscription.CreateAddNewResource(ref)
		}
	}
}

func (subscription *Subscription) Id() SubscriptionId {
	return subscription.id
}

func (subscription *Subscription) Kind() string {
	return subscription.kind
}

func (subscription *Subscription) ExternalId() string {
	return subscription.externalId
}

func (subscription *Subscription) SubscriptionOptions() map[string]string {
	return subscription.subscriptionOptions
}

func (subscription *Subscription) SetKind(kind string) {
	subscription.kind = kind
}

func (subscription *Subscription) SetExternalId(externalId string) {
	subscription.externalId = externalId
}

func (subscription *Subscription) Save() (ok bool) {
	ok = subscription.manager.Save(subscription)
	return
}

func (subscription *Subscription) Delete() (ok bool) {
	ok = subscription.manager.Delete(subscription)
	return
}

func (subscription *Subscription) ResourcesByObjectID() map[string]*SubscribedResource {
	return subscription.resourcesByObjectID
}

func (subscription *Subscription) MarshalJSON() ([]byte, error) {
	resources := make([]*SubscribedResource, 0)
	for _, resource := range subscription.resourcesByObjectID {
		resources = append(resources, resource)
	}

	aux := struct {
		Id         SubscriptionId        `json:"SubscriptionRef,omitempty"`
		ExternalId string                `json:"ExternalId,omitempty"`
		Kind       string                `json:",omitempty"`
		Resources  []*SubscribedResource `json:",omitempty"`
	}{
		Id:         subscription.id,
		ExternalId: subscription.externalId,
		Kind:       subscription.kind,
		Resources:  resources,
	}
	return json.Marshal(&aux)
}

func (subscription *Subscription) Resource(obj model.ObjectID) *SubscribedResource {
	sub, present := subscription.resourcesByObjectID[obj.String()]
	if !present {
		return nil
	}
	return sub
}

func (subscription *Subscription) Resources(now time.Time) []*SubscribedResource {
	ressources := []*SubscribedResource{}

	for _, ressource := range subscription.resourcesByObjectID {
		if ressource.SubscribedUntil.After(subscription.Clock().Now()) {
			ressources = append(ressources, ressource)
		}
	}
	return ressources
}

func (subscription *Subscription) AddNewResource(resource SubscribedResource) {
	subscription.resourcesByObjectID[resource.Reference.ObjectId.String()] = &resource
}

func (subscription *Subscription) CreateAddNewResource(reference model.Reference) *SubscribedResource {
	logger.Log.Debugf("Create subscribed resource for %v", reference.ObjectId.String())

	resource := SubscribedResource{
		Reference:        reference,
		SubscribedUntil:  subscription.Clock().Now().Add(1 * time.Minute),
		LastStates:       make(map[string]lastState),
		resourcesOptions: make(map[string]string),
	}
	subscription.resourcesByObjectID[reference.ObjectId.String()] = &resource
	return &resource
}

func (subscription *Subscription) DeleteResource(key string) {
	delete(subscription.resourcesByObjectID, key)
}

type MemorySubscriptions struct {
	model.UUIDConsumer

	mutex   *sync.RWMutex
	partner *Partner

	byIdentifier map[SubscriptionId]*Subscription
}

func (manager *MemorySubscriptions) MarshalJSON() ([]byte, error) {
	subscriptions := make([]*Subscription, 0)
	for _, subscription := range manager.byIdentifier {
		subscriptions = append(subscriptions, subscription)
	}

	return json.Marshal(subscriptions)
}

type Subscriptions interface {
	model.UUIDInterface

	New(kind string) *Subscription
	Find(id SubscriptionId) (*Subscription, bool)
	FindAll() []*Subscription
	FindOrCreateByKind(string) *Subscription
	FindByKind(string) (*Subscription, bool)
	FindSubscriptionsByKind(string) []*Subscription
	FindBroadcastSubscriptions() []*Subscription
	Save(Subscription *Subscription) bool
	Delete(Subscription *Subscription) bool
	DeleteById(id SubscriptionId)
	CancelSubscriptions()
	CancelBroadcastSubscriptions()
	CancelCollectSubscriptions()
	FindByRessourceId(id, kind string) []*Subscription
	FindByExternalId(externalId string) (*Subscription, bool)
}

func NewMemorySubscriptions(partner *Partner) *MemorySubscriptions {
	return &MemorySubscriptions{
		mutex:        &sync.RWMutex{},
		byIdentifier: make(map[SubscriptionId]*Subscription),
		partner:      partner,
	}
}

func (manager *MemorySubscriptions) New(kind string) *Subscription {
	logger.Log.Debugf("Creating subscription with kind %v", kind)
	subscription := &Subscription{
		kind:                kind,
		manager:             manager,
		resourcesByObjectID: make(map[string]*SubscribedResource),
		subscriptionOptions: make(map[string]string),
	}
	subscription.Save()
	return subscription
}

func (manager *MemorySubscriptions) FindByKind(kind string) (*Subscription, bool) {
	manager.mutex.RLock()
	defer manager.mutex.RUnlock()

	for _, subscription := range manager.byIdentifier {
		if subscription.Kind() == kind {
			return subscription, true
		}
	}
	return nil, false
}

func (manager *MemorySubscriptions) FindSubscriptionsByKind(kind string) (subscriptions []*Subscription) {
	manager.mutex.RLock()
	defer manager.mutex.RUnlock()

	for _, subscription := range manager.byIdentifier {
		if subscription.Kind() == kind {
			subscriptions = append(subscriptions, subscription)
		}
	}
	return subscriptions
}

func (manager *MemorySubscriptions) FindBroadcastSubscriptions() (subscriptions []*Subscription) {
	manager.mutex.RLock()
	defer manager.mutex.RUnlock()

	for _, subscription := range manager.byIdentifier {
		if subscription.externalId != "" {
			subscriptions = append(subscriptions, subscription)
		}
	}
	return subscriptions
}

func (manager *MemorySubscriptions) FindByExternalId(externalId string) (*Subscription, bool) {
	manager.mutex.RLock()
	defer manager.mutex.RUnlock()

	for _, subscription := range manager.byIdentifier {
		if subscription.ExternalId() == externalId {
			return subscription, true
		}
	}
	return nil, false
}

func (manager *MemorySubscriptions) FindByRessourceId(id, kind string) []*Subscription {
	manager.mutex.RLock()
	defer manager.mutex.RUnlock()
	subscriptions := []*Subscription{}

	for _, subscription := range manager.byIdentifier {
		_, ok := subscription.resourcesByObjectID[id]
		if ok && subscription.kind == kind {
			subscriptions = append(subscriptions, subscription)
		}
	}
	return subscriptions
}

func (manager *MemorySubscriptions) FindOrCreateByKind(kind string) *Subscription {
	maxResource, _ := strconv.Atoi(manager.partner.Setting("subscriptions.maximum_resources"))
	if maxResource == 1 {
		return manager.New(kind)
	}

	manager.mutex.RLock()
	for _, subscription := range manager.byIdentifier {
		if subscription.Kind() == kind && (maxResource < 1 || len(subscription.resourcesByObjectID) < maxResource) {
			manager.mutex.RUnlock()
			return subscription
		}
	}
	manager.mutex.RUnlock()

	return manager.New(kind)
}

func (manager *MemorySubscriptions) Find(id SubscriptionId) (*Subscription, bool) {
	manager.mutex.RLock()
	defer manager.mutex.RUnlock()

	subscription, ok := manager.byIdentifier[id]
	if ok {
		return subscription, true
	} else {
		return nil, false
	}
}

func (manager *MemorySubscriptions) FindAll() (subscriptions []*Subscription) {
	manager.mutex.RLock()
	defer manager.mutex.RUnlock()

	if len(manager.byIdentifier) == 0 {
		return []*Subscription{}
	}
	for _, subscription := range manager.byIdentifier {
		subscriptions = append(subscriptions, subscription)
	}
	return
}

func (manager *MemorySubscriptions) Save(subscription *Subscription) bool {
	manager.mutex.Lock()
	defer manager.mutex.Unlock()

	if subscription.Id() == "" {
		generator := manager.partner.Generator("subscription_identifier")
		subscription.id = SubscriptionId(generator.NewIdentifier(IdentifierAttributes{Id: manager.NewUUID()}))
	}

	subscription.manager = manager
	manager.byIdentifier[subscription.Id()] = subscription

	return true
}

func (manager *MemorySubscriptions) Delete(subscription *Subscription) bool {
	manager.mutex.Lock()
	defer manager.mutex.Unlock()

	delete(manager.byIdentifier, subscription.Id())
	return true
}

func (manager *MemorySubscriptions) DeleteById(id SubscriptionId) {
	manager.mutex.Lock()
	defer manager.mutex.Unlock()

	delete(manager.byIdentifier, id)
}

func (manager *MemorySubscriptions) CancelSubscriptions() {
	manager.mutex.Lock()
	defer manager.mutex.Unlock()

	for id := range manager.byIdentifier {
		delete(manager.byIdentifier, id)
	}
}

func (manager *MemorySubscriptions) CancelBroadcastSubscriptions() {
	manager.mutex.Lock()
	defer manager.mutex.Unlock()

	for id, subscription := range manager.byIdentifier {
		if subscription.externalId != "" {
			delete(manager.byIdentifier, id)
		}
	}
}

func (manager *MemorySubscriptions) CancelCollectSubscriptions() {
	manager.mutex.Lock()
	defer manager.mutex.Unlock()

	for id, subscription := range manager.byIdentifier {
		if subscription.externalId == "" {
			delete(manager.byIdentifier, id)
		}
	}
}
<commit_msg>don't show lastStates in Subscription api<commit_after>package core

import (
	"encoding/json"
	"strconv"
	"sync"
	"time"

	"github.com/af83/edwig/logger"
	"github.com/af83/edwig/model"
)

type SubscriptionId string

type Subscription struct {
	model.ClockConsumer
	manager Subscriptions

	id         SubscriptionId
	kind       string
	externalId string

	resourcesByObjectID map[string]*SubscribedResource
	subscriptionOptions map[string]string
}

type SubscribedResource struct {
	Reference        model.Reference
	RetryCount       int
	SubscribedAt     time.Time
	SubscribedUntil  time.Time
	LastStates       map[string]lastState `json:"-"`
	resourcesOptions map[string]string
}

func NewResource(ref model.Reference) SubscribedResource {
	ressource := SubscribedResource{
		Reference:        ref,
		LastStates:       make(map[string]lastState),
		resourcesOptions: make(map[string]string),
	}

	return ressource
}

func (sr *SubscribedResource) ResourcesOptions() map[string]string {
	return sr.resourcesOptions
}

type APISubscription struct {
	Kind       string
	References []model.Reference
}

func (subscription *Subscription) SetDefinition(apisub *APISubscription) {
	subscription.kind = apisub.Kind
	for _, ref := range apisub.References {
		if ref.ObjectId != nil {
			subscription.CreateAddNewResource(ref)
		}
	}
}

func (subscription *Subscription) Id() SubscriptionId {
	return subscription.id
}

func (subscription *Subscription) Kind() string {
	return subscription.kind
}

func (subscription *Subscription) ExternalId() string {
	return subscription.externalId
}

func (subscription *Subscription) SubscriptionOptions() map[string]string {
	return subscription.subscriptionOptions
}

func (subscription *Subscription) SetKind(kind string) {
	subscription.kind = kind
}

func (subscription *Subscription) SetExternalId(externalId string) {
	subscription.externalId = externalId
}

func (subscription *Subscription) Save() (ok bool) {
	ok = subscription.manager.Save(subscription)
	return
}

func (subscription *Subscription) Delete() (ok bool) {
	ok = subscription.manager.Delete(subscription)
	return
}

func (subscription *Subscription) ResourcesByObjectID() map[string]*SubscribedResource {
	return subscription.resourcesByObjectID
}

func (subscription *Subscription) MarshalJSON() ([]byte, error) {
	resources := make([]*SubscribedResource, 0)
	for _, resource := range subscription.resourcesByObjectID {
		resources = append(resources, resource)
	}

	aux := struct {
		Id         SubscriptionId        `json:"SubscriptionRef,omitempty"`
		ExternalId string                `json:"ExternalId,omitempty"`
		Kind       string                `json:",omitempty"`
		Resources  []*SubscribedResource `json:",omitempty"`
	}{
		Id:         subscription.id,
		ExternalId: subscription.externalId,
		Kind:       subscription.kind,
		Resources:  resources,
	}
	return json.Marshal(&aux)
}

func (subscription *Subscription) Resource(obj model.ObjectID) *SubscribedResource {
	sub, present := subscription.resourcesByObjectID[obj.String()]
	if !present {
		return nil
	}
	return sub
}

func (subscription *Subscription) Resources(now time.Time) []*SubscribedResource {
	ressources := []*SubscribedResource{}

	for _, ressource := range subscription.resourcesByObjectID {
		if ressource.SubscribedUntil.After(subscription.Clock().Now()) {
			ressources = append(ressources, ressource)
		}
	}
	return ressources
}

func (subscription *Subscription) AddNewResource(resource SubscribedResource) {
	subscription.resourcesByObjectID[resource.Reference.ObjectId.String()] = &resource
}

func (subscription *Subscription) CreateAddNewResource(reference model.Reference) *SubscribedResource {
	logger.Log.Debugf("Create subscribed resource for %v", reference.ObjectId.String())

	resource := SubscribedResource{
		Reference:        reference,
		SubscribedUntil:  subscription.Clock().Now().Add(1 * time.Minute),
		LastStates:       make(map[string]lastState),
		resourcesOptions: make(map[string]string),
	}
	subscription.resourcesByObjectID[reference.ObjectId.String()] = &resource
	return &resource
}

func (subscription *Subscription) DeleteResource(key string) {
	delete(subscription.resourcesByObjectID, key)
}

type MemorySubscriptions struct {
	model.UUIDConsumer

	mutex   *sync.RWMutex
	partner *Partner

	byIdentifier map[SubscriptionId]*Subscription
}

func (manager *MemorySubscriptions) MarshalJSON() ([]byte, error) {
	subscriptions := make([]*Subscription, 0)
	for _, subscription := range manager.byIdentifier {
		subscriptions = append(subscriptions, subscription)
	}

	return json.Marshal(subscriptions)
}

type Subscriptions interface {
	model.UUIDInterface

	New(kind string) *Subscription
	Find(id SubscriptionId) (*Subscription, bool)
	FindAll() []*Subscription
	FindOrCreateByKind(string) *Subscription
	FindByKind(string) (*Subscription, bool)
	FindSubscriptionsByKind(string) []*Subscription
	FindBroadcastSubscriptions() []*Subscription
	Save(Subscription *Subscription) bool
	Delete(Subscription *Subscription) bool
	DeleteById(id SubscriptionId)
	CancelSubscriptions()
	CancelBroadcastSubscriptions()
	CancelCollectSubscriptions()
	FindByRessourceId(id, kind string) []*Subscription
	FindByExternalId(externalId string) (*Subscription, bool)
}

func NewMemorySubscriptions(partner *Partner) *MemorySubscriptions {
	return &MemorySubscriptions{
		mutex:        &sync.RWMutex{},
		byIdentifier: make(map[SubscriptionId]*Subscription),
		partner:      partner,
	}
}

func (manager *MemorySubscriptions) New(kind string) *Subscription {
	logger.Log.Debugf("Creating subscription with kind %v", kind)
	subscription := &Subscription{
		kind:                kind,
		manager:             manager,
		resourcesByObjectID: make(map[string]*SubscribedResource),
		subscriptionOptions: make(map[string]string),
	}
	subscription.Save()
	return subscription
}

func (manager *MemorySubscriptions) FindByKind(kind string) (*Subscription, bool) {
	manager.mutex.RLock()
	defer manager.mutex.RUnlock()

	for _, subscription := range manager.byIdentifier {
		if subscription.Kind() == kind {
			return subscription, true
		}
	}
	return nil, false
}

func (manager *MemorySubscriptions) FindSubscriptionsByKind(kind string) (subscriptions []*Subscription) {
	manager.mutex.RLock()
	defer manager.mutex.RUnlock()

	for _, subscription := range manager.byIdentifier {
		if subscription.Kind() == kind {
			subscriptions = append(subscriptions, subscription)
		}
	}
	return subscriptions
}

func (manager *MemorySubscriptions) FindBroadcastSubscriptions() (subscriptions []*Subscription) {
	manager.mutex.RLock()
	defer manager.mutex.RUnlock()

	for _, subscription := range manager.byIdentifier {
		if subscription.externalId != "" {
			subscriptions = append(subscriptions, subscription)
		}
	}
	return subscriptions
}

func (manager *MemorySubscriptions) FindByExternalId(externalId string) (*Subscription, bool) {
	manager.mutex.RLock()
	defer manager.mutex.RUnlock()

	for _, subscription := range manager.byIdentifier {
		if subscription.ExternalId() == externalId {
			return subscription, true
		}
	}
	return nil, false
}

func (manager *MemorySubscriptions) FindByRessourceId(id, kind string) []*Subscription {
	manager.mutex.RLock()
	defer manager.mutex.RUnlock()
	subscriptions := []*Subscription{}

	for _, subscription := range manager.byIdentifier {
		_, ok := subscription.resourcesByObjectID[id]
		if ok && subscription.kind == kind {
			subscriptions = append(subscriptions, subscription)
		}
	}
	return subscriptions
}

func (manager *MemorySubscriptions) FindOrCreateByKind(kind string) *Subscription {
	maxResource, _ := strconv.Atoi(manager.partner.Setting("subscriptions.maximum_resources"))
	if maxResource == 1 {
		return manager.New(kind)
	}

	manager.mutex.RLock()
	for _, subscription := range manager.byIdentifier {
		if subscription.Kind() == kind && (maxResource < 1 || len(subscription.resourcesByObjectID) < maxResource) {
			manager.mutex.RUnlock()
			return subscription
		}
	}
	manager.mutex.RUnlock()

	return manager.New(kind)
}

func (manager *MemorySubscriptions) Find(id SubscriptionId) (*Subscription, bool) {
	manager.mutex.RLock()
	defer manager.mutex.RUnlock()

	subscription, ok := manager.byIdentifier[id]
	if ok {
		return subscription, true
	} else {
		return nil, false
	}
}

func (manager *MemorySubscriptions) FindAll() (subscriptions []*Subscription) {
	manager.mutex.RLock()
	defer manager.mutex.RUnlock()

	if len(manager.byIdentifier) == 0 {
		return []*Subscription{}
	}
	for _, subscription := range manager.byIdentifier {
		subscriptions = append(subscriptions, subscription)
	}
	return
}

func (manager *MemorySubscriptions) Save(subscription *Subscription) bool {
	manager.mutex.Lock()
	defer manager.mutex.Unlock()

	if subscription.Id() == "" {
		generator := manager.partner.Generator("subscription_identifier")
		subscription.id = SubscriptionId(generator.NewIdentifier(IdentifierAttributes{Id: manager.NewUUID()}))
	}

	subscription.manager = manager
	manager.byIdentifier[subscription.Id()] = subscription

	return true
}

func (manager *MemorySubscriptions) Delete(subscription *Subscription) bool {
	manager.mutex.Lock()
	defer manager.mutex.Unlock()

	delete(manager.byIdentifier, subscription.Id())
	return true
}

func (manager *MemorySubscriptions) DeleteById(id SubscriptionId) {
	manager.mutex.Lock()
	defer manager.mutex.Unlock()

	delete(manager.byIdentifier, id)
}

func (manager *MemorySubscriptions) CancelSubscriptions() {
	manager.mutex.Lock()
	defer manager.mutex.Unlock()

	for id := range manager.byIdentifier {
		delete(manager.byIdentifier, id)
	}
}

func (manager *MemorySubscriptions) CancelBroadcastSubscriptions() {
	manager.mutex.Lock()
	defer manager.mutex.Unlock()

	for id, subscription := range manager.byIdentifier {
		if subscription.externalId != "" {
			delete(manager.byIdentifier, id)
		}
	}
}

func (manager *MemorySubscriptions) CancelCollectSubscriptions() {
	manager.mutex.Lock()
	defer manager.mutex.Unlock()

	for id, subscription := range manager.byIdentifier {
		if subscription.externalId == "" {
			delete(manager.byIdentifier, id)
		}
	}
}
<|endoftext|>


<commit_before>package server

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/influxdata/chronograf"
)

type dashboardLinks struct {
	Self string `json:"self"` // Self link mapping to this resource
}

type dashboardResponse struct {
	chronograf.Dashboard
	Links dashboardLinks `json:"links"`
}

type getDashboardsResponse struct {
	Dashboards []dashboardResponse `json:"dashboards"`
}

func newDashboardResponse(d chronograf.Dashboard) dashboardResponse {
	base := "/chronograf/v1/dashboards"
	return dashboardResponse{
		Dashboard: d,
		Links: dashboardLinks{
			Self: fmt.Sprintf("%s/%d", base, d.ID),
		},
	}
}

// Dashboards returns all dashboards within the store
func (s *Service) Dashboards(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	dashboards, err := s.DashboardsStore.All(ctx)
	if err != nil {
		Error(w, http.StatusInternalServerError, "Error loading layouts", s.Logger)
		return
	}

	res := getDashboardsResponse{
		Dashboards: []dashboardResponse{},
	}

	for _, dashboard := range dashboards {
		res.Dashboards = append(res.Dashboards, newDashboardResponse(dashboard))
	}

	encodeJSON(w, http.StatusOK, res, s.Logger)
}

// DashboardID returns a single specified dashboard
func (s *Service) DashboardID(w http.ResponseWriter, r *http.Request) {
	id, err := paramID("id", r)
	if err != nil {
		Error(w, http.StatusUnprocessableEntity, err.Error(), s.Logger)
		return
	}

	ctx := r.Context()
	e, err := s.DashboardsStore.Get(ctx, chronograf.DashboardID(id))
	if err != nil {
		notFound(w, id, s.Logger)
		return
	}

	res := newDashboardResponse(*e)
	encodeJSON(w, http.StatusOK, res, s.Logger)
}

// NewDashboard creates and returns a new dashboard object
func (s *Service) NewDashboard(w http.ResponseWriter, r *http.Request) {
	var dashboard *chronograf.Dashboard
	if err := json.NewDecoder(r.Body).Decode(&dashboard); err != nil {
		invalidJSON(w, s.Logger)
		return
	}

	var err error
	if dashboard, err = s.DashboardsStore.Add(r.Context(), dashboard); err != nil {
		msg := fmt.Errorf("Error storing layout %v: %v", dashboard, err)
		unknownErrorWithMessage(w, msg, s.Logger)
		return
	}

	res := newDashboardResponse(*dashboard)
	w.Header().Add("Location", res.Links.Self)
	encodeJSON(w, http.StatusCreated, res, s.Logger)
}

// RemoveDashboard deletes a dashboard
func (s *Service) RemoveDashboard(w http.ResponseWriter, r *http.Request) {
	id, err := paramID("id", r)
	if err != nil {
		Error(w, http.StatusUnprocessableEntity, err.Error(), s.Logger)
		return
	}

	ctx := r.Context()
	e, err := s.DashboardsStore.Get(ctx, chronograf.DashboardID(id))
	if err != nil {
		notFound(w, id, s.Logger)
		return
	}

	if err := s.DashboardsStore.Delete(ctx, e); err != nil {
		unknownErrorWithMessage(w, err, s.Logger)
		return
	}
	w.WriteHeader(http.StatusNoContent)
}

// UpdateDashboard updates a dashboard
func (s *Service) UpdateDashboard(w http.ResponseWriter, r *http.Request) {

}
<commit_msg>add update function for dashboards api<commit_after>package server

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"

	"github.com/bouk/httprouter"
	"github.com/influxdata/chronograf"
)

type dashboardLinks struct {
	Self string `json:"self"` // Self link mapping to this resource
}

type dashboardResponse struct {
	chronograf.Dashboard
	Links dashboardLinks `json:"links"`
}

type getDashboardsResponse struct {
	Dashboards []dashboardResponse `json:"dashboards"`
}

func newDashboardResponse(d chronograf.Dashboard) dashboardResponse {
	base := "/chronograf/v1/dashboards"
	return dashboardResponse{
		Dashboard: d,
		Links: dashboardLinks{
			Self: fmt.Sprintf("%s/%d", base, d.ID),
		},
	}
}

// Dashboards returns all dashboards within the store
func (s *Service) Dashboards(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	dashboards, err := s.DashboardsStore.All(ctx)
	if err != nil {
		Error(w, http.StatusInternalServerError, "Error loading layouts", s.Logger)
		return
	}

	res := getDashboardsResponse{
		Dashboards: []dashboardResponse{},
	}

	for _, dashboard := range dashboards {
		res.Dashboards = append(res.Dashboards, newDashboardResponse(dashboard))
	}

	encodeJSON(w, http.StatusOK, res, s.Logger)
}

// DashboardID returns a single specified dashboard
func (s *Service) DashboardID(w http.ResponseWriter, r *http.Request) {
	id, err := paramID("id", r)
	if err != nil {
		Error(w, http.StatusUnprocessableEntity, err.Error(), s.Logger)
		return
	}

	ctx := r.Context()
	e, err := s.DashboardsStore.Get(ctx, chronograf.DashboardID(id))
	if err != nil {
		notFound(w, id, s.Logger)
		return
	}

	res := newDashboardResponse(*e)
	encodeJSON(w, http.StatusOK, res, s.Logger)
}

// NewDashboard creates and returns a new dashboard object
func (s *Service) NewDashboard(w http.ResponseWriter, r *http.Request) {
	var dashboard *chronograf.Dashboard
	if err := json.NewDecoder(r.Body).Decode(&dashboard); err != nil {
		invalidJSON(w, s.Logger)
		return
	}

	var err error
	if dashboard, err = s.DashboardsStore.Add(r.Context(), dashboard); err != nil {
		msg := fmt.Errorf("Error storing layout %v: %v", dashboard, err)
		unknownErrorWithMessage(w, msg, s.Logger)
		return
	}

	res := newDashboardResponse(*dashboard)
	w.Header().Add("Location", res.Links.Self)
	encodeJSON(w, http.StatusCreated, res, s.Logger)
}

// RemoveDashboard deletes a dashboard
func (s *Service) RemoveDashboard(w http.ResponseWriter, r *http.Request) {
	id, err := paramID("id", r)
	if err != nil {
		Error(w, http.StatusUnprocessableEntity, err.Error(), s.Logger)
		return
	}

	ctx := r.Context()
	e, err := s.DashboardsStore.Get(ctx, chronograf.DashboardID(id))
	if err != nil {
		notFound(w, id, s.Logger)
		return
	}

	if err := s.DashboardsStore.Delete(ctx, e); err != nil {
		unknownErrorWithMessage(w, err, s.Logger)
		return
	}
	w.WriteHeader(http.StatusNoContent)
}

// UpdateDashboard replaces a dashboard
func (s *Service) UpdateDashboard(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()
	idParam, err := strconv.Atoi(httprouter.GetParamFromContext(ctx, "id"))
	if err != nil {
		msg := fmt.Sprintf("Could not parse dashboard ID: %s", err)
		Error(w, http.StatusInternalServerError, msg, s.Logger)
	}
	id := chronograf.DashboardID(idParam)

	_, err = s.DashboardsStore.Get(ctx, id)
	if err != nil {
		Error(w, http.StatusNotFound, fmt.Sprintf("ID %s not found", id), s.Logger)
		return
	}

	var req *chronograf.Dashboard
	if err := json.NewDecoder(r.Body).Decode(req); err != nil {
		invalidJSON(w, s.Logger)
		return
	}
	req.ID = id

	if err := s.DashboardsStore.Update(ctx, req); err != nil {
		msg := fmt.Sprintf("Error updating dashboard ID %s: %v", id, err)
		Error(w, http.StatusInternalServerError, msg, s.Logger)
		return
	}

	res := newDashboardResponse(*req)
	encodeJSON(w, http.StatusOK, res, s.Logger)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"errors"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	log "github.com/sirupsen/logrus"
)

func init() {
	RegisterExporter("queue", newExporterQueue)
}

var (
	queueLabels    = []string{"cluster", "vhost", "queue", "durable", "policy", "self"}
	queueLabelKeys = []string{"vhost", "name", "durable", "policy", "state", "node", "idle_since"}

	queueGaugeVec = map[string]*prometheus.GaugeVec{
		"messages_ready":                            newGaugeVec("queue_messages_ready", "Number of messages ready to be delivered to clients.", queueLabels),
		"messages_unacknowledged":                   newGaugeVec("queue_messages_unacknowledged", "Number of messages delivered to clients but not yet acknowledged.", queueLabels),
		"messages":                                  newGaugeVec("queue_messages", "Sum of ready and unacknowledged messages (queue depth).", queueLabels),
		"messages_ready_ram":                        newGaugeVec("queue_messages_ready_ram", "Number of messages from messages_ready which are resident in ram.", queueLabels),
		"messages_unacknowledged_ram":               newGaugeVec("queue_messages_unacknowledged_ram", "Number of messages from messages_unacknowledged which are resident in ram.", queueLabels),
		"messages_ram":                              newGaugeVec("queue_messages_ram", "Total number of messages which are resident in ram.", queueLabels),
		"messages_persistent":                       newGaugeVec("queue_messages_persistent", "Total number of persistent messages in the queue (will always be 0 for transient queues).", queueLabels),
		"message_bytes":                             newGaugeVec("queue_message_bytes", "Sum of the size of all message bodies in the queue. This does not include the message properties (including headers) or any overhead.", queueLabels),
		"message_bytes_ready":                       newGaugeVec("queue_message_bytes_ready", "Like message_bytes but counting only those messages ready to be delivered to clients.", queueLabels),
		"message_bytes_unacknowledged":              newGaugeVec("queue_message_bytes_unacknowledged", "Like message_bytes but counting only those messages delivered to clients but not yet acknowledged.", queueLabels),
		"message_bytes_ram":                         newGaugeVec("queue_message_bytes_ram", "Like message_bytes but counting only those messages which are in RAM.", queueLabels),
		"message_bytes_persistent":                  newGaugeVec("queue_message_bytes_persistent", "Like message_bytes but counting only those messages which are persistent.", queueLabels),
		"consumers":                                 newGaugeVec("queue_consumers", "Number of consumers.", queueLabels),
		"consumer_utilisation":                      newGaugeVec("queue_consumer_utilisation", "Fraction of the time (between 0.0 and 1.0) that the queue is able to immediately deliver messages to consumers. This can be less than 1.0 if consumers are limited by network congestion or prefetch count.", queueLabels),
		"memory":                                    newGaugeVec("queue_memory", "Bytes of memory consumed by the Erlang process associated with the queue, including stack, heap and internal structures.", queueLabels),
		"head_message_timestamp":                    newGaugeVec("queue_head_message_timestamp", "The timestamp property of the first message in the queue, if present. Timestamps of messages only appear when they are in the paged-in state.", queueLabels), //https://github.com/rabbitmq/rabbitmq-server/pull/54
		"garbage_collection.min_heap_size":          newGaugeVec("queue_gc_min_heap", "Minimum heap size in words", queueLabels),
		"garbage_collection.min_bin_vheap_size":     newGaugeVec("queue_gc_min_vheap", "Minimum binary virtual heap size in words", queueLabels),
		"garbage_collection.fullsweep_after":        newGaugeVec("queue_gc_collections_before_fullsweep", "Maximum generational collections before fullsweep", queueLabels),
		"slave_nodes_len":                           newGaugeVec("queue_slaves_nodes_len", "Number of slave nodes attached to the queue", queueLabels),
		"synchronised_slave_nodes_len":              newGaugeVec("queue_synchronised_slave_nodes_len", "Number of slave nodes in sync to the queue", queueLabels),
		"message_stats.publish_details.rate":        newGaugeVec("queue_messages_publish_rate", "Rate at which messages are entering the server.", queueLabels),
		"message_stats.deliver_no_ack_details.rate": newGaugeVec("queue_messages_deliver_no_ack_rate", "Rate at which messages are delivered to consumers that use automatic acknowledgements.", queueLabels),
		"message_stats.deliver_details.rate":        newGaugeVec("queue_messages_deliver_rate", "Rate at which messages are delivered to consumers that use manual acknowledgements.", queueLabels),
	}
	limitsGaugeVec = map[string]*prometheus.GaugeVec{
		"max-length-bytes": newGaugeVec("queue_max_length_bytes", "Total body size for ready messages a queue can contain before it starts to drop them from its head.", queueLabels),
		"max-length":       newGaugeVec("queue_max_length", "How many (ready) messages a queue can contain before it starts to drop them from its head.", queueLabels),
	}

	queueCounterVec = map[string]*prometheus.Desc{
		"disk_reads":                   newDesc("queue_disk_reads_total", "Total number of times messages have been read from disk by this queue since it started.", queueLabels),
		"disk_writes":                  newDesc("queue_disk_writes_total", "Total number of times messages have been written to disk by this queue since it started.", queueLabels),
		"message_stats.publish":        newDesc("queue_messages_published_total", "Count of messages published.", queueLabels),
		"message_stats.confirm":        newDesc("queue_messages_confirmed_total", "Count of messages confirmed. ", queueLabels),
		"message_stats.deliver":        newDesc("queue_messages_delivered_total", "Count of messages delivered in acknowledgement mode to consumers.", queueLabels),
		"message_stats.deliver_no_ack": newDesc("queue_messages_delivered_noack_total", "Count of messages delivered in no-acknowledgement mode to consumers. ", queueLabels),
		"message_stats.get":            newDesc("queue_messages_get_total", "Count of messages delivered in acknowledgement mode in response to basic.get.", queueLabels),
		"message_stats.get_no_ack":     newDesc("queue_messages_get_noack_total", "Count of messages delivered in no-acknowledgement mode in response to basic.get.", queueLabels),
		"message_stats.redeliver":      newDesc("queue_messages_redelivered_total", "Count of subset of messages in deliver_get which had the redelivered flag set.", queueLabels),
		"message_stats.return":         newDesc("queue_messages_returned_total", "Count of messages returned to publisher as unroutable.", queueLabels),
		"message_stats.ack":            newDesc("queue_messages_ack_total", "Count of messages delivered in acknowledgement mode in response to basic.get.", queueLabels),
		"reductions":                   newDesc("queue_reductions_total", "Count of  reductions which take place on this process. .", queueLabels),
		"garbage_collection.minor_gcs": newDesc("queue_gc_minor_collections_total", "Number of minor GCs", queueLabels),
	}
)

type exporterQueue struct {
	limitsGauge         map[string]*prometheus.GaugeVec
	queueMetricsGauge   map[string]*prometheus.GaugeVec
	queueMetricsCounter map[string]*prometheus.Desc
	stateMetric         *prometheus.GaugeVec
	idleSinceMetric     *prometheus.GaugeVec
}

func newExporterQueue() Exporter {
	queueGaugeVecActual := queueGaugeVec
	queueCounterVecActual := queueCounterVec
	litmitsGaugeVecActual := limitsGaugeVec

	if len(config.ExcludeMetrics) > 0 {
		for _, metric := range config.ExcludeMetrics {
			if queueGaugeVecActual[metric] != nil {
				delete(queueGaugeVecActual, metric)
			}
			if queueCounterVecActual[metric] != nil {
				delete(queueCounterVecActual, metric)
			}
			if litmitsGaugeVecActual[metric] != nil {
				delete(litmitsGaugeVecActual, metric)
			}
		}
	}

	return exporterQueue{
		limitsGauge:         litmitsGaugeVecActual,
		queueMetricsGauge:   queueGaugeVecActual,
		queueMetricsCounter: queueCounterVecActual,
		stateMetric:         newGaugeVec("queue_state", "A metric with a value of constant '1' if the queue is in a certain state", append(queueLabels, "state")),
		idleSinceMetric:     newGaugeVec("queue_idle_since_seconds", "starttime where the queue switched to idle state; in seconds since epoch (1970).", queueLabels),
	}
}

func collectLowerMetric(metricA, metricB string, stats StatsInfo) float64 {
	mA, okA := stats.metrics[metricA]
	mB, okB := stats.metrics[metricB]

	if okA && okB {
		if mA < mB {
			return mA
		} else {
			return mB
		}
	}
	if okA {
		return mA
	}
	if okB {
		return mB
	}
	return -1.0
}

func (e exporterQueue) Collect(ctx context.Context, ch chan<- prometheus.Metric) error {
	for _, gaugevec := range e.queueMetricsGauge {
		gaugevec.Reset()
	}
	for _, m := range e.limitsGauge {
		m.Reset()
	}
	e.stateMetric.Reset()
	e.idleSinceMetric.Reset()

	if config.MaxQueues > 0 {
		// Get overview info to check total queues
		totalQueues, ok := ctx.Value(totalQueues).(int)
		if !ok {
			return errors.New("total Queue counter missing")
		}

		if totalQueues > config.MaxQueues {
			log.WithFields(log.Fields{
				"MaxQueues":   config.MaxQueues,
				"TotalQueues": totalQueues,
			}).Debug("MaxQueues exceeded.")
			return nil
		}
	}

	selfNode := ""
	if n, ok := ctx.Value(nodeName).(string); ok {
		selfNode = n
	}
	cluster := ""
	if n, ok := ctx.Value(clusterName).(string); ok {
		cluster = n
	}

	rabbitMqQueueData, err := getStatsInfo(config, "queues", queueLabelKeys)
	if err != nil {
		return err
	}

	log.WithField("queueData", rabbitMqQueueData).Debug("Queue data")
	for _, queue := range rabbitMqQueueData {
		qname := queue.labels["name"]
		vname := queue.labels["vhost"]
		if vhostIncluded := config.IncludeVHost.MatchString(vname); !vhostIncluded {
			continue
		}
		if skipVhost := config.SkipVHost.MatchString(vname); skipVhost {
			continue
		}
		if queueIncluded := config.IncludeQueues.MatchString(qname); !queueIncluded {
			continue
		}
		if queueSkipped := config.SkipQueues.MatchString(qname); queueSkipped {
			continue
		}

		self := "0"
		if queue.labels["node"] == selfNode {
			self = "1"
		}
		labelValues := []string{cluster, queue.labels["vhost"], queue.labels["name"], queue.labels["durable"], queue.labels["policy"], self}

		for key, gaugevec := range e.queueMetricsGauge {
			if value, ok := queue.metrics[key]; ok {
				// log.WithFields(log.Fields{"vhost": queue.labels["vhost"], "queue": queue.labels["name"], "key": key, "value": value}).Info("Set queue metric for key")
				gaugevec.WithLabelValues(labelValues...).Set(value)
			}
		}

		for key, countvec := range e.queueMetricsCounter {
			if value, ok := queue.metrics[key]; ok {
				ch <- prometheus.MustNewConstMetric(countvec, prometheus.CounterValue, value, labelValues...)
			} else {
				ch <- prometheus.MustNewConstMetric(countvec, prometheus.CounterValue, 0, labelValues...)
			}
		}

		state := queue.labels["state"]
		idleSince, exists := queue.labels["idle_since"]
		if exists && idleSince != "" {
			if t, err := time.Parse("2006-01-02 15:04:05", idleSince); err == nil {
				unixSeconds := float64(t.UnixNano()) / 1e9

				if state == "running" { //replace running state with idle if idle_since time is provided. Other states (flow, etc.) are not replaced
					state = "idle"
				}
				e.idleSinceMetric.WithLabelValues(labelValues...).Set(unixSeconds)
			} else {
				log.WithError(err).WithField("idle_since", idleSince).Warn("error parsing idle since time")
			}
		}
		e.stateMetric.WithLabelValues(append(labelValues, state)...).Set(1)

		if f := collectLowerMetric("arguments.x-max-length", "effective_policy_definition.max-length", queue); f >= 0 {
			limitsGaugeVec["max-length"].WithLabelValues(labelValues...).Set(f)
		}
		if f := collectLowerMetric("arguments.x-max-length-bytes", "effective_policy_definition.max-length-bytes", queue); f >= 0 {
			limitsGaugeVec["max-length-bytes"].WithLabelValues(labelValues...).Set(f)
		}

	}

	for _, metric := range e.limitsGauge {
		metric.Collect(ch)
	}
	for _, gaugevec := range e.queueMetricsGauge {
		gaugevec.Collect(ch)
	}
	e.stateMetric.Collect(ch)
	e.idleSinceMetric.Collect(ch)

	return nil
}

func (e exporterQueue) Describe(ch chan<- *prometheus.Desc) {
	for _, metric := range e.limitsGauge {
		metric.Describe(ch)
	}
	for _, gaugevec := range e.queueMetricsGauge {
		gaugevec.Describe(ch)
	}
	e.stateMetric.Describe(ch)
	e.idleSinceMetric.Describe(ch)
	for _, countervec := range e.queueMetricsCounter {
		ch <- countervec
	}
}
<commit_msg>Fix: panic when max-length metrics are excluded<commit_after>package main

import (
	"context"
	"errors"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	log "github.com/sirupsen/logrus"
)

func init() {
	RegisterExporter("queue", newExporterQueue)
}

var (
	queueLabels    = []string{"cluster", "vhost", "queue", "durable", "policy", "self"}
	queueLabelKeys = []string{"vhost", "name", "durable", "policy", "state", "node", "idle_since"}

	queueGaugeVec = map[string]*prometheus.GaugeVec{
		"messages_ready":                            newGaugeVec("queue_messages_ready", "Number of messages ready to be delivered to clients.", queueLabels),
		"messages_unacknowledged":                   newGaugeVec("queue_messages_unacknowledged", "Number of messages delivered to clients but not yet acknowledged.", queueLabels),
		"messages":                                  newGaugeVec("queue_messages", "Sum of ready and unacknowledged messages (queue depth).", queueLabels),
		"messages_ready_ram":                        newGaugeVec("queue_messages_ready_ram", "Number of messages from messages_ready which are resident in ram.", queueLabels),
		"messages_unacknowledged_ram":               newGaugeVec("queue_messages_unacknowledged_ram", "Number of messages from messages_unacknowledged which are resident in ram.", queueLabels),
		"messages_ram":                              newGaugeVec("queue_messages_ram", "Total number of messages which are resident in ram.", queueLabels),
		"messages_persistent":                       newGaugeVec("queue_messages_persistent", "Total number of persistent messages in the queue (will always be 0 for transient queues).", queueLabels),
		"message_bytes":                             newGaugeVec("queue_message_bytes", "Sum of the size of all message bodies in the queue. This does not include the message properties (including headers) or any overhead.", queueLabels),
		"message_bytes_ready":                       newGaugeVec("queue_message_bytes_ready", "Like message_bytes but counting only those messages ready to be delivered to clients.", queueLabels),
		"message_bytes_unacknowledged":              newGaugeVec("queue_message_bytes_unacknowledged", "Like message_bytes but counting only those messages delivered to clients but not yet acknowledged.", queueLabels),
		"message_bytes_ram":                         newGaugeVec("queue_message_bytes_ram", "Like message_bytes but counting only those messages which are in RAM.", queueLabels),
		"message_bytes_persistent":                  newGaugeVec("queue_message_bytes_persistent", "Like message_bytes but counting only those messages which are persistent.", queueLabels),
		"consumers":                                 newGaugeVec("queue_consumers", "Number of consumers.", queueLabels),
		"consumer_utilisation":                      newGaugeVec("queue_consumer_utilisation", "Fraction of the time (between 0.0 and 1.0) that the queue is able to immediately deliver messages to consumers. This can be less than 1.0 if consumers are limited by network congestion or prefetch count.", queueLabels),
		"memory":                                    newGaugeVec("queue_memory", "Bytes of memory consumed by the Erlang process associated with the queue, including stack, heap and internal structures.", queueLabels),
		"head_message_timestamp":                    newGaugeVec("queue_head_message_timestamp", "The timestamp property of the first message in the queue, if present. Timestamps of messages only appear when they are in the paged-in state.", queueLabels), //https://github.com/rabbitmq/rabbitmq-server/pull/54
		"garbage_collection.min_heap_size":          newGaugeVec("queue_gc_min_heap", "Minimum heap size in words", queueLabels),
		"garbage_collection.min_bin_vheap_size":     newGaugeVec("queue_gc_min_vheap", "Minimum binary virtual heap size in words", queueLabels),
		"garbage_collection.fullsweep_after":        newGaugeVec("queue_gc_collections_before_fullsweep", "Maximum generational collections before fullsweep", queueLabels),
		"slave_nodes_len":                           newGaugeVec("queue_slaves_nodes_len", "Number of slave nodes attached to the queue", queueLabels),
		"synchronised_slave_nodes_len":              newGaugeVec("queue_synchronised_slave_nodes_len", "Number of slave nodes in sync to the queue", queueLabels),
		"message_stats.publish_details.rate":        newGaugeVec("queue_messages_publish_rate", "Rate at which messages are entering the server.", queueLabels),
		"message_stats.deliver_no_ack_details.rate": newGaugeVec("queue_messages_deliver_no_ack_rate", "Rate at which messages are delivered to consumers that use automatic acknowledgements.", queueLabels),
		"message_stats.deliver_details.rate":        newGaugeVec("queue_messages_deliver_rate", "Rate at which messages are delivered to consumers that use manual acknowledgements.", queueLabels),
	}
	limitsGaugeVec = map[string]*prometheus.GaugeVec{
		"max-length-bytes": newGaugeVec("queue_max_length_bytes", "Total body size for ready messages a queue can contain before it starts to drop them from its head.", queueLabels),
		"max-length":       newGaugeVec("queue_max_length", "How many (ready) messages a queue can contain before it starts to drop them from its head.", queueLabels),
	}

	queueCounterVec = map[string]*prometheus.Desc{
		"disk_reads":                   newDesc("queue_disk_reads_total", "Total number of times messages have been read from disk by this queue since it started.", queueLabels),
		"disk_writes":                  newDesc("queue_disk_writes_total", "Total number of times messages have been written to disk by this queue since it started.", queueLabels),
		"message_stats.publish":        newDesc("queue_messages_published_total", "Count of messages published.", queueLabels),
		"message_stats.confirm":        newDesc("queue_messages_confirmed_total", "Count of messages confirmed. ", queueLabels),
		"message_stats.deliver":        newDesc("queue_messages_delivered_total", "Count of messages delivered in acknowledgement mode to consumers.", queueLabels),
		"message_stats.deliver_no_ack": newDesc("queue_messages_delivered_noack_total", "Count of messages delivered in no-acknowledgement mode to consumers. ", queueLabels),
		"message_stats.get":            newDesc("queue_messages_get_total", "Count of messages delivered in acknowledgement mode in response to basic.get.", queueLabels),
		"message_stats.get_no_ack":     newDesc("queue_messages_get_noack_total", "Count of messages delivered in no-acknowledgement mode in response to basic.get.", queueLabels),
		"message_stats.redeliver":      newDesc("queue_messages_redelivered_total", "Count of subset of messages in deliver_get which had the redelivered flag set.", queueLabels),
		"message_stats.return":         newDesc("queue_messages_returned_total", "Count of messages returned to publisher as unroutable.", queueLabels),
		"message_stats.ack":            newDesc("queue_messages_ack_total", "Count of messages delivered in acknowledgement mode in response to basic.get.", queueLabels),
		"reductions":                   newDesc("queue_reductions_total", "Count of  reductions which take place on this process. .", queueLabels),
		"garbage_collection.minor_gcs": newDesc("queue_gc_minor_collections_total", "Number of minor GCs", queueLabels),
	}
)

type exporterQueue struct {
	limitsGauge         map[string]*prometheus.GaugeVec
	queueMetricsGauge   map[string]*prometheus.GaugeVec
	queueMetricsCounter map[string]*prometheus.Desc
	stateMetric         *prometheus.GaugeVec
	idleSinceMetric     *prometheus.GaugeVec
}

func newExporterQueue() Exporter {
	queueGaugeVecActual := queueGaugeVec
	queueCounterVecActual := queueCounterVec
	litmitsGaugeVecActual := limitsGaugeVec

	if len(config.ExcludeMetrics) > 0 {
		for _, metric := range config.ExcludeMetrics {
			if queueGaugeVecActual[metric] != nil {
				delete(queueGaugeVecActual, metric)
			}
			if queueCounterVecActual[metric] != nil {
				delete(queueCounterVecActual, metric)
			}
			if litmitsGaugeVecActual[metric] != nil {
				delete(litmitsGaugeVecActual, metric)
			}
		}
	}

	return exporterQueue{
		limitsGauge:         litmitsGaugeVecActual,
		queueMetricsGauge:   queueGaugeVecActual,
		queueMetricsCounter: queueCounterVecActual,
		stateMetric:         newGaugeVec("queue_state", "A metric with a value of constant '1' if the queue is in a certain state", append(queueLabels, "state")),
		idleSinceMetric:     newGaugeVec("queue_idle_since_seconds", "starttime where the queue switched to idle state; in seconds since epoch (1970).", queueLabels),
	}
}

func collectLowerMetric(metricA, metricB string, stats StatsInfo) float64 {
	mA, okA := stats.metrics[metricA]
	mB, okB := stats.metrics[metricB]

	if okA && okB {
		if mA < mB {
			return mA
		} else {
			return mB
		}
	}
	if okA {
		return mA
	}
	if okB {
		return mB
	}
	return -1.0
}

func (e exporterQueue) Collect(ctx context.Context, ch chan<- prometheus.Metric) error {
	for _, gaugevec := range e.queueMetricsGauge {
		gaugevec.Reset()
	}
	for _, m := range e.limitsGauge {
		m.Reset()
	}
	e.stateMetric.Reset()
	e.idleSinceMetric.Reset()

	if config.MaxQueues > 0 {
		// Get overview info to check total queues
		totalQueues, ok := ctx.Value(totalQueues).(int)
		if !ok {
			return errors.New("total Queue counter missing")
		}

		if totalQueues > config.MaxQueues {
			log.WithFields(log.Fields{
				"MaxQueues":   config.MaxQueues,
				"TotalQueues": totalQueues,
			}).Debug("MaxQueues exceeded.")
			return nil
		}
	}

	selfNode := ""
	if n, ok := ctx.Value(nodeName).(string); ok {
		selfNode = n
	}
	cluster := ""
	if n, ok := ctx.Value(clusterName).(string); ok {
		cluster = n
	}

	rabbitMqQueueData, err := getStatsInfo(config, "queues", queueLabelKeys)
	if err != nil {
		return err
	}

	log.WithField("queueData", rabbitMqQueueData).Debug("Queue data")
	for _, queue := range rabbitMqQueueData {
		qname := queue.labels["name"]
		vname := queue.labels["vhost"]
		if vhostIncluded := config.IncludeVHost.MatchString(vname); !vhostIncluded {
			continue
		}
		if skipVhost := config.SkipVHost.MatchString(vname); skipVhost {
			continue
		}
		if queueIncluded := config.IncludeQueues.MatchString(qname); !queueIncluded {
			continue
		}
		if queueSkipped := config.SkipQueues.MatchString(qname); queueSkipped {
			continue
		}

		self := "0"
		if queue.labels["node"] == selfNode {
			self = "1"
		}
		labelValues := []string{cluster, queue.labels["vhost"], queue.labels["name"], queue.labels["durable"], queue.labels["policy"], self}

		for key, gaugevec := range e.queueMetricsGauge {
			if value, ok := queue.metrics[key]; ok {
				// log.WithFields(log.Fields{"vhost": queue.labels["vhost"], "queue": queue.labels["name"], "key": key, "value": value}).Info("Set queue metric for key")
				gaugevec.WithLabelValues(labelValues...).Set(value)
			}
		}

		for key, countvec := range e.queueMetricsCounter {
			if value, ok := queue.metrics[key]; ok {
				ch <- prometheus.MustNewConstMetric(countvec, prometheus.CounterValue, value, labelValues...)
			} else {
				ch <- prometheus.MustNewConstMetric(countvec, prometheus.CounterValue, 0, labelValues...)
			}
		}

		state := queue.labels["state"]
		idleSince, exists := queue.labels["idle_since"]
		if exists && idleSince != "" {
			if t, err := time.Parse("2006-01-02 15:04:05", idleSince); err == nil {
				unixSeconds := float64(t.UnixNano()) / 1e9

				if state == "running" { //replace running state with idle if idle_since time is provided. Other states (flow, etc.) are not replaced
					state = "idle"
				}
				e.idleSinceMetric.WithLabelValues(labelValues...).Set(unixSeconds)
			} else {
				log.WithError(err).WithField("idle_since", idleSince).Warn("error parsing idle since time")
			}
		}
		e.stateMetric.WithLabelValues(append(labelValues, state)...).Set(1)

		if _, ok := limitsGaugeVec["max-length"]; ok {
			if f := collectLowerMetric("arguments.x-max-length", "effective_policy_definition.max-length", queue); f >= 0 {
				limitsGaugeVec["max-length"].WithLabelValues(labelValues...).Set(f)
			}
		}

		if _, ok := limitsGaugeVec["max-length-bytes"]; ok {
			if f := collectLowerMetric("arguments.x-max-length-bytes", "effective_policy_definition.max-length-bytes", queue); f >= 0 {
				limitsGaugeVec["max-length-bytes"].WithLabelValues(labelValues...).Set(f)
			}
		}

	}

	for _, metric := range e.limitsGauge {
		metric.Collect(ch)
	}
	for _, gaugevec := range e.queueMetricsGauge {
		gaugevec.Collect(ch)
	}
	e.stateMetric.Collect(ch)
	e.idleSinceMetric.Collect(ch)

	return nil
}

func (e exporterQueue) Describe(ch chan<- *prometheus.Desc) {
	for _, metric := range e.limitsGauge {
		metric.Describe(ch)
	}
	for _, gaugevec := range e.queueMetricsGauge {
		gaugevec.Describe(ch)
	}
	e.stateMetric.Describe(ch)
	e.idleSinceMetric.Describe(ch)
	for _, countervec := range e.queueMetricsCounter {
		ch <- countervec
	}
}
<|endoftext|>