<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package helpers

import (
	"fmt"
	"testing"
	"time"

	"github.com/ghodss/yaml"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/uuid"

	eventingduck "knative.dev/eventing/pkg/apis/duck/v1alpha1"
	"knative.dev/eventing/pkg/apis/messaging/config"
	eventingtesting "knative.dev/eventing/pkg/reconciler/testing/v1beta1"
	"knative.dev/eventing/test/lib"
	"knative.dev/eventing/test/lib/cloudevents"
	"knative.dev/eventing/test/lib/duck"
	"knative.dev/eventing/test/lib/resources"
	reconciler "knative.dev/pkg/reconciler"
)

const (
	// configMapName is the name of the ConfigMap that contains the configuration for the default
	// channel CRD.
	configMapName = config.ChannelDefaultsConfigName

	// channelDefaulterKey is the key in the ConfigMap to get the name of the default
	// Channel CRD.
	channelDefaulterKey = config.ChannelDefaulterKey
)

// ChannelClusterDefaulterTestHelper is the helper function for channel_defaulter_test
func ChannelClusterDefaulterTestHelper(t *testing.T,
	channelTestRunner lib.ChannelTestRunner,
	options ...lib.SetupClientOption) {
	channelTestRunner.RunTests(t, lib.FeatureBasic, func(st *testing.T, channel metav1.TypeMeta) {
		// these tests cannot be run in parallel as they have cluster-wide impact
		client := lib.Setup(st, false, options...)
		defer lib.TearDown(client)

		if err := updateDefaultChannelCM(client, func(conf *config.ChannelDefaults) {
			setClusterDefaultChannel(conf, channel)
		}); err != nil {
			st.Fatalf("Failed to update the defaultchannel configmap: %v", err)
		}

		defaultChannelTestHelper(st, client, channel)
	})
}

// ChannelNamespaceDefaulterTestHelper is the helper function for channel_defaulter_test
func ChannelNamespaceDefaulterTestHelper(t *testing.T,
	channelTestRunner lib.ChannelTestRunner,
	options ...lib.SetupClientOption) {
	channelTestRunner.RunTests(t, lib.FeatureBasic, func(st *testing.T, channel metav1.TypeMeta) {
		// we cannot run these tests in parallel as the updateDefaultChannelCM function is not thread-safe
		// TODO(chizhg): make updateDefaultChannelCM thread-safe and run in parallel if the tests are taking too long to finish
		client := lib.Setup(st, false, options...)
		defer lib.TearDown(client)

		if err := updateDefaultChannelCM(client, func(conf *config.ChannelDefaults) {
			setNamespaceDefaultChannel(conf, client.Namespace, channel)
		}); err != nil {
			st.Fatalf("Failed to update the defaultchannel configmap: %v", err)
		}

		defaultChannelTestHelper(st, client, channel)
	})
}

func defaultChannelTestHelper(t *testing.T, client *lib.Client, expectedChannel metav1.TypeMeta) {
	channelName := "e2e-defaulter-channel"
	senderName := "e2e-defaulter-sender"
	subscriptionName := "e2e-defaulter-subscription"
	loggerPodName := "e2e-defaulter-logger-pod"

	// create channel
	client.CreateChannelWithDefaultOrFail(eventingtesting.NewChannel(channelName, client.Namespace))

	// create logger service as the subscriber
	pod := resources.EventLoggerPod(loggerPodName)
	client.CreatePodOrFail(pod, lib.WithService(loggerPodName))

	// create subscription to subscribe the channel, and forward the received events to the logger service
	client.CreateSubscriptionOrFail(
		subscriptionName,
		channelName,
		lib.ChannelTypeMeta,
		resources.WithSubscriberForSubscription(loggerPodName),
	)

	// wait for all test resources to be ready, so that we can start sending events
	client.WaitForAllTestResourcesReadyOrFail()

	// check if the defaultchannel creates exactly one underlying channel given the spec
	metaResourceList := resources.NewMetaResourceList(client.Namespace, &expectedChannel)
	objs, err := duck.GetGenericObjectList(client.Dynamic, metaResourceList, &eventingduck.SubscribableType{})
	if err != nil {
		t.Fatalf("Failed to list the underlying channels: %v", err)
	}

	// Note that since by default MT ChannelBroker creates a Broker in each namespace, there's
	// actually two channels.
	// https://github.com/knative/eventing/issues/3138
	// So, filter out the broker channel from the list before checking that there's only one.
	filteredObjs := make([]runtime.Object, 0)
	for _, o := range objs {
		if o.(*eventingduck.SubscribableType).Name != "default-kne-trigger" {
			filteredObjs = append(filteredObjs, o)
		}
	}

	if len(filteredObjs) != 1 {
		t.Logf("Got unexpected channels:")
		for i, ec := range filteredObjs {
			t.Logf("Extra channels: %d : %+v", i, ec)
		}
		t.Fatalf("The defaultchannel is expected to create 1 underlying channel, but got %d", len(filteredObjs))
	}

	// send fake CloudEvent to the channel
	body := fmt.Sprintf("TestSingleEvent %s", uuid.NewUUID())
	event := cloudevents.New(
		fmt.Sprintf(`{"msg":%q}`, body),
		cloudevents.WithSource(senderName),
	)
	client.SendFakeEventToAddressableOrFail(senderName, channelName, lib.ChannelTypeMeta, event)

	// verify the logger service receives the event
	if err := client.CheckLog(loggerPodName, lib.CheckerContains(body)); err != nil {
		t.Fatalf("String %q not found in logs of logger pod %q: %v", body, loggerPodName, err)
	}
}

// updateDefaultChannelCM will update the default channel configmap
func updateDefaultChannelCM(client *lib.Client, updateConfig func(config *config.ChannelDefaults)) error {
	cmInterface := client.Kube.Kube.CoreV1().ConfigMaps(resources.SystemNamespace)

	err := reconciler.RetryUpdateConflicts(func(attempts int) (err error) {
		// get the defaultchannel configmap
		configMap, err := cmInterface.Get(configMapName, metav1.GetOptions{})
		if err != nil {
			return err
		}
		// get the defaultchannel config value
		defaultChannelConfig, hasDefault := configMap.Data[channelDefaulterKey]
		config := &config.ChannelDefaults{}
		if hasDefault {
			if err := yaml.Unmarshal([]byte(defaultChannelConfig), config); err != nil {
				return err
			}
		}

		// update the defaultchannel config
		updateConfig(config)
		configBytes, err := yaml.Marshal(*config)
		if err != nil {
			return err
		}
		// update the defaultchannel configmap
		configMap.Data[channelDefaulterKey] = string(configBytes)
		_, err = cmInterface.Update(configMap)
		return err
	})
	if err != nil {
		return err
	}
	// In cmd/webhook.go, configMapWatcher watches the configmap changes and set the config for channeldefaulter,
	// the resync time is set to 0, which means the the resync will be delayed as long as possible (until the upstream
	// source closes the watch or times out, or you stop the controller)
	// Wait for 5 seconds to let the ConfigMap be synced up.
	// TODO(chizhg): 5 seconds is an empirical duration, and does not solve the problem from the root.
	// To make it work reliably, we may need to manually restart the controller.
	// https://github.com/knative/eventing/issues/2807
	time.Sleep(5 * time.Second)
	return nil
}

// setClusterDefaultChannel will set the default channel for cluster-wide
func setClusterDefaultChannel(cfg *config.ChannelDefaults, channel metav1.TypeMeta) {
	if cfg.ClusterDefault == nil {
		cfg.ClusterDefault = &config.ChannelTemplateSpec{}
	}
	// If we're testing with Channel, we can't default to ourselves, or badness will
	// happen. We're going to try to create Channels that are ourselves.
	if channel.Kind == "Channel" {
		channel.Kind = "InMemoryChannel"
	}
	cfg.ClusterDefault.TypeMeta = channel
}

// setNamespaceDefaultChannel will set the default channel for namespace-wide
func setNamespaceDefaultChannel(cfg *config.ChannelDefaults, namespace string, channel metav1.TypeMeta) {
	// If we're testing with Channel, we can't default to ourselves, or badness will
	// happen. We're going to try to create Channels that are ourselves.
	if channel.Kind == "Channel" {
		channel.Kind = "InMemoryChannel"
	}
	if cfg.NamespaceDefaults == nil {
		cfg.NamespaceDefaults = make(map[string]*config.ChannelTemplateSpec, 1)
	}
	namespaceDefaults := cfg.NamespaceDefaults
	if spec, exists := namespaceDefaults[namespace]; exists {
		spec.TypeMeta = channel
	} else {
		spec = &config.ChannelTemplateSpec{
			TypeMeta: channel,
		}
		namespaceDefaults[namespace] = spec
	}
}
<commit_msg>Port Default channel Test to new test images (#3284)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package helpers

import (
	"fmt"
	"testing"
	"time"

	cloudevents "github.com/cloudevents/sdk-go/v2"
	"github.com/ghodss/yaml"
	"github.com/google/uuid"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"

	eventingduck "knative.dev/eventing/pkg/apis/duck/v1alpha1"
	"knative.dev/eventing/pkg/apis/messaging/config"
	eventingtesting "knative.dev/eventing/pkg/reconciler/testing/v1beta1"
	"knative.dev/eventing/test/lib"
	"knative.dev/eventing/test/lib/duck"
	"knative.dev/eventing/test/lib/resources"
	reconciler "knative.dev/pkg/reconciler"
)

const (
	// configMapName is the name of the ConfigMap that contains the configuration for the default
	// channel CRD.
	configMapName = config.ChannelDefaultsConfigName

	// channelDefaulterKey is the key in the ConfigMap to get the name of the default
	// Channel CRD.
	channelDefaulterKey = config.ChannelDefaulterKey
)

// ChannelClusterDefaulterTestHelper is the helper function for channel_defaulter_test
func ChannelClusterDefaulterTestHelper(t *testing.T,
	channelTestRunner lib.ChannelTestRunner,
	options ...lib.SetupClientOption) {
	channelTestRunner.RunTests(t, lib.FeatureBasic, func(st *testing.T, channel metav1.TypeMeta) {
		// these tests cannot be run in parallel as they have cluster-wide impact
		client := lib.Setup(st, false, options...)
		defer lib.TearDown(client)

		if err := updateDefaultChannelCM(client, func(conf *config.ChannelDefaults) {
			setClusterDefaultChannel(conf, channel)
		}); err != nil {
			st.Fatalf("Failed to update the defaultchannel configmap: %v", err)
		}

		defaultChannelTestHelper(st, client, channel)
	})
}

// ChannelNamespaceDefaulterTestHelper is the helper function for channel_defaulter_test
func ChannelNamespaceDefaulterTestHelper(t *testing.T,
	channelTestRunner lib.ChannelTestRunner,
	options ...lib.SetupClientOption) {
	channelTestRunner.RunTests(t, lib.FeatureBasic, func(st *testing.T, channel metav1.TypeMeta) {
		// we cannot run these tests in parallel as the updateDefaultChannelCM function is not thread-safe
		// TODO(chizhg): make updateDefaultChannelCM thread-safe and run in parallel if the tests are taking too long to finish
		client := lib.Setup(st, false, options...)
		defer lib.TearDown(client)

		if err := updateDefaultChannelCM(client, func(conf *config.ChannelDefaults) {
			setNamespaceDefaultChannel(conf, client.Namespace, channel)
		}); err != nil {
			st.Fatalf("Failed to update the defaultchannel configmap: %v", err)
		}

		defaultChannelTestHelper(st, client, channel)
	})
}

func defaultChannelTestHelper(t *testing.T, client *lib.Client, expectedChannel metav1.TypeMeta) {
	channelName := "e2e-defaulter-channel"
	senderName := "e2e-defaulter-sender"
	subscriptionName := "e2e-defaulter-subscription"
	recordEventsPodName := "e2e-defaulter-recordevents-pod"

	// create channel
	client.CreateChannelWithDefaultOrFail(eventingtesting.NewChannel(channelName, client.Namespace))

	// create event logger pod and service as the subscriber
	recordEventsPod := resources.EventRecordPod(recordEventsPodName)
	client.CreatePodOrFail(recordEventsPod, lib.WithService(recordEventsPodName))
	eventTracker, err := client.NewEventInfoStore(recordEventsPodName, t.Logf)
	if err != nil {
		t.Fatalf("Pod tracker failed: %v", err)
	}
	defer eventTracker.Cleanup()

	// create subscription to subscribe the channel, and forward the received events to the logger service
	client.CreateSubscriptionOrFail(
		subscriptionName,
		channelName,
		lib.ChannelTypeMeta,
		resources.WithSubscriberForSubscription(recordEventsPodName),
	)

	// wait for all test resources to be ready, so that we can start sending events
	client.WaitForAllTestResourcesReadyOrFail()

	// check if the defaultchannel creates exactly one underlying channel given the spec
	metaResourceList := resources.NewMetaResourceList(client.Namespace, &expectedChannel)
	objs, err := duck.GetGenericObjectList(client.Dynamic, metaResourceList, &eventingduck.SubscribableType{})
	if err != nil {
		t.Fatalf("Failed to list the underlying channels: %v", err)
	}

	// Note that since by default MT ChannelBroker creates a Broker in each namespace, there's
	// actually two channels.
	// https://github.com/knative/eventing/issues/3138
	// So, filter out the broker channel from the list before checking that there's only one.
	filteredObjs := make([]runtime.Object, 0)
	for _, o := range objs {
		if o.(*eventingduck.SubscribableType).Name != "default-kne-trigger" {
			filteredObjs = append(filteredObjs, o)
		}
	}

	if len(filteredObjs) != 1 {
		t.Logf("Got unexpected channels:")
		for i, ec := range filteredObjs {
			t.Logf("Extra channels: %d : %+v", i, ec)
		}
		t.Fatalf("The defaultchannel is expected to create 1 underlying channel, but got %d", len(filteredObjs))
	}

	// send CloudEvent to the channel
	event := cloudevents.NewEvent()
	event.SetID("dummy")
	eventSource := fmt.Sprintf("http://%s.svc/", senderName)
	event.SetSource(eventSource)
	event.SetType(lib.DefaultEventType)
	body := fmt.Sprintf(`{"msg":"TestSingleEvent %s"}`, uuid.New().String())
	if err := event.SetData(cloudevents.ApplicationJSON, []byte(body)); err != nil {
		t.Fatalf("Cannot set the payload of the event: %s", err.Error())
	}
	client.SendEventToAddressable(senderName, channelName, lib.ChannelTypeMeta, event)

	// verify the logger service receives the event
	eventTracker.AssertWaitMatchSourceData(t, recordEventsPodName, eventSource, body, 1, 1)
}

// updateDefaultChannelCM will update the default channel configmap
func updateDefaultChannelCM(client *lib.Client, updateConfig func(config *config.ChannelDefaults)) error {
	cmInterface := client.Kube.Kube.CoreV1().ConfigMaps(resources.SystemNamespace)

	err := reconciler.RetryUpdateConflicts(func(attempts int) (err error) {
		// get the defaultchannel configmap
		configMap, err := cmInterface.Get(configMapName, metav1.GetOptions{})
		if err != nil {
			return err
		}
		// get the defaultchannel config value
		defaultChannelConfig, hasDefault := configMap.Data[channelDefaulterKey]
		config := &config.ChannelDefaults{}
		if hasDefault {
			if err := yaml.Unmarshal([]byte(defaultChannelConfig), config); err != nil {
				return err
			}
		}

		// update the defaultchannel config
		updateConfig(config)
		configBytes, err := yaml.Marshal(*config)
		if err != nil {
			return err
		}
		// update the defaultchannel configmap
		configMap.Data[channelDefaulterKey] = string(configBytes)
		_, err = cmInterface.Update(configMap)
		return err
	})
	if err != nil {
		return err
	}
	// In cmd/webhook.go, configMapWatcher watches the configmap changes and set the config for channeldefaulter,
	// the resync time is set to 0, which means the the resync will be delayed as long as possible (until the upstream
	// source closes the watch or times out, or you stop the controller)
	// Wait for 5 seconds to let the ConfigMap be synced up.
	// TODO(chizhg): 5 seconds is an empirical duration, and does not solve the problem from the root.
	// To make it work reliably, we may need to manually restart the controller.
	// https://github.com/knative/eventing/issues/2807
	time.Sleep(5 * time.Second)
	return nil
}

// setClusterDefaultChannel will set the default channel for cluster-wide
func setClusterDefaultChannel(cfg *config.ChannelDefaults, channel metav1.TypeMeta) {
	if cfg.ClusterDefault == nil {
		cfg.ClusterDefault = &config.ChannelTemplateSpec{}
	}
	// If we're testing with Channel, we can't default to ourselves, or badness will
	// happen. We're going to try to create Channels that are ourselves.
	if channel.Kind == "Channel" {
		channel.Kind = "InMemoryChannel"
	}
	cfg.ClusterDefault.TypeMeta = channel
}

// setNamespaceDefaultChannel will set the default channel for namespace-wide
func setNamespaceDefaultChannel(cfg *config.ChannelDefaults, namespace string, channel metav1.TypeMeta) {
	// If we're testing with Channel, we can't default to ourselves, or badness will
	// happen. We're going to try to create Channels that are ourselves.
	if channel.Kind == "Channel" {
		channel.Kind = "InMemoryChannel"
	}
	if cfg.NamespaceDefaults == nil {
		cfg.NamespaceDefaults = make(map[string]*config.ChannelTemplateSpec, 1)
	}
	namespaceDefaults := cfg.NamespaceDefaults
	if spec, exists := namespaceDefaults[namespace]; exists {
		spec.TypeMeta = channel
	} else {
		spec = &config.ChannelTemplateSpec{
			TypeMeta: channel,
		}
		namespaceDefaults[namespace] = spec
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Ashley Jeffs
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, sub to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// +build !wasm

package metrics

import (
	"fmt"
	"net/http"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/Jeffail/benthos/lib/log"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/client_golang/prometheus/push"
)

//------------------------------------------------------------------------------

// PromGauge is a representation of a single metric stat. Interactions with this
// stat are thread safe.
type PromGauge struct {
	ctr prometheus.Gauge
}

// Incr increments a metric by an amount.
func (p *PromGauge) Incr(count int64) error {
	p.ctr.Add(float64(count))
	return nil
}

// Decr decrements a metric by an amount.
func (p *PromGauge) Decr(count int64) error {
	p.ctr.Add(float64(-count))
	return nil
}

// Set sets a gauge metric.
func (p *PromGauge) Set(value int64) error {
	p.ctr.Set(float64(value))
	return nil
}

// PromCounter is a representation of a single metric stat. Interactions with
// this stat are thread safe.
type PromCounter struct {
	ctr prometheus.Counter
}

// Incr increments a metric by an amount.
func (p *PromCounter) Incr(count int64) error {
	p.ctr.Add(float64(count))
	return nil
}

// PromTiming is a representation of a single metric stat. Interactions with
// this stat are thread safe.
type PromTiming struct {
	sum prometheus.Observer
}

// Timing sets a timing metric.
func (p *PromTiming) Timing(val int64) error {
	p.sum.Observe(float64(val))
	return nil
}

//------------------------------------------------------------------------------

// PromCounterVec creates StatCounters with dynamic labels.
type PromCounterVec struct {
	ctr *prometheus.CounterVec
}

// With returns a StatCounter with a set of label values.
func (p *PromCounterVec) With(labelValues ...string) StatCounter {
	return &PromCounter{
		ctr: p.ctr.WithLabelValues(labelValues...),
	}
}

// PromTimingVec creates StatTimers with dynamic labels.
type PromTimingVec struct {
	sum *prometheus.SummaryVec
}

// With returns a StatTimer with a set of label values.
func (p *PromTimingVec) With(labelValues ...string) StatTimer {
	return &PromTiming{
		sum: p.sum.WithLabelValues(labelValues...),
	}
}

// PromGaugeVec creates StatGauges with dynamic labels.
type PromGaugeVec struct {
	ctr *prometheus.GaugeVec
}

// With returns a StatGauge with a set of label values.
func (p *PromGaugeVec) With(labelValues ...string) StatGauge {
	return &PromGauge{
		ctr: p.ctr.WithLabelValues(labelValues...),
	}
}

//------------------------------------------------------------------------------

// Prometheus is a stats object with capability to hold internal stats as a JSON
// endpoint.
type Prometheus struct {
	log        log.Modular
	closedChan chan struct{}
	running    int32

	config PrometheusConfig
	prefix string

	counters map[string]*prometheus.CounterVec
	gauges   map[string]*prometheus.GaugeVec
	timers   map[string]*prometheus.SummaryVec

	sync.Mutex
}

// NewPrometheus creates and returns a new Prometheus object.
func NewPrometheus(config Config, opts ...func(Type)) (Type, error) {
	p := &Prometheus{
		log:        log.Noop(),
		running:    1,
		closedChan: make(chan struct{}),
		config:     config.Prometheus,
		prefix:     toPromName(config.Prefix),
		counters:   map[string]*prometheus.CounterVec{},
		gauges:     map[string]*prometheus.GaugeVec{},
		timers:     map[string]*prometheus.SummaryVec{},
	}

	for _, opt := range opts {
		opt(p)
	}

	if len(p.config.PushURL) > 0 && len(p.config.PushInterval) > 0 {
		interval, err := time.ParseDuration(p.config.PushInterval)
		if err != nil {
			return nil, fmt.Errorf("failed to parse push interval: %v", err)
		}
		go func() {
			for {
				select {
				case <-p.closedChan:
					return
				case <-time.After(interval):
					if err = push.New(p.config.PushURL, p.config.PushJobName).Gatherer(prometheus.DefaultGatherer).Push(); err != nil {
						p.log.Errorf("Failed to push metrics: %v\n", err)
					}
				}
			}
		}()
	}

	return p, nil
}

//------------------------------------------------------------------------------

// HandlerFunc returns an http.HandlerFunc for scraping metrics.
func (p *Prometheus) HandlerFunc() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		promhttp.Handler().ServeHTTP(w, r)
	}
}

//------------------------------------------------------------------------------

func toPromName(dotSepName string) string {
	dotSepName = strings.Replace(dotSepName, "_", "__", -1)
	dotSepName = strings.Replace(dotSepName, "-", "__", -1)
	return strings.Replace(dotSepName, ".", "_", -1)
}

// GetCounter returns a stat counter object for a path.
func (p *Prometheus) GetCounter(path string) StatCounter {
	stat := toPromName(path)

	var ctr *prometheus.CounterVec

	p.Lock()
	var exists bool
	if ctr, exists = p.counters[stat]; !exists {
		ctr = prometheus.NewCounterVec(prometheus.CounterOpts{
			Namespace: p.prefix,
			Name:      stat,
			Help:      "Benthos Counter metric",
		}, nil)
		prometheus.MustRegister(ctr)
		p.counters[stat] = ctr
	}
	p.Unlock()

	return &PromCounter{
		ctr: ctr.WithLabelValues(),
	}
}

// GetTimer returns a stat timer object for a path.
func (p *Prometheus) GetTimer(path string) StatTimer {
	stat := toPromName(path)

	var tmr *prometheus.SummaryVec

	p.Lock()
	var exists bool
	if tmr, exists = p.timers[stat]; !exists {
		tmr = prometheus.NewSummaryVec(prometheus.SummaryOpts{
			Namespace: p.prefix,
			Name:      stat,
			Help:      "Benthos Timing metric",
		}, nil)
		prometheus.MustRegister(tmr)
		p.timers[stat] = tmr
	}
	p.Unlock()

	return &PromTiming{
		sum: tmr.WithLabelValues(),
	}
}

// GetGauge returns a stat gauge object for a path.
func (p *Prometheus) GetGauge(path string) StatGauge {
	stat := toPromName(path)

	var ctr *prometheus.GaugeVec

	p.Lock()
	var exists bool
	if ctr, exists = p.gauges[stat]; !exists {
		ctr = prometheus.NewGaugeVec(prometheus.GaugeOpts{
			Namespace: p.prefix,
			Name:      stat,
			Help:      "Benthos Gauge metric",
		}, nil)
		prometheus.MustRegister(ctr)
		p.gauges[stat] = ctr
	}
	p.Unlock()

	return &PromGauge{
		ctr: ctr.WithLabelValues(),
	}
}

// GetCounterVec returns an editable counter stat for a given path with labels,
// these labels must be consistent with any other metrics registered on the same
// path.
func (p *Prometheus) GetCounterVec(path string, labelNames []string) StatCounterVec {
	stat := toPromName(path)

	var ctr *prometheus.CounterVec

	p.Lock()
	var exists bool
	if ctr, exists = p.counters[stat]; !exists {
		ctr = prometheus.NewCounterVec(prometheus.CounterOpts{
			Namespace: p.prefix,
			Name:      stat,
			Help:      "Benthos Counter metric",
		}, labelNames)
		prometheus.MustRegister(ctr)
		p.counters[stat] = ctr
	}
	p.Unlock()

	return &PromCounterVec{
		ctr: ctr,
	}
}

// GetTimerVec returns an editable timer stat for a given path with labels,
// these labels must be consistent with any other metrics registered on the same
// path.
func (p *Prometheus) GetTimerVec(path string, labelNames []string) StatTimerVec {
	stat := toPromName(path)

	var tmr *prometheus.SummaryVec

	p.Lock()
	var exists bool
	if tmr, exists = p.timers[stat]; !exists {
		tmr = prometheus.NewSummaryVec(prometheus.SummaryOpts{
			Namespace: p.prefix,
			Name:      stat,
			Help:      "Benthos Timing metric",
		}, labelNames)
		prometheus.MustRegister(tmr)
		p.timers[stat] = tmr
	}
	p.Unlock()

	return &PromTimingVec{
		sum: tmr,
	}
}

// GetGaugeVec returns an editable gauge stat for a given path with labels,
// these labels must be consistent with any other metrics registered on the same
// path.
func (p *Prometheus) GetGaugeVec(path string, labelNames []string) StatGaugeVec {
	stat := toPromName(path)

	var ctr *prometheus.GaugeVec

	p.Lock()
	var exists bool
	if ctr, exists = p.gauges[stat]; !exists {
		ctr = prometheus.NewGaugeVec(prometheus.GaugeOpts{
			Namespace: p.prefix,
			Name:      stat,
			Help:      "Benthos Gauge metric",
		}, labelNames)
		prometheus.MustRegister(ctr)
		p.gauges[stat] = ctr
	}
	p.Unlock()

	return &PromGaugeVec{
		ctr: ctr,
	}
}

// SetLogger does nothing.
func (p *Prometheus) SetLogger(log log.Modular) {
	p.log = log
}

// Close stops the Prometheus object from aggregating metrics and cleans up
// resources.
func (p *Prometheus) Close() error {
	if atomic.CompareAndSwapInt32(&p.running, 1, 0) {
		close(p.closedChan)
	}
	if len(p.config.PushURL) > 0 {
		return push.New(p.config.PushURL, p.config.PushJobName).Gatherer(prometheus.DefaultGatherer).Push()
	}
	return nil
}

//------------------------------------------------------------------------------
<commit_msg>[metrics/prometheus]: Add default summary objectives<commit_after>// Copyright (c) 2018 Ashley Jeffs
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, sub to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// +build !wasm

package metrics

import (
	"fmt"
	"net/http"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/Jeffail/benthos/lib/log"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/prometheus/client_golang/prometheus/push"
)

//------------------------------------------------------------------------------

// PromGauge is a representation of a single metric stat. Interactions with this
// stat are thread safe.
type PromGauge struct {
	ctr prometheus.Gauge
}

// Incr increments a metric by an amount.
func (p *PromGauge) Incr(count int64) error {
	p.ctr.Add(float64(count))
	return nil
}

// Decr decrements a metric by an amount.
func (p *PromGauge) Decr(count int64) error {
	p.ctr.Add(float64(-count))
	return nil
}

// Set sets a gauge metric.
func (p *PromGauge) Set(value int64) error {
	p.ctr.Set(float64(value))
	return nil
}

// PromCounter is a representation of a single metric stat. Interactions with
// this stat are thread safe.
type PromCounter struct {
	ctr prometheus.Counter
}

// Incr increments a metric by an amount.
func (p *PromCounter) Incr(count int64) error {
	p.ctr.Add(float64(count))
	return nil
}

// PromTiming is a representation of a single metric stat. Interactions with
// this stat are thread safe.
type PromTiming struct {
	sum prometheus.Observer
}

// Timing sets a timing metric.
func (p *PromTiming) Timing(val int64) error {
	p.sum.Observe(float64(val))
	return nil
}

//------------------------------------------------------------------------------

// PromCounterVec creates StatCounters with dynamic labels.
type PromCounterVec struct {
	ctr *prometheus.CounterVec
}

// With returns a StatCounter with a set of label values.
func (p *PromCounterVec) With(labelValues ...string) StatCounter {
	return &PromCounter{
		ctr: p.ctr.WithLabelValues(labelValues...),
	}
}

// PromTimingVec creates StatTimers with dynamic labels.
type PromTimingVec struct {
	sum *prometheus.SummaryVec
}

// With returns a StatTimer with a set of label values.
func (p *PromTimingVec) With(labelValues ...string) StatTimer {
	return &PromTiming{
		sum: p.sum.WithLabelValues(labelValues...),
	}
}

// PromGaugeVec creates StatGauges with dynamic labels.
type PromGaugeVec struct {
	ctr *prometheus.GaugeVec
}

// With returns a StatGauge with a set of label values.
func (p *PromGaugeVec) With(labelValues ...string) StatGauge {
	return &PromGauge{
		ctr: p.ctr.WithLabelValues(labelValues...),
	}
}

//------------------------------------------------------------------------------

// Prometheus is a stats object with capability to hold internal stats as a JSON
// endpoint.
type Prometheus struct {
	log        log.Modular
	closedChan chan struct{}
	running    int32

	config PrometheusConfig
	prefix string

	counters map[string]*prometheus.CounterVec
	gauges   map[string]*prometheus.GaugeVec
	timers   map[string]*prometheus.SummaryVec

	sync.Mutex
}

// NewPrometheus creates and returns a new Prometheus object.
func NewPrometheus(config Config, opts ...func(Type)) (Type, error) {
	p := &Prometheus{
		log:        log.Noop(),
		running:    1,
		closedChan: make(chan struct{}),
		config:     config.Prometheus,
		prefix:     toPromName(config.Prefix),
		counters:   map[string]*prometheus.CounterVec{},
		gauges:     map[string]*prometheus.GaugeVec{},
		timers:     map[string]*prometheus.SummaryVec{},
	}

	for _, opt := range opts {
		opt(p)
	}

	if len(p.config.PushURL) > 0 && len(p.config.PushInterval) > 0 {
		interval, err := time.ParseDuration(p.config.PushInterval)
		if err != nil {
			return nil, fmt.Errorf("failed to parse push interval: %v", err)
		}
		go func() {
			for {
				select {
				case <-p.closedChan:
					return
				case <-time.After(interval):
					if err = push.New(p.config.PushURL, p.config.PushJobName).Gatherer(prometheus.DefaultGatherer).Push(); err != nil {
						p.log.Errorf("Failed to push metrics: %v\n", err)
					}
				}
			}
		}()
	}

	return p, nil
}

//------------------------------------------------------------------------------

// HandlerFunc returns an http.HandlerFunc for scraping metrics.
func (p *Prometheus) HandlerFunc() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		promhttp.Handler().ServeHTTP(w, r)
	}
}

//------------------------------------------------------------------------------

func toPromName(dotSepName string) string {
	dotSepName = strings.Replace(dotSepName, "_", "__", -1)
	dotSepName = strings.Replace(dotSepName, "-", "__", -1)
	return strings.Replace(dotSepName, ".", "_", -1)
}

// GetCounter returns a stat counter object for a path.
func (p *Prometheus) GetCounter(path string) StatCounter {
	stat := toPromName(path)

	var ctr *prometheus.CounterVec

	p.Lock()
	var exists bool
	if ctr, exists = p.counters[stat]; !exists {
		ctr = prometheus.NewCounterVec(prometheus.CounterOpts{
			Namespace: p.prefix,
			Name:      stat,
			Help:      "Benthos Counter metric",
		}, nil)
		prometheus.MustRegister(ctr)
		p.counters[stat] = ctr
	}
	p.Unlock()

	return &PromCounter{
		ctr: ctr.WithLabelValues(),
	}
}

// GetTimer returns a stat timer object for a path.
func (p *Prometheus) GetTimer(path string) StatTimer {
	stat := toPromName(path)

	var tmr *prometheus.SummaryVec

	p.Lock()
	var exists bool
	if tmr, exists = p.timers[stat]; !exists {
		tmr = prometheus.NewSummaryVec(prometheus.SummaryOpts{
			Namespace:  p.prefix,
			Name:       stat,
			Help:       "Benthos Timing metric",
			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
		}, nil)
		prometheus.MustRegister(tmr)
		p.timers[stat] = tmr
	}
	p.Unlock()

	return &PromTiming{
		sum: tmr.WithLabelValues(),
	}
}

// GetGauge returns a stat gauge object for a path.
func (p *Prometheus) GetGauge(path string) StatGauge {
	stat := toPromName(path)

	var ctr *prometheus.GaugeVec

	p.Lock()
	var exists bool
	if ctr, exists = p.gauges[stat]; !exists {
		ctr = prometheus.NewGaugeVec(prometheus.GaugeOpts{
			Namespace: p.prefix,
			Name:      stat,
			Help:      "Benthos Gauge metric",
		}, nil)
		prometheus.MustRegister(ctr)
		p.gauges[stat] = ctr
	}
	p.Unlock()

	return &PromGauge{
		ctr: ctr.WithLabelValues(),
	}
}

// GetCounterVec returns an editable counter stat for a given path with labels,
// these labels must be consistent with any other metrics registered on the same
// path.
func (p *Prometheus) GetCounterVec(path string, labelNames []string) StatCounterVec {
	stat := toPromName(path)

	var ctr *prometheus.CounterVec

	p.Lock()
	var exists bool
	if ctr, exists = p.counters[stat]; !exists {
		ctr = prometheus.NewCounterVec(prometheus.CounterOpts{
			Namespace: p.prefix,
			Name:      stat,
			Help:      "Benthos Counter metric",
		}, labelNames)
		prometheus.MustRegister(ctr)
		p.counters[stat] = ctr
	}
	p.Unlock()

	return &PromCounterVec{
		ctr: ctr,
	}
}

// GetTimerVec returns an editable timer stat for a given path with labels,
// these labels must be consistent with any other metrics registered on the same
// path.
func (p *Prometheus) GetTimerVec(path string, labelNames []string) StatTimerVec {
	stat := toPromName(path)

	var tmr *prometheus.SummaryVec

	p.Lock()
	var exists bool
	if tmr, exists = p.timers[stat]; !exists {
		tmr = prometheus.NewSummaryVec(prometheus.SummaryOpts{
			Namespace:  p.prefix,
			Name:       stat,
			Help:       "Benthos Timing metric",
			Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
		}, labelNames)
		prometheus.MustRegister(tmr)
		p.timers[stat] = tmr
	}
	p.Unlock()

	return &PromTimingVec{
		sum: tmr,
	}
}

// GetGaugeVec returns an editable gauge stat for a given path with labels,
// these labels must be consistent with any other metrics registered on the same
// path.
func (p *Prometheus) GetGaugeVec(path string, labelNames []string) StatGaugeVec {
	stat := toPromName(path)

	var ctr *prometheus.GaugeVec

	p.Lock()
	var exists bool
	if ctr, exists = p.gauges[stat]; !exists {
		ctr = prometheus.NewGaugeVec(prometheus.GaugeOpts{
			Namespace: p.prefix,
			Name:      stat,
			Help:      "Benthos Gauge metric",
		}, labelNames)
		prometheus.MustRegister(ctr)
		p.gauges[stat] = ctr
	}
	p.Unlock()

	return &PromGaugeVec{
		ctr: ctr,
	}
}

// SetLogger does nothing.
func (p *Prometheus) SetLogger(log log.Modular) {
	p.log = log
}

// Close stops the Prometheus object from aggregating metrics and cleans up
// resources.
func (p *Prometheus) Close() error {
	if atomic.CompareAndSwapInt32(&p.running, 1, 0) {
		close(p.closedChan)
	}
	if len(p.config.PushURL) > 0 {
		return push.New(p.config.PushURL, p.config.PushJobName).Gatherer(prometheus.DefaultGatherer).Push()
	}
	return nil
}

//------------------------------------------------------------------------------
<|endoftext|>


<commit_before>/***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# The Initial Developer of the Original Code is the Mozilla Foundation.
# Portions created by the Initial Developer are Copyright (C) 2015
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Reid (mark@mozilla.com)
# ***** END LICENSE BLOCK *****/

/*

A command-line utility for listing files on Amazon S3, filtered by dimension.

*/
package main

import (
	"flag"
	"fmt"
	"github.com/crowdmob/goamz/aws"
	"github.com/crowdmob/goamz/s3"
	"github.com/mozilla-services/data-pipeline/heka/plugins/s3splitfile"
	"os"
	"strings"
	"time"
)

func main() {
	flagSchema := flag.String("schema", "", "Filename of the schema to use as a filter")
	flagBucket := flag.String("bucket", "default-bucket", "S3 Bucket name")
	flagBucketPrefix := flag.String("bucket-prefix", "", "S3 Bucket path prefix")
	flagAWSKey := flag.String("aws-key", "DUMMY", "AWS Key")
	flagAWSSecretKey := flag.String("aws-secret-key", "DUMMY", "AWS Secret Key")
	flagAWSRegion := flag.String("aws-region", "us-west-2", "AWS Region")
	flagDryRun := flag.Bool("dry-run", false, "Don't actually do anything, just output what would be done")
	flagVerbose := flag.Bool("verbose", false, "Print detailed info")
	flag.Parse()

	if flag.NArg() != 0 {
		flag.PrintDefaults()
		os.Exit(1)
	}

	var err error
	var schema s3splitfile.Schema
	schema, err = s3splitfile.LoadSchema(*flagSchema)
	if err != nil {
		fmt.Printf("schema: %s\n", err)
		os.Exit(2)
	}

	if *flagDryRun {
		fmt.Printf("Dry Run: Would have listed files in s3://%s/%s according to filter schema %s\n",
			*flagBucket, *flagBucketPrefix, *flagSchema)
		os.Exit(0)
	}

	var b *s3.Bucket

	prefix := strings.Trim(*flagBucketPrefix, "/")
	if prefix != "" {
		prefix += "/"
	}

	// Initialize the S3 bucket
	auth := aws.Auth{AccessKey: *flagAWSKey, SecretKey: *flagAWSSecretKey}
	s := s3.New(auth, aws.Regions[*flagAWSRegion])
	b = s.Bucket(*flagBucket)

	var errCount int
	var totalCount int
	var totalSize int64

	startTime := time.Now().UTC()

	// List the keys as we see them
	for k := range s3splitfile.S3Iterator(b, prefix, schema) {
		if k.Err != nil {
			fmt.Printf("ERROR fetching key: %s\n", k.Err)
			errCount++
		} else {
			totalCount++
			totalSize += k.Key.Size
			fmt.Printf("%s\n", k.Key.Key)
		}
	}

	duration := time.Now().UTC().Sub(startTime).Seconds()

	if *flagVerbose {
		fmt.Printf("Filter matched %d files totaling %s in %.02fs (%d errors)\n",
			totalCount, s3splitfile.PrettySize(totalSize), duration, errCount)
	}
}
<commit_msg>Print a sensible message on invalid aws region.<commit_after>/***** BEGIN LICENSE BLOCK *****
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this file,
# You can obtain one at http://mozilla.org/MPL/2.0/.
#
# The Initial Developer of the Original Code is the Mozilla Foundation.
# Portions created by the Initial Developer are Copyright (C) 2015
# the Initial Developer. All Rights Reserved.
#
# Contributor(s):
#   Mark Reid (mark@mozilla.com)
# ***** END LICENSE BLOCK *****/

/*

A command-line utility for listing files on Amazon S3, filtered by dimension.

*/
package main

import (
	"flag"
	"fmt"
	"github.com/crowdmob/goamz/aws"
	"github.com/crowdmob/goamz/s3"
	"github.com/mozilla-services/data-pipeline/heka/plugins/s3splitfile"
	"os"
	"strings"
	"time"
)

func main() {
	flagSchema := flag.String("schema", "", "Filename of the schema to use as a filter")
	flagBucket := flag.String("bucket", "default-bucket", "S3 Bucket name")
	flagBucketPrefix := flag.String("bucket-prefix", "", "S3 Bucket path prefix")
	flagAWSKey := flag.String("aws-key", "DUMMY", "AWS Key")
	flagAWSSecretKey := flag.String("aws-secret-key", "DUMMY", "AWS Secret Key")
	flagAWSRegion := flag.String("aws-region", "us-west-2", "AWS Region")
	flagDryRun := flag.Bool("dry-run", false, "Don't actually do anything, just output what would be done")
	flagVerbose := flag.Bool("verbose", false, "Print detailed info")
	flag.Parse()

	if flag.NArg() != 0 {
		flag.PrintDefaults()
		os.Exit(1)
	}

	var err error
	var schema s3splitfile.Schema
	schema, err = s3splitfile.LoadSchema(*flagSchema)
	if err != nil {
		fmt.Printf("schema: %s\n", err)
		os.Exit(2)
	}

	if *flagDryRun {
		fmt.Printf("Dry Run: Would have listed files in s3://%s/%s according to filter schema %s\n",
			*flagBucket, *flagBucketPrefix, *flagSchema)
		os.Exit(0)
	}

	var b *s3.Bucket

	prefix := strings.Trim(*flagBucketPrefix, "/")
	if prefix != "" {
		prefix += "/"
	}

	// Initialize the S3 bucket
	auth := aws.Auth{AccessKey: *flagAWSKey, SecretKey: *flagAWSSecretKey}
	region, ok := aws.Regions[*flagAWSRegion]
	if !ok {
		fmt.Printf("Parameter 'aws-region' must be a valid AWS Region\n")
		os.Exit(4)
	}
	s := s3.New(auth, region)
	b = s.Bucket(*flagBucket)

	var errCount int
	var totalCount int
	var totalSize int64

	startTime := time.Now().UTC()

	// List the keys as we see them
	for k := range s3splitfile.S3Iterator(b, prefix, schema) {
		if k.Err != nil {
			fmt.Printf("ERROR fetching key: %s\n", k.Err)
			errCount++
		} else {
			totalCount++
			totalSize += k.Key.Size
			fmt.Printf("%s\n", k.Key.Key)
		}
	}

	duration := time.Now().UTC().Sub(startTime).Seconds()

	if *flagVerbose {
		fmt.Printf("Filter matched %d files totaling %s in %.02fs (%d errors)\n",
			totalCount, s3splitfile.PrettySize(totalSize), duration, errCount)
	}
}
<|endoftext|>


<commit_before>package consts

type ReplicationState uint32

const (
	_ ReplicationState = iota
	OldReplicationPrimary
	OldReplicationSecondary
	OldReplicationBootstrapping
	// Don't add anything here. This was meant to be in two separate const
	// blocks, to reset iota, but it didn't happen that way, so iota carries
	// over. Adding anything to this Old block would cause the rest of the
	// values to change below.

	ReplicationUnknown            ReplicationState = 0
	ReplicationPerformancePrimary ReplicationState = 1 << iota
	ReplicationPerformanceSecondary
	OldSplitReplicationBootstrapping
	ReplicationDRPrimary
	ReplicationDRSecondary
	ReplicationPerformanceBootstrapping
	ReplicationDRBootstrapping
	ReplicationPerformanceDisabled
	ReplicationDRDisabled
)

func (r ReplicationState) string() string {
	switch r {
	case ReplicationPerformanceSecondary:
		return "secondary"
	case ReplicationPerformancePrimary:
		return "primary"
	case ReplicationPerformanceBootstrapping:
		return "bootstrapping"
	case ReplicationPerformanceDisabled:
		return "disabled"
	case ReplicationDRPrimary:
		return "primary"
	case ReplicationDRSecondary:
		return "secondary"
	case ReplicationDRBootstrapping:
		return "bootstrapping"
	case ReplicationDRDisabled:
		return "disabled"
	}

	return "unknown"
}

func (r ReplicationState) GetDRString() string {
	switch {
	case r.HasState(ReplicationDRBootstrapping):
		return ReplicationDRBootstrapping.string()
	case r.HasState(ReplicationDRPrimary):
		return ReplicationDRPrimary.string()
	case r.HasState(ReplicationDRSecondary):
		return ReplicationDRSecondary.string()
	case r.HasState(ReplicationDRDisabled):
		return ReplicationDRDisabled.string()
	default:
		return "unknown"
	}
}

func (r ReplicationState) GetPerformanceString() string {
	switch {
	case r.HasState(ReplicationPerformanceBootstrapping):
		return ReplicationPerformanceBootstrapping.string()
	case r.HasState(ReplicationPerformancePrimary):
		return ReplicationPerformancePrimary.string()
	case r.HasState(ReplicationPerformanceSecondary):
		return ReplicationPerformanceSecondary.string()
	case r.HasState(ReplicationPerformanceDisabled):
		return ReplicationPerformanceDisabled.string()
	default:
		return "unknown"
	}
}

func (r ReplicationState) HasState(flag ReplicationState) bool { return r&flag != 0 }
func (r *ReplicationState) AddState(flag ReplicationState)     { *r |= flag }
func (r *ReplicationState) ClearState(flag ReplicationState)   { *r &= ^flag }
func (r *ReplicationState) ToggleState(flag ReplicationState)  { *r ^= flag }
<commit_msg>Update comment to replication consts<commit_after>package consts

type ReplicationState uint32

const (
	_ ReplicationState = iota
	OldReplicationPrimary
	OldReplicationSecondary
	OldReplicationBootstrapping
	// Don't add anything here. Adding anything to this Old block would cause
	// the rest of the values to change below. This was done originally to
	// ensure no overlap between old and new values.

	ReplicationUnknown            ReplicationState = 0
	ReplicationPerformancePrimary ReplicationState = 1 << iota
	ReplicationPerformanceSecondary
	OldSplitReplicationBootstrapping
	ReplicationDRPrimary
	ReplicationDRSecondary
	ReplicationPerformanceBootstrapping
	ReplicationDRBootstrapping
	ReplicationPerformanceDisabled
	ReplicationDRDisabled
)

func (r ReplicationState) string() string {
	switch r {
	case ReplicationPerformanceSecondary:
		return "secondary"
	case ReplicationPerformancePrimary:
		return "primary"
	case ReplicationPerformanceBootstrapping:
		return "bootstrapping"
	case ReplicationPerformanceDisabled:
		return "disabled"
	case ReplicationDRPrimary:
		return "primary"
	case ReplicationDRSecondary:
		return "secondary"
	case ReplicationDRBootstrapping:
		return "bootstrapping"
	case ReplicationDRDisabled:
		return "disabled"
	}

	return "unknown"
}

func (r ReplicationState) GetDRString() string {
	switch {
	case r.HasState(ReplicationDRBootstrapping):
		return ReplicationDRBootstrapping.string()
	case r.HasState(ReplicationDRPrimary):
		return ReplicationDRPrimary.string()
	case r.HasState(ReplicationDRSecondary):
		return ReplicationDRSecondary.string()
	case r.HasState(ReplicationDRDisabled):
		return ReplicationDRDisabled.string()
	default:
		return "unknown"
	}
}

func (r ReplicationState) GetPerformanceString() string {
	switch {
	case r.HasState(ReplicationPerformanceBootstrapping):
		return ReplicationPerformanceBootstrapping.string()
	case r.HasState(ReplicationPerformancePrimary):
		return ReplicationPerformancePrimary.string()
	case r.HasState(ReplicationPerformanceSecondary):
		return ReplicationPerformanceSecondary.string()
	case r.HasState(ReplicationPerformanceDisabled):
		return ReplicationPerformanceDisabled.string()
	default:
		return "unknown"
	}
}

func (r ReplicationState) HasState(flag ReplicationState) bool { return r&flag != 0 }
func (r *ReplicationState) AddState(flag ReplicationState)     { *r |= flag }
func (r *ReplicationState) ClearState(flag ReplicationState)   { *r &= ^flag }
func (r *ReplicationState) ToggleState(flag ReplicationState)  { *r ^= flag }
<|endoftext|>


<commit_before>package sudoku

import (
	"testing"
)

func TestHumanSolveSearcher(t *testing.T) {

	grid := NewGrid()
	grid.LoadSDK(TEST_GRID)

	searcher := newHumanSolveSearcher(grid, nil, DefaultHumanSolveOptions())

	if searcher.Len() != 1 {
		t.Error("Expected new frontier to have exactly one item in it, but got", searcher.Len())
	}

	if searcher.grid == nil {
		t.Error("No grid in frontier")
	}

	basePotentialNextStep := searcher.NextPossibleStep()

	if basePotentialNextStep == nil {
		t.Error("Didn'get base potential next step")
	}

	baseGrid := basePotentialNextStep.Grid()

	if baseGrid.DataString() != grid.DataString() {
		t.Error("the grid in the base item in the frontier was not right. Got", baseGrid.DataString(), "wanted", grid.DataString())
	}

	if searcher.Len() != 0 {
		t.Error("Getting the base potential next step should have emptied it, but len is", searcher.Len())
	}

	nInRowTechnique := techniquesByName["Necessary In Row"]

	simpleFillStep := &SolveStep{
		Technique: nInRowTechnique,
		TargetCells: CellSlice{
			grid.Cell(0, 0),
		},
		TargetNums: IntSlice{1},
	}

	if simpleFillStep.Technique == nil {
		t.Fatal("couldn't find necessary in row technique")
	}

	simpleFillStepItem := basePotentialNextStep.AddStep(simpleFillStep)

	if simpleFillStepItem == nil {
		t.Fatal("Adding fill step didn't return anything")
	}

	if simpleFillStepItem.heapIndex != -1 {
		t.Fatal("Adding completed item to frontier didn't have -1 index")
	}

	if len(searcher.completedItems) != 1 {
		t.Error("Expected the completed item to go into CompletedItems,but it's empty")
	}

	if len(searcher.itemsToExplore) != 0 {
		t.Error("Expected the completed item to go into COmpletedItems, but it apparently went into items.")
	}

	//TODO: this is a fragile test. Whenever we change twiddlers it will change.
	//Curerntly 5 for common numbers, 22.5 for Human Likelihood, and 1 for chained steps.
	expectedGoodness := 112.5

	if simpleFillStepItem.Goodness() != expectedGoodness {
		t.Error("Goodness of simple fill step was wrong. Execpted", expectedGoodness, "got", simpleFillStepItem.Goodness(), simpleFillStepItem.explainGoodness(0))
	}

	cell := simpleFillStepItem.Grid().Cell(0, 0)

	if cell.Number() != 1 {
		t.Error("Cell in grid was not set correctly. Got", cell.Number(), "wanted 1")
	}

	nonFillStep := &SolveStep{
		Technique: techniquesByName["Pointing Pair Row"],
		TargetCells: CellSlice{
			grid.Cell(0, 1),
		},
		TargetNums: IntSlice{2},
	}

	if nonFillStep.Technique == nil {
		t.Fatal("Couldn't find pointing pair row techhnique")
	}

	nonFillStepItem := basePotentialNextStep.AddStep(nonFillStep)

	if nonFillStepItem == nil {
		t.Fatal("Adding non fill step didn't return a frontier object")
	}

	//TODO: now that we only have one item on the actual frontier, we should
	//extend the test to add another non-fill step so we can test twiddling
	//moving the order.

	if searcher.Len() != 1 {
		t.Error("Frontier had wrong length after adding one complete and one incomplete items. Got", searcher.Len(), "expected 1")
	}

	if searcher.itemsToExplore[0] != nonFillStepItem {
		t.Error("We though that simpleFillStep should be at the end of the queue but it wasn't.")
	}

	expensiveStep := &SolveStep{
		Technique: techniquesByName["Hidden Quad Block"],
		TargetCells: CellSlice{
			grid.Cell(0, 2),
		},
		TargetNums: IntSlice{3},
	}

	expensiveStepItem := basePotentialNextStep.AddStep(expensiveStep)

	if searcher.Len() != 2 {
		t.Error("Wrong length after adding two items to frontier. Got", searcher.Len(), "expected 2")
	}

	if searcher.itemsToExplore[1] != nonFillStepItem {
		t.Error("We expected the expensive step to be worse", searcher.String())
	}

	expensiveStepItem.Twiddle(0.00000000000000001, "Very small amount to make this #1")

	if searcher.itemsToExplore[1] != expensiveStepItem {
		t.Error("Even after twiddling up guess step by a lot it still wasn't in the top position in frontier", searcher.itemsToExplore[0], searcher.itemsToExplore[1])
	}

	poppedItem := searcher.NextPossibleStep()

	if poppedItem != expensiveStepItem {
		t.Error("Expected popped item to be the non-fill step now that its goodness is higher, but got", poppedItem)
	}

	if searcher.Len() != 1 {
		t.Error("Wrong frontier length after popping item. Got", searcher.Len(), "expected 1")
	}

	poppedItem = searcher.NextPossibleStep()
	//Should be nonFillStepItem

	currentGoodness := nonFillStepItem.Goodness()

	completedNonFillStemItem := nonFillStepItem.AddStep(simpleFillStep)

	if completedNonFillStemItem.Goodness() == currentGoodness {
		t.Error("Adding a step to end of nonfill step didn't change goodness.")
	}

	if searcher.Len() != 0 {
		t.Error("Adding an item gave wrong len. Got", searcher.Len(), "wanted 0")
	}

	if len(searcher.completedItems) != 2 {
		t.Error("Got wrong number of completed items. Got", len(searcher.completedItems), "expected 2")
	}

	steps := completedNonFillStemItem.Steps()

	if len(steps) != 2 {
		t.Error("Expected two steps back, got", len(steps))
	}

	if steps[0] != nonFillStepItem.step {
		t.Error("Expected first step to be the step of nonFillStepItem. Got", steps[0])
	}

}
<commit_msg>Removed a completed TODO<commit_after>package sudoku

import (
	"testing"
)

func TestHumanSolveSearcher(t *testing.T) {

	grid := NewGrid()
	grid.LoadSDK(TEST_GRID)

	searcher := newHumanSolveSearcher(grid, nil, DefaultHumanSolveOptions())

	if searcher.Len() != 1 {
		t.Error("Expected new frontier to have exactly one item in it, but got", searcher.Len())
	}

	if searcher.grid == nil {
		t.Error("No grid in frontier")
	}

	basePotentialNextStep := searcher.NextPossibleStep()

	if basePotentialNextStep == nil {
		t.Error("Didn'get base potential next step")
	}

	baseGrid := basePotentialNextStep.Grid()

	if baseGrid.DataString() != grid.DataString() {
		t.Error("the grid in the base item in the frontier was not right. Got", baseGrid.DataString(), "wanted", grid.DataString())
	}

	if searcher.Len() != 0 {
		t.Error("Getting the base potential next step should have emptied it, but len is", searcher.Len())
	}

	nInRowTechnique := techniquesByName["Necessary In Row"]

	simpleFillStep := &SolveStep{
		Technique: nInRowTechnique,
		TargetCells: CellSlice{
			grid.Cell(0, 0),
		},
		TargetNums: IntSlice{1},
	}

	if simpleFillStep.Technique == nil {
		t.Fatal("couldn't find necessary in row technique")
	}

	simpleFillStepItem := basePotentialNextStep.AddStep(simpleFillStep)

	if simpleFillStepItem == nil {
		t.Fatal("Adding fill step didn't return anything")
	}

	if simpleFillStepItem.heapIndex != -1 {
		t.Fatal("Adding completed item to frontier didn't have -1 index")
	}

	if len(searcher.completedItems) != 1 {
		t.Error("Expected the completed item to go into CompletedItems,but it's empty")
	}

	if len(searcher.itemsToExplore) != 0 {
		t.Error("Expected the completed item to go into COmpletedItems, but it apparently went into items.")
	}

	//TODO: this is a fragile test. Whenever we change twiddlers it will change.
	//Curerntly 5 for common numbers, 22.5 for Human Likelihood, and 1 for chained steps.
	expectedGoodness := 112.5

	if simpleFillStepItem.Goodness() != expectedGoodness {
		t.Error("Goodness of simple fill step was wrong. Execpted", expectedGoodness, "got", simpleFillStepItem.Goodness(), simpleFillStepItem.explainGoodness(0))
	}

	cell := simpleFillStepItem.Grid().Cell(0, 0)

	if cell.Number() != 1 {
		t.Error("Cell in grid was not set correctly. Got", cell.Number(), "wanted 1")
	}

	nonFillStep := &SolveStep{
		Technique: techniquesByName["Pointing Pair Row"],
		TargetCells: CellSlice{
			grid.Cell(0, 1),
		},
		TargetNums: IntSlice{2},
	}

	if nonFillStep.Technique == nil {
		t.Fatal("Couldn't find pointing pair row techhnique")
	}

	nonFillStepItem := basePotentialNextStep.AddStep(nonFillStep)

	if nonFillStepItem == nil {
		t.Fatal("Adding non fill step didn't return a frontier object")
	}

	if searcher.Len() != 1 {
		t.Error("Frontier had wrong length after adding one complete and one incomplete items. Got", searcher.Len(), "expected 1")
	}

	if searcher.itemsToExplore[0] != nonFillStepItem {
		t.Error("We though that simpleFillStep should be at the end of the queue but it wasn't.")
	}

	expensiveStep := &SolveStep{
		Technique: techniquesByName["Hidden Quad Block"],
		TargetCells: CellSlice{
			grid.Cell(0, 2),
		},
		TargetNums: IntSlice{3},
	}

	expensiveStepItem := basePotentialNextStep.AddStep(expensiveStep)

	if searcher.Len() != 2 {
		t.Error("Wrong length after adding two items to frontier. Got", searcher.Len(), "expected 2")
	}

	if searcher.itemsToExplore[1] != nonFillStepItem {
		t.Error("We expected the expensive step to be worse", searcher.String())
	}

	expensiveStepItem.Twiddle(0.00000000000000001, "Very small amount to make this #1")

	if searcher.itemsToExplore[1] != expensiveStepItem {
		t.Error("Even after twiddling up guess step by a lot it still wasn't in the top position in frontier", searcher.itemsToExplore[0], searcher.itemsToExplore[1])
	}

	poppedItem := searcher.NextPossibleStep()

	if poppedItem != expensiveStepItem {
		t.Error("Expected popped item to be the non-fill step now that its goodness is higher, but got", poppedItem)
	}

	if searcher.Len() != 1 {
		t.Error("Wrong frontier length after popping item. Got", searcher.Len(), "expected 1")
	}

	poppedItem = searcher.NextPossibleStep()
	//Should be nonFillStepItem

	currentGoodness := nonFillStepItem.Goodness()

	completedNonFillStemItem := nonFillStepItem.AddStep(simpleFillStep)

	if completedNonFillStemItem.Goodness() == currentGoodness {
		t.Error("Adding a step to end of nonfill step didn't change goodness.")
	}

	if searcher.Len() != 0 {
		t.Error("Adding an item gave wrong len. Got", searcher.Len(), "wanted 0")
	}

	if len(searcher.completedItems) != 2 {
		t.Error("Got wrong number of completed items. Got", len(searcher.completedItems), "expected 2")
	}

	steps := completedNonFillStemItem.Steps()

	if len(steps) != 2 {
		t.Error("Expected two steps back, got", len(steps))
	}

	if steps[0] != nonFillStepItem.step {
		t.Error("Expected first step to be the step of nonFillStepItem. Got", steps[0])
	}

}
<|endoftext|>


<commit_before>package hcl2shim

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/zclconf/go-cty/cty/convert"

	"github.com/zclconf/go-cty/cty"
)

// FlatmapValueFromHCL2 converts a value from HCL2 (really, from the cty dynamic
// types library that HCL2 uses) to a map compatible with what would be
// produced by the "flatmap" package.
//
// The type of the given value informs the structure of the resulting map.
// The value must be of an object type or this function will panic.
//
// Flatmap values can only represent maps when they are of primitive types,
// so the given value must not have any maps of complex types or the result
// is undefined.
func FlatmapValueFromHCL2(v cty.Value) map[string]string {
	if v.IsNull() {
		return nil
	}

	if !v.Type().IsObjectType() {
		panic(fmt.Sprintf("HCL2ValueFromFlatmap called on %#v", v.Type()))
	}

	m := make(map[string]string)
	flatmapValueFromHCL2Map(m, "", v)
	return m
}

func flatmapValueFromHCL2Value(m map[string]string, key string, val cty.Value) {
	ty := val.Type()
	switch {
	case ty.IsPrimitiveType():
		flatmapValueFromHCL2Primitive(m, key, val)
	case ty.IsObjectType() || ty.IsMapType():
		flatmapValueFromHCL2Map(m, key+".", val)
	case ty.IsTupleType() || ty.IsListType() || ty.IsSetType():
		flatmapValueFromHCL2Seq(m, key+".", val)
	default:
		panic(fmt.Sprintf("cannot encode %s to flatmap", ty.FriendlyName()))
	}
}

func flatmapValueFromHCL2Primitive(m map[string]string, key string, val cty.Value) {
	if !val.IsKnown() {
		m[key] = UnknownVariableValue
		return
	}
	if val.IsNull() {
		// Omit entirely
		return
	}

	var err error
	val, err = convert.Convert(val, cty.String)
	if err != nil {
		// Should not be possible, since all primitive types can convert to string.
		panic(fmt.Sprintf("invalid primitive encoding to flatmap: %s", err))
	}
	m[key] = val.AsString()
}

func flatmapValueFromHCL2Map(m map[string]string, prefix string, val cty.Value) {
	if val.IsNull() {
		// Omit entirely
		return
	}
	if !val.IsKnown() {
		switch {
		case val.Type().IsObjectType():
			// Whole objects can't be unknown in flatmap, so instead we'll
			// just write all of the attribute values out as unknown.
			for name, aty := range val.Type().AttributeTypes() {
				flatmapValueFromHCL2Value(m, prefix+name, cty.UnknownVal(aty))
			}
		default:
			m[prefix+"%"] = UnknownVariableValue
		}
		return
	}

	len := 0
	for it := val.ElementIterator(); it.Next(); {
		ak, av := it.Element()
		name := ak.AsString()
		flatmapValueFromHCL2Value(m, prefix+name, av)
		len++
	}
	if !val.Type().IsObjectType() { // objects don't have an explicit count included, since their attribute count is fixed
		m[prefix+"%"] = strconv.Itoa(len)
	}
}

func flatmapValueFromHCL2Seq(m map[string]string, prefix string, val cty.Value) {
	if val.IsNull() {
		// Omit entirely
		return
	}
	if !val.IsKnown() {
		m[prefix+"#"] = UnknownVariableValue
		return
	}

	// For sets this won't actually generate exactly what helper/schema would've
	// generated, because we don't have access to the set key function it
	// would've used. However, in practice it doesn't actually matter what the
	// keys are as long as they are unique, so we'll just generate sequential
	// indexes for them as if it were a list.
	//
	// An important implication of this, however, is that the set ordering will
	// not be consistent across mutations and so different keys may be assigned
	// to the same value when round-tripping. Since this shim is intended to
	// be short-lived and not used for round-tripping, we accept this.
	i := 0
	for it := val.ElementIterator(); it.Next(); {
		_, av := it.Element()
		key := prefix + strconv.Itoa(i)
		flatmapValueFromHCL2Value(m, key, av)
		i++
	}
	m[prefix+"#"] = strconv.Itoa(i)
}

// HCL2ValueFromFlatmap converts a map compatible with what would be produced
// by the "flatmap" package to a HCL2 (really, the cty dynamic types library
// that HCL2 uses) object type.
//
// The intended result type must be provided in order to guide how the
// map contents are decoded. This must be an object type or this function
// will panic.
//
// Flatmap values can only represent maps when they are of primitive types,
// so the given type must not have any maps of complex types or the result
// is undefined.
//
// The result may contain null values if the given map does not contain keys
// for all of the different key paths implied by the given type.
func HCL2ValueFromFlatmap(m map[string]string, ty cty.Type) (cty.Value, error) {
	if m == nil {
		return cty.NullVal(ty), nil
	}
	if !ty.IsObjectType() {
		panic(fmt.Sprintf("HCL2ValueFromFlatmap called on %#v", ty))
	}

	return hcl2ValueFromFlatmapObject(m, "", ty.AttributeTypes())
}

func hcl2ValueFromFlatmapValue(m map[string]string, key string, ty cty.Type) (cty.Value, error) {
	var val cty.Value
	var err error
	switch {
	case ty.IsPrimitiveType():
		val, err = hcl2ValueFromFlatmapPrimitive(m, key, ty)
	case ty.IsObjectType():
		val, err = hcl2ValueFromFlatmapObject(m, key+".", ty.AttributeTypes())
	case ty.IsTupleType():
		val, err = hcl2ValueFromFlatmapTuple(m, key+".", ty.TupleElementTypes())
	case ty.IsMapType():
		val, err = hcl2ValueFromFlatmapMap(m, key+".", ty)
	case ty.IsListType():
		val, err = hcl2ValueFromFlatmapList(m, key+".", ty)
	case ty.IsSetType():
		val, err = hcl2ValueFromFlatmapSet(m, key+".", ty)
	default:
		err = fmt.Errorf("cannot decode %s from flatmap", ty.FriendlyName())
	}

	if err != nil {
		return cty.DynamicVal, err
	}
	return val, nil
}

func hcl2ValueFromFlatmapPrimitive(m map[string]string, key string, ty cty.Type) (cty.Value, error) {
	rawVal, exists := m[key]
	if !exists {
		return cty.NullVal(ty), nil
	}
	if rawVal == UnknownVariableValue {
		return cty.UnknownVal(ty), nil
	}

	var err error
	val := cty.StringVal(rawVal)
	val, err = convert.Convert(val, ty)
	if err != nil {
		// This should never happen for _valid_ input, but flatmap data might
		// be tampered with by the user and become invalid.
		return cty.DynamicVal, fmt.Errorf("invalid value for %q in state: %s", key, err)
	}

	return val, nil
}

func hcl2ValueFromFlatmapObject(m map[string]string, prefix string, atys map[string]cty.Type) (cty.Value, error) {
	vals := make(map[string]cty.Value)
	for name, aty := range atys {
		val, err := hcl2ValueFromFlatmapValue(m, prefix+name, aty)
		if err != nil {
			return cty.DynamicVal, err
		}
		vals[name] = val
	}
	return cty.ObjectVal(vals), nil
}

func hcl2ValueFromFlatmapTuple(m map[string]string, prefix string, etys []cty.Type) (cty.Value, error) {
	var vals []cty.Value

	countStr, exists := m[prefix+"#"]
	if !exists {
		return cty.NullVal(cty.Tuple(etys)), nil
	}
	if countStr == UnknownVariableValue {
		return cty.UnknownVal(cty.Tuple(etys)), nil
	}

	count, err := strconv.Atoi(countStr)
	if err != nil {
		return cty.DynamicVal, fmt.Errorf("invalid count value for %q in state: %s", prefix, err)
	}
	if count != len(etys) {
		return cty.DynamicVal, fmt.Errorf("wrong number of values for %q in state: got %d, but need %d", prefix, count, len(etys))
	}

	vals = make([]cty.Value, len(etys))
	for i, ety := range etys {
		key := prefix + strconv.Itoa(i)
		val, err := hcl2ValueFromFlatmapValue(m, key, ety)
		if err != nil {
			return cty.DynamicVal, err
		}
		vals[i] = val
	}
	return cty.TupleVal(vals), nil
}

func hcl2ValueFromFlatmapMap(m map[string]string, prefix string, ty cty.Type) (cty.Value, error) {
	vals := make(map[string]cty.Value)
	ety := ty.ElementType()

	// We actually don't really care about the "count" of a map for our
	// purposes here, but we do need to check if it _exists_ in order to
	// recognize the difference between null (not set at all) and empty.
	if strCount, exists := m[prefix+"%"]; !exists {
		return cty.NullVal(ty), nil
	} else if strCount == UnknownVariableValue {
		return cty.UnknownVal(ty), nil
	}

	for fullKey := range m {
		if !strings.HasPrefix(fullKey, prefix) {
			continue
		}

		// The flatmap format doesn't allow us to distinguish between keys
		// that contain periods and nested objects, so by convention a
		// map is only ever of primitive type in flatmap, and we just assume
		// that the remainder of the raw key (dots and all) is the key we
		// want in the result value.
		key := fullKey[len(prefix):]
		if key == "%" {
			// Ignore the "count" key
			continue
		}

		val, err := hcl2ValueFromFlatmapValue(m, fullKey, ety)
		if err != nil {
			return cty.DynamicVal, err
		}
		vals[key] = val
	}

	if len(vals) == 0 {
		return cty.MapValEmpty(ety), nil
	}
	return cty.MapVal(vals), nil
}

func hcl2ValueFromFlatmapList(m map[string]string, prefix string, ty cty.Type) (cty.Value, error) {
	var vals []cty.Value

	countStr, exists := m[prefix+"#"]
	if !exists {
		return cty.NullVal(ty), nil
	}
	if countStr == UnknownVariableValue {
		return cty.UnknownVal(ty), nil
	}

	count, err := strconv.Atoi(countStr)
	if err != nil {
		return cty.DynamicVal, fmt.Errorf("invalid count value for %q in state: %s", prefix, err)
	}

	ety := ty.ElementType()
	if count == 0 {
		return cty.ListValEmpty(ety), nil
	}

	vals = make([]cty.Value, count)
	for i := 0; i < count; i++ {
		key := prefix + strconv.Itoa(i)
		val, err := hcl2ValueFromFlatmapValue(m, key, ety)
		if err != nil {
			return cty.DynamicVal, err
		}
		vals[i] = val
	}

	return cty.ListVal(vals), nil
}

func hcl2ValueFromFlatmapSet(m map[string]string, prefix string, ty cty.Type) (cty.Value, error) {
	var vals []cty.Value
	ety := ty.ElementType()

	// We actually don't really care about the "count" of a set for our
	// purposes here, but we do need to check if it _exists_ in order to
	// recognize the difference between null (not set at all) and empty.
	if strCount, exists := m[prefix+"#"]; !exists {
		return cty.NullVal(ty), nil
	} else if strCount == UnknownVariableValue {
		return cty.UnknownVal(ty), nil
	}

	for fullKey := range m {
		if !strings.HasPrefix(fullKey, prefix) {
			continue
		}
		subKey := fullKey[len(prefix):]
		if subKey == "#" {
			// Ignore the "count" key
			continue
		}
		key := fullKey
		if dot := strings.IndexByte(subKey, '.'); dot != -1 {
			key = fullKey[:dot+len(prefix)]
		}

		// The flatmap format doesn't allow us to distinguish between keys
		// that contain periods and nested objects, so by convention a
		// map is only ever of primitive type in flatmap, and we just assume
		// that the remainder of the raw key (dots and all) is the key we
		// want in the result value.

		val, err := hcl2ValueFromFlatmapValue(m, key, ety)
		if err != nil {
			return cty.DynamicVal, err
		}
		vals = append(vals, val)
	}

	if len(vals) == 0 {
		return cty.SetValEmpty(ety), nil
	}
	return cty.SetVal(vals), nil
}
<commit_msg>config/hcl2shim: Treat DynamicVal like any other unknown value in flatmap<commit_after>package hcl2shim

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/zclconf/go-cty/cty/convert"

	"github.com/zclconf/go-cty/cty"
)

// FlatmapValueFromHCL2 converts a value from HCL2 (really, from the cty dynamic
// types library that HCL2 uses) to a map compatible with what would be
// produced by the "flatmap" package.
//
// The type of the given value informs the structure of the resulting map.
// The value must be of an object type or this function will panic.
//
// Flatmap values can only represent maps when they are of primitive types,
// so the given value must not have any maps of complex types or the result
// is undefined.
func FlatmapValueFromHCL2(v cty.Value) map[string]string {
	if v.IsNull() {
		return nil
	}

	if !v.Type().IsObjectType() {
		panic(fmt.Sprintf("HCL2ValueFromFlatmap called on %#v", v.Type()))
	}

	m := make(map[string]string)
	flatmapValueFromHCL2Map(m, "", v)
	return m
}

func flatmapValueFromHCL2Value(m map[string]string, key string, val cty.Value) {
	ty := val.Type()
	switch {
	case ty.IsPrimitiveType() || ty == cty.DynamicPseudoType:
		flatmapValueFromHCL2Primitive(m, key, val)
	case ty.IsObjectType() || ty.IsMapType():
		flatmapValueFromHCL2Map(m, key+".", val)
	case ty.IsTupleType() || ty.IsListType() || ty.IsSetType():
		flatmapValueFromHCL2Seq(m, key+".", val)
	default:
		panic(fmt.Sprintf("cannot encode %s to flatmap", ty.FriendlyName()))
	}
}

func flatmapValueFromHCL2Primitive(m map[string]string, key string, val cty.Value) {
	if !val.IsKnown() {
		m[key] = UnknownVariableValue
		return
	}
	if val.IsNull() {
		// Omit entirely
		return
	}

	var err error
	val, err = convert.Convert(val, cty.String)
	if err != nil {
		// Should not be possible, since all primitive types can convert to string.
		panic(fmt.Sprintf("invalid primitive encoding to flatmap: %s", err))
	}
	m[key] = val.AsString()
}

func flatmapValueFromHCL2Map(m map[string]string, prefix string, val cty.Value) {
	if val.IsNull() {
		// Omit entirely
		return
	}
	if !val.IsKnown() {
		switch {
		case val.Type().IsObjectType():
			// Whole objects can't be unknown in flatmap, so instead we'll
			// just write all of the attribute values out as unknown.
			for name, aty := range val.Type().AttributeTypes() {
				flatmapValueFromHCL2Value(m, prefix+name, cty.UnknownVal(aty))
			}
		default:
			m[prefix+"%"] = UnknownVariableValue
		}
		return
	}

	len := 0
	for it := val.ElementIterator(); it.Next(); {
		ak, av := it.Element()
		name := ak.AsString()
		flatmapValueFromHCL2Value(m, prefix+name, av)
		len++
	}
	if !val.Type().IsObjectType() { // objects don't have an explicit count included, since their attribute count is fixed
		m[prefix+"%"] = strconv.Itoa(len)
	}
}

func flatmapValueFromHCL2Seq(m map[string]string, prefix string, val cty.Value) {
	if val.IsNull() {
		// Omit entirely
		return
	}
	if !val.IsKnown() {
		m[prefix+"#"] = UnknownVariableValue
		return
	}

	// For sets this won't actually generate exactly what helper/schema would've
	// generated, because we don't have access to the set key function it
	// would've used. However, in practice it doesn't actually matter what the
	// keys are as long as they are unique, so we'll just generate sequential
	// indexes for them as if it were a list.
	//
	// An important implication of this, however, is that the set ordering will
	// not be consistent across mutations and so different keys may be assigned
	// to the same value when round-tripping. Since this shim is intended to
	// be short-lived and not used for round-tripping, we accept this.
	i := 0
	for it := val.ElementIterator(); it.Next(); {
		_, av := it.Element()
		key := prefix + strconv.Itoa(i)
		flatmapValueFromHCL2Value(m, key, av)
		i++
	}
	m[prefix+"#"] = strconv.Itoa(i)
}

// HCL2ValueFromFlatmap converts a map compatible with what would be produced
// by the "flatmap" package to a HCL2 (really, the cty dynamic types library
// that HCL2 uses) object type.
//
// The intended result type must be provided in order to guide how the
// map contents are decoded. This must be an object type or this function
// will panic.
//
// Flatmap values can only represent maps when they are of primitive types,
// so the given type must not have any maps of complex types or the result
// is undefined.
//
// The result may contain null values if the given map does not contain keys
// for all of the different key paths implied by the given type.
func HCL2ValueFromFlatmap(m map[string]string, ty cty.Type) (cty.Value, error) {
	if m == nil {
		return cty.NullVal(ty), nil
	}
	if !ty.IsObjectType() {
		panic(fmt.Sprintf("HCL2ValueFromFlatmap called on %#v", ty))
	}

	return hcl2ValueFromFlatmapObject(m, "", ty.AttributeTypes())
}

func hcl2ValueFromFlatmapValue(m map[string]string, key string, ty cty.Type) (cty.Value, error) {
	var val cty.Value
	var err error
	switch {
	case ty.IsPrimitiveType():
		val, err = hcl2ValueFromFlatmapPrimitive(m, key, ty)
	case ty.IsObjectType():
		val, err = hcl2ValueFromFlatmapObject(m, key+".", ty.AttributeTypes())
	case ty.IsTupleType():
		val, err = hcl2ValueFromFlatmapTuple(m, key+".", ty.TupleElementTypes())
	case ty.IsMapType():
		val, err = hcl2ValueFromFlatmapMap(m, key+".", ty)
	case ty.IsListType():
		val, err = hcl2ValueFromFlatmapList(m, key+".", ty)
	case ty.IsSetType():
		val, err = hcl2ValueFromFlatmapSet(m, key+".", ty)
	default:
		err = fmt.Errorf("cannot decode %s from flatmap", ty.FriendlyName())
	}

	if err != nil {
		return cty.DynamicVal, err
	}
	return val, nil
}

func hcl2ValueFromFlatmapPrimitive(m map[string]string, key string, ty cty.Type) (cty.Value, error) {
	rawVal, exists := m[key]
	if !exists {
		return cty.NullVal(ty), nil
	}
	if rawVal == UnknownVariableValue {
		return cty.UnknownVal(ty), nil
	}

	var err error
	val := cty.StringVal(rawVal)
	val, err = convert.Convert(val, ty)
	if err != nil {
		// This should never happen for _valid_ input, but flatmap data might
		// be tampered with by the user and become invalid.
		return cty.DynamicVal, fmt.Errorf("invalid value for %q in state: %s", key, err)
	}

	return val, nil
}

func hcl2ValueFromFlatmapObject(m map[string]string, prefix string, atys map[string]cty.Type) (cty.Value, error) {
	vals := make(map[string]cty.Value)
	for name, aty := range atys {
		val, err := hcl2ValueFromFlatmapValue(m, prefix+name, aty)
		if err != nil {
			return cty.DynamicVal, err
		}
		vals[name] = val
	}
	return cty.ObjectVal(vals), nil
}

func hcl2ValueFromFlatmapTuple(m map[string]string, prefix string, etys []cty.Type) (cty.Value, error) {
	var vals []cty.Value

	countStr, exists := m[prefix+"#"]
	if !exists {
		return cty.NullVal(cty.Tuple(etys)), nil
	}
	if countStr == UnknownVariableValue {
		return cty.UnknownVal(cty.Tuple(etys)), nil
	}

	count, err := strconv.Atoi(countStr)
	if err != nil {
		return cty.DynamicVal, fmt.Errorf("invalid count value for %q in state: %s", prefix, err)
	}
	if count != len(etys) {
		return cty.DynamicVal, fmt.Errorf("wrong number of values for %q in state: got %d, but need %d", prefix, count, len(etys))
	}

	vals = make([]cty.Value, len(etys))
	for i, ety := range etys {
		key := prefix + strconv.Itoa(i)
		val, err := hcl2ValueFromFlatmapValue(m, key, ety)
		if err != nil {
			return cty.DynamicVal, err
		}
		vals[i] = val
	}
	return cty.TupleVal(vals), nil
}

func hcl2ValueFromFlatmapMap(m map[string]string, prefix string, ty cty.Type) (cty.Value, error) {
	vals := make(map[string]cty.Value)
	ety := ty.ElementType()

	// We actually don't really care about the "count" of a map for our
	// purposes here, but we do need to check if it _exists_ in order to
	// recognize the difference between null (not set at all) and empty.
	if strCount, exists := m[prefix+"%"]; !exists {
		return cty.NullVal(ty), nil
	} else if strCount == UnknownVariableValue {
		return cty.UnknownVal(ty), nil
	}

	for fullKey := range m {
		if !strings.HasPrefix(fullKey, prefix) {
			continue
		}

		// The flatmap format doesn't allow us to distinguish between keys
		// that contain periods and nested objects, so by convention a
		// map is only ever of primitive type in flatmap, and we just assume
		// that the remainder of the raw key (dots and all) is the key we
		// want in the result value.
		key := fullKey[len(prefix):]
		if key == "%" {
			// Ignore the "count" key
			continue
		}

		val, err := hcl2ValueFromFlatmapValue(m, fullKey, ety)
		if err != nil {
			return cty.DynamicVal, err
		}
		vals[key] = val
	}

	if len(vals) == 0 {
		return cty.MapValEmpty(ety), nil
	}
	return cty.MapVal(vals), nil
}

func hcl2ValueFromFlatmapList(m map[string]string, prefix string, ty cty.Type) (cty.Value, error) {
	var vals []cty.Value

	countStr, exists := m[prefix+"#"]
	if !exists {
		return cty.NullVal(ty), nil
	}
	if countStr == UnknownVariableValue {
		return cty.UnknownVal(ty), nil
	}

	count, err := strconv.Atoi(countStr)
	if err != nil {
		return cty.DynamicVal, fmt.Errorf("invalid count value for %q in state: %s", prefix, err)
	}

	ety := ty.ElementType()
	if count == 0 {
		return cty.ListValEmpty(ety), nil
	}

	vals = make([]cty.Value, count)
	for i := 0; i < count; i++ {
		key := prefix + strconv.Itoa(i)
		val, err := hcl2ValueFromFlatmapValue(m, key, ety)
		if err != nil {
			return cty.DynamicVal, err
		}
		vals[i] = val
	}

	return cty.ListVal(vals), nil
}

func hcl2ValueFromFlatmapSet(m map[string]string, prefix string, ty cty.Type) (cty.Value, error) {
	var vals []cty.Value
	ety := ty.ElementType()

	// We actually don't really care about the "count" of a set for our
	// purposes here, but we do need to check if it _exists_ in order to
	// recognize the difference between null (not set at all) and empty.
	if strCount, exists := m[prefix+"#"]; !exists {
		return cty.NullVal(ty), nil
	} else if strCount == UnknownVariableValue {
		return cty.UnknownVal(ty), nil
	}

	for fullKey := range m {
		if !strings.HasPrefix(fullKey, prefix) {
			continue
		}
		subKey := fullKey[len(prefix):]
		if subKey == "#" {
			// Ignore the "count" key
			continue
		}
		key := fullKey
		if dot := strings.IndexByte(subKey, '.'); dot != -1 {
			key = fullKey[:dot+len(prefix)]
		}

		// The flatmap format doesn't allow us to distinguish between keys
		// that contain periods and nested objects, so by convention a
		// map is only ever of primitive type in flatmap, and we just assume
		// that the remainder of the raw key (dots and all) is the key we
		// want in the result value.

		val, err := hcl2ValueFromFlatmapValue(m, key, ety)
		if err != nil {
			return cty.DynamicVal, err
		}
		vals = append(vals, val)
	}

	if len(vals) == 0 {
		return cty.SetValEmpty(ety), nil
	}
	return cty.SetVal(vals), nil
}
<|endoftext|>


<commit_before>package raymond

import "testing"

var evalTests = []Test{
	{
		"only content",
		"this is content",
		nil, nil, nil, nil,
		"this is content",
	},
	{
		"checks path in parent contexts",
		"{{#a}}{{one}}{{#b}}{{one}}{{two}}{{one}}{{/b}}{{/a}}",
		map[string]interface{}{"a": map[string]int{"one": 1}, "b": map[string]int{"two": 2}},
		nil, nil, nil,
		"1121",
	},
	{
		"block params",
		"{{#foo as |bar|}}{{bar}}{{/foo}}{{bar}}",
		map[string]string{"foo": "baz", "bar": "bat"},
		nil, nil, nil,
		"bazbat",
	},
	{
		"block params on array",
		"{{#foo as |bar i|}}{{i}}.{{bar}} {{/foo}}",
		map[string][]string{"foo": {"baz", "bar", "bat"}},
		nil, nil, nil,
		"0.baz 1.bar 2.bat ",
	},
	{
		"nested block params",
		"{{#foos as |foo iFoo|}}{{#wats as |wat iWat|}}{{iFoo}}.{{iWat}}.{{foo}}-{{wat}} {{/wats}}{{/foos}}",
		map[string][]string{"foos": {"baz", "bar"}, "wats": {"the", "phoque"}},
		nil, nil, nil,
		"0.0.baz-the 0.1.baz-phoque 1.0.bar-the 1.1.bar-phoque ",
	},
	{
		"block params with path reference",
		"{{#foo as |bar|}}{{bar.baz}}{{/foo}}",
		map[string]map[string]string{"foo": {"baz": "bat"}},
		nil, nil, nil,
		"bat",
	},
	{
		"falsy block evaluation",
		"{{#foo}}bar{{/foo}} baz",
		map[string]interface{}{"foo": false},
		nil, nil, nil,
		" baz",
	},

	// @todo Test with a struct for data
	// @todo Test with a "../../path" (depth 2 path) while context is only depth 1
}

func TestEval(t *testing.T) {
	launchTests(t, evalTests)
}

var evalErrors = []Test{
	{
		"functions with wrong number of arguments",
		`{{foo "bar"}}`,
		map[string]interface{}{"foo": func(a string, b string) string { return "foo" }},
		nil, nil, nil,
		"Helper called with wrong number of arguments, needed 2 but got 1",
	},
	{
		"functions returning a non boolean second value",
		"{{foo}}",
		map[string]interface{}{"foo": func() (string, string) { return "foo", "bar" }},
		nil, nil, nil,
		"Second returned value of helper function must be a boolean",
	},
	{
		"functions with wrong number of returned values (1)",
		"{{foo}}",
		map[string]interface{}{"foo": func() {}},
		nil, nil, nil,
		"Helper function must return a string or a SafeString",
	},
	{
		"functions with wrong number of returned values (2)",
		"{{foo}}",
		map[string]interface{}{"foo": func() (string, bool, string) { return "foo", true, "bar" }},
		nil, nil, nil,
		"Helper function must not return more than two values",
	},
}

func TestEvalErrors(t *testing.T) {
	launchErrorTests(t, evalErrors)
}
<commit_msg>Fixes test<commit_after>package raymond

import "testing"

var evalTests = []Test{
	{
		"only content",
		"this is content",
		nil, nil, nil, nil,
		"this is content",
	},
	{
		"checks path in parent contexts",
		"{{#a}}{{one}}{{#b}}{{one}}{{two}}{{one}}{{/b}}{{/a}}",
		map[string]interface{}{"a": map[string]int{"one": 1}, "b": map[string]int{"two": 2}},
		nil, nil, nil,
		"1121",
	},
	{
		"block params",
		"{{#foo as |bar|}}{{bar}}{{/foo}}{{bar}}",
		map[string]string{"foo": "baz", "bar": "bat"},
		nil, nil, nil,
		"bazbat",
	},
	{
		"block params on array",
		"{{#foo as |bar i|}}{{i}}.{{bar}} {{/foo}}",
		map[string][]string{"foo": {"baz", "bar", "bat"}},
		nil, nil, nil,
		"0.baz 1.bar 2.bat ",
	},
	{
		"nested block params",
		"{{#foos as |foo iFoo|}}{{#wats as |wat iWat|}}{{iFoo}}.{{iWat}}.{{foo}}-{{wat}} {{/wats}}{{/foos}}",
		map[string][]string{"foos": {"baz", "bar"}, "wats": {"the", "phoque"}},
		nil, nil, nil,
		"0.0.baz-the 0.1.baz-phoque 1.0.bar-the 1.1.bar-phoque ",
	},
	{
		"block params with path reference",
		"{{#foo as |bar|}}{{bar.baz}}{{/foo}}",
		map[string]map[string]string{"foo": {"baz": "bat"}},
		nil, nil, nil,
		"bat",
	},
	{
		"falsy block evaluation",
		"{{#foo}}bar{{/foo}} baz",
		map[string]interface{}{"foo": false},
		nil, nil, nil,
		" baz",
	},

	// @todo Test with a struct for data
	// @todo Test with a "../../path" (depth 2 path) while context is only depth 1
}

func TestEval(t *testing.T) {
	launchTests(t, evalTests)
}

var evalErrors = []Test{
	{
		"functions with wrong number of arguments",
		`{{foo "bar"}}`,
		map[string]interface{}{"foo": func(a string, b string) string { return "foo" }},
		nil, nil, nil,
		"Helper foo called with wrong number of arguments, needed 2 but got 1",
	},
	{
		"functions returning a non boolean second value",
		"{{foo}}",
		map[string]interface{}{"foo": func() (string, string) { return "foo", "bar" }},
		nil, nil, nil,
		"Second returned value of helper function must be a boolean",
	},
	{
		"functions with wrong number of returned values (1)",
		"{{foo}}",
		map[string]interface{}{"foo": func() {}},
		nil, nil, nil,
		"Helper function must return a string or a SafeString",
	},
	{
		"functions with wrong number of returned values (2)",
		"{{foo}}",
		map[string]interface{}{"foo": func() (string, bool, string) { return "foo", true, "bar" }},
		nil, nil, nil,
		"Helper function must not return more than two values",
	},
}

func TestEvalErrors(t *testing.T) {
	launchErrorTests(t, evalErrors)
}
<|endoftext|>


<commit_before>package gflowparser

import (
	"reflect"
	"testing"

	"github.com/flowdev/gflowparser/data"
	"github.com/flowdev/gflowparser/svg"
)

// func parserToSVGData(flowDat data.Flow) *svg.Flow {
func TestParserToSVGData(t *testing.T) {
	specs := []struct {
		name     string
		given    data.Flow
		expected *svg.Flow
	}{
		{
			name: "simple 1",
			given: data.Flow{
				Parts: [][]interface{}{
					{
						data.Arrow{
							FromPort: &data.Port{Name: "a"},
							Data:     []data.Type{data.Type{LocalType: "b"}},
						},
						data.Component{
							Decl: data.CompDecl{
								Name:      "c",
								Type:      data.Type{LocalType: "c"},
								VagueType: true,
							},
						},
					},
				},
			},
			expected: &svg.Flow{
				Shapes: [][]interface{}{
					{
						&svg.Arrow{
							HasSrcOp: false, SrcPort: "a",
							DataType: "(b)",
							HasDstOp: true, DstPort: "",
						},
						&svg.Op{
							Main: &svg.Rect{
								Text: []string{"c", "c"},
							},
							Plugins: []*svg.Plugin{},
						},
					},
				},
			},
		},
	}

	for _, spec := range specs {
		t.Logf("Testing spec: %s\n", spec.name)
		gotAll := parserToSVGData(spec.given)
		if len(gotAll.Shapes) != len(spec.expected.Shapes) {
			t.Errorf("Expected %d part lines, got: %d",
				len(spec.expected.Shapes), len(gotAll.Shapes))
			continue
		}
		for i, expectedLine := range spec.expected.Shapes {
			t.Logf("Testing part line: %d\n", i+1)
			gotLine := gotAll.Shapes[i]
			if len(gotLine) != len(expectedLine) {
				t.Errorf("Expected %d parts in line %d, got: %d",
					len(expectedLine), i+1, len(gotLine))
				continue
			}
			for j, expectedPart := range expectedLine {
				t.Logf("Testing part: %d\n", j+1)
				gotPart := gotLine[j]
				checkValue(gotPart, expectedPart, t)
			}
		}
	}
}

func checkValue(got, expected interface{}, t *testing.T) {
	if expected != nil && got == nil {
		t.Errorf("Expected a value.")
	} else if !reflect.DeepEqual(got, expected) {
		t.Logf("The acutal value isn't equal to the expected one:")
		t.Logf("Expected value of type '%T', got '%T'.", expected, got)
		t.Errorf("Expected value '%#v', got '%#v'.", expected, got)
	}
}
<commit_msg>Add test for parserToSVGData<commit_after>package gflowparser

import (
	"reflect"
	"testing"

	"github.com/flowdev/gflowparser/data"
	"github.com/flowdev/gflowparser/svg"
)

// func parserToSVGData(flowDat data.Flow) *svg.Flow {
func TestParserToSVGData(t *testing.T) {
	specs := []struct {
		name     string
		given    data.Flow
		expected *svg.Flow
	}{
		{
			name: "simple",
			given: data.Flow{
				Parts: [][]interface{}{
					{
						data.Arrow{
							FromPort: &data.Port{Name: "a"},
							Data:     []data.Type{data.Type{LocalType: "b"}},
						},
						data.Component{
							Decl: data.CompDecl{
								Name:      "c",
								Type:      data.Type{LocalType: "c"},
								VagueType: true,
							},
						},
					},
				},
			},
			expected: &svg.Flow{
				Shapes: [][]interface{}{
					{
						&svg.Arrow{
							HasSrcOp: false, SrcPort: "a",
							DataType: "(b)",
							HasDstOp: true, DstPort: "",
						},
						&svg.Op{
							Main: &svg.Rect{
								Text: []string{"c", "c"},
							},
							Plugins: []*svg.Plugin{},
						},
					},
				},
			},
		}, {
			name: "full arrows",
			given: data.Flow{
				Parts: [][]interface{}{
					{
						data.Arrow{
							FromPort: &data.Port{Name: "a"},
							Data:     []data.Type{data.Type{LocalType: "b"}},
							ToPort:   &data.Port{Name: "in"},
						},
						data.Component{
							Decl: data.CompDecl{
								Name:      "c",
								Type:      data.Type{LocalType: "c"},
								VagueType: true,
							},
						},
						data.Arrow{
							FromPort: &data.Port{Name: "out"},
							Data:     []data.Type{data.Type{LocalType: "b"}},
							ToPort:   &data.Port{Name: "in"},
						},
						data.Component{
							Decl: data.CompDecl{
								Name:      "d",
								Type:      data.Type{LocalType: "D"},
								VagueType: false,
							},
						},
						data.Arrow{
							FromPort: &data.Port{Name: "out"},
							Data:     []data.Type{data.Type{LocalType: "b"}},
							ToPort:   &data.Port{Name: "out"},
						},
					},
				},
			},
			expected: &svg.Flow{
				Shapes: [][]interface{}{
					{
						&svg.Arrow{
							HasSrcOp: false, SrcPort: "a",
							DataType: "(b)",
							HasDstOp: true, DstPort: "in",
						},
						&svg.Op{
							Main: &svg.Rect{
								Text: []string{"c", "c"},
							},
							Plugins: []*svg.Plugin{},
						},
						&svg.Arrow{
							HasSrcOp: true, SrcPort: "out",
							DataType: "(b)",
							HasDstOp: true, DstPort: "in",
						},
						&svg.Op{
							Main: &svg.Rect{
								Text: []string{"d", "D"},
							},
							Plugins: []*svg.Plugin{},
						},
						&svg.Arrow{
							HasSrcOp: true, SrcPort: "out",
							DataType: "(b)",
							HasDstOp: false, DstPort: "out",
						},
					},
				},
			},
		},
	}

	for _, spec := range specs {
		t.Logf("Testing spec: %s\n", spec.name)
		gotAll := parserToSVGData(spec.given)
		if len(gotAll.Shapes) != len(spec.expected.Shapes) {
			t.Errorf("Expected %d part lines, got: %d",
				len(spec.expected.Shapes), len(gotAll.Shapes))
			continue
		}
		for i, expectedLine := range spec.expected.Shapes {
			t.Logf("Testing part line: %d\n", i+1)
			gotLine := gotAll.Shapes[i]
			if len(gotLine) != len(expectedLine) {
				t.Errorf("Expected %d parts in line %d, got: %d",
					len(expectedLine), i+1, len(gotLine))
				continue
			}
			for j, expectedPart := range expectedLine {
				t.Logf("Testing part: %d\n", j+1)
				gotPart := gotLine[j]
				checkValue(gotPart, expectedPart, t)
			}
		}
	}
}

func checkValue(got, expected interface{}, t *testing.T) {
	if expected != nil && got == nil {
		t.Errorf("Expected a value.")
	} else if !reflect.DeepEqual(got, expected) {
		t.Logf("The acutal value isn't equal to the expected one:")
		t.Logf("Expected value of type '%T', got '%T'.", expected, got)
		t.Errorf("Expected value '%#v', got '%#v'.", expected, got)
	}
}
<|endoftext|>


<commit_before>package termite

import (
	"fmt"
	"github.com/hanwen/termite/attr"
	"github.com/hanwen/termite/stats"
	"os"
)

type AttrRequest struct {
	Origin string
	Name   string
}

type AttrResponse struct {
	Attrs []*attr.FileAttr
}

type UpdateRequest struct {
	Files []*attr.FileAttr
}

type UpdateResponse struct {

}

type MirrorStatusRequest struct {

}

type MirrorStatusResponse struct {
	Root         string
	Granted      int
	Running      []string
	ShuttingDown bool
	WaitingTasks int
	IdleFses     int
	RpcTimings   []string
}

type WorkerStatusRequest struct {

}

type WorkerStatusResponse struct {
	MirrorStatus []MirrorStatusResponse
	Version      string
	MaxJobCount  int
	ShuttingDown bool

	// In chronological order.
	CpuStats            []stats.CpuStat
	TotalCpu            stats.CpuStat
	ContentCacheHitRate float64
	PhaseNames          []string
	PhaseCounts         []int
}

type Timing struct {
	Name string
	Dt   float64
}

type WorkResponse struct {
	Exit   os.Waitmsg
	Stderr string
	Stdout string

	Timings []Timing

	LastTime int64

	*attr.FileSet
	TaskIds  []int
	WorkerId string
}

type WorkRequest struct {
	// Id of connection streaming stdin.
	TaskId       int
	StdinId      string
	Debug        bool
	WritableRoot string
	Binary       string
	Argv         []string
	Env          []string
	Dir          string
	RanLocally   bool

	// If set, must run on worker. Used for debugging.
	Worker string
}

func (me *WorkRequest) Summary() string {
	return fmt.Sprintf("Stdin %s Cmd %s Id %d", me.StdinId, me.Argv, me.TaskId)
}

type CreateMirrorRequest struct {
	RpcId        string
	RevRpcId     string
	WritableRoot string
	// Max number of processes to reserve.
	MaxJobCount int
}

type CreateMirrorResponse struct {
	GrantedJobCount int
}

type ShutdownRequest struct {
	Restart bool
}

type ShutdownResponse struct {

}

type LogRequest struct {
	Whence int
	Off    int64
	Size   int64
}

type LogResponse struct {
	Data []byte
}
<commit_msg>Document fields.<commit_after>package termite

import (
	"fmt"
	"github.com/hanwen/termite/attr"
	"github.com/hanwen/termite/stats"
	"os"
)

type AttrRequest struct {
	Name   string

	// Worker asking for the request. Useful for debugging.
	Origin string
}

type AttrResponse struct {
	Attrs []*attr.FileAttr
}

type UpdateRequest struct {
	Files []*attr.FileAttr
}

type UpdateResponse struct {

}

type MirrorStatusRequest struct {

}

type MirrorStatusResponse struct {
	Root         string
	Granted      int
	Running      []string
	ShuttingDown bool
	WaitingTasks int
	IdleFses     int
	RpcTimings   []string
}

type WorkerStatusRequest struct {

}

type WorkerStatusResponse struct {
	MirrorStatus []MirrorStatusResponse
	Version      string
	MaxJobCount  int
	ShuttingDown bool

	// In chronological order.
	CpuStats            []stats.CpuStat
	TotalCpu            stats.CpuStat
	ContentCacheHitRate float64
	PhaseNames          []string
	PhaseCounts         []int
}

type Timing struct {
	Name string
	Dt   float64
}

type WorkResponse struct {
	Exit   os.Waitmsg
	Stderr string
	Stdout string

	Timings []Timing

	LastTime int64

	// Reaped files, if any
	*attr.FileSet

	// Task ids for which the fileset contains data.
	TaskIds  []int

	// Worker where this was processed.
	WorkerId string
}

type WorkRequest struct {
	// Unique id of this request.
	TaskId       int

	// Id of connection streaming stdin.
	StdinId      string
	Debug        bool
	Binary       string
	Argv         []string
	Env          []string
	Dir          string

	// Signal that a command ran locally.  Used for logging in the master.
	RanLocally   bool

	// If set, must run on this worker. Used for debugging.
	Worker string
}

func (me *WorkRequest) Summary() string {
	return fmt.Sprintf("Stdin %s Cmd %s Id %d", me.StdinId, me.Argv, me.TaskId)
}

type CreateMirrorRequest struct {
	// Ids of connections to use for RPC
	RpcId        string
	RevRpcId     string

	// The writable root for the mirror.
	WritableRoot string

	// Max number of processes to reserve.
	MaxJobCount int
}

type CreateMirrorResponse struct {
	GrantedJobCount int
}

type ShutdownRequest struct {
	Restart bool
}

type ShutdownResponse struct {

}

type LogRequest struct {
	Whence int
	Off    int64
	Size   int64
}

type LogResponse struct {
	Data []byte
}
<|endoftext|>