<commit_before>package mpawselasticache

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/crowdmob/goamz/aws"
	"github.com/crowdmob/goamz/cloudwatch"
	mp "github.com/mackerelio/go-mackerel-plugin"
)

var metricsdefMemcached = []string{
	"CPUUtilization", "SwapUsage", "FreeableMemory", "NetworkBytesIn", "NetworkBytesOut",
	"BytesUsedForCacheItems", "BytesReadIntoMemcached", "BytesWrittenOutFromMemcached", "CasBadval", "CasHits",
	"CasMisses", "CmdFlush", "CmdGet", "CmdSet", "CurrConnections",
	"CurrItems", "DecrHits", "DecrMisses", "DeleteHits", "DeleteMisses",
	"Evictions", "GetHits", "GetMisses", "IncrHits", "IncrMisses",
	"Reclaimed", "BytesUsedForHash", "CmdConfigGet", "CmdConfigSet", "CmdTouch",
	"CurrConfig", "EvictedUnfetched", "ExpiredUnfetched", "SlabsMoved", "TouchHits",
	"TouchMisses", "NewConnections", "NewItems", "UnusedMemory",
}

var metricsdefRedis = []string{
	"CPUUtilization", "SwapUsage", "FreeableMemory", "NetworkBytesIn", "NetworkBytesOut",
	"CurrConnections", "Evictions", "Reclaimed", "NewConnections", "BytesUsedForCache",
	"CacheHits", "CacheMisses", "ReplicationLag", "GetTypeCmds", "SetTypeCmds",
	"KeyBasedCmds", "StringBasedCmds", "HashBasedCmds", "ListBasedCmds", "SetBasedCmds",
	"SortedSetBasedCmds", "CurrItems",
}

var graphdefMemcached = map[string]mp.Graphs{
	"ecache.CPUUtilization": {
		Label: "ECache CPU Utilization",
		Unit:  "percentage",
		Metrics: []mp.Metrics{
			{Name: "CPUUtilization", Label: "CPUUtilization"},
		},
	},
	"ecache.SwapUsage": {
		Label: "ECache Swap Usage",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "SwapUsage", Label: "SwapUsage"},
		},
	},
	"ecache.FreeableMemory": {
		Label: "ECache Freeable Memory",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "FreeableMemory", Label: "FreeableMemory"},
		},
	},
	"ecache.NetworkTraffic": {
		Label: "ECache Network Traffic",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "NetworkBytesIn", Label: "NetworkBytesIn"},
			{Name: "NetworkBytesOut", Label: "NetworkBytesOut"},
		},
	},
	"ecache.Command": {
		Label: "ECache Command",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "CmdGet", Label: "CmdGet"},
			{Name: "CmdSet", Label: "CmdSet"},
			{Name: "CmdFlush", Label: "CmdFlush"},
			{Name: "CmdTouch", Label: "CmdTouch"},
			{Name: "CmdConfigGet", Label: "CmdConfigGet"},
			{Name: "CmdConfigSet", Label: "CmdConfigSet"},
		},
	},
	"ecache.CacheHitAndMiss": {
		Label: "ECache Hits/Misses",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "GetHits", Label: "GetHits"},
			{Name: "GetMisses", Label: "GetMisses"},
			{Name: "DeleteHits", Label: "DeleteHits"},
			{Name: "DeleteMisses", Label: "DeleteMisses"},
			{Name: "IncrHits", Label: "IncrHits"},
			{Name: "IncrMisses", Label: "IncrMisses"},
			{Name: "DecrHits", Label: "DecrHits"},
			{Name: "DecrMisses", Label: "DecrMisses"},
			{Name: "CasBadval", Label: "CasBadval"},
			{Name: "CasHits", Label: "CasHits"},
			{Name: "CasMisses", Label: "CasMisses"},
			{Name: "TouchHits", Label: "TouchHits"},
			{Name: "TouchMisses", Label: "TouchMisses"},
		},
	},
	"ecache.Evictions": {
		Label: "ECache Evictions",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "Evictions", Label: "Evictions"},
		},
	},
	"ecache.Unfetched": {
		Label: "ECache Unfetched",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "EvictedUnfetched", Label: "EvictedUnfetched"},
			{Name: "ExpiredUnfetched", Label: "ExpiredUnfetched"},
		},
	},
	"ecache.Bytes": {
		Label: "ECache Traffics",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "BytesReadIntoMemcached", Label: "BytesReadIntoMemcached"},
			{Name: "BytesWrittenOutFromMemcached", Label: "BytesWrittenOutFromMemcached"},
		},
	},
	"ecache.Connections": {
		Label: "ECache Connections",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "CurrConnections", Label: "CurrConnections"},
			{Name: "NewConnections", Label: "NewConnections"},
		},
	},
	"ecache.Items": {
		Label: "ECache Items",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "CurrItems", Label: "CurrItems"},
			{Name: "NewItems", Label: "NewItems"},
			{Name: "Reclaimed", Label: "Reclaimed"},
			{Name: "CurrConfig", Label: "CurrConfig"},
			{Name: "SlabsMoved", Label: "SlabsMoved"},
		},
	},
	"ecache.MemoryUsage": {
		Label: "ECache Memory Usage",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "UnusedMemory", Label: "UnusedMemory"},
			{Name: "BytesUsedForHash", Label: "BytesUsedForHash"},
			{Name: "BytesUsedForCacheItems", Label: "BytesUsedForCacheItems"},
		},
	},
}

var graphdefRedis = map[string]mp.Graphs{
	"ecache.CPUUtilization": {
		Label: "ECache CPU Utilization",
		Unit:  "percentage",
		Metrics: []mp.Metrics{
			{Name: "CPUUtilization", Label: "CPUUtilization"},
		},
	},
	"ecache.SwapUsage": {
		Label: "ECache Swap Usage",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "SwapUsage", Label: "SwapUsage"},
		},
	},
	"ecache.FreeableMemory": {
		Label: "ECache Freeable Memory",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "FreeableMemory", Label: "FreeableMemory"},
		},
	},
	"ecache.NetworkTraffic": {
		Label: "ECache Network Traffic",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "NetworkBytesIn", Label: "NetworkBytesIn"},
			{Name: "NetworkBytesOut", Label: "NetworkBytesOut"},
		},
	},
	"ecache.Command": {
		Label: "ECache Command",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "GetTypeCmds", Label: "GetTypeCmds"},
			{Name: "SetTypeCmds", Label: "SetTypeCmds"},
			{Name: "KeyBasedCmds", Label: "KeyBasedCmds"},
			{Name: "StringBasedCmds", Label: "StringBasedCmds"},
			{Name: "HashBasedCmds", Label: "HashBasedCmds"},
			{Name: "ListBasedCmds", Label: "ListBasedCmds"},
			{Name: "SetBasedCmds", Label: "SetBasedCmds"},
			{Name: "SortedSetBasedCmds", Label: "SortedSetBasedCmds"},
		},
	},
	"ecache.CacheHitAndMiss": {
		Label: "ECache Hits/Misses",
		Unit:  "float",
		Metrics: []mp.Metrics{
			{Name: "CacheHits", Label: "CacheHits"},
			{Name: "CacheMisses", Label: "CacheMisses"},
		},
	},
	"ecache.Evictions": {
		Label: "ECache Evictions",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "Evictions", Label: "Evictions"},
		},
	},
	"ecache.Bytes": {
		Label: "ECache Traffics",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "BytesUsedForCache", Label: "BytesUsedForCache"},
		},
	},
	"ecache.Connections": {
		Label: "ECache Connections",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "CurrConnections", Label: "CurrConnections"},
			{Name: "NewConnections", Label: "NewConnections"},
		},
	},
	"ecache.Items": {
		Label: "ECache Items",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "CurrItems", Label: "CurrItems"},
			{Name: "Reclaimed", Label: "Reclaimed"},
		},
	},
}

// ECachePlugin mackerel plugin for elasticache
type ECachePlugin struct {
	Region          string
	AccessKeyID     string
	SecretAccessKey string
	CacheClusterID  string
	CacheNodeID     string
	ElastiCacheType string
	CacheMetrics    []string
}

func getLastPoint(cloudWatch *cloudwatch.CloudWatch, dimensions *[]cloudwatch.Dimension, metricName string) (float64, error) {
	now := time.Now()

	response, err := cloudWatch.GetMetricStatistics(&cloudwatch.GetMetricStatisticsRequest{
		Dimensions: *dimensions,
		StartTime:  now.Add(time.Duration(180) * time.Second * -1), // 3 mins (to fetch at least 1 data-point)
		EndTime:    now,
		MetricName: metricName,
		Period:     60,
		Statistics: []string{"Average"},
		Namespace:  "AWS/ElastiCache",
	})
	if err != nil {
		return 0, err
	}

	datapoints := response.GetMetricStatisticsResult.Datapoints
	if len(datapoints) == 0 {
		return 0, errors.New("fetched no datapoints")
	}

	latest := time.Unix(0, 0)
	var latestVal float64
	for _, dp := range datapoints {
		if dp.Timestamp.Before(latest) {
			continue
		}

		latest = dp.Timestamp
		latestVal = dp.Average
	}

	return latestVal, nil
}

// FetchMetrics fetch elasticache values
func (p ECachePlugin) FetchMetrics() (map[string]float64, error) {
	auth, err := aws.GetAuth(p.AccessKeyID, p.SecretAccessKey, "", time.Now())
	if err != nil {
		return nil, err
	}

	cloudWatch, err := cloudwatch.NewCloudWatch(auth, aws.Regions[p.Region].CloudWatchServicepoint)
	if err != nil {
		return nil, err
	}

	stat := make(map[string]float64)

	perInstances := &[]cloudwatch.Dimension{
		{
			Name:  "CacheClusterId",
			Value: p.CacheClusterID,
		},
		{
			Name:  "CacheNodeId",
			Value: p.CacheNodeID,
		},
	}

	for _, met := range p.CacheMetrics {
		v, err := getLastPoint(cloudWatch, perInstances, met)
		if err == nil {
			stat[met] = v
		} else {
			log.Printf("%s: %s", met, err)
		}
	}

	return stat, nil
}

// GraphDefinition graph definition
func (p ECachePlugin) GraphDefinition() map[string]mp.Graphs {
	switch p.ElastiCacheType {
	case "memcached":
		return graphdefMemcached
	case "redis":
		return graphdefRedis
	default:
		log.Printf("elasticache-type is 'memcached' or 'redis'.")
		return nil
	}
}

// Do the plugin
func Do() {
	optRegion := flag.String("region", "", "AWS Region")
	optAccessKeyID := flag.String("access-key-id", "", "AWS Access Key ID")
	optSecretAccessKey := flag.String("secret-access-key", "", "AWS Secret Access Key")
	optCacheClusterID := flag.String("cache-cluster-id", "", "Cache Cluster Id")
	optCacheNodeID := flag.String("cache-node-id", "0001", "Cache Node Id")
	optElastiCacheType := flag.String("elasticache-type", "", "ElastiCache type")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	var ecache ECachePlugin

	if *optRegion == "" {
		ecache.Region = aws.InstanceRegion()
	} else {
		ecache.Region = *optRegion
	}

	ecache.AccessKeyID = *optAccessKeyID
	ecache.SecretAccessKey = *optSecretAccessKey
	ecache.CacheClusterID = *optCacheClusterID
	ecache.CacheNodeID = *optCacheNodeID
	ecache.ElastiCacheType = *optElastiCacheType
	switch ecache.ElastiCacheType {
	case "memcached":
		ecache.CacheMetrics = metricsdefMemcached
	case "redis":
		ecache.CacheMetrics = metricsdefRedis
	default:
		log.Printf("elasticache-type is 'memcached' or 'redis'.")
		os.Exit(1)
	}

	helper := mp.NewMackerelPlugin(ecache)
	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.Tempfile = fmt.Sprintf("/tmp/mackerel-plugin-aws-elasticache-%s-%s", *optCacheClusterID, *optCacheNodeID)
	}

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<commit_msg>[aws-elasticache] migrate from goamz to aws-sdk-go<commit_after>package mpawselasticache

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	mp "github.com/mackerelio/go-mackerel-plugin"
)

var metricsdefMemcached = []string{
	"CPUUtilization", "SwapUsage", "FreeableMemory", "NetworkBytesIn", "NetworkBytesOut",
	"BytesUsedForCacheItems", "BytesReadIntoMemcached", "BytesWrittenOutFromMemcached", "CasBadval", "CasHits",
	"CasMisses", "CmdFlush", "CmdGet", "CmdSet", "CurrConnections",
	"CurrItems", "DecrHits", "DecrMisses", "DeleteHits", "DeleteMisses",
	"Evictions", "GetHits", "GetMisses", "IncrHits", "IncrMisses",
	"Reclaimed", "BytesUsedForHash", "CmdConfigGet", "CmdConfigSet", "CmdTouch",
	"CurrConfig", "EvictedUnfetched", "ExpiredUnfetched", "SlabsMoved", "TouchHits",
	"TouchMisses", "NewConnections", "NewItems", "UnusedMemory",
}

var metricsdefRedis = []string{
	"CPUUtilization", "SwapUsage", "FreeableMemory", "NetworkBytesIn", "NetworkBytesOut",
	"CurrConnections", "Evictions", "Reclaimed", "NewConnections", "BytesUsedForCache",
	"CacheHits", "CacheMisses", "ReplicationLag", "GetTypeCmds", "SetTypeCmds",
	"KeyBasedCmds", "StringBasedCmds", "HashBasedCmds", "ListBasedCmds", "SetBasedCmds",
	"SortedSetBasedCmds", "CurrItems",
}

var graphdefMemcached = map[string]mp.Graphs{
	"ecache.CPUUtilization": {
		Label: "ECache CPU Utilization",
		Unit:  "percentage",
		Metrics: []mp.Metrics{
			{Name: "CPUUtilization", Label: "CPUUtilization"},
		},
	},
	"ecache.SwapUsage": {
		Label: "ECache Swap Usage",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "SwapUsage", Label: "SwapUsage"},
		},
	},
	"ecache.FreeableMemory": {
		Label: "ECache Freeable Memory",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "FreeableMemory", Label: "FreeableMemory"},
		},
	},
	"ecache.NetworkTraffic": {
		Label: "ECache Network Traffic",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "NetworkBytesIn", Label: "NetworkBytesIn"},
			{Name: "NetworkBytesOut", Label: "NetworkBytesOut"},
		},
	},
	"ecache.Command": {
		Label: "ECache Command",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "CmdGet", Label: "CmdGet"},
			{Name: "CmdSet", Label: "CmdSet"},
			{Name: "CmdFlush", Label: "CmdFlush"},
			{Name: "CmdTouch", Label: "CmdTouch"},
			{Name: "CmdConfigGet", Label: "CmdConfigGet"},
			{Name: "CmdConfigSet", Label: "CmdConfigSet"},
		},
	},
	"ecache.CacheHitAndMiss": {
		Label: "ECache Hits/Misses",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "GetHits", Label: "GetHits"},
			{Name: "GetMisses", Label: "GetMisses"},
			{Name: "DeleteHits", Label: "DeleteHits"},
			{Name: "DeleteMisses", Label: "DeleteMisses"},
			{Name: "IncrHits", Label: "IncrHits"},
			{Name: "IncrMisses", Label: "IncrMisses"},
			{Name: "DecrHits", Label: "DecrHits"},
			{Name: "DecrMisses", Label: "DecrMisses"},
			{Name: "CasBadval", Label: "CasBadval"},
			{Name: "CasHits", Label: "CasHits"},
			{Name: "CasMisses", Label: "CasMisses"},
			{Name: "TouchHits", Label: "TouchHits"},
			{Name: "TouchMisses", Label: "TouchMisses"},
		},
	},
	"ecache.Evictions": {
		Label: "ECache Evictions",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "Evictions", Label: "Evictions"},
		},
	},
	"ecache.Unfetched": {
		Label: "ECache Unfetched",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "EvictedUnfetched", Label: "EvictedUnfetched"},
			{Name: "ExpiredUnfetched", Label: "ExpiredUnfetched"},
		},
	},
	"ecache.Bytes": {
		Label: "ECache Traffics",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "BytesReadIntoMemcached", Label: "BytesReadIntoMemcached"},
			{Name: "BytesWrittenOutFromMemcached", Label: "BytesWrittenOutFromMemcached"},
		},
	},
	"ecache.Connections": {
		Label: "ECache Connections",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "CurrConnections", Label: "CurrConnections"},
			{Name: "NewConnections", Label: "NewConnections"},
		},
	},
	"ecache.Items": {
		Label: "ECache Items",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "CurrItems", Label: "CurrItems"},
			{Name: "NewItems", Label: "NewItems"},
			{Name: "Reclaimed", Label: "Reclaimed"},
			{Name: "CurrConfig", Label: "CurrConfig"},
			{Name: "SlabsMoved", Label: "SlabsMoved"},
		},
	},
	"ecache.MemoryUsage": {
		Label: "ECache Memory Usage",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "UnusedMemory", Label: "UnusedMemory"},
			{Name: "BytesUsedForHash", Label: "BytesUsedForHash"},
			{Name: "BytesUsedForCacheItems", Label: "BytesUsedForCacheItems"},
		},
	},
}

var graphdefRedis = map[string]mp.Graphs{
	"ecache.CPUUtilization": {
		Label: "ECache CPU Utilization",
		Unit:  "percentage",
		Metrics: []mp.Metrics{
			{Name: "CPUUtilization", Label: "CPUUtilization"},
		},
	},
	"ecache.SwapUsage": {
		Label: "ECache Swap Usage",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "SwapUsage", Label: "SwapUsage"},
		},
	},
	"ecache.FreeableMemory": {
		Label: "ECache Freeable Memory",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "FreeableMemory", Label: "FreeableMemory"},
		},
	},
	"ecache.NetworkTraffic": {
		Label: "ECache Network Traffic",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "NetworkBytesIn", Label: "NetworkBytesIn"},
			{Name: "NetworkBytesOut", Label: "NetworkBytesOut"},
		},
	},
	"ecache.Command": {
		Label: "ECache Command",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "GetTypeCmds", Label: "GetTypeCmds"},
			{Name: "SetTypeCmds", Label: "SetTypeCmds"},
			{Name: "KeyBasedCmds", Label: "KeyBasedCmds"},
			{Name: "StringBasedCmds", Label: "StringBasedCmds"},
			{Name: "HashBasedCmds", Label: "HashBasedCmds"},
			{Name: "ListBasedCmds", Label: "ListBasedCmds"},
			{Name: "SetBasedCmds", Label: "SetBasedCmds"},
			{Name: "SortedSetBasedCmds", Label: "SortedSetBasedCmds"},
		},
	},
	"ecache.CacheHitAndMiss": {
		Label: "ECache Hits/Misses",
		Unit:  "float",
		Metrics: []mp.Metrics{
			{Name: "CacheHits", Label: "CacheHits"},
			{Name: "CacheMisses", Label: "CacheMisses"},
		},
	},
	"ecache.Evictions": {
		Label: "ECache Evictions",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "Evictions", Label: "Evictions"},
		},
	},
	"ecache.Bytes": {
		Label: "ECache Traffics",
		Unit:  "bytes",
		Metrics: []mp.Metrics{
			{Name: "BytesUsedForCache", Label: "BytesUsedForCache"},
		},
	},
	"ecache.Connections": {
		Label: "ECache Connections",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "CurrConnections", Label: "CurrConnections"},
			{Name: "NewConnections", Label: "NewConnections"},
		},
	},
	"ecache.Items": {
		Label: "ECache Items",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "CurrItems", Label: "CurrItems"},
			{Name: "Reclaimed", Label: "Reclaimed"},
		},
	},
}

// ECachePlugin mackerel plugin for elasticache
type ECachePlugin struct {
	Region          string
	AccessKeyID     string
	SecretAccessKey string
	CacheClusterID  string
	CacheNodeID     string
	ElastiCacheType string
	CacheMetrics    []string
}

func getLastPoint(cloudWatch *cloudwatch.CloudWatch, dimensions []*cloudwatch.Dimension, metricName string) (float64, error) {
	now := time.Now()

	response, err := cloudWatch.GetMetricStatistics(&cloudwatch.GetMetricStatisticsInput{
		Dimensions: dimensions,
		StartTime:  aws.Time(now.Add(time.Duration(180) * time.Second * -1)), // 3 mins (to fetch at least 1 data-point)
		EndTime:    aws.Time(now),
		MetricName: aws.String(metricName),
		Period:     aws.Int64(60),
		Statistics: []*string{aws.String("Average")},
		Namespace:  aws.String("AWS/ElastiCache"),
	})
	if err != nil {
		return 0, err
	}

	datapoints := response.Datapoints
	if len(datapoints) == 0 {
		return 0, errors.New("fetched no datapoints")
	}

	latest := new(time.Time)
	var latestVal float64
	for _, dp := range datapoints {
		if dp.Timestamp.Before(*latest) {
			continue
		}

		latest = dp.Timestamp
		latestVal = *dp.Average
	}

	return latestVal, nil
}

// FetchMetrics fetch elasticache values
func (p ECachePlugin) FetchMetrics() (map[string]float64, error) {
	sess, err := session.NewSession()
	if err != nil {
		return nil, err
	}

	config := aws.NewConfig()
	if p.AccessKeyID != "" && p.SecretAccessKey != "" {
		config = config.WithCredentials(credentials.NewStaticCredentials(p.AccessKeyID, p.SecretAccessKey, ""))
	}
	if p.Region != "" {
		config = config.WithRegion(p.Region)
	}

	cloudWatch := cloudwatch.New(sess, config)

	stat := make(map[string]float64)

	perInstances := []*cloudwatch.Dimension{
		{
			Name:  aws.String("CacheClusterId"),
			Value: aws.String(p.CacheClusterID),
		},
		{
			Name:  aws.String("CacheNodeId"),
			Value: aws.String(p.CacheNodeID),
		},
	}

	for _, met := range p.CacheMetrics {
		v, err := getLastPoint(cloudWatch, perInstances, met)
		if err == nil {
			stat[met] = v
		} else {
			log.Printf("%s: %s", met, err)
		}
	}

	return stat, nil
}

// GraphDefinition graph definition
func (p ECachePlugin) GraphDefinition() map[string]mp.Graphs {
	switch p.ElastiCacheType {
	case "memcached":
		return graphdefMemcached
	case "redis":
		return graphdefRedis
	default:
		log.Printf("elasticache-type is 'memcached' or 'redis'.")
		return nil
	}
}

// Do the plugin
func Do() {
	optRegion := flag.String("region", "", "AWS Region")
	optAccessKeyID := flag.String("access-key-id", "", "AWS Access Key ID")
	optSecretAccessKey := flag.String("secret-access-key", "", "AWS Secret Access Key")
	optCacheClusterID := flag.String("cache-cluster-id", "", "Cache Cluster Id")
	optCacheNodeID := flag.String("cache-node-id", "0001", "Cache Node Id")
	optElastiCacheType := flag.String("elasticache-type", "", "ElastiCache type")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	var ecache ECachePlugin

	ecache.Region = *optRegion
	ecache.AccessKeyID = *optAccessKeyID
	ecache.SecretAccessKey = *optSecretAccessKey
	ecache.CacheClusterID = *optCacheClusterID
	ecache.CacheNodeID = *optCacheNodeID
	ecache.ElastiCacheType = *optElastiCacheType
	switch ecache.ElastiCacheType {
	case "memcached":
		ecache.CacheMetrics = metricsdefMemcached
	case "redis":
		ecache.CacheMetrics = metricsdefRedis
	default:
		log.Printf("elasticache-type is 'memcached' or 'redis'.")
		os.Exit(1)
	}

	helper := mp.NewMackerelPlugin(ecache)
	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.Tempfile = fmt.Sprintf("/tmp/mackerel-plugin-aws-elasticache-%s-%s", *optCacheClusterID, *optCacheNodeID)
	}

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		helper.OutputDefinitions()
	} else {
		helper.OutputValues()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"net/url"
	"os"
	"text/tabwriter"

	"github.com/dustin/go-humanize"
)

var verbose = flag.Bool("v", false, "verbose")
var short = flag.Bool("short", false, "short form")

const defaultTemplate = `{{range .}}
{{.DBName}}:
  Space Used:       {{.SpaceUsed|bytes}}
  Last Sequence:    {{.SpaceUsed|comma}}
  Header Position:  {{.HeaderPos|comma}}
  Document Count:   {{.DocCount|comma}}
  Deleted Count:    {{.DeletedCount|comma}}
{{end}}
`

const shortTemplate = `dbname	docs	space used
------	----	----------
{{range .}}{{.DBName}}	{{.DocCount|comma}}	{{.SpaceUsed|bytes}}
{{end}}`

type dbinfo struct {
	DBName       string
	SpaceUsed    json.Number `json:"space_used"`
	LastSeq      json.Number `json:"last_seq"`
	HeaderPos    json.Number `json:"header_pos"`
	DocCount     json.Number `json:"doc_count"`
	DeletedCount json.Number `json:"deleted_count"`
	Error        string
}

var funcMap = template.FuncMap{
	"comma": func(n json.Number) string {
		nint, err := n.Int64()
		maybeFatal(err, "Invalid int64: %v: %v", n, err)
		return humanize.Comma(nint)
	},
	"bytes": func(n json.Number) string {
		nint, err := n.Int64()
		maybeFatal(err, "Invalid int64: %v: %v", n, err)
		return humanize.Bytes(uint64(nint))
	}}

var tmpl = template.Must(template.New("").Funcs(funcMap).Parse(defaultTemplate))

var shortTmpl = template.Must(template.New("").Funcs(funcMap).Parse(shortTemplate))

func init() {
	log.SetFlags(log.Lmicroseconds)
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %v [-v] http://serieslyhost:3133/ [dbnames...]\n",
			os.Args[0])
		flag.PrintDefaults()
	}
}

func maybeFatal(err error, fmt string, args ...interface{}) {
	if err != nil {
		log.Fatalf(fmt, args...)
	}
}

func listDatabases(u url.URL) []string {
	u.Path = "/_all_dbs"
	res, err := http.Get(u.String())
	maybeFatal(err, "Error listing databases: %v", err)
	defer res.Body.Close()

	rv := []string{}
	d := json.NewDecoder(res.Body)
	err = d.Decode(&rv)
	maybeFatal(err, "Error decoding database list: %v", err)
	return rv
}

func fetchDBInfo(u url.URL, which string) (dbinfo, error) {
	u.Path = "/" + which
	rv := dbinfo{}
	res, err := http.Get(u.String())
	if err != nil {
		return rv, err
	}
	if res.StatusCode != 200 {
		return rv, fmt.Errorf("HTTP error:  %v", res.Status)
	}
	defer res.Body.Close()

	err = json.NewDecoder(res.Body).Decode(&rv)
	return rv, err
}

func vlog(s string, a ...interface{}) {
	if *verbose {
		log.Printf(s, a...)
	}
}

func describe(base url.URL, dbs ...string) <-chan dbinfo {
	rv := make(chan dbinfo)
	go func() {
		defer close(rv)
		for _, db := range dbs {
			di, err := fetchDBInfo(base, db)
			maybeFatal(err, "Couldn't fetch info for %v: %v", db, err)
			di.DBName = db
			rv <- di
		}
	}()
	return rv
}

func main() {
	flag.Parse()

	if flag.NArg() < 1 {
		flag.Usage()
		os.Exit(64)
	}

	base, err := url.Parse(flag.Arg(0))
	maybeFatal(err, "Couldn't parse URL: %v", err)

	dbs := flag.Args()[1:]
	if len(dbs) == 0 {
		dbs = listDatabases(*base)
	}

	t := tmpl
	if *short {
		t = shortTmpl
	}

	tw := tabwriter.NewWriter(os.Stdout, 8, 8, 2, ' ', 0)
	t.Execute(tw, describe(*base, dbs...))
	defer tw.Flush()
}
<commit_msg>Configurable info templates<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"text/tabwriter"

	"github.com/dustin/go-humanize"
)

var (
	verbose      = flag.Bool("v", false, "verbose")
	short        = flag.Bool("short", false, "short form")
	templateSrc  = flag.String("t", "", "Display template")
	templateFile = flag.String("T", "", "Display template filename")
)

const defaultTemplate = `{{range .}}
{{.DBName}}:
  Space Used:       {{.SpaceUsed|bytes}}
  Last Sequence:    {{.SpaceUsed|comma}}
  Header Position:  {{.HeaderPos|comma}}
  Document Count:   {{.DocCount|comma}}
  Deleted Count:    {{.DeletedCount|comma}}
{{end}}
`

const shortTemplate = `dbname	docs	space used
------	----	----------
{{range .}}{{.DBName}}	{{.DocCount|comma}}	{{.SpaceUsed|bytes}}
{{end}}`

type dbinfo struct {
	DBName       string
	SpaceUsed    json.Number `json:"space_used"`
	LastSeq      json.Number `json:"last_seq"`
	HeaderPos    json.Number `json:"header_pos"`
	DocCount     json.Number `json:"doc_count"`
	DeletedCount json.Number `json:"deleted_count"`
	Error        string
}

var funcMap = template.FuncMap{
	"comma": func(n json.Number) string {
		nint, err := n.Int64()
		maybeFatal(err, "Invalid int64: %v: %v", n, err)
		return humanize.Comma(nint)
	},
	"bytes": func(n json.Number) string {
		nint, err := n.Int64()
		maybeFatal(err, "Invalid int64: %v: %v", n, err)
		return humanize.Bytes(uint64(nint))
	}}

func init() {
	log.SetFlags(log.Lmicroseconds)
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %v [-v] http://serieslyhost:3133/ [dbnames...]\n",
			os.Args[0])
		flag.PrintDefaults()
	}
}

func maybeFatal(err error, fmt string, args ...interface{}) {
	if err != nil {
		log.Fatalf(fmt, args...)
	}
}

func listDatabases(u url.URL) []string {
	u.Path = "/_all_dbs"
	res, err := http.Get(u.String())
	maybeFatal(err, "Error listing databases: %v", err)
	defer res.Body.Close()

	rv := []string{}
	d := json.NewDecoder(res.Body)
	err = d.Decode(&rv)
	maybeFatal(err, "Error decoding database list: %v", err)
	return rv
}

func fetchDBInfo(u url.URL, which string) (dbinfo, error) {
	u.Path = "/" + which
	rv := dbinfo{}
	res, err := http.Get(u.String())
	if err != nil {
		return rv, err
	}
	if res.StatusCode != 200 {
		return rv, fmt.Errorf("HTTP error:  %v", res.Status)
	}
	defer res.Body.Close()

	err = json.NewDecoder(res.Body).Decode(&rv)
	return rv, err
}

func vlog(s string, a ...interface{}) {
	if *verbose {
		log.Printf(s, a...)
	}
}

func describe(base url.URL, dbs ...string) <-chan dbinfo {
	rv := make(chan dbinfo)
	go func() {
		defer close(rv)
		for _, db := range dbs {
			di, err := fetchDBInfo(base, db)
			maybeFatal(err, "Couldn't fetch info for %v: %v", db, err)
			di.DBName = db
			rv <- di
		}
	}()
	return rv
}

func getTemplate(tdefault string) *template.Template {
	tmplstr := *templateSrc
	if tmplstr == "" {
		switch *templateFile {
		case "":
			tmplstr = tdefault
		case "-":
			td, err := ioutil.ReadAll(os.Stdin)
			maybeFatal(err, "Error reading template from stdin: %v", err)
			tmplstr = string(td)
		default:
			td, err := ioutil.ReadFile(*templateFile)
			maybeFatal(err, "Error reading template file: %v", err)
			tmplstr = string(td)
		}
	}

	tmpl, err := template.New("").Funcs(funcMap).Parse(tmplstr)
	maybeFatal(err, "Error parsing template: %v", err)
	return tmpl
}

func main() {
	flag.Parse()

	if flag.NArg() < 1 {
		flag.Usage()
		os.Exit(64)
	}

	base, err := url.Parse(flag.Arg(0))
	maybeFatal(err, "Couldn't parse URL: %v", err)

	dbs := flag.Args()[1:]
	if len(dbs) == 0 {
		dbs = listDatabases(*base)
	}

	tsrc := defaultTemplate
	if *short {
		tsrc = shortTemplate
	}

	tmpl := getTemplate(tsrc)

	tw := tabwriter.NewWriter(os.Stdout, 8, 8, 2, ' ', 0)
	tmpl.Execute(tw, describe(*base, dbs...))
	defer tw.Flush()
}
<|endoftext|>


<commit_before><commit_msg>Don't assume local client for GetXferRequest and GetDPNBag<commit_after><|endoftext|>


<commit_before>// Copyright 2009 The Go9p Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The clnt package provides definitions and functions used to implement
// a 9P2000 file client.
package clnt

import (
	"code.google.com/p/go9p/p"
	"fmt"
	"log"
	"net"
	"sync"
	"syscall"
)

// Debug flags
const (
	DbgPrintFcalls  = (1 << iota) // print all 9P messages on stderr
	DbgPrintPackets               // print the raw packets on stderr
	DbgLogFcalls                  // keep the last N 9P messages (can be accessed over http)
	DbgLogPackets                 // keep the last N 9P messages (can be accessed over http)
)

type StatsOps interface {
	statsRegister()
	statsUnregister()
}

// The Clnt type represents a 9P2000 client. The client is connected to
// a 9P2000 file server and its methods can be used to access and manipulate
// the files exported by the server.
type Clnt struct {
	sync.Mutex
	Debuglevel int    // =0 don't print anything, >0 print Fcalls, >1 print raw packets
	Msize      uint32 // Maximum size of the 9P messages
	Dotu       bool   // If true, 9P2000.u protocol is spoken
	Root       *Fid   // Fid that points to the rood directory
	Id         string // Used when printing debug messages
	Log        *p.Logger

	conn     net.Conn
	tagpool  *pool
	fidpool  *pool
	reqout   chan *Req
	done     chan bool
	reqfirst *Req
	reqlast  *Req
	err      error

	reqchan chan *Req
	tchan   chan *p.Fcall

	next, prev *Clnt
}

// A Fid type represents a file on the server. Fids are used for the
// low level methods that correspond directly to the 9P2000 message requests
type Fid struct {
	sync.Mutex
	Clnt   *Clnt // Client the fid belongs to
	Iounit uint32
	p.Qid         // The Qid description for the file
	Mode   uint8  // Open mode (one of p.O* values) (if file is open)
	Fid    uint32 // Fid number
	p.User        // The user the fid belongs to
	walked bool   // true if the fid points to a walked file on the server
}

// The file is similar to the Fid, but is used in the high-level client
// interface.
type File struct {
	fid    *Fid
	offset uint64
}

type pool struct {
	sync.Mutex
	need  int
	nchan chan uint32
	maxid uint32
	imap  []byte
}

type Req struct {
	sync.Mutex
	Clnt       *Clnt
	Tc         *p.Fcall
	Rc         *p.Fcall
	Err        error
	Done       chan *Req
	tag        uint16
	prev, next *Req
	fid        *Fid
}

type ClntList struct {
	sync.Mutex
	clntList, clntLast *Clnt
}

var clnts *ClntList
var DefaultDebuglevel int
var DefaultLogger *p.Logger

func (clnt *Clnt) Rpcnb(r *Req) error {
	var tag uint16

	if r.Tc.Type == p.Tversion {
		tag = p.NOTAG
	} else {
		tag = r.tag
	}

	p.SetTag(r.Tc, tag)
	clnt.Lock()
	if clnt.err != nil {
		clnt.Unlock()
		return clnt.err
	}

	if clnt.reqlast != nil {
		clnt.reqlast.next = r
	} else {
		clnt.reqfirst = r
	}

	r.prev = clnt.reqlast
	clnt.reqlast = r
	clnt.Unlock()

	clnt.reqout <- r
	return nil
}

func (clnt *Clnt) Rpc(tc *p.Fcall) (rc *p.Fcall, err error) {
	r := clnt.ReqAlloc()
	r.Tc = tc
	r.Done = make(chan *Req)
	err = clnt.Rpcnb(r)
	if err != nil {
		return
	}

	<-r.Done
	rc = r.Rc
	err = r.Err
	clnt.ReqFree(r)
	return
}

func (clnt *Clnt) recv() {
	var err error

	err = nil
	buf := make([]byte, clnt.Msize*8)
	pos := 0
	for {
		if len(buf) < int(clnt.Msize) {
			b := make([]byte, clnt.Msize*8)
			copy(b, buf[0:pos])
			buf = b
			b = nil
		}

		n, oerr := clnt.conn.Read(buf[pos:])
		if oerr != nil || n == 0 {
			err = &p.Error{oerr.Error(), p.EIO}
			clnt.Lock()
			clnt.err = err
			clnt.Unlock()
			goto closed
		}

		pos += n
		for pos > 4 {
			sz, _ := p.Gint32(buf)
			if pos < int(sz) {
				if len(buf) < int(sz) {
					b := make([]byte, clnt.Msize*8)
					copy(b, buf[0:pos])
					buf = b
					b = nil
				}

				break
			}

			fc, err, fcsize := p.Unpack(buf, clnt.Dotu)
			clnt.Lock()
			if err != nil {
				clnt.err = err
				clnt.conn.Close()
				clnt.Unlock()
				goto closed
			}

			if clnt.Debuglevel > 0 {
				clnt.logFcall(fc)
				if clnt.Debuglevel&DbgPrintPackets != 0 {
					log.Println("}-}", clnt.Id, fmt.Sprint(fc.Pkt))
				}

				if clnt.Debuglevel&DbgPrintFcalls != 0 {
					log.Println("}}}", clnt.Id, fc.String())
				}
			}

			var r *Req = nil
			for r = clnt.reqfirst; r != nil; r = r.next {
				if r.Tc.Tag == fc.Tag {
					break
				}
			}

			if r == nil {
				clnt.err = &p.Error{"unexpected response", p.EINVAL}
				clnt.conn.Close()
				clnt.Unlock()
				goto closed
			}

			r.Rc = fc
			if r.prev != nil {
				r.prev.next = r.next
			} else {
				clnt.reqfirst = r.next
			}

			if r.next != nil {
				r.next.prev = r.prev
			} else {
				clnt.reqlast = r.prev
			}
			clnt.Unlock()

			if r.Tc.Type != r.Rc.Type-1 {
				if r.Rc.Type != p.Rerror {
					r.Err = &p.Error{"invalid response", p.EINVAL}
					log.Println(fmt.Sprintf("TTT %v", r.Tc))
					log.Println(fmt.Sprintf("RRR %v", r.Rc))
				} else {
					if r.Err == nil {
						r.Err = &p.Error{r.Rc.Error, syscall.Errno(r.Rc.Errornum)}
					}
				}
			}

			if r.Done != nil {
				r.Done <- r
			}

			pos -= fcsize
			buf = buf[fcsize:]
		}
	}

closed:
	clnt.done <- true

	/* send error to all pending requests */
	clnt.Lock()
	r := clnt.reqfirst
	clnt.reqfirst = nil
	clnt.reqlast = nil
	if err == nil {
		err = clnt.err
	}
	clnt.Unlock()
	for ; r != nil; r = r.next {
		r.Err = err
		if r.Done != nil {
			r.Done <- r
		}
	}

	clnts.Lock()
	if clnt.prev != nil {
		clnt.prev.next = clnt.next
	} else {
		clnts.clntList = clnt.next
	}

	if clnt.next != nil {
		clnt.next.prev = clnt.prev
	} else {
		clnts.clntLast = clnt.prev
	}
	clnts.Unlock()

	if sop, ok := (interface{}(clnt)).(StatsOps); ok {
		sop.statsUnregister()
	}
}

func (clnt *Clnt) send() {
	for {
		select {
		case <-clnt.done:
			return

		case req := <-clnt.reqout:
			if clnt.Debuglevel > 0 {
				clnt.logFcall(req.Tc)
				if clnt.Debuglevel&DbgPrintPackets != 0 {
					log.Println("{-{", clnt.Id, fmt.Sprint(req.Tc.Pkt))
				}

				if clnt.Debuglevel&DbgPrintFcalls != 0 {
					log.Println("{{{", clnt.Id, req.Tc.String())
				}
			}

			for buf := req.Tc.Pkt; len(buf) > 0; {
				n, err := clnt.conn.Write(buf)
				if err != nil {
					/* just close the socket, will get signal on clnt.done */
					clnt.conn.Close()
					break
				}

				buf = buf[n:]
			}
		}
	}
}

// Creates and initializes a new Clnt object. Doesn't send any data
// on the wire.
func NewClnt(c net.Conn, msize uint32, dotu bool) *Clnt {
	clnt := new(Clnt)
	clnt.conn = c
	clnt.Msize = msize
	clnt.Dotu = dotu
	clnt.Debuglevel = DefaultDebuglevel
	clnt.Log = DefaultLogger
	clnt.tagpool = newPool(uint32(p.NOTAG))
	clnt.fidpool = newPool(p.NOFID)
	clnt.reqout = make(chan *Req)
	clnt.done = make(chan bool)
	clnt.reqchan = make(chan *Req, 16)
	clnt.tchan = make(chan *p.Fcall, 16)

	go clnt.recv()
	go clnt.send()

	clnts.Lock()
	if clnts.clntLast != nil {
		clnts.clntLast.next = clnt
	} else {
		clnts.clntList = clnt
	}

	clnt.prev = clnts.clntLast
	clnts.clntLast = clnt
	clnts.Unlock()

	if sop, ok := (interface{}(clnt)).(StatsOps); ok {
		sop.statsRegister()
	}

	return clnt
}

// Establishes a new socket connection to the 9P server and creates
// a client object for it. Negotiates the dialect and msize for the
// connection. Returns a Clnt object, or Error.
func Connect(c net.Conn, msize uint32, dotu bool) (*Clnt, error) {
	clnt := NewClnt(c, msize, dotu)
	clnt.Id = c.RemoteAddr().String() + ":"
	ver := "9P2000"
	if clnt.Dotu {
		ver = "9P2000.u"
	}

	tc := p.NewFcall(clnt.Msize)
	err := p.PackTversion(tc, clnt.Msize, ver)
	if err != nil {
		return nil, err
	}

	rc, err := clnt.Rpc(tc)
	if err != nil {
		return nil, err
	}

	if rc.Msize < clnt.Msize {
		clnt.Msize = rc.Msize
	}

	clnt.Dotu = rc.Version == "9P2000.u" && clnt.Dotu
	return clnt, nil
}

// Creates a new Fid object for the client
func (clnt *Clnt) FidAlloc() *Fid {
	fid := new(Fid)
	fid.Fid = clnt.fidpool.getId()
	fid.Clnt = clnt

	return fid
}

func (clnt *Clnt) NewFcall() *p.Fcall {
	select {
	case tc := <-clnt.tchan:
		return tc
	default:
	}
	return p.NewFcall(clnt.Msize)
}

func (clnt *Clnt) FreeFcall(fc *p.Fcall) {
	if fc != nil && len(fc.Buf) >= int(clnt.Msize) {
		select {
		case clnt.tchan <- fc:
			break
		default:
		}
	}
}

func (clnt *Clnt) ReqAlloc() *Req {
	var req *Req
	select {
	case req = <-clnt.reqchan:
		break
	default:
		req = new(Req)
		req.Clnt = clnt
		req.tag = uint16(clnt.tagpool.getId())
	}
	return req
}

func (clnt *Clnt) ReqFree(req *Req) {
	clnt.FreeFcall(req.Tc)
	req.Tc = nil
	req.Rc = nil
	req.Err = nil
	req.Done = nil
	req.next = nil
	req.prev = nil

	select {
	case clnt.reqchan <- req:
		break
	default:
		clnt.tagpool.putId(uint32(req.tag))
	}
}

func (clnt *Clnt) logFcall(fc *p.Fcall) {
	if clnt.Debuglevel&DbgLogPackets != 0 {
		pkt := make([]byte, len(fc.Pkt))
		copy(pkt, fc.Pkt)
		clnt.Log.Log(pkt, clnt, DbgLogPackets)
	}

	if clnt.Debuglevel&DbgLogFcalls != 0 {
		f := new(p.Fcall)
		*f = *fc
		f.Pkt = nil
		clnt.Log.Log(f, clnt, DbgLogFcalls)
	}
}

func init() {
	clnts = new(ClntList)
	if sop, ok := (interface{}(clnts)).(StatsOps); ok {
		sop.statsRegister()
	}
}
<commit_msg>fix for Issue #22<commit_after>// Copyright 2009 The Go9p Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The clnt package provides definitions and functions used to implement
// a 9P2000 file client.
package clnt

import (
	"code.google.com/p/go9p/p"
	"fmt"
	"log"
	"net"
	"sync"
	"syscall"
)

// Debug flags
const (
	DbgPrintFcalls  = (1 << iota) // print all 9P messages on stderr
	DbgPrintPackets               // print the raw packets on stderr
	DbgLogFcalls                  // keep the last N 9P messages (can be accessed over http)
	DbgLogPackets                 // keep the last N 9P messages (can be accessed over http)
)

type StatsOps interface {
	statsRegister()
	statsUnregister()
}

// The Clnt type represents a 9P2000 client. The client is connected to
// a 9P2000 file server and its methods can be used to access and manipulate
// the files exported by the server.
type Clnt struct {
	sync.Mutex
	Debuglevel int    // =0 don't print anything, >0 print Fcalls, >1 print raw packets
	Msize      uint32 // Maximum size of the 9P messages
	Dotu       bool   // If true, 9P2000.u protocol is spoken
	Root       *Fid   // Fid that points to the rood directory
	Id         string // Used when printing debug messages
	Log        *p.Logger

	conn     net.Conn
	tagpool  *pool
	fidpool  *pool
	reqout   chan *Req
	done     chan bool
	reqfirst *Req
	reqlast  *Req
	err      error

	reqchan chan *Req
	tchan   chan *p.Fcall

	next, prev *Clnt
}

// A Fid type represents a file on the server. Fids are used for the
// low level methods that correspond directly to the 9P2000 message requests
type Fid struct {
	sync.Mutex
	Clnt   *Clnt // Client the fid belongs to
	Iounit uint32
	p.Qid         // The Qid description for the file
	Mode   uint8  // Open mode (one of p.O* values) (if file is open)
	Fid    uint32 // Fid number
	p.User        // The user the fid belongs to
	walked bool   // true if the fid points to a walked file on the server
}

// The file is similar to the Fid, but is used in the high-level client
// interface.
type File struct {
	fid    *Fid
	offset uint64
}

type pool struct {
	sync.Mutex
	need  int
	nchan chan uint32
	maxid uint32
	imap  []byte
}

type Req struct {
	sync.Mutex
	Clnt       *Clnt
	Tc         *p.Fcall
	Rc         *p.Fcall
	Err        error
	Done       chan *Req
	tag        uint16
	prev, next *Req
	fid        *Fid
}

type ClntList struct {
	sync.Mutex
	clntList, clntLast *Clnt
}

var clnts *ClntList
var DefaultDebuglevel int
var DefaultLogger *p.Logger

func (clnt *Clnt) Rpcnb(r *Req) error {
	var tag uint16

	if r.Tc.Type == p.Tversion {
		tag = p.NOTAG
	} else {
		tag = r.tag
	}

	p.SetTag(r.Tc, tag)
	clnt.Lock()
	if clnt.err != nil {
		clnt.Unlock()
		return clnt.err
	}

	if clnt.reqlast != nil {
		clnt.reqlast.next = r
	} else {
		clnt.reqfirst = r
	}

	r.prev = clnt.reqlast
	clnt.reqlast = r
	clnt.Unlock()

	clnt.reqout <- r
	return nil
}

func (clnt *Clnt) Rpc(tc *p.Fcall) (rc *p.Fcall, err error) {
	r := clnt.ReqAlloc()
	r.Tc = tc
	r.Done = make(chan *Req)
	err = clnt.Rpcnb(r)
	if err != nil {
		return
	}

	<-r.Done
	rc = r.Rc
	err = r.Err
	clnt.ReqFree(r)
	return
}

func (clnt *Clnt) recv() {
	var err error

	err = nil
	buf := make([]byte, clnt.Msize*8)
	pos := 0
	for {
		if len(buf) < int(clnt.Msize) {
			b := make([]byte, clnt.Msize*8)
			copy(b, buf[0:pos])
			buf = b
			b = nil
		}

		n, oerr := clnt.conn.Read(buf[pos:])
		if oerr != nil || n == 0 {
			err = &p.Error{oerr.Error(), p.EIO}
			clnt.Lock()
			clnt.err = err
			clnt.Unlock()
			goto closed
		}

		pos += n
		for pos > 4 {
			sz, _ := p.Gint32(buf)
			if pos < int(sz) {
				if len(buf) < int(sz) {
					b := make([]byte, clnt.Msize*8)
					copy(b, buf[0:pos])
					buf = b
					b = nil
				}

				break
			}

			fc, err, fcsize := p.Unpack(buf, clnt.Dotu)
			clnt.Lock()
			if err != nil {
				clnt.err = err
				clnt.conn.Close()
				clnt.Unlock()
				goto closed
			}

			if clnt.Debuglevel > 0 {
				clnt.logFcall(fc)
				if clnt.Debuglevel&DbgPrintPackets != 0 {
					log.Println("}-}", clnt.Id, fmt.Sprint(fc.Pkt))
				}

				if clnt.Debuglevel&DbgPrintFcalls != 0 {
					log.Println("}}}", clnt.Id, fc.String())
				}
			}

			var r *Req = nil
			for r = clnt.reqfirst; r != nil; r = r.next {
				if r.Tc.Tag == fc.Tag {
					break
				}
			}

			if r == nil {
				clnt.err = &p.Error{"unexpected response", p.EINVAL}
				clnt.conn.Close()
				clnt.Unlock()
				goto closed
			}

			r.Rc = fc
			if r.prev != nil {
				r.prev.next = r.next
			} else {
				clnt.reqfirst = r.next
			}

			if r.next != nil {
				r.next.prev = r.prev
			} else {
				clnt.reqlast = r.prev
			}
			clnt.Unlock()

			if r.Tc.Type != r.Rc.Type-1 {
				if r.Rc.Type != p.Rerror {
					r.Err = &p.Error{"invalid response", p.EINVAL}
					log.Println(fmt.Sprintf("TTT %v", r.Tc))
					log.Println(fmt.Sprintf("RRR %v", r.Rc))
				} else {
					if r.Err == nil {
						r.Err = &p.Error{r.Rc.Error, syscall.Errno(r.Rc.Errornum)}
					}
				}
			}

			if r.Done != nil {
				r.Done <- r
			}

			pos -= fcsize
			buf = buf[fcsize:]
		}
	}

closed:
	clnt.done <- true

	/* send error to all pending requests */
	clnt.Lock()
	r := clnt.reqfirst
	clnt.reqfirst = nil
	clnt.reqlast = nil
	if err == nil {
		err = clnt.err
	}
	clnt.Unlock()
	for ; r != nil; r = r.next {
		r.Err = err
		if r.Done != nil {
			r.Done <- r
		}
	}

	clnts.Lock()
	if clnt.prev != nil {
		clnt.prev.next = clnt.next
	} else {
		clnts.clntList = clnt.next
	}

	if clnt.next != nil {
		clnt.next.prev = clnt.prev
	} else {
		clnts.clntLast = clnt.prev
	}
	clnts.Unlock()

	if sop, ok := (interface{}(clnt)).(StatsOps); ok {
		sop.statsUnregister()
	}
}

func (clnt *Clnt) send() {
	for {
		select {
		case <-clnt.done:
			return

		case req := <-clnt.reqout:
			if clnt.Debuglevel > 0 {
				clnt.logFcall(req.Tc)
				if clnt.Debuglevel&DbgPrintPackets != 0 {
					log.Println("{-{", clnt.Id, fmt.Sprint(req.Tc.Pkt))
				}

				if clnt.Debuglevel&DbgPrintFcalls != 0 {
					log.Println("{{{", clnt.Id, req.Tc.String())
				}
			}

			for buf := req.Tc.Pkt; len(buf) > 0; {
				n, err := clnt.conn.Write(buf)
				if err != nil {
					/* just close the socket, will get signal on clnt.done */
					clnt.conn.Close()
					break
				}

				buf = buf[n:]
			}
		}
	}
}

// Creates and initializes a new Clnt object. Doesn't send any data
// on the wire.
func NewClnt(c net.Conn, msize uint32, dotu bool) *Clnt {
	clnt := new(Clnt)
	clnt.conn = c
	clnt.Msize = msize
	clnt.Dotu = dotu
	clnt.Debuglevel = DefaultDebuglevel
	clnt.Log = DefaultLogger
	clnt.Id = c.RemoteAddr().String() + ":"
	clnt.tagpool = newPool(uint32(p.NOTAG))
	clnt.fidpool = newPool(p.NOFID)
	clnt.reqout = make(chan *Req)
	clnt.done = make(chan bool)
	clnt.reqchan = make(chan *Req, 16)
	clnt.tchan = make(chan *p.Fcall, 16)

	go clnt.recv()
	go clnt.send()

	clnts.Lock()
	if clnts.clntLast != nil {
		clnts.clntLast.next = clnt
	} else {
		clnts.clntList = clnt
	}

	clnt.prev = clnts.clntLast
	clnts.clntLast = clnt
	clnts.Unlock()

	if sop, ok := (interface{}(clnt)).(StatsOps); ok {
		sop.statsRegister()
	}

	return clnt
}

// Establishes a new socket connection to the 9P server and creates
// a client object for it. Negotiates the dialect and msize for the
// connection. Returns a Clnt object, or Error.
func Connect(c net.Conn, msize uint32, dotu bool) (*Clnt, error) {
	clnt := NewClnt(c, msize, dotu)
	ver := "9P2000"
	if clnt.Dotu {
		ver = "9P2000.u"
	}

	tc := p.NewFcall(clnt.Msize)
	err := p.PackTversion(tc, clnt.Msize, ver)
	if err != nil {
		return nil, err
	}

	rc, err := clnt.Rpc(tc)
	if err != nil {
		return nil, err
	}

	if rc.Msize < clnt.Msize {
		clnt.Msize = rc.Msize
	}

	clnt.Dotu = rc.Version == "9P2000.u" && clnt.Dotu
	return clnt, nil
}

// Creates a new Fid object for the client
func (clnt *Clnt) FidAlloc() *Fid {
	fid := new(Fid)
	fid.Fid = clnt.fidpool.getId()
	fid.Clnt = clnt

	return fid
}

func (clnt *Clnt) NewFcall() *p.Fcall {
	select {
	case tc := <-clnt.tchan:
		return tc
	default:
	}
	return p.NewFcall(clnt.Msize)
}

func (clnt *Clnt) FreeFcall(fc *p.Fcall) {
	if fc != nil && len(fc.Buf) >= int(clnt.Msize) {
		select {
		case clnt.tchan <- fc:
			break
		default:
		}
	}
}

func (clnt *Clnt) ReqAlloc() *Req {
	var req *Req
	select {
	case req = <-clnt.reqchan:
		break
	default:
		req = new(Req)
		req.Clnt = clnt
		req.tag = uint16(clnt.tagpool.getId())
	}
	return req
}

func (clnt *Clnt) ReqFree(req *Req) {
	clnt.FreeFcall(req.Tc)
	req.Tc = nil
	req.Rc = nil
	req.Err = nil
	req.Done = nil
	req.next = nil
	req.prev = nil

	select {
	case clnt.reqchan <- req:
		break
	default:
		clnt.tagpool.putId(uint32(req.tag))
	}
}

func (clnt *Clnt) logFcall(fc *p.Fcall) {
	if clnt.Debuglevel&DbgLogPackets != 0 {
		pkt := make([]byte, len(fc.Pkt))
		copy(pkt, fc.Pkt)
		clnt.Log.Log(pkt, clnt, DbgLogPackets)
	}

	if clnt.Debuglevel&DbgLogFcalls != 0 {
		f := new(p.Fcall)
		*f = *fc
		f.Pkt = nil
		clnt.Log.Log(f, clnt, DbgLogFcalls)
	}
}

func init() {
	clnts = new(ClntList)
	if sop, ok := (interface{}(clnts)).(StatsOps); ok {
		sop.statsRegister()
	}
}
<|endoftext|>


<commit_before>package pack

import (
	//"fmt"
	"reflect"
	"fmt"
	"encoding/json"
)

type pointer struct {
	addr reflect.Value
	ttype reflect.Type

}

func (p *pointer) Set(val interface{}) {
	po := *p
	switch vv := val.(type) {
	case string:
		fmt.Println("is string", vv)
		po.addr.SetString(vv)
	case int:
		po.addr.SetInt(int64(vv))
	case float64:
		po.addr.SetFloat(vv)
	case []interface{}:

	default:
		fmt.Println("todo feature")
	}
}

func Packer(b *[]byte, class interface{}) {

	v := reflect.ValueOf(class)

	indirect_type := reflect.Indirect(v).Type()

	//store addr
	name_addr := make(map[string]pointer)
	el := v.Elem()
	for i := 0; i < el.NumField(); i++ {
		name_addr[indirect_type.Field(i).Name] =
			pointer{v.Elem().Field(i), indirect_type}
	}

	var f interface{}
	json.Unmarshal(*b, &f)
	m := f.(map[string]interface{})

	for k, v := range m {
		if q, ok := name_addr[k]; ok {
			p := &q
			p.Set(v)
		}
	}
}
<commit_msg>ADD bool type handler<commit_after>package pack

import (
	//"fmt"
	"reflect"
	"fmt"
	"encoding/json"
)

type pointer struct {
	addr reflect.Value
	ttype reflect.Type

}

func (p *pointer) Set(val interface{}) {
	po := *p
	switch vv := val.(type) {
	case string:
		fmt.Println("is string", vv)
		po.addr.SetString(vv)
	case int:
		po.addr.SetInt(int64(vv))
	case float64:
		po.addr.SetFloat(vv)
	case bool:
		po.addr.SetBool(vv)
	case []interface{}:

	default:
		fmt.Println("todo feature")
	}
}

func Packer(b *[]byte, class interface{}) {

	v := reflect.ValueOf(class)

	indirect_type := reflect.Indirect(v).Type()

	//store addr
	name_addr := make(map[string]pointer)
	el := v.Elem()
	for i := 0; i < el.NumField(); i++ {
		name_addr[indirect_type.Field(i).Name] =
			pointer{v.Elem().Field(i), indirect_type}
	}

	var f interface{}
	json.Unmarshal(*b, &f)
	m := f.(map[string]interface{})

	for k, v := range m {
		if q, ok := name_addr[k]; ok {
			p := &q
			p.Set(v)
		}
	}
}
<|endoftext|>


<commit_before>package topgun_test

import (
	. "github.com/concourse/concourse/topgun/common"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("A pipeline-provided resource type", func() {
	BeforeEach(func() {
		Deploy("deployments/concourse.yml", "-o", "operations/no-gc.yml")
	})

	It("does not result in redundant containers when running resource actions", func() {
		By("setting a pipeline")
		Fly.Run("set-pipeline", "-n", "-c", "pipelines/custom-types.yml", "-p", "pipe")
		Fly.Run("unpause-pipeline", "-p", "pipe")

		By("triggering the build")
		buildSession := Fly.Start("trigger-job", "-w", "-j", "pipe/get-10m")
		<-buildSession.Exited
		Expect(buildSession.ExitCode()).To(Equal(1))

		By("expecting a container for the resource check and task image check")
		Expect(ContainersBy("type", "check")).To(HaveLen(2))

		By("expecting a container for the resource check, build get, build task image check, build task image get, and build task")
		expectedContainersBefore := 5
		Expect(FlyTable("containers")).Should(HaveLen(expectedContainersBefore))

		By("triggering the build again")
		buildSession = Fly.Start("trigger-job", "-w", "-j", "pipe/get-10m")
		<-buildSession.Exited
		Expect(buildSession.ExitCode()).To(Equal(1))

		By("expecting only one additional check container for the task's image check")
		Expect(ContainersBy("type", "check")).To(HaveLen(3))

		By("expecting to only have new containers for build task image check and build task")
		Expect(FlyTable("containers")).Should(HaveLen(expectedContainersBefore + 2))
	})
})

var _ = Describe("Tagged resource types", func() {
	BeforeEach(func() {
		Deploy("deployments/concourse.yml", "-o", "operations/tagged-worker.yml")

		By("setting a pipeline with tagged custom types")
		Fly.Run("set-pipeline", "-n", "-c", "pipelines/tagged-custom-types.yml", "-p", "pipe")
		Fly.Run("unpause-pipeline", "-p", "pipe")
	})

	It("is able to be used with tagged workers", func() {
		By("running a check which uses the tagged custom resource")
		Eventually(Fly.Start("check-resource", "-r", "pipe/10m")).Should(gexec.Exit(0))

		By("triggering a build which uses the tagged custom resource")
		buildSession := Fly.Start("trigger-job", "-w", "-j", "pipe/get-10m")
		<-buildSession.Exited
		Expect(buildSession.ExitCode()).To(Equal(0))
	})
})
<commit_msg>Revert "update test due to in-DB check change"<commit_after>package topgun_test

import (
	. "github.com/concourse/concourse/topgun/common"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("A pipeline-provided resource type", func() {
	BeforeEach(func() {
		Deploy("deployments/concourse.yml", "-o", "operations/no-gc.yml")
	})

	It("does not result in redundant containers when running resource actions", func() {
		By("setting a pipeline")
		Fly.Run("set-pipeline", "-n", "-c", "pipelines/custom-types.yml", "-p", "pipe")
		Fly.Run("unpause-pipeline", "-p", "pipe")

		By("triggering the build")
		buildSession := Fly.Start("trigger-job", "-w", "-j", "pipe/get-10m")
		<-buildSession.Exited
		Expect(buildSession.ExitCode()).To(Equal(1))

		By("expecting a container for the resource check, resource type check, and task image check")
		Expect(ContainersBy("type", "check")).To(HaveLen(3))

		By("expecting a container for the resource check, resource type check, build resource image get, build get, build task image check, build task image get, and build task")
		expectedContainersBefore := 7
		Expect(FlyTable("containers")).Should(HaveLen(expectedContainersBefore))

		By("triggering the build again")
		buildSession = Fly.Start("trigger-job", "-w", "-j", "pipe/get-10m")
		<-buildSession.Exited
		Expect(buildSession.ExitCode()).To(Equal(1))

		By("expecting only one additional check container for the task's image check")
		Expect(ContainersBy("type", "check")).To(HaveLen(4))

		By("expecting to only have new containers for build task image check and build task")
		Expect(FlyTable("containers")).Should(HaveLen(expectedContainersBefore + 2))
	})
})

var _ = Describe("Tagged resource types", func() {
	BeforeEach(func() {
		Deploy("deployments/concourse.yml", "-o", "operations/tagged-worker.yml")

		By("setting a pipeline with tagged custom types")
		Fly.Run("set-pipeline", "-n", "-c", "pipelines/tagged-custom-types.yml", "-p", "pipe")
		Fly.Run("unpause-pipeline", "-p", "pipe")
	})

	It("is able to be used with tagged workers", func() {
		By("running a check which uses the tagged custom resource")
		Eventually(Fly.Start("check-resource", "-r", "pipe/10m")).Should(gexec.Exit(0))

		By("triggering a build which uses the tagged custom resource")
		buildSession := Fly.Start("trigger-job", "-w", "-j", "pipe/get-10m")
		<-buildSession.Exited
		Expect(buildSession.ExitCode()).To(Equal(0))
	})
})
<|endoftext|>


<commit_before>package statsd

import (
	"os"
	"sort"
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestPipelineWithGlobalTags(t *testing.T) {
	ts, client := newClientAndTestServer(t,
		"udp",
		"localhost:8765",
		[]string{"tag1", "tag2"},
		WithTags([]string{"tag1", "tag2"}),
	)

	ts.sendAllAndAssert(t, client)
}

func TestKownEnvTags(t *testing.T) {
	entityIDEnvName := "DD_ENTITY_ID"
	ddEnvName := "DD_ENV"
	ddServiceName := "DD_SERVICE"
	ddVersionName := "DD_VERSION"

	defer func() { os.Unsetenv(entityIDEnvName) }()
	defer func() { os.Unsetenv(ddEnvName) }()
	defer func() { os.Unsetenv(ddServiceName) }()
	defer func() { os.Unsetenv(ddVersionName) }()

	os.Setenv(entityIDEnvName, "test_id")
	os.Setenv(ddEnvName, "test_env")
	os.Setenv(ddServiceName, "test_service")
	os.Setenv(ddVersionName, "test_version")

	expectedTags := []string{"dd.internal.entity_id:test_id", "env:test_env", "service:test_service", "version:test_version"}
	ts, client := newClientAndTestServer(t,
		"udp",
		"localhost:8765",
		expectedTags,
	)

	sort.Strings(client.tags)
	assert.Equal(t, expectedTags, client.tags)
	ts.sendAllAndAssert(t, client)
}

func TestKnownEnvTagsWithCustomTags(t *testing.T) {
	entityIDEnvName := "DD_ENTITY_ID"
	ddEnvName := "DD_ENV"
	ddServiceName := "DD_SERVICE"
	ddVersionName := "DD_VERSION"

	defer func() { os.Unsetenv(entityIDEnvName) }()
	defer func() { os.Unsetenv(ddEnvName) }()
	defer func() { os.Unsetenv(ddServiceName) }()
	defer func() { os.Unsetenv(ddVersionName) }()

	os.Setenv(entityIDEnvName, "test_id")
	os.Setenv(ddEnvName, "test_env")
	os.Setenv(ddServiceName, "test_service")
	os.Setenv(ddVersionName, "test_version")

	expectedTags := []string{"tag1", "tag2", "dd.internal.entity_id:test_id", "env:test_env",
		"service:test_service", "version:test_version"}
	ts, client := newClientAndTestServer(t,
		"udp",
		"localhost:8765",
		expectedTags,
		WithTags([]string{"tag1", "tag2"}),
	)

	ts.sendAllAndAssert(t, client)

	sort.Strings(expectedTags)
	sort.Strings(client.tags)
	assert.Equal(t, expectedTags, client.tags)
}

func TestKnownEnvTagsEmptyString(t *testing.T) {
	entityIDEnvName := "DD_ENTITY_ID"
	ddEnvName := "DD_ENV"
	ddServiceName := "DD_SERVICE"
	ddVersionName := "DD_VERSION"

	defer func() { os.Unsetenv(entityIDEnvName) }()
	defer func() { os.Unsetenv(ddEnvName) }()
	defer func() { os.Unsetenv(ddServiceName) }()
	defer func() { os.Unsetenv(ddVersionName) }()

	os.Setenv(entityIDEnvName, "")
	os.Setenv(ddEnvName, "")
	os.Setenv(ddServiceName, "")
	os.Setenv(ddVersionName, "")

	ts, client := newClientAndTestServer(t,
		"udp",
		"localhost:8765",
		nil,
	)

	assert.Len(t, client.tags, 0)
	ts.sendAllAndAssert(t, client)
}

func TestPipelineWithGlobalTagsAndEnv(t *testing.T) {
	orig := os.Getenv("DD_ENV")
	os.Setenv("DD_ENV", "test")
	defer os.Setenv("DD_ENV", orig)

	ts, client := newClientAndTestServer(t,
		"udp",
		"localhost:8765",
		[]string{"tag1", "tag2", "env:test"},
		WithTags([]string{"tag1", "tag2"}),
	)

	ts.sendAllAndAssert(t, client)
}

func TestTelemetryAllOptions(t *testing.T) {
	orig := os.Getenv("DD_ENV")
	os.Setenv("DD_ENV", "test")
	defer os.Setenv("DD_ENV", orig)

	ts, client := newClientAndTestServer(t,
		"udp",
		"localhost:8765",
		[]string{"tag1", "tag2", "env:test"},
		WithExtendedClientSideAggregation(),
		WithTags([]string{"tag1", "tag2"}),
		WithNamespace("test_namespace"),
	)

	ts.sendAllAndAssert(t, client)
}

type testCase struct {
	opt      []Option
	testFunc func(*testing.T, *testServer, *Client)
}

func getTestMap() map[string]testCase {
	return map[string]testCase{
		"Default": testCase{
			[]Option{},
			func(t *testing.T, ts *testServer, client *Client) {
				ts.sendAllAndAssert(t, client)
			},
		},
		"Default without aggregation": testCase{
			[]Option{
				WithoutClientSideAggregation(),
			},
			func(t *testing.T, ts *testServer, client *Client) {
				ts.sendAllAndAssert(t, client)
			},
		},
		"With namespace": testCase{
			[]Option{
				WithNamespace("test_namespace"),
			},
			func(t *testing.T, ts *testServer, client *Client) {
				ts.sendAllAndAssert(t, client)
			},
		},
		"With namespace dot": testCase{
			[]Option{
				WithNamespace("test_namespace."),
			},
			func(t *testing.T, ts *testServer, client *Client) {
				ts.sendAllAndAssert(t, client)
			},
		},
		"With max messages per payload": testCase{
			[]Option{
				WithMaxMessagesPerPayload(5),
				WithWorkersCount(1),
			},
			func(t *testing.T, ts *testServer, client *Client) {
				ts.sendAllAndAssert(t, client)
				// We send 4 non aggregated metrics, 1 service_check and 1 event. So 2 reads (5 items per
				// payload). Then we flush the aggregator that will send 5 metrics, so 1 read. Finally,
				// the telemetry is 18 metrics flushed at a different time so 4 more payload for a
				// total of 8 reads on the network
				ts.assertNbRead(t, 8)
			},
		},
		"With max messages per payload + WithoutClientSideAggregation": testCase{
			[]Option{
				WithMaxMessagesPerPayload(5),
				WithoutClientSideAggregation(),
				WithWorkersCount(1),
			},
			func(t *testing.T, ts *testServer, client *Client) {
				ts.sendAllAndAssert(t, client)
				// We send 9 non aggregated metrics, 1 service_check and 1 event. So 3 reads (5 items
				// per payload). Then the telemetry is 18 metrics flushed at a different time so 4 more
				// payload for a total of 8 reads on the network
				ts.assertNbRead(t, 7)
			},
		},
		"ChannelMode without client side aggregation": testCase{
			[]Option{
				WithoutClientSideAggregation(),
				WithChannelMode(),
			},
			func(t *testing.T, ts *testServer, client *Client) {
				ts.sendAllAndAssert(t, client)
			},
		},
		"Basic client side aggregation": testCase{
			[]Option{},
			func(t *testing.T, ts *testServer, client *Client) {
				expectedMetrics := ts.sendAllMetricsForBasicAggregation(client)
				ts.assert(t, client, expectedMetrics)
			},
		},
		"Extended client side aggregation": testCase{
			[]Option{
				WithExtendedClientSideAggregation(),
			},
			func(t *testing.T, ts *testServer, client *Client) {
				expectedMetrics := ts.sendAllMetricsForExtendedAggregation(client)
				ts.assert(t, client, expectedMetrics)
			},
		},
		"Basic client side aggregation + ChannelMode": testCase{
			[]Option{
				WithChannelMode(),
			},
			func(t *testing.T, ts *testServer, client *Client) {
				expectedMetrics := ts.sendAllMetricsForBasicAggregation(client)
				ts.assert(t, client, expectedMetrics)
			},
		},
		"Extended client side aggregation + ChannelMode": testCase{
			[]Option{
				WithExtendedClientSideAggregation(),
				WithChannelMode(),
			},
			func(t *testing.T, ts *testServer, client *Client) {
				expectedMetrics := ts.sendAllMetricsForExtendedAggregation(client)
				ts.assert(t, client, expectedMetrics)
			},
		},
	}
}

func TestFullPipelineUDP(t *testing.T) {
	for testName, c := range getTestMap() {
		t.Run(testName, func(t *testing.T) {
			ts, client := newClientAndTestServer(t,
				"udp",
				"localhost:8765",
				nil,
				c.opt...,
			)
			c.testFunc(t, ts, client)
		})
	}
}
<commit_msg>Fix flakiness for 'WithMaxMessagesPerPayload' test<commit_after>package statsd

import (
	"os"
	"sort"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestPipelineWithGlobalTags(t *testing.T) {
	ts, client := newClientAndTestServer(t,
		"udp",
		"localhost:8765",
		[]string{"tag1", "tag2"},
		WithTags([]string{"tag1", "tag2"}),
	)

	ts.sendAllAndAssert(t, client)
}

func TestKownEnvTags(t *testing.T) {
	entityIDEnvName := "DD_ENTITY_ID"
	ddEnvName := "DD_ENV"
	ddServiceName := "DD_SERVICE"
	ddVersionName := "DD_VERSION"

	defer func() { os.Unsetenv(entityIDEnvName) }()
	defer func() { os.Unsetenv(ddEnvName) }()
	defer func() { os.Unsetenv(ddServiceName) }()
	defer func() { os.Unsetenv(ddVersionName) }()

	os.Setenv(entityIDEnvName, "test_id")
	os.Setenv(ddEnvName, "test_env")
	os.Setenv(ddServiceName, "test_service")
	os.Setenv(ddVersionName, "test_version")

	expectedTags := []string{"dd.internal.entity_id:test_id", "env:test_env", "service:test_service", "version:test_version"}
	ts, client := newClientAndTestServer(t,
		"udp",
		"localhost:8765",
		expectedTags,
	)

	sort.Strings(client.tags)
	assert.Equal(t, expectedTags, client.tags)
	ts.sendAllAndAssert(t, client)
}

func TestKnownEnvTagsWithCustomTags(t *testing.T) {
	entityIDEnvName := "DD_ENTITY_ID"
	ddEnvName := "DD_ENV"
	ddServiceName := "DD_SERVICE"
	ddVersionName := "DD_VERSION"

	defer func() { os.Unsetenv(entityIDEnvName) }()
	defer func() { os.Unsetenv(ddEnvName) }()
	defer func() { os.Unsetenv(ddServiceName) }()
	defer func() { os.Unsetenv(ddVersionName) }()

	os.Setenv(entityIDEnvName, "test_id")
	os.Setenv(ddEnvName, "test_env")
	os.Setenv(ddServiceName, "test_service")
	os.Setenv(ddVersionName, "test_version")

	expectedTags := []string{"tag1", "tag2", "dd.internal.entity_id:test_id", "env:test_env",
		"service:test_service", "version:test_version"}
	ts, client := newClientAndTestServer(t,
		"udp",
		"localhost:8765",
		expectedTags,
		WithTags([]string{"tag1", "tag2"}),
	)

	ts.sendAllAndAssert(t, client)

	sort.Strings(expectedTags)
	sort.Strings(client.tags)
	assert.Equal(t, expectedTags, client.tags)
}

func TestKnownEnvTagsEmptyString(t *testing.T) {
	entityIDEnvName := "DD_ENTITY_ID"
	ddEnvName := "DD_ENV"
	ddServiceName := "DD_SERVICE"
	ddVersionName := "DD_VERSION"

	defer func() { os.Unsetenv(entityIDEnvName) }()
	defer func() { os.Unsetenv(ddEnvName) }()
	defer func() { os.Unsetenv(ddServiceName) }()
	defer func() { os.Unsetenv(ddVersionName) }()

	os.Setenv(entityIDEnvName, "")
	os.Setenv(ddEnvName, "")
	os.Setenv(ddServiceName, "")
	os.Setenv(ddVersionName, "")

	ts, client := newClientAndTestServer(t,
		"udp",
		"localhost:8765",
		nil,
	)

	assert.Len(t, client.tags, 0)
	ts.sendAllAndAssert(t, client)
}

func TestPipelineWithGlobalTagsAndEnv(t *testing.T) {
	orig := os.Getenv("DD_ENV")
	os.Setenv("DD_ENV", "test")
	defer os.Setenv("DD_ENV", orig)

	ts, client := newClientAndTestServer(t,
		"udp",
		"localhost:8765",
		[]string{"tag1", "tag2", "env:test"},
		WithTags([]string{"tag1", "tag2"}),
	)

	ts.sendAllAndAssert(t, client)
}

func TestTelemetryAllOptions(t *testing.T) {
	orig := os.Getenv("DD_ENV")
	os.Setenv("DD_ENV", "test")
	defer os.Setenv("DD_ENV", orig)

	ts, client := newClientAndTestServer(t,
		"udp",
		"localhost:8765",
		[]string{"tag1", "tag2", "env:test"},
		WithExtendedClientSideAggregation(),
		WithTags([]string{"tag1", "tag2"}),
		WithNamespace("test_namespace"),
	)

	ts.sendAllAndAssert(t, client)
}

type testCase struct {
	opt      []Option
	testFunc func(*testing.T, *testServer, *Client)
}

func getTestMap() map[string]testCase {
	return map[string]testCase{
		"Default": testCase{
			[]Option{},
			func(t *testing.T, ts *testServer, client *Client) {
				ts.sendAllAndAssert(t, client)
			},
		},
		"Default without aggregation": testCase{
			[]Option{
				WithoutClientSideAggregation(),
			},
			func(t *testing.T, ts *testServer, client *Client) {
				ts.sendAllAndAssert(t, client)
			},
		},
		"With namespace": testCase{
			[]Option{
				WithNamespace("test_namespace"),
			},
			func(t *testing.T, ts *testServer, client *Client) {
				ts.sendAllAndAssert(t, client)
			},
		},
		"With namespace dot": testCase{
			[]Option{
				WithNamespace("test_namespace."),
			},
			func(t *testing.T, ts *testServer, client *Client) {
				ts.sendAllAndAssert(t, client)
			},
		},
		"With max messages per payload": testCase{
			[]Option{
				WithMaxMessagesPerPayload(5),
				// Make sure we hit the maxMessagesPerPayload before hitting the flush timeout
				WithBufferFlushInterval(1 * time.Second),
				WithWorkersCount(1),
			},
			func(t *testing.T, ts *testServer, client *Client) {
				ts.sendAllAndAssert(t, client)
				// We send 4 non aggregated metrics, 1 service_check and 1 event. So 2 reads (5 items per
				// payload). Then we flush the aggregator that will send 5 metrics, so 1 read. Finally,
				// the telemetry is 18 metrics flushed at a different time so 4 more payload for a
				// total of 8 reads on the network
				ts.assertNbRead(t, 8)
			},
		},
		"With max messages per payload + WithoutClientSideAggregation": testCase{
			[]Option{
				WithMaxMessagesPerPayload(5),
				// Make sure we hit the maxMessagesPerPayload before hitting the flush timeout
				WithBufferFlushInterval(1 * time.Second),
				WithoutClientSideAggregation(),
				WithWorkersCount(1),
			},
			func(t *testing.T, ts *testServer, client *Client) {
				ts.sendAllAndAssert(t, client)
				// We send 9 non aggregated metrics, 1 service_check and 1 event. So 3 reads (5 items
				// per payload). Then the telemetry is 18 metrics flushed at a different time so 4 more
				// payload for a total of 8 reads on the network
				ts.assertNbRead(t, 7)
			},
		},
		"ChannelMode without client side aggregation": testCase{
			[]Option{
				WithoutClientSideAggregation(),
				WithChannelMode(),
			},
			func(t *testing.T, ts *testServer, client *Client) {
				ts.sendAllAndAssert(t, client)
			},
		},
		"Basic client side aggregation": testCase{
			[]Option{},
			func(t *testing.T, ts *testServer, client *Client) {
				expectedMetrics := ts.sendAllMetricsForBasicAggregation(client)
				ts.assert(t, client, expectedMetrics)
			},
		},
		"Extended client side aggregation": testCase{
			[]Option{
				WithExtendedClientSideAggregation(),
			},
			func(t *testing.T, ts *testServer, client *Client) {
				expectedMetrics := ts.sendAllMetricsForExtendedAggregation(client)
				ts.assert(t, client, expectedMetrics)
			},
		},
		"Basic client side aggregation + ChannelMode": testCase{
			[]Option{
				WithChannelMode(),
			},
			func(t *testing.T, ts *testServer, client *Client) {
				expectedMetrics := ts.sendAllMetricsForBasicAggregation(client)
				ts.assert(t, client, expectedMetrics)
			},
		},
		"Extended client side aggregation + ChannelMode": testCase{
			[]Option{
				WithExtendedClientSideAggregation(),
				WithChannelMode(),
			},
			func(t *testing.T, ts *testServer, client *Client) {
				expectedMetrics := ts.sendAllMetricsForExtendedAggregation(client)
				ts.assert(t, client, expectedMetrics)
			},
		},
	}
}

func TestFullPipelineUDP(t *testing.T) {
	for testName, c := range getTestMap() {
		t.Run(testName, func(t *testing.T) {
			ts, client := newClientAndTestServer(t,
				"udp",
				"localhost:8765",
				nil,
				c.opt...,
			)
			c.testFunc(t, ts, client)
		})
	}
}
<|endoftext|>


<commit_before>package graph

import (
	"fmt"
	"regexp"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/emirpasic/gods/maps/hashmap"
	"github.com/gin-gonic/gin"
	cmodel "github.com/open-falcon/falcon-plus/common/model"
	h "github.com/open-falcon/falcon-plus/modules/api/app/helper"
	m "github.com/open-falcon/falcon-plus/modules/api/app/model/graph"
	u "github.com/open-falcon/falcon-plus/modules/api/app/utils"
)

type APIGrafanaMainQueryInputs struct {
	Limit int    `json:"limit"  form:"limit"`
	Query string `json:"query"  form:"query"`
}

type APIGrafanaMainQueryOutputs struct {
	Expandable bool   `json:"expandable"`
	Text       string `json:"text"`
}

//for return a host list for api test
func repsonseDefault(limit int) (result []APIGrafanaMainQueryOutputs) {
	result = []APIGrafanaMainQueryOutputs{}
	//for get right table name
	enpsHelp := m.Endpoint{}
	enps := []m.Endpoint{}
	db.Graph.Table(enpsHelp.TableName()).Limit(limit).Scan(&enps)
	for _, h := range enps {
		result = append(result, APIGrafanaMainQueryOutputs{
			Expandable: true,
			Text:       h.Endpoint,
		})
	}
	return
}

//for find host list & grafana template searching, regexp support
func responseHostsRegexp(limit int, regexpKey string) (result []APIGrafanaMainQueryOutputs) {
	result = []APIGrafanaMainQueryOutputs{}
	//for get right table name
	enpsHelp := m.Endpoint{}
	enps := []m.Endpoint{}
	db.Graph.Table(enpsHelp.TableName()).Where("endpoint regexp ?", regexpKey).Limit(limit).Scan(&enps)
	for _, h := range enps {
		result = append(result, APIGrafanaMainQueryOutputs{
			Expandable: true,
			Text:       h.Endpoint,
		})
	}
	return
}

//for resolve mixed query with endpoint & counter of query string
func cutEndpointCounterHelp(regexpKey string) (hosts []string, counter string) {
	r, _ := regexp.Compile("^{?([^#}]+)}?#(.+)")
	matchedList := r.FindAllStringSubmatch(regexpKey, 1)
	if len(matchedList) != 0 {
		if len(matchedList[0]) > 1 {
			//get hosts
			hostsTmp := matchedList[0][1]
			counterTmp := matchedList[0][2]
			hosts = strings.Split(hostsTmp, ",")
			counter = strings.Replace(counterTmp, "#", "\\.", -1)
		}
	} else {
		log.Errorf("grafana query inputs error: %v", regexpKey)
	}
	return
}

func expandableChecking(counter string, counterSearchKeyWord string) (expsub string, needexp bool) {
	re := regexp.MustCompile("(\\.\\+|\\.\\*)\\s*$")
	counterSearchKeyWord = re.ReplaceAllString(counterSearchKeyWord, "")
	counterSearchKeyWord = strings.Replace(counterSearchKeyWord, "\\.", ".", -1)
	expCheck := strings.Replace(counter, counterSearchKeyWord, "", -1)
	if expCheck == "" {
		needexp = false
		expsub = expCheck
	} else {
		needexp = true
		re = regexp.MustCompile("^\\.")
		expsubArr := strings.Split(re.ReplaceAllString(expCheck, ""), ".")
		switch len(expsubArr) {
		case 0:
			expsub = ""
		case 1:
			expsub = expsubArr[0]
			needexp = false
		default:
			expsub = expsubArr[0]
			needexp = true
		}
	}
	return
}

/* add additional items (ex. $ & %)
   $ means metric is stop on here. no need expand any more.
   % means a wirecard string.
   also clean defecate metrics
*/
func addAddItionalItems(items []APIGrafanaMainQueryOutputs, regexpKey string) (result []APIGrafanaMainQueryOutputs) {
	flag := false
	mapset := hashmap.New()
	for _, i := range items {
		if !i.Expandable {
			flag = true
		}
		if val, exist := mapset.Get(i.Text); exist {
			if val != i.Expandable && i.Expandable {
				mapset.Put(i.Text, i.Expandable)
			}
		} else {
			mapset.Put(i.Text, i.Expandable)
		}
	}
	result = make([]APIGrafanaMainQueryOutputs, mapset.Size())
	for idx, ctmp := range mapset.Keys() {
		c := ctmp.(string)
		val, _ := mapset.Get(c)
		result[idx] = APIGrafanaMainQueryOutputs{
			Text:       c,
			Expandable: val.(bool),
		}
	}
	if flag {
		result = append(result, APIGrafanaMainQueryOutputs{
			Text:       "$",
			Expandable: false,
		})
	}
	if len(strings.Split(regexpKey, "\\.")) > 0 {
		result = append(result, APIGrafanaMainQueryOutputs{
			Text:       "%",
			Expandable: false,
		})
	}
	return
}

func findEndpointIdByEndpointList(hosts []string) []int64 {
	//for get right table name
	enpsHelp := m.Endpoint{}
	enps := []m.Endpoint{}
	hostStr, _ := u.ArrStringsToString(hosts)
	log.Debugf("hostStr: %v", hostStr)
	log.Debugf("endpoint in (%s)", hostStr)
	db.Graph.Table(enpsHelp.TableName()).Where(fmt.Sprintf("endpoint in (%s)", hostStr)).Scan(&enps)
	hostIds := make([]int64, len(enps))
	for indx, h := range enps {
		hostIds[indx] = int64(h.ID)
	}
	return hostIds
}

//for reture counter list of endpoints
func responseCounterRegexp(regexpKey string) (result []APIGrafanaMainQueryOutputs) {
	result = []APIGrafanaMainQueryOutputs{}
	hosts, counter := cutEndpointCounterHelp(regexpKey)
	if len(hosts) == 0 || counter == "" {
		return
	}
	hostIds := findEndpointIdByEndpointList(hosts)
	//if not any endpoint matched
	if len(hostIds) == 0 {
		return
	}
	idConcact, _ := u.ArrInt64ToString(hostIds)
	//for get right table name
	countHelp := m.EndpointCounter{}
	counters := []m.EndpointCounter{}
	db.Graph.Table(countHelp.TableName()).Where(fmt.Sprintf("endpoint_id IN (%s) AND counter regexp '%s'", idConcact, counter)).Scan(&counters)
	//if not any counter matched
	if len(counters) == 0 {
		return
	}
	for _, c := range counters {
		expsub, needexp := expandableChecking(c.Counter, counter)
		result = append(result, APIGrafanaMainQueryOutputs{
			Text:       expsub,
			Expandable: needexp,
		})
	}
	result = addAddItionalItems(result, regexpKey)
	return
}

func GrafanaMainQuery(c *gin.Context) {
	inputs := APIGrafanaMainQueryInputs{}
	inputs.Limit = 1000
	inputs.Query = "!N!"
	if err := c.Bind(&inputs); err != nil {
		h.JSONR(c, badstatus, err.Error())
		return
	}
	log.Debugf("got query string: %s", inputs.Query)
	output := []APIGrafanaMainQueryOutputs{}
	if inputs.Query == "!N!" {
		output = repsonseDefault(inputs.Limit)
	} else if !strings.Contains(inputs.Query, "#") {
		output = responseHostsRegexp(inputs.Limit, inputs.Query)
	} else if strings.Contains(inputs.Query, "#") && !strings.Contains(inputs.Query, "#select metric") {
		output = responseCounterRegexp(inputs.Query)
	}
	c.JSON(200, output)
	return
}

type APIGrafanaRenderInput struct {
	Target        []string `json:"target" form:"target"  binding:"required"`
	From          int64    `json:"from" form:"from" binding:"required"`
	Until         int64    `json:"until" form:"until" binding:"required"`
	Format        string   `json:"format" form:"format"`
	MaxDataPoints int64    `json:"maxDataPoints" form:"maxDataPoints"`
	Step          int      `json:"step" form:"step"`
	ConsolFun     string   `json:"consolFun" form:"consolFun"`
}

func GrafanaRender(c *gin.Context) {
	inputs := APIGrafanaRenderInput{}
	//set default step is 60
	inputs.Step = 60
	inputs.ConsolFun = "AVERAGE"
	if err := c.Bind(&inputs); err != nil {
		h.JSONR(c, badstatus, err.Error())
		return
	}
	respList := []*cmodel.GraphQueryResponse{}
	for _, target := range inputs.Target {
		hosts, counter := cutEndpointCounterHelp(target)
		//clean characters
		log.Debug(counter)
		re := regexp.MustCompile("\\\\.\\$\\s*$")
		flag := re.MatchString(counter)
		counter = re.ReplaceAllString(counter, "")
		counter = strings.Replace(counter, "\\.%", ".+", -1)
		ecHelp := m.EndpointCounter{}
		counters := []m.EndpointCounter{}
		hostIds := findEndpointIdByEndpointList(hosts)
		if flag {
			db.Graph.Table(ecHelp.TableName()).Select("distinct counter").Where(fmt.Sprintf("endpoint_id IN (%s) AND counter = '%s'", u.ArrInt64ToStringMust(hostIds), counter)).Scan(&counters)
		} else {
			db.Graph.Table(ecHelp.TableName()).Select("distinct counter").Where(fmt.Sprintf("endpoint_id IN (%s) AND counter regexp '%s'", u.ArrInt64ToStringMust(hostIds), counter)).Scan(&counters)
		}
		if len(counters) == 0 {
			h.JSONR(c, []interface{}{})
			return
		}
		counterArr := make([]string, len(counters))
		for indx, c := range counters {
			counterArr[indx] = c.Counter
		}
		for _, host := range hosts {
			for _, c := range counterArr {
				resp, err := fetchData(host, c, inputs.ConsolFun, inputs.From, inputs.Until, inputs.Step)
				if err != nil {
					log.Debugf("query graph got error with: %v", inputs)
				} else {
					respList = append(respList, resp)
				}
			}
		}
	}
	c.JSON(200, respList)
	return
}
<commit_msg>API：修复当grafana graph有多个查询时，因为其中一个未匹配到导致其他正常的查询也无法渲染视图<commit_after>package graph

import (
	"fmt"
	"regexp"
	"strings"

	log "github.com/Sirupsen/logrus"
	"github.com/emirpasic/gods/maps/hashmap"
	"github.com/gin-gonic/gin"
	cmodel "github.com/open-falcon/falcon-plus/common/model"
	h "github.com/open-falcon/falcon-plus/modules/api/app/helper"
	m "github.com/open-falcon/falcon-plus/modules/api/app/model/graph"
	u "github.com/open-falcon/falcon-plus/modules/api/app/utils"
)

type APIGrafanaMainQueryInputs struct {
	Limit int    `json:"limit"  form:"limit"`
	Query string `json:"query"  form:"query"`
}

type APIGrafanaMainQueryOutputs struct {
	Expandable bool   `json:"expandable"`
	Text       string `json:"text"`
}

//for return a host list for api test
func repsonseDefault(limit int) (result []APIGrafanaMainQueryOutputs) {
	result = []APIGrafanaMainQueryOutputs{}
	//for get right table name
	enpsHelp := m.Endpoint{}
	enps := []m.Endpoint{}
	db.Graph.Table(enpsHelp.TableName()).Limit(limit).Scan(&enps)
	for _, h := range enps {
		result = append(result, APIGrafanaMainQueryOutputs{
			Expandable: true,
			Text:       h.Endpoint,
		})
	}
	return
}

//for find host list & grafana template searching, regexp support
func responseHostsRegexp(limit int, regexpKey string) (result []APIGrafanaMainQueryOutputs) {
	result = []APIGrafanaMainQueryOutputs{}
	//for get right table name
	enpsHelp := m.Endpoint{}
	enps := []m.Endpoint{}
	db.Graph.Table(enpsHelp.TableName()).Where("endpoint regexp ?", regexpKey).Limit(limit).Scan(&enps)
	for _, h := range enps {
		result = append(result, APIGrafanaMainQueryOutputs{
			Expandable: true,
			Text:       h.Endpoint,
		})
	}
	return
}

//for resolve mixed query with endpoint & counter of query string
func cutEndpointCounterHelp(regexpKey string) (hosts []string, counter string) {
	r, _ := regexp.Compile("^{?([^#}]+)}?#(.+)")
	matchedList := r.FindAllStringSubmatch(regexpKey, 1)
	if len(matchedList) != 0 {
		if len(matchedList[0]) > 1 {
			//get hosts
			hostsTmp := matchedList[0][1]
			counterTmp := matchedList[0][2]
			hosts = strings.Split(hostsTmp, ",")
			counter = strings.Replace(counterTmp, "#", "\\.", -1)
		}
	} else {
		log.Errorf("grafana query inputs error: %v", regexpKey)
	}
	return
}

func expandableChecking(counter string, counterSearchKeyWord string) (expsub string, needexp bool) {
	re := regexp.MustCompile("(\\.\\+|\\.\\*)\\s*$")
	counterSearchKeyWord = re.ReplaceAllString(counterSearchKeyWord, "")
	counterSearchKeyWord = strings.Replace(counterSearchKeyWord, "\\.", ".", -1)
	expCheck := strings.Replace(counter, counterSearchKeyWord, "", -1)
	if expCheck == "" {
		needexp = false
		expsub = expCheck
	} else {
		needexp = true
		re = regexp.MustCompile("^\\.")
		expsubArr := strings.Split(re.ReplaceAllString(expCheck, ""), ".")
		switch len(expsubArr) {
		case 0:
			expsub = ""
		case 1:
			expsub = expsubArr[0]
			needexp = false
		default:
			expsub = expsubArr[0]
			needexp = true
		}
	}
	return
}

/* add additional items (ex. $ & %)
   $ means metric is stop on here. no need expand any more.
   % means a wirecard string.
   also clean defecate metrics
*/
func addAddItionalItems(items []APIGrafanaMainQueryOutputs, regexpKey string) (result []APIGrafanaMainQueryOutputs) {
	flag := false
	mapset := hashmap.New()
	for _, i := range items {
		if !i.Expandable {
			flag = true
		}
		if val, exist := mapset.Get(i.Text); exist {
			if val != i.Expandable && i.Expandable {
				mapset.Put(i.Text, i.Expandable)
			}
		} else {
			mapset.Put(i.Text, i.Expandable)
		}
	}
	result = make([]APIGrafanaMainQueryOutputs, mapset.Size())
	for idx, ctmp := range mapset.Keys() {
		c := ctmp.(string)
		val, _ := mapset.Get(c)
		result[idx] = APIGrafanaMainQueryOutputs{
			Text:       c,
			Expandable: val.(bool),
		}
	}
	if flag {
		result = append(result, APIGrafanaMainQueryOutputs{
			Text:       "$",
			Expandable: false,
		})
	}
	if len(strings.Split(regexpKey, "\\.")) > 0 {
		result = append(result, APIGrafanaMainQueryOutputs{
			Text:       "%",
			Expandable: false,
		})
	}
	return
}

func findEndpointIdByEndpointList(hosts []string) []int64 {
	//for get right table name
	enpsHelp := m.Endpoint{}
	enps := []m.Endpoint{}
	hostStr, _ := u.ArrStringsToString(hosts)
	log.Debugf("hostStr: %v", hostStr)
	log.Debugf("endpoint in (%s)", hostStr)
	db.Graph.Table(enpsHelp.TableName()).Where(fmt.Sprintf("endpoint in (%s)", hostStr)).Scan(&enps)
	hostIds := make([]int64, len(enps))
	for indx, h := range enps {
		hostIds[indx] = int64(h.ID)
	}
	return hostIds
}

//for reture counter list of endpoints
func responseCounterRegexp(regexpKey string) (result []APIGrafanaMainQueryOutputs) {
	result = []APIGrafanaMainQueryOutputs{}
	hosts, counter := cutEndpointCounterHelp(regexpKey)
	if len(hosts) == 0 || counter == "" {
		return
	}
	hostIds := findEndpointIdByEndpointList(hosts)
	//if not any endpoint matched
	if len(hostIds) == 0 {
		return
	}
	idConcact, _ := u.ArrInt64ToString(hostIds)
	//for get right table name
	countHelp := m.EndpointCounter{}
	counters := []m.EndpointCounter{}
	db.Graph.Table(countHelp.TableName()).Where(fmt.Sprintf("endpoint_id IN (%s) AND counter regexp '%s'", idConcact, counter)).Scan(&counters)
	//if not any counter matched
	if len(counters) == 0 {
		return
	}
	for _, c := range counters {
		expsub, needexp := expandableChecking(c.Counter, counter)
		result = append(result, APIGrafanaMainQueryOutputs{
			Text:       expsub,
			Expandable: needexp,
		})
	}
	result = addAddItionalItems(result, regexpKey)
	return
}

func GrafanaMainQuery(c *gin.Context) {
	inputs := APIGrafanaMainQueryInputs{}
	inputs.Limit = 1000
	inputs.Query = "!N!"
	if err := c.Bind(&inputs); err != nil {
		h.JSONR(c, badstatus, err.Error())
		return
	}
	log.Debugf("got query string: %s", inputs.Query)
	output := []APIGrafanaMainQueryOutputs{}
	if inputs.Query == "!N!" {
		output = repsonseDefault(inputs.Limit)
	} else if !strings.Contains(inputs.Query, "#") {
		output = responseHostsRegexp(inputs.Limit, inputs.Query)
	} else if strings.Contains(inputs.Query, "#") && !strings.Contains(inputs.Query, "#select metric") {
		output = responseCounterRegexp(inputs.Query)
	}
	c.JSON(200, output)
	return
}

type APIGrafanaRenderInput struct {
	Target        []string `json:"target" form:"target"  binding:"required"`
	From          int64    `json:"from" form:"from" binding:"required"`
	Until         int64    `json:"until" form:"until" binding:"required"`
	Format        string   `json:"format" form:"format"`
	MaxDataPoints int64    `json:"maxDataPoints" form:"maxDataPoints"`
	Step          int      `json:"step" form:"step"`
	ConsolFun     string   `json:"consolFun" form:"consolFun"`
}

func GrafanaRender(c *gin.Context) {
	inputs := APIGrafanaRenderInput{}
	//set default step is 60
	inputs.Step = 60
	inputs.ConsolFun = "AVERAGE"
	if err := c.Bind(&inputs); err != nil {
		h.JSONR(c, badstatus, err.Error())
		return
	}
	respList := []*cmodel.GraphQueryResponse{}
	for _, target := range inputs.Target {
		hosts, counter := cutEndpointCounterHelp(target)
		//clean characters
		log.Debug(counter)
		re := regexp.MustCompile("\\\\.\\$\\s*$")
		flag := re.MatchString(counter)
		counter = re.ReplaceAllString(counter, "")
		counter = strings.Replace(counter, "\\.%", ".+", -1)
		ecHelp := m.EndpointCounter{}
		counters := []m.EndpointCounter{}
		hostIds := findEndpointIdByEndpointList(hosts)
		if flag {
			db.Graph.Table(ecHelp.TableName()).Select("distinct counter").Where(fmt.Sprintf("endpoint_id IN (%s) AND counter = '%s'", u.ArrInt64ToStringMust(hostIds), counter)).Scan(&counters)
		} else {
			db.Graph.Table(ecHelp.TableName()).Select("distinct counter").Where(fmt.Sprintf("endpoint_id IN (%s) AND counter regexp '%s'", u.ArrInt64ToStringMust(hostIds), counter)).Scan(&counters)
		}
		if len(counters) == 0 {
			// 没有匹配到的继续执行，避免当grafana graph有多个查询时，其他正常的查询也无法渲染视图
			continue
		}
		counterArr := make([]string, len(counters))
		for indx, c := range counters {
			counterArr[indx] = c.Counter
		}
		for _, host := range hosts {
			for _, c := range counterArr {
				resp, err := fetchData(host, c, inputs.ConsolFun, inputs.From, inputs.Until, inputs.Step)
				if err != nil {
					log.Debugf("query graph got error with: %v", inputs)
				} else {
					respList = append(respList, resp)
				}
			}
		}
	}
	c.JSON(200, respList)
	return
}
<|endoftext|>


<commit_before>// +build !nofuse

package readonly

import (
	"bytes"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"path"
	"sync"
	"testing"

	core "github.com/ipfs/go-ipfs/core"
	coreapi "github.com/ipfs/go-ipfs/core/coreapi"
	iface "github.com/ipfs/go-ipfs/core/coreapi/interface"
	coremock "github.com/ipfs/go-ipfs/core/mock"

	u "gx/ipfs/QmNohiVssaPw3KVLZik59DBVGTSm2dGvYT9eoXt5DQ36Yz/go-ipfs-util"
	ci "gx/ipfs/QmNvHv84aH2qZafDuSdKJCQ1cvPZ1kmQmyD4YtzjUHuk9v/go-testutil/ci"
	importer "gx/ipfs/QmQXze9tG878pa4Euya4rrDpyTNX3kQe4dhCaBzBozGgpe/go-unixfs/importer"
	uio "gx/ipfs/QmQXze9tG878pa4Euya4rrDpyTNX3kQe4dhCaBzBozGgpe/go-unixfs/io"
	chunker "gx/ipfs/QmR4QQVkBZsZENRjYFVi8dEtPL3daZRNKk24m4r6WKJHNm/go-ipfs-chunker"
	fstest "gx/ipfs/QmSJBsmLP1XMjv8hxYg2rUMdPDB7YUpyBo9idjrJ6Cmq6F/fuse/fs/fstestutil"
	dag "gx/ipfs/QmTQdH4848iTVCJmKXYyRiK72HufWTLYQQ8iN3JaQ8K1Hq/go-merkledag"
	files "gx/ipfs/QmXWZCd8jfaHmt4UDSnjKmGcrQMw95bDGWqEeVLVJjoANX/go-ipfs-files"
	ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format"
)

func maybeSkipFuseTests(t *testing.T) {
	if ci.NoFuse() {
		t.Skip("Skipping FUSE tests")
	}
}

func randObj(t *testing.T, nd *core.IpfsNode, size int64) (ipld.Node, []byte) {
	buf := make([]byte, size)
	u.NewTimeSeededRand().Read(buf)
	read := bytes.NewReader(buf)
	obj, err := importer.BuildTrickleDagFromReader(nd.DAG, chunker.DefaultSplitter(read))
	if err != nil {
		t.Fatal(err)
	}

	return obj, buf
}

func setupIpfsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.Mount) {
	maybeSkipFuseTests(t)

	var err error
	if node == nil {
		node, err = coremock.NewMockNode()
		if err != nil {
			t.Fatal(err)
		}
	}

	fs := NewFileSystem(node)
	mnt, err := fstest.MountedT(t, fs, nil)
	if err != nil {
		t.Fatal(err)
	}

	return node, mnt
}

// Test writing an object and reading it back through fuse
func TestIpfsBasicRead(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	nd, mnt := setupIpfsTest(t, nil)
	defer mnt.Close()

	fi, data := randObj(t, nd, 10000)
	k := fi.Cid()
	fname := path.Join(mnt.Dir, k.String())
	rbuf, err := ioutil.ReadFile(fname)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(rbuf, data) {
		t.Fatal("Incorrect Read!")
	}
}

func getPaths(t *testing.T, ipfs *core.IpfsNode, name string, n *dag.ProtoNode) []string {
	if len(n.Links()) == 0 {
		return []string{name}
	}
	var out []string
	for _, lnk := range n.Links() {
		child, err := lnk.GetNode(ipfs.Context(), ipfs.DAG)
		if err != nil {
			t.Fatal(err)
		}

		childpb, ok := child.(*dag.ProtoNode)
		if !ok {
			t.Fatal(dag.ErrNotProtobuf)
		}

		sub := getPaths(t, ipfs, path.Join(name, lnk.Name), childpb)
		out = append(out, sub...)
	}
	return out
}

// Perform a large number of concurrent reads to stress the system
func TestIpfsStressRead(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	nd, mnt := setupIpfsTest(t, nil)
	defer mnt.Close()

	api, err := coreapi.NewCoreAPI(nd)
	if err != nil {
		t.Fatal(err)
	}

	var nodes []ipld.Node
	var paths []string

	nobj := 50
	ndiriter := 50

	// Make a bunch of objects
	for i := 0; i < nobj; i++ {
		fi, _ := randObj(t, nd, rand.Int63n(50000))
		nodes = append(nodes, fi)
		paths = append(paths, fi.Cid().String())
	}

	// Now make a bunch of dirs
	for i := 0; i < ndiriter; i++ {
		db := uio.NewDirectory(nd.DAG)
		for j := 0; j < 1+rand.Intn(10); j++ {
			name := fmt.Sprintf("child%d", j)

			err := db.AddChild(nd.Context(), name, nodes[rand.Intn(len(nodes))])
			if err != nil {
				t.Fatal(err)
			}
		}
		newdir, err := db.GetNode()
		if err != nil {
			t.Fatal(err)
		}

		err = nd.DAG.Add(nd.Context(), newdir)
		if err != nil {
			t.Fatal(err)
		}

		nodes = append(nodes, newdir)
		npaths := getPaths(t, nd, newdir.Cid().String(), newdir.(*dag.ProtoNode))
		paths = append(paths, npaths...)
	}

	// Now read a bunch, concurrently
	wg := sync.WaitGroup{}
	errs := make(chan error)

	for s := 0; s < 4; s++ {
		wg.Add(1)
		go func() {
			defer wg.Done()

			for i := 0; i < 2000; i++ {
				item, _ := iface.ParsePath(paths[rand.Intn(len(paths))])
				fname := path.Join(mnt.Dir, item.String())
				rbuf, err := ioutil.ReadFile(fname)
				if err != nil {
					errs <- err
				}

				read, err := api.Unixfs().Get(nd.Context(), item)
				if err != nil {
					errs <- err
				}

				data, err := ioutil.ReadAll(read.(files.File))
				if err != nil {
					errs <- err
				}

				if !bytes.Equal(rbuf, data) {
					errs <- errors.New("incorrect read")
				}
			}
		}()
	}

	go func() {
		wg.Wait()
		close(errs)
	}()

	for err := range errs {
		if err != nil {
			t.Fatal(err)
		}
	}
}

// Test writing a file and reading it back
func TestIpfsBasicDirRead(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	nd, mnt := setupIpfsTest(t, nil)
	defer mnt.Close()

	// Make a 'file'
	fi, data := randObj(t, nd, 10000)

	// Make a directory and put that file in it
	db := uio.NewDirectory(nd.DAG)
	err := db.AddChild(nd.Context(), "actual", fi)
	if err != nil {
		t.Fatal(err)
	}

	d1nd, err := db.GetNode()
	if err != nil {
		t.Fatal(err)
	}

	err = nd.DAG.Add(nd.Context(), d1nd)
	if err != nil {
		t.Fatal(err)
	}

	dirname := path.Join(mnt.Dir, d1nd.Cid().String())
	fname := path.Join(dirname, "actual")
	rbuf, err := ioutil.ReadFile(fname)
	if err != nil {
		t.Fatal(err)
	}

	dirents, err := ioutil.ReadDir(dirname)
	if err != nil {
		t.Fatal(err)
	}
	if len(dirents) != 1 {
		t.Fatal("Bad directory entry count")
	}
	if dirents[0].Name() != "actual" {
		t.Fatal("Bad directory entry")
	}

	if !bytes.Equal(rbuf, data) {
		t.Fatal("Incorrect Read!")
	}
}

// Test to make sure the filesystem reports file sizes correctly
func TestFileSizeReporting(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	nd, mnt := setupIpfsTest(t, nil)
	defer mnt.Close()

	fi, data := randObj(t, nd, 10000)
	k := fi.Cid()

	fname := path.Join(mnt.Dir, k.String())

	finfo, err := os.Stat(fname)
	if err != nil {
		t.Fatal(err)
	}

	if finfo.Size() != int64(len(data)) {
		t.Fatal("Read incorrect size from stat!")
	}
}
<commit_msg>Fixed and cleaned up TestIpfsStressRead<commit_after>// +build !nofuse

package readonly

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"path"
	"strconv"
	"strings"
	"testing"

	core "github.com/ipfs/go-ipfs/core"
	coreapi "github.com/ipfs/go-ipfs/core/coreapi"
	iface "github.com/ipfs/go-ipfs/core/coreapi/interface"
	coremock "github.com/ipfs/go-ipfs/core/mock"

	u "gx/ipfs/QmNohiVssaPw3KVLZik59DBVGTSm2dGvYT9eoXt5DQ36Yz/go-ipfs-util"
	ci "gx/ipfs/QmNvHv84aH2qZafDuSdKJCQ1cvPZ1kmQmyD4YtzjUHuk9v/go-testutil/ci"
	importer "gx/ipfs/QmQXze9tG878pa4Euya4rrDpyTNX3kQe4dhCaBzBozGgpe/go-unixfs/importer"
	uio "gx/ipfs/QmQXze9tG878pa4Euya4rrDpyTNX3kQe4dhCaBzBozGgpe/go-unixfs/io"
	chunker "gx/ipfs/QmR4QQVkBZsZENRjYFVi8dEtPL3daZRNKk24m4r6WKJHNm/go-ipfs-chunker"
	fstest "gx/ipfs/QmSJBsmLP1XMjv8hxYg2rUMdPDB7YUpyBo9idjrJ6Cmq6F/fuse/fs/fstestutil"
	dag "gx/ipfs/QmTQdH4848iTVCJmKXYyRiK72HufWTLYQQ8iN3JaQ8K1Hq/go-merkledag"
	files "gx/ipfs/QmXWZCd8jfaHmt4UDSnjKmGcrQMw95bDGWqEeVLVJjoANX/go-ipfs-files"
	ipld "gx/ipfs/QmcKKBwfz6FyQdHR2jsXrrF6XeSBXYL86anmWNewpFpoF5/go-ipld-format"
)

func maybeSkipFuseTests(t *testing.T) {
	if ci.NoFuse() {
		t.Skip("Skipping FUSE tests")
	}
}

func randObj(t *testing.T, nd *core.IpfsNode, size int64) (ipld.Node, []byte) {
	buf := make([]byte, size)
	u.NewTimeSeededRand().Read(buf)
	read := bytes.NewReader(buf)
	obj, err := importer.BuildTrickleDagFromReader(nd.DAG, chunker.DefaultSplitter(read))
	if err != nil {
		t.Fatal(err)
	}

	return obj, buf
}

func setupIpfsTest(t *testing.T, node *core.IpfsNode) (*core.IpfsNode, *fstest.Mount) {
	maybeSkipFuseTests(t)

	var err error
	if node == nil {
		node, err = coremock.NewMockNode()
		if err != nil {
			t.Fatal(err)
		}
	}

	fs := NewFileSystem(node)
	mnt, err := fstest.MountedT(t, fs, nil)
	if err != nil {
		t.Fatal(err)
	}

	return node, mnt
}

// Test writing an object and reading it back through fuse
func TestIpfsBasicRead(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	nd, mnt := setupIpfsTest(t, nil)
	defer mnt.Close()

	fi, data := randObj(t, nd, 10000)
	k := fi.Cid()
	fname := path.Join(mnt.Dir, k.String())
	rbuf, err := ioutil.ReadFile(fname)
	if err != nil {
		t.Fatal(err)
	}

	if !bytes.Equal(rbuf, data) {
		t.Fatal("Incorrect Read!")
	}
}

func getPaths(t *testing.T, ipfs *core.IpfsNode, name string, n *dag.ProtoNode) []string {
	if len(n.Links()) == 0 {
		return []string{name}
	}
	var out []string
	for _, lnk := range n.Links() {
		child, err := lnk.GetNode(ipfs.Context(), ipfs.DAG)
		if err != nil {
			t.Fatal(err)
		}

		childpb, ok := child.(*dag.ProtoNode)
		if !ok {
			t.Fatal(dag.ErrNotProtobuf)
		}

		sub := getPaths(t, ipfs, path.Join(name, lnk.Name), childpb)
		out = append(out, sub...)
	}
	return out
}

// Perform a large number of concurrent reads to stress the system
func TestIpfsStressRead(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	nd, mnt := setupIpfsTest(t, nil)
	defer mnt.Close()

	api, err := coreapi.NewCoreAPI(nd)
	if err != nil {
		t.Fatal(err)
	}

	var nodes []ipld.Node
	var paths []string

	nobj := 50
	ndiriter := 50

	// Make a bunch of objects
	for i := 0; i < nobj; i++ {
		fi, _ := randObj(t, nd, rand.Int63n(50000))
		nodes = append(nodes, fi)
		paths = append(paths, fi.Cid().String())
	}

	// Now make a bunch of dirs
	for i := 0; i < ndiriter; i++ {
		db := uio.NewDirectory(nd.DAG)
		for j := 0; j < 1+rand.Intn(10); j++ {
			name := fmt.Sprintf("child%d", j)

			err := db.AddChild(nd.Context(), name, nodes[rand.Intn(len(nodes))])
			if err != nil {
				t.Fatal(err)
			}
		}
		newdir, err := db.GetNode()
		if err != nil {
			t.Fatal(err)
		}

		err = nd.DAG.Add(nd.Context(), newdir)
		if err != nil {
			t.Fatal(err)
		}

		nodes = append(nodes, newdir)
		npaths := getPaths(t, nd, newdir.Cid().String(), newdir.(*dag.ProtoNode))
		paths = append(paths, npaths...)
	}

	t.Parallel()

	for s := 0; s < 4; s++ {
		t.Run(strconv.Itoa(s), func(t *testing.T) {
			for i := 0; i < 2000; i++ {
				item, err := iface.ParsePath(paths[rand.Intn(len(paths))])
				if err != nil {
					t.Fatal(err)
				}

				relpath := strings.Replace(item.String(), "/ipfs/", "/", 1)
				fname := path.Join(mnt.Dir, relpath)

				rbuf, err := ioutil.ReadFile(fname)
				if err != nil {
					t.Fatal(err)
				}

				//nd.Context() is never closed which leads to
				//hitting 8128 goroutine limit in go test -race mode
				ctx, cancelFunc := context.WithCancel(context.Background())

				read, err := api.Unixfs().Get(ctx, item)
				if err != nil {
					t.Fatal(err)
				}

				data, err := ioutil.ReadAll(read.(files.File))
				if err != nil {
					t.Fatal(err)
				}

				cancelFunc()

				if !bytes.Equal(rbuf, data) {
					t.Fatal("incorrect read")
				}
			}
		})
	}
}

// Test writing a file and reading it back
func TestIpfsBasicDirRead(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	nd, mnt := setupIpfsTest(t, nil)
	defer mnt.Close()

	// Make a 'file'
	fi, data := randObj(t, nd, 10000)

	// Make a directory and put that file in it
	db := uio.NewDirectory(nd.DAG)
	err := db.AddChild(nd.Context(), "actual", fi)
	if err != nil {
		t.Fatal(err)
	}

	d1nd, err := db.GetNode()
	if err != nil {
		t.Fatal(err)
	}

	err = nd.DAG.Add(nd.Context(), d1nd)
	if err != nil {
		t.Fatal(err)
	}

	dirname := path.Join(mnt.Dir, d1nd.Cid().String())
	fname := path.Join(dirname, "actual")
	rbuf, err := ioutil.ReadFile(fname)
	if err != nil {
		t.Fatal(err)
	}

	dirents, err := ioutil.ReadDir(dirname)
	if err != nil {
		t.Fatal(err)
	}
	if len(dirents) != 1 {
		t.Fatal("Bad directory entry count")
	}
	if dirents[0].Name() != "actual" {
		t.Fatal("Bad directory entry")
	}

	if !bytes.Equal(rbuf, data) {
		t.Fatal("Incorrect Read!")
	}
}

// Test to make sure the filesystem reports file sizes correctly
func TestFileSizeReporting(t *testing.T) {
	if testing.Short() {
		t.SkipNow()
	}
	nd, mnt := setupIpfsTest(t, nil)
	defer mnt.Close()

	fi, data := randObj(t, nd, 10000)
	k := fi.Cid()

	fname := path.Join(mnt.Dir, k.String())

	finfo, err := os.Stat(fname)
	if err != nil {
		t.Fatal(err)
	}

	if finfo.Size() != int64(len(data)) {
		t.Fatal("Read incorrect size from stat!")
	}
}
<|endoftext|>