<commit_before>package convert

import (
	"crypto/md5"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"github.com/suapapa/go_hangul/encoding/cp949"
	"strconv"
	"strings"
	"time"
)

func Int(val interface{}) int64 {
	if val != nil {
		switch val.(type) {
		case float64:
			return int64(val.(float64))
		case int64:
			return val.(int64)
		default:
			ret, err := strconv.ParseInt(String(val), 10, 64)
			if err != nil {
				return 0
			} else {
				return ret
			}
		}
	}
	return 0
}

func IntWith(val interface{}, defaultValue int64) int64 {
	if val != nil {
		switch val.(type) {
		case float64:
			return int64(val.(float64))
		case int64:
			return val.(int64)
		default:
			ret, err := strconv.ParseInt(String(val), 10, 64)
			if err != nil {
				return defaultValue
			} else {
				return ret
			}
		}
	}
	return defaultValue
}

func Float(val interface{}) float64 {
	if val != nil {
		switch val.(type) {
		case float64:
			return val.(float64)
		case int64:
			return float64(val.(int64))
		default:
			ret, err := strconv.ParseFloat(String(val), 64)
			if err != nil {
				return 0
			} else {
				return float64(ret)
			}
		}
	}
	return 0
}

func FloatWith(val interface{}, defaultValue float64) float64 {
	if val != nil {
		switch val.(type) {
		case float64:
			return val.(float64)
		case int64:
			return float64(val.(int64))
		default:
			ret, err := strconv.ParseFloat(String(val), 64)
			if err != nil {
				return defaultValue
			} else {
				return float64(ret)
			}
		}
	}
	return defaultValue
}

func String(val interface{}) string {
	switch val.(type) {
	case string:
		return val.(string)
	case []byte:
		return string(val.([]byte))
	case time.Time:
		t := val.(time.Time)
		return fmt.Sprintf("%4.4d-%2.2d-%2.2d %2.2d:%2.2d:%2.2d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
	case float32:
		if int64(val.(float32)*1000000) == int64(val.(float32))*1000000 {
			return fmt.Sprintf("%v", int64(val.(float32)))
		} else {
			return fmt.Sprintf("%v", val)
		}
	case float64:
		if int64(val.(float64)*1000000) == int64(val.(float64))*1000000 {
			return fmt.Sprintf("%v", int64(val.(float64)))
		} else {
			return fmt.Sprintf("%v", val)
		}
	case nil:
		return ""
	default:
		if val == nil {
			return ""
		} else {
			return fmt.Sprintf("%v", val)
		}
	}
}

func QueryString(val string) string {
	return strings.Join(strings.Split(val, "'"), "''")
}

func MD5(src string) string {
	return fmt.Sprintf("%x", md5.Sum([]byte(src)))
}

func SHA256(src string) string {
	hasher := sha256.New()
	hasher.Write([]byte(src))
	return hex.EncodeToString(hasher.Sum(nil))
}

func Time(val interface{}) *time.Time {
	if val != nil {
		v := String(val)
		if len(v) > 0 {
			if t, err := time.Parse("2006-01-02 15:04:05.000", v); err == nil && t.Year() > 0 {
				return &t
			} else if t, err := time.Parse("2006-01-02 15:04:05", v); err == nil && t.Year() > 0 {
				return &t
			} else if t, err := time.Parse("2006-01-02", v); err == nil && t.Year() > 0 {
				return &t
			} else if t, err := time.Parse("20060102150405", v); err == nil && t.Year() > 0 {
				return &t
			} else if t, err := time.Parse("20060102", v); err == nil && t.Year() > 0 {
				return &t
			} else if t, err := time.Parse("01-02-06", v); err == nil && t.Year() > 0 {
				return &t
			} else if t, err := time.Parse("01-02-2006", v); err == nil && t.Year() > 0 {
				return &t
			} else if t, err := time.Parse("15:04:05.000", v); err == nil {
				return &t
			} else if t, err := time.Parse("15:04:05", v); err == nil {
				return &t
			}
		}
	}
	return nil
}

func UTF8(ustr string) (str string) {
	defer func() {
		if r := recover(); r != nil {
			ustr = str
			return
		}
	}()

	bytes, err := cp949.From([]byte(ustr))
	if err != nil {
		str = ustr
	} else {
		str = string(bytes)
	}
	return
}

func EUCKR(str string) (ustr string) {
	defer func() {
		if r := recover(); r != nil {
			ustr = str
			return
		}
	}()

	ubytes, err := cp949.To([]byte(str))
	if err != nil {
		ustr = str
	} else {
		ustr = string(ubytes)
	}
	return
}
<commit_msg>add *time.Time<commit_after>package convert

import (
	"crypto/md5"
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"github.com/suapapa/go_hangul/encoding/cp949"
	"strconv"
	"strings"
	"time"
)

func Int(val interface{}) int64 {
	if val != nil {
		switch val.(type) {
		case float64:
			return int64(val.(float64))
		case int64:
			return val.(int64)
		default:
			ret, err := strconv.ParseInt(String(val), 10, 64)
			if err != nil {
				return 0
			} else {
				return ret
			}
		}
	}
	return 0
}

func IntWith(val interface{}, defaultValue int64) int64 {
	if val != nil {
		switch val.(type) {
		case float64:
			return int64(val.(float64))
		case int64:
			return val.(int64)
		default:
			ret, err := strconv.ParseInt(String(val), 10, 64)
			if err != nil {
				return defaultValue
			} else {
				return ret
			}
		}
	}
	return defaultValue
}

func Float(val interface{}) float64 {
	if val != nil {
		switch val.(type) {
		case float64:
			return val.(float64)
		case int64:
			return float64(val.(int64))
		default:
			ret, err := strconv.ParseFloat(String(val), 64)
			if err != nil {
				return 0
			} else {
				return float64(ret)
			}
		}
	}
	return 0
}

func FloatWith(val interface{}, defaultValue float64) float64 {
	if val != nil {
		switch val.(type) {
		case float64:
			return val.(float64)
		case int64:
			return float64(val.(int64))
		default:
			ret, err := strconv.ParseFloat(String(val), 64)
			if err != nil {
				return defaultValue
			} else {
				return float64(ret)
			}
		}
	}
	return defaultValue
}

func String(val interface{}) string {
	switch val.(type) {
	case string:
		return val.(string)
	case []byte:
		return string(val.([]byte))
	case *time.Time:
		t := val.(*time.Time)
		return fmt.Sprintf("%4.4d-%2.2d-%2.2d %2.2d:%2.2d:%2.2d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
	case time.Time:
		t := val.(time.Time)
		return fmt.Sprintf("%4.4d-%2.2d-%2.2d %2.2d:%2.2d:%2.2d", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute(), t.Second())
	case float32:
		if int64(val.(float32)*1000000) == int64(val.(float32))*1000000 {
			return fmt.Sprintf("%v", int64(val.(float32)))
		} else {
			return fmt.Sprintf("%v", val)
		}
	case float64:
		if int64(val.(float64)*1000000) == int64(val.(float64))*1000000 {
			return fmt.Sprintf("%v", int64(val.(float64)))
		} else {
			return fmt.Sprintf("%v", val)
		}
	case nil:
		return ""
	default:
		if val == nil {
			return ""
		} else {
			return fmt.Sprintf("%v", val)
		}
	}
}

func QueryString(val string) string {
	return strings.Join(strings.Split(val, "'"), "''")
}

func MD5(src string) string {
	return fmt.Sprintf("%x", md5.Sum([]byte(src)))
}

func SHA256(src string) string {
	hasher := sha256.New()
	hasher.Write([]byte(src))
	return hex.EncodeToString(hasher.Sum(nil))
}

func Time(val interface{}) *time.Time {
	if val != nil {
		v := String(val)
		if len(v) > 0 {
			if t, err := time.Parse("2006-01-02 15:04:05.000", v); err == nil && t.Year() > 0 {
				return &t
			} else if t, err := time.Parse("2006-01-02 15:04:05", v); err == nil && t.Year() > 0 {
				return &t
			} else if t, err := time.Parse("2006-01-02", v); err == nil && t.Year() > 0 {
				return &t
			} else if t, err := time.Parse("20060102150405", v); err == nil && t.Year() > 0 {
				return &t
			} else if t, err := time.Parse("20060102", v); err == nil && t.Year() > 0 {
				return &t
			} else if t, err := time.Parse("01-02-06", v); err == nil && t.Year() > 0 {
				return &t
			} else if t, err := time.Parse("01-02-2006", v); err == nil && t.Year() > 0 {
				return &t
			} else if t, err := time.Parse("15:04:05.000", v); err == nil {
				return &t
			} else if t, err := time.Parse("15:04:05", v); err == nil {
				return &t
			}
		}
	}
	return nil
}

func UTF8(ustr string) (str string) {
	defer func() {
		if r := recover(); r != nil {
			ustr = str
			return
		}
	}()

	bytes, err := cp949.From([]byte(ustr))
	if err != nil {
		str = ustr
	} else {
		str = string(bytes)
	}
	return
}

func EUCKR(str string) (ustr string) {
	defer func() {
		if r := recover(); r != nil {
			ustr = str
			return
		}
	}()

	ubytes, err := cp949.To([]byte(str))
	if err != nil {
		ustr = str
	} else {
		ustr = string(ubytes)
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Delete every object in the bucket with the prefix `garbage/`. (These are
// objects produced by `comeback gc`.)

package main

import (
	"fmt"
	"log"
	"time"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	"github.com/jacobsa/syncutil"
	"golang.org/x/net/context"
)

var cmdDeleteGarbage = &Command{
	Name: "delete_garbage",
	Run:  runDeleteGarbage,
}

////////////////////////////////////////////////////////////////////////
// Delete garbage
////////////////////////////////////////////////////////////////////////

func runDeleteGarbage(args []string) {
	// Die on error.
	var err error
	defer func() {
		if err != nil {
			log.Fatalln(err)
		}
	}()

	// Grab dependencies.
	bucket := getBucket()

	b := syncutil.NewBundle(context.Background())

	// List all garbage objects.
	objects := make(chan *gcs.Object, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(objects)
		err = gcsutil.ListPrefix(ctx, bucket, garbagePrefix, objects)
		if err != nil {
			err = fmt.Errorf("ListPrefix: %v", err)
			return
		}

		return
	})

	// Count the objects passing through, periodically printing a status update.
	// Convert to names.
	var count uint64
	toDelete := make(chan string)
	b.Add(func(ctx context.Context) (err error) {
		defer close(toDelete)
		ticker := time.Tick(2 * time.Second)

		for o := range objects {
			count++

			// Print a status update?
			select {
			case <-ticker:
				log.Printf("%d names seen so far.", count)

			default:
			}

			// Pass on the name.
			select {
			case <-ctx.Done():
				err = ctx.Err()
				return

			case toDelete <- o.Name:
			}
		}

		return
	})

	// Delete the objects.
	b.Add(func(ctx context.Context) (err error) {
		err = deleteObjects(ctx, bucket, toDelete)
		if err != nil {
			err = fmt.Errorf("deleteObjects: %v", err)
			return
		}

		return
	})

	err = b.Join()
	if err != nil {
		return
	}

	// Print a summary.
	log.Printf("Deleted %d objects.", count)
}
<commit_msg>Fixed cmdDeleteGarbage.<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Delete every object in the bucket with the prefix `garbage/`. (These are
// objects produced by `comeback gc`.)

package main

import (
	"fmt"
	"log"
	"time"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	"github.com/jacobsa/syncutil"
	"golang.org/x/net/context"
)

var cmdDeleteGarbage = &Command{
	Name: "delete_garbage",
	Run:  runDeleteGarbage,
}

////////////////////////////////////////////////////////////////////////
// Delete garbage
////////////////////////////////////////////////////////////////////////

func runDeleteGarbage(ctx context.Context, args []string) (err error) {
	b := syncutil.NewBundle(ctx)

	// Grab dependencies.
	bucket := getBucket()

	// List all garbage objects.
	objects := make(chan *gcs.Object, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(objects)
		err = gcsutil.ListPrefix(ctx, bucket, garbagePrefix, objects)
		if err != nil {
			err = fmt.Errorf("ListPrefix: %v", err)
			return
		}

		return
	})

	// Count the objects passing through, periodically printing a status update.
	// Convert to names.
	var count uint64
	toDelete := make(chan string)
	b.Add(func(ctx context.Context) (err error) {
		defer close(toDelete)
		ticker := time.Tick(2 * time.Second)

		for o := range objects {
			count++

			// Print a status update?
			select {
			case <-ticker:
				log.Printf("%d names seen so far.", count)

			default:
			}

			// Pass on the name.
			select {
			case <-ctx.Done():
				err = ctx.Err()
				return

			case toDelete <- o.Name:
			}
		}

		return
	})

	// Delete the objects.
	b.Add(func(ctx context.Context) (err error) {
		err = deleteObjects(ctx, bucket, toDelete)
		if err != nil {
			err = fmt.Errorf("deleteObjects: %v", err)
			return
		}

		return
	})

	err = b.Join()
	if err != nil {
		return
	}

	// Print a summary.
	log.Printf("Deleted %d objects.", count)

	return
}
<|endoftext|>


<commit_before>// Copyright 2019, OpenCensus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tailsampling

import (
	"sync"
	"testing"
	"time"

	"github.com/census-instrumentation/opencensus-service/data"
	"github.com/census-instrumentation/opencensus-service/internal/collector/processor"
	"github.com/census-instrumentation/opencensus-service/internal/collector/processor/idbatcher"
	"github.com/census-instrumentation/opencensus-service/internal/collector/sampling"
	tracetranslator "github.com/census-instrumentation/opencensus-service/translator/trace"

	tracepb "github.com/census-instrumentation/opencensus-proto/gen-go/trace/v1"
	"go.uber.org/zap"
)

const (
	defaultTestDecisionWait = 30 * time.Second
)

func TestSequentialTraceArrival(t *testing.T) {
	traceIds, batches := generateIdsAndBatches(128)
	sp, _ := NewTailSamplingSpanProcessor(newTestPolicy(), uint64(2*len(traceIds)), 64, defaultTestDecisionWait, zap.NewNop())
	tsp := sp.(*tailSamplingSpanProcessor)
	for _, batch := range batches {
		tsp.ProcessSpans(batch, "test")
	}

	for i := range traceIds {
		d, ok := tsp.idToTrace.Load(traceKey(traceIds[i]))
		v := d.(*sampling.TraceData)
		if !ok {
			t.Fatal("Missing expected traceId")
		} else if v.SpanCount != int64(i+1) {
			t.Fatalf("Incorrect number of spans for entry %d, got %d, want %d", i, v.SpanCount, i+1)
		}
	}
}

func TestConcurrentTraceArrival(t *testing.T) {
	traceIds, batches := generateIdsAndBatches(128)

	var wg sync.WaitGroup
	sp, _ := NewTailSamplingSpanProcessor(newTestPolicy(), uint64(2*len(traceIds)), 64, defaultTestDecisionWait, zap.NewNop())
	tsp := sp.(*tailSamplingSpanProcessor)
	for _, batch := range batches {
		// Add the same traceId twice.
		wg.Add(2)
		go func(td data.TraceData, sf string) {
			tsp.ProcessSpans(td, sf)
			wg.Done()
		}(batch, "test-0")
		go func(td data.TraceData, sf string) {
			tsp.ProcessSpans(td, sf)
			wg.Done()
		}(batch, "test-1")
	}

	wg.Wait()

	for i := range traceIds {
		d, ok := tsp.idToTrace.Load(traceKey(traceIds[i]))
		v := d.(*sampling.TraceData)
		if !ok {
			t.Fatal("Missing expected traceId")
		} else if v.SpanCount != int64(i+1)*2 {
			t.Fatalf("Incorrect number of spans for entry %d, got %d, want %d", i, v.SpanCount, i+1)
		}
	}
}

func TestConcurrentTraceMapSize(t *testing.T) {
	traceIds, batches := generateIdsAndBatches(210)
	const maxSize = 100
	var wg sync.WaitGroup
	sp, _ := NewTailSamplingSpanProcessor(newTestPolicy(), uint64(maxSize), 64, defaultTestDecisionWait, zap.NewNop())
	tsp := sp.(*tailSamplingSpanProcessor)
	for _, batch := range batches {
		wg.Add(1)
		go func(td data.TraceData, sf string) {
			tsp.ProcessSpans(td, sf)
			wg.Done()
		}(batch, "test")
	}

	wg.Wait()

	for i := 0; i < len(traceIds)-maxSize; i++ {
		if _, ok := tsp.idToTrace.Load(traceKey(traceIds[i])); ok {
			t.Fatalf("Found unexpected traceId[%d] still on map (id: %v)", i, traceIds[i])
		}
	}
}

func TestSamplingPolicyTypicalPath(t *testing.T) {
	const maxSize = 100
	const decisionWaitSeconds = 5
	decisionWait := time.Second * decisionWaitSeconds
	msp := &mockSpanProcessor{}
	mpe := &mockPolicyEvaluator{}
	testPolicy := []*Policy{
		{
			Name:        "test",
			Evaluator:   mpe,
			Destination: msp,
		},
	}
	sp, _ := NewTailSamplingSpanProcessor(testPolicy, maxSize, 64, decisionWait, zap.NewNop())
	tsp := sp.(*tailSamplingSpanProcessor)

	// For this test  explicitly control the timer calls and batcher.
	mtt := &manualTTicker{}
	tsp.policyTicker = mtt
	tsp.decisionBatcher = newSyncIDBatcher(decisionWaitSeconds)

	_, batches := generateIdsAndBatches(210)
	currItem := 0
	numSpansPerBatchWindow := 10
	// First evaluations shouldn't have anything to evaluate, until decision wait time passed.
	for evalNum := 0; evalNum < decisionWaitSeconds; evalNum++ {
		for ; currItem < numSpansPerBatchWindow*(evalNum+1); currItem++ {
			tsp.ProcessSpans(batches[currItem], "test")
			if !mtt.Started {
				t.Fatalf("Time ticker was expected to have started")
			}
		}
		tsp.samplingPolicyOnTick()
		if msp.TotalSpans != 0 || mpe.EvaluationCount != 0 {
			t.Fatalf("policy for initial items was evaluated before decision wait period")
		}
	}

	// Now the first batch that waited the decision period.
	mpe.NextDecision = sampling.Sampled
	tsp.samplingPolicyOnTick()
	if msp.TotalSpans == 0 || mpe.EvaluationCount == 0 {
		t.Fatalf("policy should have been evaluated totalspans == %d and evaluationcount == %d", msp.TotalSpans, mpe.EvaluationCount)
	}

	if msp.TotalSpans != numSpansPerBatchWindow {
		t.Fatalf("not all spans of first window were accounted for: got %d, want %d", msp.TotalSpans, numSpansPerBatchWindow)
	}

	// Late span of a sampled trace should be sent directly down the pipeline exporter
	tsp.ProcessSpans(batches[0], "test")
	expectedNumWithLateSpan := numSpansPerBatchWindow + 1
	if msp.TotalSpans != expectedNumWithLateSpan {
		t.Fatalf("late span was not accounted for: got %d, want %d", msp.TotalSpans, expectedNumWithLateSpan)
	}
	if mpe.LateArrivingSpansCount != 1 {
		t.Fatalf("policy was not notified of the late span")
	}
}

func generateIdsAndBatches(numIds int) ([][]byte, []data.TraceData) {
	traceIds := make([][]byte, numIds, numIds)
	for i := 0; i < numIds; i++ {
		traceIds[i] = tracetranslator.UInt64ToByteTraceID(1, uint64(i+1))
	}

	tds := []data.TraceData{}
	for i := range traceIds {
		spans := make([]*tracepb.Span, i+1)
		for j := range spans {
			spans[j] = &tracepb.Span{
				TraceId: traceIds[i],
				SpanId:  tracetranslator.UInt64ToByteSpanID(uint64(i + 1)),
			}
		}

		// Send each span in a separate batch
		for _, span := range spans {
			td := data.TraceData{
				Spans: []*tracepb.Span{span},
			}
			tds = append(tds, td)
		}
	}

	return traceIds, tds
}

func newTestPolicy() []*Policy {
	return []*Policy{
		{
			Name:        "test",
			Evaluator:   sampling.NewAlwaysSample(),
			Destination: &mockSpanProcessor{},
		},
	}
}

type mockPolicyEvaluator struct {
	NextDecision           sampling.Decision
	NextError              error
	EvaluationCount        int
	LateArrivingSpansCount int
	OnDroppedSpansCount    int
}

var _ (sampling.PolicyEvaluator) = (*mockPolicyEvaluator)(nil)

func (m *mockPolicyEvaluator) OnLateArrivingSpans(earlyDecision sampling.Decision, spans []*tracepb.Span) error {
	m.LateArrivingSpansCount++
	return m.NextError
}
func (m *mockPolicyEvaluator) Evaluate(traceID []byte, trace *sampling.TraceData) (sampling.Decision, error) {
	m.EvaluationCount++
	return m.NextDecision, m.NextError
}
func (m *mockPolicyEvaluator) OnDroppedSpans(traceID []byte, trace *sampling.TraceData) (sampling.Decision, error) {
	m.OnDroppedSpansCount++
	return m.NextDecision, m.NextError
}

type manualTTicker struct {
	Started bool
}

var _ tTicker = (*manualTTicker)(nil)

func (t *manualTTicker) Start(d time.Duration) {
	t.Started = true
}
func (t *manualTTicker) OnTick() {
}
func (t *manualTTicker) Stop() {
}

type syncIDBatcher struct {
	sync.Mutex
	openBatch idbatcher.Batch
	batchPipe chan idbatcher.Batch
}

var _ (idbatcher.Batcher) = (*syncIDBatcher)(nil)

func newSyncIDBatcher(numBatches uint64) idbatcher.Batcher {
	batches := make(chan idbatcher.Batch, numBatches)
	for i := uint64(0); i < numBatches; i++ {
		batches <- nil
	}
	return &syncIDBatcher{
		batchPipe: batches,
	}
}
func (s *syncIDBatcher) AddToCurrentBatch(id idbatcher.ID) {
	s.Lock()
	s.openBatch = append(s.openBatch, id)
	s.Unlock()
}
func (s *syncIDBatcher) CloseCurrentAndTakeFirstBatch() (idbatcher.Batch, bool) {
	s.Lock()
	defer s.Unlock()
	firstBatch := <-s.batchPipe
	s.batchPipe <- s.openBatch
	s.openBatch = nil
	return firstBatch, true
}
func (s *syncIDBatcher) Stop() {
}

type mockSpanProcessor struct {
	TotalSpans int
}

var _ processor.SpanProcessor = &mockSpanProcessor{}

func (p *mockSpanProcessor) ProcessSpans(td data.TraceData, spanFormat string) error {
	batchSize := len(td.Spans)
	p.TotalSpans += batchSize
	return nil
}
<commit_msg>Fix test checking idToTrace map used in tail sampling (#486)<commit_after>// Copyright 2019, OpenCensus Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tailsampling

import (
	"sync"
	"testing"
	"time"

	"github.com/census-instrumentation/opencensus-service/data"
	"github.com/census-instrumentation/opencensus-service/internal/collector/processor"
	"github.com/census-instrumentation/opencensus-service/internal/collector/processor/idbatcher"
	"github.com/census-instrumentation/opencensus-service/internal/collector/sampling"
	tracetranslator "github.com/census-instrumentation/opencensus-service/translator/trace"

	tracepb "github.com/census-instrumentation/opencensus-proto/gen-go/trace/v1"
	"go.uber.org/zap"
)

const (
	defaultTestDecisionWait = 30 * time.Second
)

func TestSequentialTraceArrival(t *testing.T) {
	traceIds, batches := generateIdsAndBatches(128)
	sp, _ := NewTailSamplingSpanProcessor(newTestPolicy(), uint64(2*len(traceIds)), 64, defaultTestDecisionWait, zap.NewNop())
	tsp := sp.(*tailSamplingSpanProcessor)
	for _, batch := range batches {
		tsp.ProcessSpans(batch, "test")
	}

	for i := range traceIds {
		d, ok := tsp.idToTrace.Load(traceKey(traceIds[i]))
		v := d.(*sampling.TraceData)
		if !ok {
			t.Fatal("Missing expected traceId")
		} else if v.SpanCount != int64(i+1) {
			t.Fatalf("Incorrect number of spans for entry %d, got %d, want %d", i, v.SpanCount, i+1)
		}
	}
}

func TestConcurrentTraceArrival(t *testing.T) {
	traceIds, batches := generateIdsAndBatches(128)

	var wg sync.WaitGroup
	sp, _ := NewTailSamplingSpanProcessor(newTestPolicy(), uint64(2*len(traceIds)), 64, defaultTestDecisionWait, zap.NewNop())
	tsp := sp.(*tailSamplingSpanProcessor)
	for _, batch := range batches {
		// Add the same traceId twice.
		wg.Add(2)
		go func(td data.TraceData, sf string) {
			tsp.ProcessSpans(td, sf)
			wg.Done()
		}(batch, "test-0")
		go func(td data.TraceData, sf string) {
			tsp.ProcessSpans(td, sf)
			wg.Done()
		}(batch, "test-1")
	}

	wg.Wait()

	for i := range traceIds {
		d, ok := tsp.idToTrace.Load(traceKey(traceIds[i]))
		v := d.(*sampling.TraceData)
		if !ok {
			t.Fatal("Missing expected traceId")
		} else if v.SpanCount != int64(i+1)*2 {
			t.Fatalf("Incorrect number of spans for entry %d, got %d, want %d", i, v.SpanCount, i+1)
		}
	}
}

func TestSequentialTraceMapSize(t *testing.T) {
	traceIds, batches := generateIdsAndBatches(210)
	const maxSize = 100
	sp, _ := NewTailSamplingSpanProcessor(newTestPolicy(), uint64(maxSize), 64, defaultTestDecisionWait, zap.NewNop())
	tsp := sp.(*tailSamplingSpanProcessor)
	for _, batch := range batches {
		tsp.ProcessSpans(batch, "test")
	}

	// On sequential insertion it is possible to know exactly which traces should be still on the map.
	for i := 0; i < len(traceIds)-maxSize; i++ {
		if _, ok := tsp.idToTrace.Load(traceKey(traceIds[i])); ok {
			t.Fatalf("Found unexpected traceId[%d] still on map (id: %v)", i, traceIds[i])
		}
	}
}

func TestConcurrentTraceMapSize(t *testing.T) {
	_, batches := generateIdsAndBatches(210)
	const maxSize = 100
	var wg sync.WaitGroup
	sp, _ := NewTailSamplingSpanProcessor(newTestPolicy(), uint64(maxSize), 64, defaultTestDecisionWait, zap.NewNop())
	tsp := sp.(*tailSamplingSpanProcessor)
	for _, batch := range batches {
		wg.Add(1)
		go func(td data.TraceData, sf string) {
			tsp.ProcessSpans(td, sf)
			wg.Done()
		}(batch, "test")
	}

	wg.Wait()

	// Since we can't guarantee the order of insertion the only thing that can be checked is
	// if the number of traces on the map matches the expected value.
	cnt := 0
	tsp.idToTrace.Range(func(_ interface{}, _ interface{}) bool {
		cnt++
		return true
	})
	if cnt != maxSize {
		t.Fatalf("got %d traces on idToTrace, want %d", cnt, maxSize)
	}
}

func TestSamplingPolicyTypicalPath(t *testing.T) {
	const maxSize = 100
	const decisionWaitSeconds = 5
	decisionWait := time.Second * decisionWaitSeconds
	msp := &mockSpanProcessor{}
	mpe := &mockPolicyEvaluator{}
	testPolicy := []*Policy{
		{
			Name:        "test",
			Evaluator:   mpe,
			Destination: msp,
		},
	}
	sp, _ := NewTailSamplingSpanProcessor(testPolicy, maxSize, 64, decisionWait, zap.NewNop())
	tsp := sp.(*tailSamplingSpanProcessor)

	// For this test  explicitly control the timer calls and batcher.
	mtt := &manualTTicker{}
	tsp.policyTicker = mtt
	tsp.decisionBatcher = newSyncIDBatcher(decisionWaitSeconds)

	_, batches := generateIdsAndBatches(210)
	currItem := 0
	numSpansPerBatchWindow := 10
	// First evaluations shouldn't have anything to evaluate, until decision wait time passed.
	for evalNum := 0; evalNum < decisionWaitSeconds; evalNum++ {
		for ; currItem < numSpansPerBatchWindow*(evalNum+1); currItem++ {
			tsp.ProcessSpans(batches[currItem], "test")
			if !mtt.Started {
				t.Fatalf("Time ticker was expected to have started")
			}
		}
		tsp.samplingPolicyOnTick()
		if msp.TotalSpans != 0 || mpe.EvaluationCount != 0 {
			t.Fatalf("policy for initial items was evaluated before decision wait period")
		}
	}

	// Now the first batch that waited the decision period.
	mpe.NextDecision = sampling.Sampled
	tsp.samplingPolicyOnTick()
	if msp.TotalSpans == 0 || mpe.EvaluationCount == 0 {
		t.Fatalf("policy should have been evaluated totalspans == %d and evaluationcount == %d", msp.TotalSpans, mpe.EvaluationCount)
	}

	if msp.TotalSpans != numSpansPerBatchWindow {
		t.Fatalf("not all spans of first window were accounted for: got %d, want %d", msp.TotalSpans, numSpansPerBatchWindow)
	}

	// Late span of a sampled trace should be sent directly down the pipeline exporter
	tsp.ProcessSpans(batches[0], "test")
	expectedNumWithLateSpan := numSpansPerBatchWindow + 1
	if msp.TotalSpans != expectedNumWithLateSpan {
		t.Fatalf("late span was not accounted for: got %d, want %d", msp.TotalSpans, expectedNumWithLateSpan)
	}
	if mpe.LateArrivingSpansCount != 1 {
		t.Fatalf("policy was not notified of the late span")
	}
}

func generateIdsAndBatches(numIds int) ([][]byte, []data.TraceData) {
	traceIds := make([][]byte, numIds, numIds)
	for i := 0; i < numIds; i++ {
		traceIds[i] = tracetranslator.UInt64ToByteTraceID(1, uint64(i+1))
	}

	tds := []data.TraceData{}
	for i := range traceIds {
		spans := make([]*tracepb.Span, i+1)
		for j := range spans {
			spans[j] = &tracepb.Span{
				TraceId: traceIds[i],
				SpanId:  tracetranslator.UInt64ToByteSpanID(uint64(i + 1)),
			}
		}

		// Send each span in a separate batch
		for _, span := range spans {
			td := data.TraceData{
				Spans: []*tracepb.Span{span},
			}
			tds = append(tds, td)
		}
	}

	return traceIds, tds
}

func newTestPolicy() []*Policy {
	return []*Policy{
		{
			Name:        "test",
			Evaluator:   sampling.NewAlwaysSample(),
			Destination: &mockSpanProcessor{},
		},
	}
}

type mockPolicyEvaluator struct {
	NextDecision           sampling.Decision
	NextError              error
	EvaluationCount        int
	LateArrivingSpansCount int
	OnDroppedSpansCount    int
}

var _ (sampling.PolicyEvaluator) = (*mockPolicyEvaluator)(nil)

func (m *mockPolicyEvaluator) OnLateArrivingSpans(earlyDecision sampling.Decision, spans []*tracepb.Span) error {
	m.LateArrivingSpansCount++
	return m.NextError
}
func (m *mockPolicyEvaluator) Evaluate(traceID []byte, trace *sampling.TraceData) (sampling.Decision, error) {
	m.EvaluationCount++
	return m.NextDecision, m.NextError
}
func (m *mockPolicyEvaluator) OnDroppedSpans(traceID []byte, trace *sampling.TraceData) (sampling.Decision, error) {
	m.OnDroppedSpansCount++
	return m.NextDecision, m.NextError
}

type manualTTicker struct {
	Started bool
}

var _ tTicker = (*manualTTicker)(nil)

func (t *manualTTicker) Start(d time.Duration) {
	t.Started = true
}
func (t *manualTTicker) OnTick() {
}
func (t *manualTTicker) Stop() {
}

type syncIDBatcher struct {
	sync.Mutex
	openBatch idbatcher.Batch
	batchPipe chan idbatcher.Batch
}

var _ (idbatcher.Batcher) = (*syncIDBatcher)(nil)

func newSyncIDBatcher(numBatches uint64) idbatcher.Batcher {
	batches := make(chan idbatcher.Batch, numBatches)
	for i := uint64(0); i < numBatches; i++ {
		batches <- nil
	}
	return &syncIDBatcher{
		batchPipe: batches,
	}
}
func (s *syncIDBatcher) AddToCurrentBatch(id idbatcher.ID) {
	s.Lock()
	s.openBatch = append(s.openBatch, id)
	s.Unlock()
}
func (s *syncIDBatcher) CloseCurrentAndTakeFirstBatch() (idbatcher.Batch, bool) {
	s.Lock()
	defer s.Unlock()
	firstBatch := <-s.batchPipe
	s.batchPipe <- s.openBatch
	s.openBatch = nil
	return firstBatch, true
}
func (s *syncIDBatcher) Stop() {
}

type mockSpanProcessor struct {
	TotalSpans int
}

var _ processor.SpanProcessor = &mockSpanProcessor{}

func (p *mockSpanProcessor) ProcessSpans(td data.TraceData, spanFormat string) error {
	batchSize := len(td.Spans)
	p.TotalSpans += batchSize
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tests

import (
	"log"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/suite"
	piazza "github.com/venicegeo/pz-gocommon"
	"github.com/venicegeo/pz-gocommon/elasticsearch"
	pzlogger "github.com/venicegeo/pz-logger/lib"
)

const MOCKING = true

type LoggerTester struct {
	suite.Suite

	esi    elasticsearch.IIndex
	sys    *piazza.SystemConfig
	logger pzlogger.IClient
}

func (suite *LoggerTester) setupFixture() {
	t := suite.T()
	assert := assert.New(t)

	var required []piazza.ServiceName
	if MOCKING {
		required = []piazza.ServiceName{}
	} else {
		required = []piazza.ServiceName{piazza.PzElasticSearch}
	}
	sys, err := piazza.NewSystemConfig(piazza.PzLogger, required)
	assert.NoError(err)
	suite.sys = sys

	esi, err := elasticsearch.NewIndexInterface(sys, "loggertest$", MOCKING)
	assert.NoError(err)
	suite.esi = esi

	_ = sys.StartServer(pzlogger.CreateHandlers(sys, esi))

	if MOCKING {
		logger, err := pzlogger.NewMockClient(sys)
		assert.NoError(err)
		suite.logger = logger
	} else {
		logger, err := pzlogger.NewClient(sys)
		assert.NoError(err)
		suite.logger = logger
	}
}

func (suite *LoggerTester) teardownFixture() {
	//TODO: kill the go routine running the server

	suite.esi.Close()
	suite.esi.Delete()
}

func TestRunSuite(t *testing.T) {
	s := &LoggerTester{}
	suite.Run(t, s)
}

func (suite *LoggerTester) getLastMessage() string {
	t := suite.T()
	assert := assert.New(t)

	logger := suite.logger

	format := elasticsearch.QueryFormat{Size: 100, From: 0, Order: elasticsearch.SortDescending, Key: "stamp"}
	ms, err := logger.GetFromMessages(format, map[string]string{})
	assert.NoError(err)
	assert.True(len(ms) > 0)

	return ms[len(ms)-1].String()
}

func (suite *LoggerTester) Test01Elasticsearch() {
	t := suite.T()
	assert := assert.New(t)

	suite.setupFixture()
	defer suite.teardownFixture()

	version := suite.esi.GetVersion()
	assert.Contains("2.2.0", version)
}

func (suite *LoggerTester) Test02One() {
	t := suite.T()
	assert := assert.New(t)

	suite.setupFixture()
	defer suite.teardownFixture()

	logger := suite.logger

	var err error

	data1 := pzlogger.Message{
		Service:  "log-tester",
		Address:  "128.1.2.3",
		Stamp:    time.Now().Unix(),
		Severity: "Info",
		Message:  "The quick brown fox",
	}

	data2 := pzlogger.Message{
		Service:  "log-tester",
		Address:  "128.0.0.0",
		Stamp:    time.Now().Unix(),
		Severity: "Fatal",
		Message:  "The quick brown fox",
	}

	{
		err = logger.LogMessage(&data1)
		assert.NoError(err, "PostToMessages")
	}

	//	time.Sleep(1 * time.Second)

	{
		actualMssg := suite.getLastMessage()
		expectedMssg := data1.String()
		assert.EqualValues(actualMssg, expectedMssg)
	}

	{
		err = logger.LogMessage(&data2)
		assert.NoError(err, "PostToMessages")
	}

	time.Sleep(4 * time.Second)

	{
		actualMssg := suite.getLastMessage()
		expectedMssg := data2.String()
		assert.EqualValues(actualMssg, expectedMssg)
	}

	{
		stats, err := logger.GetFromAdminStats()
		assert.NoError(err, "GetFromAdminStats")
		assert.Equal(2, stats.NumMessages, "stats check")
		assert.WithinDuration(time.Now(), stats.StartTime, 30*time.Second, "service start time too long ago")
	}
}

func (suite *LoggerTester) Test03Help() {
	t := suite.T()
	assert := assert.New(t)

	err := suite.logger.Log("mocktest", "0.0.0.0", pzlogger.SeverityInfo, time.Now(), "message from logger unit test via piazza.Log()")
	assert.NoError(err, "pzService.Log()")
}

func (suite *LoggerTester) Test04Admin() {
	t := suite.T()
	assert := assert.New(t)

	suite.setupFixture()
	defer suite.teardownFixture()

	logger := suite.logger

	_, err := logger.GetFromAdminStats()
	assert.NoError(err, "GetFromAdminStats")
}

func (suite *LoggerTester) Test05Pagination() {
	t := suite.T()
	assert := assert.New(t)

	if MOCKING {
		t.Skip("Skipping test, because mocking.")
	}

	suite.setupFixture()
	defer suite.teardownFixture()

	logger := suite.logger

	logger.SetService("myservice", "1.2.3.4")

	err := logger.Debug("d")
	assert.NoError(err)
	time.Sleep(1 * time.Second)
	err = logger.Info("i")
	assert.NoError(err)
	time.Sleep(1 * time.Second)
	err = logger.Warn("w")
	assert.NoError(err)
	time.Sleep(1 * time.Second)
	err = logger.Error("e")
	assert.NoError(err)
	time.Sleep(1 * time.Second)
	err = logger.Fatal("f")
	assert.NoError(err)
	time.Sleep(1 * time.Second)

	format := elasticsearch.QueryFormat{Size: 1, From: 0, Key: "stamp", Order: elasticsearch.SortDescending}
	ms, err := logger.GetFromMessages(format, map[string]string{})
	assert.NoError(err)
	assert.Len(ms, 1)
	assert.EqualValues(pzlogger.SeverityFatal, ms[0].Severity)

	format = elasticsearch.QueryFormat{Size: 3, From: 2, Key: "stamp", Order: elasticsearch.SortDescending}
	ms, err = logger.GetFromMessages(format, map[string]string{})
	assert.NoError(err)
	assert.Len(ms, 3)
	assert.EqualValues(pzlogger.SeverityWarning, ms[0].Severity)
	assert.EqualValues(pzlogger.SeverityInfo, ms[1].Severity)
	assert.EqualValues(pzlogger.SeverityDebug, ms[2].Severity)
}

func (suite *LoggerTester) Test06OtherParams() {
	t := suite.T()
	assert := assert.New(t)

	if MOCKING {
		t.Skip("Skipping test, because mocking.")
	}

	suite.setupFixture()
	defer suite.teardownFixture()

	logger := suite.logger

	logger.SetService("myservice", "1.2.3.4")

	var testData = []pzlogger.Message{
		{
			Address:  "gnemud7smfv/10.254.0.66",
			Message:  "Received Message to Relay on topic Request-Job with key f3b63085-b482-4ae8-8297-3c7d1fcfff7d",
			Service:  "Dispatcher",
			Severity: "Info",
			Stamp:    1461181461,
		}, {
			Address:  "gnfbnqsn5m9/10.254.0.14",
			Message:  "Processed Update Status for Job 6d0ea538-4382-4ea5-9669-56519b8c8f58 with Status Success",
			Service:  "JobManager",
			Severity: "Info",
			Stamp:    1461181378,
		}, {
			Address:  "0.0.0.0",
			Message:  "generated 1: 09d4ec60-ea61-4066-8697-5568a47f84bf",
			Service:  "pz-uuidgen",
			Severity: "Info",
			Stamp:    1461181362,
		}, {
			Address:  "gnfbnqsn5m9/10.254.0.14",
			Message:  "Handling Job with Topic Create-Job for Job ID 09d4ec60-ea61-4066-8697-5568a47f84bf",
			Service:  "JobManager",
			Severity: "Info",
			Stamp:    1461181362,
		}, {
			Address:  "gnfbnqsn5m9/10.254.0.14",
			Message:  "Handling Job with Topic Update-Job for Job ID be4ce034-1187-4a4f-95a9-a9c31826248b",
			Service:  "JobManager",
			Severity: "Info",
			Stamp:    1461181283,
		},
	}

	for _, e := range testData {
		// log.Printf("%d, %v\n", i, e)
		err := logger.LogMessage(&e)
		assert.NoError(err)
	}

	time.Sleep(1 * time.Second)

	format := elasticsearch.QueryFormat{
		Size: 100, From: 0,
		Order: elasticsearch.SortDescending,
		Key:   "stamp"}

	msgs, err := logger.GetFromMessages(format,
		map[string]string{
			"service":  "JobManager",
			"contains": "Success",
		})
	assert.NoError(err)
	assert.Len(msgs, 1)

	for _, msg := range msgs {
		log.Printf("%v\n", msg)
	}

	msgs, err = logger.GetFromMessages(format,
		map[string]string{
			"before": "1461181461",
			"after":  "1461181362",
		})

	assert.NoError(err)
	assert.Len(msgs, 4)

	for _, msg := range msgs {
		log.Printf("%v\n", msg)
	}

}
<commit_msg>3813 - add sleep() for elasticsearch<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tests

import (
	"log"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/suite"
	piazza "github.com/venicegeo/pz-gocommon"
	"github.com/venicegeo/pz-gocommon/elasticsearch"
	pzlogger "github.com/venicegeo/pz-logger/lib"
)

const MOCKING = true

func sleep() {
	time.Sleep(1 * time.Second)
}

type LoggerTester struct {
	suite.Suite

	esi    elasticsearch.IIndex
	sys    *piazza.SystemConfig
	logger pzlogger.IClient
}

func (suite *LoggerTester) setupFixture() {
	t := suite.T()
	assert := assert.New(t)

	var required []piazza.ServiceName
	if MOCKING {
		required = []piazza.ServiceName{}
	} else {
		required = []piazza.ServiceName{piazza.PzElasticSearch}
	}
	sys, err := piazza.NewSystemConfig(piazza.PzLogger, required)
	assert.NoError(err)
	suite.sys = sys

	esi, err := elasticsearch.NewIndexInterface(sys, "loggertest$", MOCKING)
	assert.NoError(err)
	suite.esi = esi

	_ = sys.StartServer(pzlogger.CreateHandlers(sys, esi))

	if MOCKING {
		logger, err := pzlogger.NewMockClient(sys)
		assert.NoError(err)
		suite.logger = logger
	} else {
		logger, err := pzlogger.NewClient(sys)
		assert.NoError(err)
		suite.logger = logger
	}
}

func (suite *LoggerTester) teardownFixture() {
	//TODO: kill the go routine running the server

	suite.esi.Close()
	suite.esi.Delete()
}

func TestRunSuite(t *testing.T) {
	s := &LoggerTester{}
	suite.Run(t, s)
}

func (suite *LoggerTester) getLastMessage() string {
	t := suite.T()
	assert := assert.New(t)

	logger := suite.logger

	format := elasticsearch.QueryFormat{Size: 100, From: 0, Order: elasticsearch.SortDescending, Key: "stamp"}
	ms, err := logger.GetFromMessages(format, map[string]string{})
	assert.NoError(err)
	assert.True(len(ms) > 0)

	return ms[len(ms)-1].String()
}

func (suite *LoggerTester) Test01Elasticsearch() {
	t := suite.T()
	assert := assert.New(t)

	suite.setupFixture()
	defer suite.teardownFixture()

	version := suite.esi.GetVersion()
	assert.Contains("2.2.0", version)
}

func (suite *LoggerTester) Test02One() {
	t := suite.T()
	assert := assert.New(t)

	suite.setupFixture()
	defer suite.teardownFixture()

	logger := suite.logger

	var err error

	data1 := pzlogger.Message{
		Service:  "log-tester",
		Address:  "128.1.2.3",
		Stamp:    time.Now().Unix(),
		Severity: "Info",
		Message:  "The quick brown fox",
	}

	data2 := pzlogger.Message{
		Service:  "log-tester",
		Address:  "128.0.0.0",
		Stamp:    time.Now().Unix(),
		Severity: "Fatal",
		Message:  "The quick brown fox",
	}

	{
		err = logger.LogMessage(&data1)
		assert.NoError(err, "PostToMessages")
	}

	sleep()

	{
		actualMssg := suite.getLastMessage()
		expectedMssg := data1.String()
		assert.EqualValues(actualMssg, expectedMssg)
	}

	{
		err = logger.LogMessage(&data2)
		assert.NoError(err, "PostToMessages")
	}

	sleep()

	{
		actualMssg := suite.getLastMessage()
		expectedMssg := data2.String()
		assert.EqualValues(actualMssg, expectedMssg)
	}

	{
		stats, err := logger.GetFromAdminStats()
		assert.NoError(err, "GetFromAdminStats")
		assert.Equal(2, stats.NumMessages, "stats check")
		assert.WithinDuration(time.Now(), stats.StartTime, 30*time.Second, "service start time too long ago")
	}
}

func (suite *LoggerTester) Test03Help() {
	t := suite.T()
	assert := assert.New(t)

	err := suite.logger.Log("mocktest", "0.0.0.0", pzlogger.SeverityInfo, time.Now(), "message from logger unit test via piazza.Log()")
	assert.NoError(err, "pzService.Log()")
}

func (suite *LoggerTester) Test04Admin() {
	t := suite.T()
	assert := assert.New(t)

	suite.setupFixture()
	defer suite.teardownFixture()

	logger := suite.logger

	_, err := logger.GetFromAdminStats()
	assert.NoError(err, "GetFromAdminStats")
}

func (suite *LoggerTester) Test05Pagination() {
	t := suite.T()
	assert := assert.New(t)

	if MOCKING {
		t.Skip("Skipping test, because mocking.")
	}

	suite.setupFixture()
	defer suite.teardownFixture()

	logger := suite.logger

	logger.SetService("myservice", "1.2.3.4")

	err := logger.Debug("d")
	assert.NoError(err)
	sleep()
	err = logger.Info("i")
	assert.NoError(err)
	sleep()
	err = logger.Warn("w")
	assert.NoError(err)
	sleep()
	err = logger.Error("e")
	assert.NoError(err)
	sleep()
	err = logger.Fatal("f")
	assert.NoError(err)
	sleep()

	format := elasticsearch.QueryFormat{Size: 1, From: 0, Key: "stamp", Order: elasticsearch.SortDescending}
	ms, err := logger.GetFromMessages(format, map[string]string{})
	assert.NoError(err)
	assert.Len(ms, 1)
	assert.EqualValues(pzlogger.SeverityFatal, ms[0].Severity)

	format = elasticsearch.QueryFormat{Size: 3, From: 2, Key: "stamp", Order: elasticsearch.SortDescending}
	ms, err = logger.GetFromMessages(format, map[string]string{})
	assert.NoError(err)
	assert.Len(ms, 3)
	assert.EqualValues(pzlogger.SeverityWarning, ms[0].Severity)
	assert.EqualValues(pzlogger.SeverityInfo, ms[1].Severity)
	assert.EqualValues(pzlogger.SeverityDebug, ms[2].Severity)
}

func (suite *LoggerTester) Test06OtherParams() {
	t := suite.T()
	assert := assert.New(t)

	if MOCKING {
		t.Skip("Skipping test, because mocking.")
	}

	suite.setupFixture()
	defer suite.teardownFixture()

	logger := suite.logger

	logger.SetService("myservice", "1.2.3.4")

	var testData = []pzlogger.Message{
		{
			Address:  "gnemud7smfv/10.254.0.66",
			Message:  "Received Message to Relay on topic Request-Job with key f3b63085-b482-4ae8-8297-3c7d1fcfff7d",
			Service:  "Dispatcher",
			Severity: "Info",
			Stamp:    1461181461,
		}, {
			Address:  "gnfbnqsn5m9/10.254.0.14",
			Message:  "Processed Update Status for Job 6d0ea538-4382-4ea5-9669-56519b8c8f58 with Status Success",
			Service:  "JobManager",
			Severity: "Info",
			Stamp:    1461181378,
		}, {
			Address:  "0.0.0.0",
			Message:  "generated 1: 09d4ec60-ea61-4066-8697-5568a47f84bf",
			Service:  "pz-uuidgen",
			Severity: "Info",
			Stamp:    1461181362,
		}, {
			Address:  "gnfbnqsn5m9/10.254.0.14",
			Message:  "Handling Job with Topic Create-Job for Job ID 09d4ec60-ea61-4066-8697-5568a47f84bf",
			Service:  "JobManager",
			Severity: "Info",
			Stamp:    1461181362,
		}, {
			Address:  "gnfbnqsn5m9/10.254.0.14",
			Message:  "Handling Job with Topic Update-Job for Job ID be4ce034-1187-4a4f-95a9-a9c31826248b",
			Service:  "JobManager",
			Severity: "Info",
			Stamp:    1461181283,
		},
	}

	for _, e := range testData {
		// log.Printf("%d, %v\n", i, e)
		err := logger.LogMessage(&e)
		assert.NoError(err)
	}

	sleep()

	format := elasticsearch.QueryFormat{
		Size: 100, From: 0,
		Order: elasticsearch.SortDescending,
		Key:   "stamp"}

	msgs, err := logger.GetFromMessages(format,
		map[string]string{
			"service":  "JobManager",
			"contains": "Success",
		})
	assert.NoError(err)
	assert.Len(msgs, 1)

	for _, msg := range msgs {
		log.Printf("%v\n", msg)
	}

	msgs, err = logger.GetFromMessages(format,
		map[string]string{
			"before": "1461181461",
			"after":  "1461181362",
		})

	assert.NoError(err)
	assert.Len(msgs, 4)

	for _, msg := range msgs {
		log.Printf("%v\n", msg)
	}

}
<|endoftext|>


<commit_before>package collectors

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/StackExchange/scollector/metadata"
	"github.com/StackExchange/scollector/opentsdb"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_procstats_linux})
}

var uptimeRE = regexp.MustCompile(`(\S+)\s+(\S+)`)
var meminfoRE = regexp.MustCompile(`(\w+):\s+(\d+)\s+(\w+)`)
var vmstatRE = regexp.MustCompile(`(\w+)\s+(\d+)`)
var statRE = regexp.MustCompile(`(\w+)\s+(.*)`)
var statCpuRE = regexp.MustCompile(`cpu(\d+)`)
var loadavgRE = regexp.MustCompile(`(\S+)\s+(\S+)\s+(\S+)\s+(\d+)/(\d+)\s+`)
var inoutRE = regexp.MustCompile(`(.*)(in|out)`)

var CPU_FIELDS = []string{
	"user",
	"nice",
	"system",
	"idle",
	"iowait",
	"irq",
	"softirq",
	"steal",
	"guest",
	"guest_nice",
}

func c_procstats_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	var Error error
	if err := readLine("/proc/uptime", func(s string) error {
		m := uptimeRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}
		Add(&md, "linux.uptime_total", m[1], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.uptime_now", m[2], nil, metadata.Unknown, metadata.None, "")
		return nil
	}); err != nil {
		Error = err
	}
	mem := make(map[string]float64)
	if err := readLine("/proc/meminfo", func(s string) error {
		m := meminfoRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}
		i, err := strconv.ParseFloat(m[2], 64)
		if err != nil {
			return err
		}
		mem[m[1]] = i
		Add(&md, "linux.mem."+strings.ToLower(m[1]), m[2], nil, metadata.Unknown, metadata.None, "")
		return nil
	}); err != nil {
		Error = err
	}
	Add(&md, osMemTotal, int(mem["MemTotal"])*1024, nil, metadata.Unknown, metadata.None, "")
	Add(&md, osMemFree, int(mem["MemFree"])*1024, nil, metadata.Unknown, metadata.None, "")
	Add(&md, osMemUsed, (int(mem["MemTotal"])-(int(mem["MemFree"])+int(mem["Buffers"])+int(mem["Cached"])))*1024, nil, metadata.Unknown, metadata.None, "")
	if mem["MemTotal"] != 0 {
		Add(&md, osMemPctFree, (mem["MemFree"]+mem["Buffers"]+mem["Cached"])/mem["MemTotal"]*100, nil, metadata.Unknown, metadata.None, "")
	}

	if err := readLine("/proc/vmstat", func(s string) error {
		m := vmstatRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}

		switch m[1] {
		case "pgpgin", "pgpgout", "pswpin", "pswpout", "pgfault", "pgmajfault":
			mio := inoutRE.FindStringSubmatch(m[1])
			if mio != nil {
				Add(&md, "linux.mem."+mio[1], m[2], opentsdb.TagSet{"direction": mio[2]}, metadata.Unknown, metadata.None, "")
			} else {
				Add(&md, "linux.mem."+m[1], m[2], nil, metadata.Unknown, metadata.None, "")
			}
		}
		return nil
	}); err != nil {
		Error = err
	}
	num_cores := 0
	var t_util float64
	if err := readLine("/proc/stat", func(s string) error {
		m := statRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}
		if strings.HasPrefix(m[1], "cpu") {
			metric_percpu := ""
			tag_cpu := ""
			cpu_m := statCpuRE.FindStringSubmatch(m[1])
			if cpu_m != nil {
				num_cores += 1
				metric_percpu = ".percpu"
				tag_cpu = cpu_m[1]
			}
			fields := strings.Fields(m[2])
			for i, value := range fields {
				if i >= len(CPU_FIELDS) {
					break
				}
				tags := opentsdb.TagSet{
					"type": CPU_FIELDS[i],
				}
				if tag_cpu != "" {
					tags["cpu"] = tag_cpu
				}
				Add(&md, "linux.cpu"+metric_percpu, value, tags, metadata.Unknown, metadata.None, "")
			}
			if metric_percpu == "" {
				if len(fields) < 3 {
					return nil
				}
				user, err := strconv.ParseFloat(fields[0], 64)
				if err != nil {
					return nil
				}
				nice, err := strconv.ParseFloat(fields[1], 64)
				if err != nil {
					return nil
				}
				system, err := strconv.ParseFloat(fields[2], 64)
				if err != nil {
					return nil
				}
				t_util = user + nice + system
			}
		} else if m[1] == "intr" {
			Add(&md, "linux.intr", strings.Fields(m[2])[0], nil, metadata.Unknown, metadata.None, "")
		} else if m[1] == "ctxt" {
			Add(&md, "linux.ctxt", m[2], nil, metadata.Unknown, metadata.None, "")
		} else if m[1] == "processes" {
			Add(&md, "linux.processes", m[2], nil, metadata.Unknown, metadata.None, "")
		} else if m[1] == "procs_blocked" {
			Add(&md, "linux.procs_blocked", m[2], nil, metadata.Unknown, metadata.None, "")
		}
		return nil
	}); err != nil {
		Error = err
	}
	if num_cores != 0 && t_util != 0 {
		Add(&md, osCPU, t_util/float64(num_cores), nil, metadata.Unknown, metadata.None, "")
	}
	if err := readLine("/proc/loadavg", func(s string) error {
		m := loadavgRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}
		Add(&md, "linux.loadavg_1_min", m[1], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.loadavg_5_min", m[2], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.loadavg_15_min", m[3], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.loadavg_runnable", m[4], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.loadavg_total_threads", m[5], nil, metadata.Unknown, metadata.None, "")
		return nil
	}); err != nil {
		Error = err
	}
	if err := readLine("/proc/sys/kernel/random/entropy_avail", func(s string) error {
		Add(&md, "linux.entropy_avail", strings.TrimSpace(s), nil, metadata.Unknown, metadata.None, "")
		return nil
	}); err != nil {
		Error = err
	}
	num_cpus := 0
	if err := readLine("/proc/interrupts", func(s string) error {
		cols := strings.Fields(s)
		if num_cpus == 0 {
			num_cpus = len(cols)
			return nil
		} else if len(cols) < 2 {
			return nil
		}
		irq_type := strings.TrimRight(cols[0], ":")
		if !IsAlNum(irq_type) {
			return nil
		}
		if IsDigit(irq_type) {
			if cols[len(cols)-2] == "PCI-MSI-edge" && strings.Contains(cols[len(cols)-1], "eth") {
				irq_type = cols[len(cols)-1]
			} else {
				// Interrupt type is just a number, ignore.
				return nil
			}
		}
		for i, val := range cols[1:] {
			if i >= num_cpus {
				// All values read, remaining cols contain textual description.
				break
			}
			if !IsDigit(val) {
				// Something is weird, there should only be digit values.
				return fmt.Errorf("interrupts: unexpected value: %v", val)
				break
			}
			Add(&md, "linux.interrupts", val, opentsdb.TagSet{"type": irq_type, "cpu": strconv.Itoa(i)}, metadata.Unknown, metadata.None, "")
		}
		return nil
	}); err != nil {
		Error = err
	}
	if err := readLine("/proc/net/sockstat", func(s string) error {
		cols := strings.Fields(s)
		switch cols[0] {
		case "sockets:":
			if len(cols) < 3 {
				return fmt.Errorf("sockstat: error parsing sockets line")
			}
			Add(&md, "linux.net.sockets.used", cols[2], nil, metadata.Unknown, metadata.None, "")
		case "TCP:":
			if len(cols) < 11 {
				return fmt.Errorf("sockstat: error parsing tcp line")
			}
			Add(&md, "linux.net.sockets.tcp_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.tcp_orphaned", cols[4], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.tcp_time_wait", cols[6], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.tcp_allocated", cols[8], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.tcp_mem", cols[10], nil, metadata.Unknown, metadata.None, "")
		case "UDP:":
			if len(cols) < 5 {
				return fmt.Errorf("sockstat: error parsing udp line")
			}
			Add(&md, "linux.net.sockets.udp_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.udp_mem", cols[4], nil, metadata.Unknown, metadata.None, "")
		case "UDPLITE:":
			if len(cols) < 3 {
				return fmt.Errorf("sockstat: error parsing udplite line")
			}
			Add(&md, "linux.net.sockets.udplite_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
		case "RAW:":
			if len(cols) < 3 {
				return fmt.Errorf("sockstat: error parsing raw line")
			}
			Add(&md, "linux.net.sockets.raw_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
		case "FRAG:":
			if len(cols) < 5 {
				return fmt.Errorf("sockstat: error parsing frag line")
			}
			Add(&md, "linux.net.sockets.frag_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.frag_mem", cols[4], nil, metadata.Unknown, metadata.None, "")
		}
		return nil
	}); err != nil {
		Error = err
	}
	ln := 0
	var headers []string
	if err := readLine("/proc/net/netstat", func(s string) error {
		cols := strings.Fields(s)
		if ln%2 == 0 {
			headers = cols
		} else {
			if len(cols) < 1 || len(cols) != len(headers) {
				return fmt.Errorf("netstat: parsing failed")
			}
			root := strings.ToLower(strings.TrimSuffix(headers[0], "Ext:"))
			for i, v := range cols[1:] {
				i += 1
				m := "linux.net.stat." + root + "." + strings.TrimPrefix(strings.ToLower(headers[i]), "tcp")
				Add(&md, m, v, nil, metadata.Unknown, metadata.None, "")
			}
		}
		ln += 1
		return nil
	}); err != nil {
		Error = err
	}
	ln = 0
	metric := "linux.net.stat."
	if err := readLine("/proc/net/snmp", func(s string) error {
		ln++
		if ln%2 != 0 {
			f := strings.Fields(s)
			if len(f) < 2 {
				return fmt.Errorf("Failed to parse header line")
			}
			headers = f
		} else {
			values := strings.Fields(s)
			if len(values) != len(headers) {
				return fmt.Errorf("Mismatched header and value length")
			}
			proto := strings.ToLower(strings.TrimSuffix(values[0], ":"))
			for i, v := range values {
				if i == 0 {
					continue
				}
				var stype metadata.RateType = metadata.Counter
				stat := strings.ToLower(headers[i])
				if strings.HasPrefix(stat, "rto") {
					stype = metadata.Gauge
					continue
				}
				Add(&md, metric+proto+"."+stat, v, nil, stype, metadata.None, "")
			}
		}
		return nil
	}); err != nil {
		Error = err
	}
	return md, Error
}
<commit_msg>cmd/scollector: Get rid of metric variable, get rid of continue<commit_after>package collectors

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/StackExchange/scollector/metadata"
	"github.com/StackExchange/scollector/opentsdb"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_procstats_linux})
}

var uptimeRE = regexp.MustCompile(`(\S+)\s+(\S+)`)
var meminfoRE = regexp.MustCompile(`(\w+):\s+(\d+)\s+(\w+)`)
var vmstatRE = regexp.MustCompile(`(\w+)\s+(\d+)`)
var statRE = regexp.MustCompile(`(\w+)\s+(.*)`)
var statCpuRE = regexp.MustCompile(`cpu(\d+)`)
var loadavgRE = regexp.MustCompile(`(\S+)\s+(\S+)\s+(\S+)\s+(\d+)/(\d+)\s+`)
var inoutRE = regexp.MustCompile(`(.*)(in|out)`)

var CPU_FIELDS = []string{
	"user",
	"nice",
	"system",
	"idle",
	"iowait",
	"irq",
	"softirq",
	"steal",
	"guest",
	"guest_nice",
}

func c_procstats_linux() (opentsdb.MultiDataPoint, error) {
	var md opentsdb.MultiDataPoint
	var Error error
	if err := readLine("/proc/uptime", func(s string) error {
		m := uptimeRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}
		Add(&md, "linux.uptime_total", m[1], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.uptime_now", m[2], nil, metadata.Unknown, metadata.None, "")
		return nil
	}); err != nil {
		Error = err
	}
	mem := make(map[string]float64)
	if err := readLine("/proc/meminfo", func(s string) error {
		m := meminfoRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}
		i, err := strconv.ParseFloat(m[2], 64)
		if err != nil {
			return err
		}
		mem[m[1]] = i
		Add(&md, "linux.mem."+strings.ToLower(m[1]), m[2], nil, metadata.Unknown, metadata.None, "")
		return nil
	}); err != nil {
		Error = err
	}
	Add(&md, osMemTotal, int(mem["MemTotal"])*1024, nil, metadata.Unknown, metadata.None, "")
	Add(&md, osMemFree, int(mem["MemFree"])*1024, nil, metadata.Unknown, metadata.None, "")
	Add(&md, osMemUsed, (int(mem["MemTotal"])-(int(mem["MemFree"])+int(mem["Buffers"])+int(mem["Cached"])))*1024, nil, metadata.Unknown, metadata.None, "")
	if mem["MemTotal"] != 0 {
		Add(&md, osMemPctFree, (mem["MemFree"]+mem["Buffers"]+mem["Cached"])/mem["MemTotal"]*100, nil, metadata.Unknown, metadata.None, "")
	}

	if err := readLine("/proc/vmstat", func(s string) error {
		m := vmstatRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}

		switch m[1] {
		case "pgpgin", "pgpgout", "pswpin", "pswpout", "pgfault", "pgmajfault":
			mio := inoutRE.FindStringSubmatch(m[1])
			if mio != nil {
				Add(&md, "linux.mem."+mio[1], m[2], opentsdb.TagSet{"direction": mio[2]}, metadata.Unknown, metadata.None, "")
			} else {
				Add(&md, "linux.mem."+m[1], m[2], nil, metadata.Unknown, metadata.None, "")
			}
		}
		return nil
	}); err != nil {
		Error = err
	}
	num_cores := 0
	var t_util float64
	if err := readLine("/proc/stat", func(s string) error {
		m := statRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}
		if strings.HasPrefix(m[1], "cpu") {
			metric_percpu := ""
			tag_cpu := ""
			cpu_m := statCpuRE.FindStringSubmatch(m[1])
			if cpu_m != nil {
				num_cores += 1
				metric_percpu = ".percpu"
				tag_cpu = cpu_m[1]
			}
			fields := strings.Fields(m[2])
			for i, value := range fields {
				if i >= len(CPU_FIELDS) {
					break
				}
				tags := opentsdb.TagSet{
					"type": CPU_FIELDS[i],
				}
				if tag_cpu != "" {
					tags["cpu"] = tag_cpu
				}
				Add(&md, "linux.cpu"+metric_percpu, value, tags, metadata.Unknown, metadata.None, "")
			}
			if metric_percpu == "" {
				if len(fields) < 3 {
					return nil
				}
				user, err := strconv.ParseFloat(fields[0], 64)
				if err != nil {
					return nil
				}
				nice, err := strconv.ParseFloat(fields[1], 64)
				if err != nil {
					return nil
				}
				system, err := strconv.ParseFloat(fields[2], 64)
				if err != nil {
					return nil
				}
				t_util = user + nice + system
			}
		} else if m[1] == "intr" {
			Add(&md, "linux.intr", strings.Fields(m[2])[0], nil, metadata.Unknown, metadata.None, "")
		} else if m[1] == "ctxt" {
			Add(&md, "linux.ctxt", m[2], nil, metadata.Unknown, metadata.None, "")
		} else if m[1] == "processes" {
			Add(&md, "linux.processes", m[2], nil, metadata.Unknown, metadata.None, "")
		} else if m[1] == "procs_blocked" {
			Add(&md, "linux.procs_blocked", m[2], nil, metadata.Unknown, metadata.None, "")
		}
		return nil
	}); err != nil {
		Error = err
	}
	if num_cores != 0 && t_util != 0 {
		Add(&md, osCPU, t_util/float64(num_cores), nil, metadata.Unknown, metadata.None, "")
	}
	if err := readLine("/proc/loadavg", func(s string) error {
		m := loadavgRE.FindStringSubmatch(s)
		if m == nil {
			return nil
		}
		Add(&md, "linux.loadavg_1_min", m[1], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.loadavg_5_min", m[2], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.loadavg_15_min", m[3], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.loadavg_runnable", m[4], nil, metadata.Unknown, metadata.None, "")
		Add(&md, "linux.loadavg_total_threads", m[5], nil, metadata.Unknown, metadata.None, "")
		return nil
	}); err != nil {
		Error = err
	}
	if err := readLine("/proc/sys/kernel/random/entropy_avail", func(s string) error {
		Add(&md, "linux.entropy_avail", strings.TrimSpace(s), nil, metadata.Unknown, metadata.None, "")
		return nil
	}); err != nil {
		Error = err
	}
	num_cpus := 0
	if err := readLine("/proc/interrupts", func(s string) error {
		cols := strings.Fields(s)
		if num_cpus == 0 {
			num_cpus = len(cols)
			return nil
		} else if len(cols) < 2 {
			return nil
		}
		irq_type := strings.TrimRight(cols[0], ":")
		if !IsAlNum(irq_type) {
			return nil
		}
		if IsDigit(irq_type) {
			if cols[len(cols)-2] == "PCI-MSI-edge" && strings.Contains(cols[len(cols)-1], "eth") {
				irq_type = cols[len(cols)-1]
			} else {
				// Interrupt type is just a number, ignore.
				return nil
			}
		}
		for i, val := range cols[1:] {
			if i >= num_cpus {
				// All values read, remaining cols contain textual description.
				break
			}
			if !IsDigit(val) {
				// Something is weird, there should only be digit values.
				return fmt.Errorf("interrupts: unexpected value: %v", val)
				break
			}
			Add(&md, "linux.interrupts", val, opentsdb.TagSet{"type": irq_type, "cpu": strconv.Itoa(i)}, metadata.Unknown, metadata.None, "")
		}
		return nil
	}); err != nil {
		Error = err
	}
	if err := readLine("/proc/net/sockstat", func(s string) error {
		cols := strings.Fields(s)
		switch cols[0] {
		case "sockets:":
			if len(cols) < 3 {
				return fmt.Errorf("sockstat: error parsing sockets line")
			}
			Add(&md, "linux.net.sockets.used", cols[2], nil, metadata.Unknown, metadata.None, "")
		case "TCP:":
			if len(cols) < 11 {
				return fmt.Errorf("sockstat: error parsing tcp line")
			}
			Add(&md, "linux.net.sockets.tcp_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.tcp_orphaned", cols[4], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.tcp_time_wait", cols[6], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.tcp_allocated", cols[8], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.tcp_mem", cols[10], nil, metadata.Unknown, metadata.None, "")
		case "UDP:":
			if len(cols) < 5 {
				return fmt.Errorf("sockstat: error parsing udp line")
			}
			Add(&md, "linux.net.sockets.udp_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.udp_mem", cols[4], nil, metadata.Unknown, metadata.None, "")
		case "UDPLITE:":
			if len(cols) < 3 {
				return fmt.Errorf("sockstat: error parsing udplite line")
			}
			Add(&md, "linux.net.sockets.udplite_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
		case "RAW:":
			if len(cols) < 3 {
				return fmt.Errorf("sockstat: error parsing raw line")
			}
			Add(&md, "linux.net.sockets.raw_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
		case "FRAG:":
			if len(cols) < 5 {
				return fmt.Errorf("sockstat: error parsing frag line")
			}
			Add(&md, "linux.net.sockets.frag_in_use", cols[2], nil, metadata.Unknown, metadata.None, "")
			Add(&md, "linux.net.sockets.frag_mem", cols[4], nil, metadata.Unknown, metadata.None, "")
		}
		return nil
	}); err != nil {
		Error = err
	}
	ln := 0
	var headers []string
	if err := readLine("/proc/net/netstat", func(s string) error {
		cols := strings.Fields(s)
		if ln%2 == 0 {
			headers = cols
		} else {
			if len(cols) < 1 || len(cols) != len(headers) {
				return fmt.Errorf("netstat: parsing failed")
			}
			root := strings.ToLower(strings.TrimSuffix(headers[0], "Ext:"))
			for i, v := range cols[1:] {
				i += 1
				m := "linux.net.stat." + root + "." + strings.TrimPrefix(strings.ToLower(headers[i]), "tcp")
				Add(&md, m, v, nil, metadata.Unknown, metadata.None, "")
			}
		}
		ln += 1
		return nil
	}); err != nil {
		Error = err
	}
	ln = 0
	if err := readLine("/proc/net/snmp", func(s string) error {
		ln++
		if ln%2 != 0 {
			f := strings.Fields(s)
			if len(f) < 2 {
				return fmt.Errorf("Failed to parse header line")
			}
			headers = f
		} else {
			values := strings.Fields(s)
			if len(values) != len(headers) {
				return fmt.Errorf("Mismatched header and value length")
			}
			proto := strings.ToLower(strings.TrimSuffix(values[0], ":"))
			for i, v := range values {
				if i == 0 {
					continue
				}
				var stype metadata.RateType = metadata.Counter
				stat := strings.ToLower(headers[i])
				if strings.HasPrefix(stat, "rto") {
					stype = metadata.Gauge
				}
				Add(&md, "linux.net.stat."+proto+"."+stat, v, nil, stype, metadata.None, "")
			}
		}
		return nil
	}); err != nil {
		Error = err
	}
	return md, Error
}
<|endoftext|>


<commit_before>package core

import (
	"encoding/json"
	"time"

	dapi "./docker-api"
)

func StatsController() {
	var status int                              // HTTP status returned
	var body string                             // HTTP body returned
	var err error                               // Error handling
	var tmpContainerArray []dapi.ContainerShort // Temporary container array
	var tmpDAPIContainerS dapi.ContainerStats   // Temporary DAPI container stats

	for {
		// Get container list
		l.Debug("StatsController: Get tmpContainerArray")
		status, body = HTTPReq("/containers/json?all=1")
		if status != 200 {
			l.Error("StatsController: Can't get container list, status:", status)
			time.Sleep(time.Second * 5)
			continue
		}

		// Parse returned json
		err = json.Unmarshal([]byte(body), &tmpContainerArray)
		if err != nil {
			l.Error("StatsController: Parsing container list error:", err)
			time.Sleep(time.Second * 5)
			continue
		}
		l.Debug("StatsController: Get tmpContainerArray OK")

		// If no container => stop
		if len(tmpContainerArray) == 0 {
			time.Sleep(time.Second * time.Duration(DGConfig.DockerGuard.StorageControllerPause))
			continue
		}

		for i := 0; i < len(tmpContainerArray); i++ {
			l.Silly("StatsController: Get", tmpContainerArray[i], "storage usage")

			// Get container stats
			status, body = HTTPReq("/containers/" + tmpContainerArray[i].ID + "/stats?stream=0")
			if status != 200 {
				l.Error("StatsController: Can't get docker container (", tmpContainerArray[i].ID, ") stats, status:", status)
				continue
			} else {
				// Parse container returned json
				err = json.Unmarshal([]byte(body), &tmpDAPIContainerS)
				if err != nil {
					l.Error("StatsController: Parsing docker container (", tmpContainerArray[i].ID, ") stats:", err)
					continue
				}
			}

			// Add values to map
			if status == 200 {
				l.Debug("StatsController: Add values to map")
				SetContainerMemoryUsed(tmpContainerArray[i].ID, float64(tmpDAPIContainerS.MemoryStats.Usage))
				SetContainerNetBandwithRX(tmpContainerArray[i].ID, float64(tmpDAPIContainerS.Network.RxBytes))
				SetContainerNetBandwithTX(tmpContainerArray[i].ID, float64(tmpDAPIContainerS.Network.TxBytes))
				SetContainerCPUUsage(tmpContainerArray[i].ID, float64(tmpDAPIContainerS.CPUStats.CPUUsage.TotalUsage))
				ContainerResetTime(tmpContainerArray[i].ID)
				l.Debug("StatsController: Add values to map OK")
			}

			// Pause 1sec * StorageControllerInterval
			time.Sleep(time.Second * time.Duration(DGConfig.DockerGuard.StorageControllerInterval))
		}

		// Pause 1sec * StorageControllerPause
		l.Silly("StatsController: End getting containers storage usage")
		time.Sleep(time.Second * time.Duration(DGConfig.DockerGuard.StorageControllerPause))
	}

}
<commit_msg>Get asynchronously stats => BETTER PERFORMANCE OMG!<commit_after>package core

import (
	"encoding/json"
	"sync"
	"time"

	dapi "./docker-api"
)

func StatsController() {
	var status int                               // HTTP status returned
	var body string                              // HTTP body returned
	var err error                                // Error handling
	var tmpContainerArray []*dapi.ContainerShort // Temporary container array
	var wg sync.WaitGroup

	for {
		// Get container list
		l.Debug("StatsController: Get tmpContainerArray")
		status, body = HTTPReq("/containers/json?all=1")
		if status != 200 {
			l.Error("StatsController: Can't get container list, status:", status)
			time.Sleep(time.Second * 5)
			continue
		}

		// Parse returned json
		err = json.Unmarshal([]byte(body), &tmpContainerArray)
		if err != nil {
			l.Error("StatsController: Parsing container list error:", err)
			time.Sleep(time.Second * 5)
			continue
		}
		l.Debug("StatsController: Get tmpContainerArray OK")

		// If no container => stop
		if len(tmpContainerArray) == 0 {
			time.Sleep(time.Second * time.Duration(DGConfig.DockerGuard.StorageControllerPause))
			continue
		}

		// Get containers' stats
		for i := 0; i < len(tmpContainerArray); i++ {
			tmpContainer := tmpContainerArray[i]

			// Get asynchronously stats
			wg.Add(1)
			go GetStats(tmpContainer, &wg)

			// Pause 1sec * StorageControllerInterval
			time.Sleep(time.Second * time.Duration(DGConfig.DockerGuard.StorageControllerInterval))
		}

		// Wait stats getters
		wg.Wait()

		// Pause 1sec * StorageControllerPause
		l.Silly("StatsController: End getting containers storage usage")
		time.Sleep(time.Second * time.Duration(DGConfig.DockerGuard.StorageControllerPause))
	}

}

/*
	Get container's stats
*/
func GetStats(container *dapi.ContainerShort, wg *sync.WaitGroup) {
	var tmpDAPIContainerS dapi.ContainerStats // Temporary DAPI container stats
	var status int                            // HTTP status returned
	var body string                           // HTTP body returned
	var err error                             // Error handling

	defer func() {
		wg.Done()
	}()

	// Get container stats
	status, body = HTTPReq("/containers/" + container.ID + "/stats?stream=0")
	if status != 200 {
		l.Error("StatsController: Can't get docker container (", container.ID, ") stats, status:", status)
		return
	} else {
		// Parse container returned json
		err = json.Unmarshal([]byte(body), &tmpDAPIContainerS)
		if err != nil {
			l.Error("StatsController: Parsing docker container (", container.ID, ") stats:", err)
			return
		}
	}

	// Add values to map
	if status == 200 {
		l.Debug("StatsController: Add values to map")
		SetContainerMemoryUsed(container.ID, float64(tmpDAPIContainerS.MemoryStats.Usage))
		SetContainerNetBandwithRX(container.ID, float64(tmpDAPIContainerS.Network.RxBytes))
		SetContainerNetBandwithTX(container.ID, float64(tmpDAPIContainerS.Network.TxBytes))
		SetContainerCPUUsage(container.ID, float64(tmpDAPIContainerS.CPUStats.CPUUsage.TotalUsage))
		ContainerResetTime(container.ID)
		l.Debug("StatsController: Add values to map OK")
	}
}
<|endoftext|>


<commit_before>package core

import (
	"github.com/emicklei/melrose/notify"
)

var debugEnabled = false

func IsDebug() bool {
	return debugEnabled
}

func ToggleDebug() bool {
	debugEnabled = !debugEnabled
	return debugEnabled
}

type Watch struct {
	Context Context
	Target  interface{}
}

func (w Watch) Evaluate(c Condition) error {
	// TODO check c?
	w.S()
	return nil
}

// S is part of Sequenceable
func (w Watch) S() Sequence {
	beats, bars := w.Context.Control().BeatsAndBars()
	in := NewInspect(w.Context, w.Target)
	in.Properties["bar"] = bars
	in.Properties["beat"] = beats
	notify.Print(notify.Infof("%s", in.String()))
	return EmptySequence
}
<commit_msg>make playable<commit_after>package core

import (
	"time"

	"github.com/emicklei/melrose/notify"
)

var debugEnabled = false

func IsDebug() bool {
	return debugEnabled
}

func ToggleDebug() bool {
	debugEnabled = !debugEnabled
	return debugEnabled
}

type Watch struct {
	Context Context
	Target  interface{}
}

func (w Watch) Play(ctx Context, at time.Time) error {
	w.S()
	return nil
}

func (w Watch) Stop(ctx Context) error {
	return nil
}

func (w Watch) Evaluate(ctx Context) error {
	// TODO check c?
	w.S()
	return nil
}

// S is part of Sequenceable
func (w Watch) S() Sequence {
	beats, bars := w.Context.Control().BeatsAndBars()
	in := NewInspect(w.Context, w.Target)
	in.Properties["bar"] = bars
	in.Properties["beat"] = beats
	notify.Print(notify.Infof("%s", in.String()))
	return EmptySequence
}
<|endoftext|>


<commit_before>package core

import (
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

// the tag used to denote the name of the question
const tagName = "survey"

func WriteAnswer(t interface{}, name string, v interface{}) (err error) {
	// the target to write to
	target := reflect.ValueOf(t)
	// the value to write from
	value := reflect.ValueOf(v)

	// make sure we are writing to a pointer
	if target.Kind() != reflect.Ptr {
		return errors.New("you must pass a pointer as the target of a Write operation")
	}
	// the object "inside" of the target pointer
	elem := target.Elem()

	// handle the special types
	switch elem.Kind() {
	// if we are writing to a struct
	case reflect.Struct:
		// get the name of the field that matches the string we  were given
		fieldIndex, err := findFieldIndex(elem, name)
		// if something went wrong
		if err != nil {
			// bubble up
			return err
		}

		// copy the value over to the field
		return copy(elem.Field(fieldIndex), value)
	case reflect.Map:
		mapType := reflect.TypeOf(t).Elem()
		if mapType.Key().Kind() != reflect.String || mapType.Elem().Kind() != reflect.Interface {
			return errors.New("answer maps must be of type map[string]interface")
		}
		mt := *t.(*map[string]interface{})
		mt[name] = value.Interface()
		return nil
	}
	// otherwise just copy the value to the target
	return copy(elem, value)
}

// BUG(AlecAivazis): the current implementation might cause weird conflicts if there are
// two fields with same name that only differ by casing.
func findFieldIndex(s reflect.Value, name string) (int, error) {
	// the type of the value
	sType := s.Type()

	// first look for matching tags so we can overwrite matching field names
	for i := 0; i < sType.NumField(); i++ {
		// the field we are current scanning
		field := sType.Field(i)

		// the value of the survey tag
		tag := field.Tag.Get(tagName)
		// if the tag matches the name we are looking for
		if tag != "" && tag == name {
			// then we found our index
			return i, nil
		}
	}

	// then look for matching names
	for i := 0; i < sType.NumField(); i++ {
		// the field we are current scanning
		field := sType.Field(i)

		// if the name of the field matches what we're looking for
		if strings.ToLower(field.Name) == strings.ToLower(name) {
			return i, nil
		}
	}

	// we didn't find the field
	return -1, fmt.Errorf("could not find field matching %v", name)
}

// Write takes a value and copies it to the target
func copy(t reflect.Value, v reflect.Value) (err error) {
	// if something ends up panicing we need to catch it in a deferred func
	defer func() {
		if r := recover(); r != nil {
			// if we paniced with an error
			if _, ok := r.(error); ok {
				// cast the result to an error object
				err = r.(error)
			} else if _, ok := r.(string); ok {
				// otherwise we could have paniced with a string so wrap it in an error
				err = errors.New(r.(string))
			}
		}
	}()

	// attempt to copy the underlying value to the target
	if v.Kind() == reflect.String && v.Type() != t.Type() {
		var castVal interface{}
		var casterr error
		vString := v.Interface().(string)

		switch t.Kind() {
		case reflect.Bool:
			castVal, casterr = strconv.ParseBool(vString)
		case reflect.Int:
			castVal, casterr = strconv.Atoi(vString)
		case reflect.Int8:
			var val64 int64
			val64, casterr = strconv.ParseInt(vString, 10, 8)
			if casterr == nil {
				castVal = int8(val64)
			}
		case reflect.Int16:
			var val64 int64
			val64, casterr = strconv.ParseInt(vString, 10, 16)
			if casterr == nil {
				castVal = int16(val64)
			}
		case reflect.Int32:
			var val64 int64
			val64, casterr = strconv.ParseInt(vString, 10, 32)
			if casterr == nil {
				castVal = int32(val64)
			}
		case reflect.Int64:
			castVal, casterr = strconv.ParseInt(vString, 10, 64)
		case reflect.Uint:
			var val64 uint64
			val64, casterr = strconv.ParseUint(vString, 10, 8)
			if casterr == nil {
				castVal = uint(val64)
			}
		case reflect.Uint8:
			var val64 uint64
			val64, casterr = strconv.ParseUint(vString, 10, 8)
			if casterr == nil {
				castVal = uint8(val64)
			}
		case reflect.Uint16:
			var val64 uint64
			val64, casterr = strconv.ParseUint(vString, 10, 16)
			if casterr == nil {
				castVal = uint16(val64)
			}
		case reflect.Uint32:
			var val64 uint64
			val64, casterr = strconv.ParseUint(vString, 10, 32)
			if casterr == nil {
				castVal = uint32(val64)
			}
		case reflect.Uint64:
			castVal, casterr = strconv.ParseUint(vString, 10, 64)
		case reflect.Float32:
			var val64 float64
			val64, casterr = strconv.ParseFloat(vString, 32)
			if casterr == nil {
				castVal = float32(val64)
			}
		case reflect.Float64:
			castVal, casterr = strconv.ParseFloat(vString, 64)
		}

		if casterr != nil {
			return casterr
		}

		t.Set(reflect.ValueOf(castVal))
		return
	}

	t.Set(v)

	// we're done
	return
}
<commit_msg>added default error when converting string to field type fails<commit_after>package core

import (
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"strings"
)

// the tag used to denote the name of the question
const tagName = "survey"

func WriteAnswer(t interface{}, name string, v interface{}) (err error) {
	// the target to write to
	target := reflect.ValueOf(t)
	// the value to write from
	value := reflect.ValueOf(v)

	// make sure we are writing to a pointer
	if target.Kind() != reflect.Ptr {
		return errors.New("you must pass a pointer as the target of a Write operation")
	}
	// the object "inside" of the target pointer
	elem := target.Elem()

	// handle the special types
	switch elem.Kind() {
	// if we are writing to a struct
	case reflect.Struct:
		// get the name of the field that matches the string we  were given
		fieldIndex, err := findFieldIndex(elem, name)
		// if something went wrong
		if err != nil {
			// bubble up
			return err
		}

		// copy the value over to the field
		return copy(elem.Field(fieldIndex), value)
	case reflect.Map:
		mapType := reflect.TypeOf(t).Elem()
		if mapType.Key().Kind() != reflect.String || mapType.Elem().Kind() != reflect.Interface {
			return errors.New("answer maps must be of type map[string]interface")
		}
		mt := *t.(*map[string]interface{})
		mt[name] = value.Interface()
		return nil
	}
	// otherwise just copy the value to the target
	return copy(elem, value)
}

// BUG(AlecAivazis): the current implementation might cause weird conflicts if there are
// two fields with same name that only differ by casing.
func findFieldIndex(s reflect.Value, name string) (int, error) {
	// the type of the value
	sType := s.Type()

	// first look for matching tags so we can overwrite matching field names
	for i := 0; i < sType.NumField(); i++ {
		// the field we are current scanning
		field := sType.Field(i)

		// the value of the survey tag
		tag := field.Tag.Get(tagName)
		// if the tag matches the name we are looking for
		if tag != "" && tag == name {
			// then we found our index
			return i, nil
		}
	}

	// then look for matching names
	for i := 0; i < sType.NumField(); i++ {
		// the field we are current scanning
		field := sType.Field(i)

		// if the name of the field matches what we're looking for
		if strings.ToLower(field.Name) == strings.ToLower(name) {
			return i, nil
		}
	}

	// we didn't find the field
	return -1, fmt.Errorf("could not find field matching %v", name)
}

// Write takes a value and copies it to the target
func copy(t reflect.Value, v reflect.Value) (err error) {
	// if something ends up panicing we need to catch it in a deferred func
	defer func() {
		if r := recover(); r != nil {
			// if we paniced with an error
			if _, ok := r.(error); ok {
				// cast the result to an error object
				err = r.(error)
			} else if _, ok := r.(string); ok {
				// otherwise we could have paniced with a string so wrap it in an error
				err = errors.New(r.(string))
			}
		}
	}()

	// attempt to copy the underlying value to the target
	if v.Kind() == reflect.String && v.Type() != t.Type() {
		var castVal interface{}
		var casterr error
		vString := v.Interface().(string)

		switch t.Kind() {
		case reflect.Bool:
			castVal, casterr = strconv.ParseBool(vString)
		case reflect.Int:
			castVal, casterr = strconv.Atoi(vString)
		case reflect.Int8:
			var val64 int64
			val64, casterr = strconv.ParseInt(vString, 10, 8)
			if casterr == nil {
				castVal = int8(val64)
			}
		case reflect.Int16:
			var val64 int64
			val64, casterr = strconv.ParseInt(vString, 10, 16)
			if casterr == nil {
				castVal = int16(val64)
			}
		case reflect.Int32:
			var val64 int64
			val64, casterr = strconv.ParseInt(vString, 10, 32)
			if casterr == nil {
				castVal = int32(val64)
			}
		case reflect.Int64:
			castVal, casterr = strconv.ParseInt(vString, 10, 64)
		case reflect.Uint:
			var val64 uint64
			val64, casterr = strconv.ParseUint(vString, 10, 8)
			if casterr == nil {
				castVal = uint(val64)
			}
		case reflect.Uint8:
			var val64 uint64
			val64, casterr = strconv.ParseUint(vString, 10, 8)
			if casterr == nil {
				castVal = uint8(val64)
			}
		case reflect.Uint16:
			var val64 uint64
			val64, casterr = strconv.ParseUint(vString, 10, 16)
			if casterr == nil {
				castVal = uint16(val64)
			}
		case reflect.Uint32:
			var val64 uint64
			val64, casterr = strconv.ParseUint(vString, 10, 32)
			if casterr == nil {
				castVal = uint32(val64)
			}
		case reflect.Uint64:
			castVal, casterr = strconv.ParseUint(vString, 10, 64)
		case reflect.Float32:
			var val64 float64
			val64, casterr = strconv.ParseFloat(vString, 32)
			if casterr == nil {
				castVal = float32(val64)
			}
		case reflect.Float64:
			castVal, casterr = strconv.ParseFloat(vString, 64)
		default:
			return fmt.Errorf("Unable to convert from string to type %s", t.Kind())
		}

		if casterr != nil {
			return casterr
		}

		t.Set(reflect.ValueOf(castVal))
		return
	}

	t.Set(v)

	// we're done
	return
}
<|endoftext|>


<commit_before>// Copyright 2012 The LevelDB-Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package memfs

import (
	"io"
	"os"
	"sort"
	"strings"
	"testing"

	"code.google.com/p/leveldb-go/leveldb/db"
)

func normalize(name string) string {
	if os.PathSeparator == '/' {
		return name
	}
	return strings.Replace(name, "/", string(os.PathSeparator), -1)
}

func TestBasics(t *testing.T) {
	var (
		fs   = New()
		f, g db.File
		fi   os.FileInfo
		buf  [16]byte
		n    int
	)

	// Create a top-level file.
	f, err := fs.Create(normalize("/foo"))
	if err != nil {
		t.Fatalf("1a: Create: %v", err)
	}
	f.Close()

	// Create a child of that file. It should fail, since /foo is not a directory.
	_, err = fs.Create(normalize("/foo/x"))
	if err == nil {
		t.Fatalf("2a: Create: got nil error, want non-nil")
	}

	// Create a third-level file. It should fail, since /bar has not been created.
	// Similarly, opening that file should fail.
	_, err = fs.Create(normalize("/bar/baz/y"))
	if err == nil {
		t.Fatalf("3a: Create: got nil error, want non-nil")
	}
	_, err = fs.Open(normalize("/bar/baz/y"))
	if err == nil {
		t.Fatalf("3b: Open: got nil error, want non-nil")
	}

	// Make the /bar/baz directory; create a third-level file. Creation should now succeed.
	err = fs.MkdirAll(normalize("/bar/baz/"), 0755)
	if err != nil {
		t.Fatalf("4a: MkdirAll: %v", err)
	}
	f, err = fs.Create(normalize("/bar/baz/y"))
	if err != nil {
		t.Fatalf("4b: Create: %v", err)
	}
	fi, err = f.Stat()
	if err != nil {
		t.Fatalf("4c: Stat: %v", err)
	}
	if got, want := fi.Name(), "y"; got != want {
		t.Fatalf("4d: Name: got %q, want %q", got, want)
	}

	// Write some data; read it back.
	_, err = f.Write([]byte("ABCDE"))
	if err != nil {
		t.Fatalf("5a: Write: %v", err)
	}
	err = f.Close()
	if err != nil {
		t.Fatalf("5b: Close: %v", err)
	}
	g, err = fs.Open(normalize("/bar/baz/y"))
	if err != nil {
		t.Fatalf("5c: Open: %v", err)
	}
	n, err = io.ReadFull(g, buf[:5])
	if err != nil {
		t.Fatalf("5d: ReadFull: %v", err)
	}
	if n != 5 {
		t.Fatalf("5e: ReadFull: got %d, want %d", n, 5)
	}
	n, err = g.ReadAt(buf[5:8], 0)
	if err != nil {
		t.Fatalf("5f: ReadAt: %v", err)
	}
	if n != 3 {
		t.Fatalf("5g: ReadAt: got %d, want %d", n, 3)
	}
	if got, want := string(buf[:8]), "ABCDEABC"; got != want {
		t.Fatalf("5h: ReadAt: got %q, want %q", got, want)
	}
	err = g.Close()
	if err != nil {
		t.Fatalf("5i: Close: %v", err)
	}

	// Remove the file twice. The first should succeed, the second should fail.
	err = fs.Remove(normalize("/bar/baz/y"))
	if err != nil {
		t.Fatalf("6a: Remove: %v", err)
	}
	err = fs.Remove(normalize("/bar/baz/y"))
	if err == nil {
		t.Fatalf("6b: Remove: got nil error, want non-nil")
	}
	_, err = fs.Open(normalize("/bar/baz/y"))
	if err == nil {
		t.Fatalf("6c: Open: got nil error, want non-nil")
	}

	// Rename /foo to /goo. Trying to open /foo should succeed before the rename and
	// fail afterwards, and vice versa for /goo.
	f, err = fs.Open(normalize("/foo"))
	if err != nil {
		t.Fatalf("7a: Open: %v", err)
	}
	f.Close()
	_, err = fs.Open(normalize("/goo"))
	if err == nil {
		t.Fatalf("7b: Open: got nil error, want non-nil")
	}
	err = fs.Rename(normalize("/foo"), normalize("/goo"))
	if err != nil {
		t.Fatalf("7c: Rename: %v", err)
	}
	_, err = fs.Open(normalize("/foo"))
	if err == nil {
		t.Fatalf("7d: Open: got nil error, want non-nil")
	}
	f, err = fs.Open(normalize("/goo"))
	if err != nil {
		t.Fatalf("7e: Open: %v", err)
	}
	f.Close()

	// Create /bar/baz/z and rename /bar/baz to /bar/caz.
	f, err = fs.Create(normalize("/bar/baz/z"))
	if err != nil {
		t.Fatalf("8a: Create: %v", err)
	}
	f.Close()
	f, err = fs.Open(normalize("/bar/baz/z"))
	if err != nil {
		t.Fatalf("8b: Open: %v", err)
	}
	f.Close()
	_, err = fs.Open(normalize("/bar/caz/z"))
	if err == nil {
		t.Fatalf("8c: Open: got nil error, want non-nil")
	}
	err = fs.Rename(normalize("/bar/baz"), normalize("/bar/caz"))
	if err != nil {
		t.Fatalf("8d: Rename: %v", err)
	}
	_, err = fs.Open(normalize("/bar/baz/z"))
	if err == nil {
		t.Fatalf("8e: Open: got nil error, want non-nil")
	}
	f, err = fs.Open(normalize("/bar/caz/z"))
	if err != nil {
		t.Fatalf("8f: Open: %v", err)
	}
	f.Close()
}

func TestList(t *testing.T) {
	fs := New()

	dirnames := []string{
		"/bar",
		"/foo/2",
	}
	for _, dirname := range dirnames {
		err := fs.MkdirAll(dirname, 0755)
		if err != nil {
			t.Fatalf("MkdirAll %q: %v", dirname, err)
		}
	}

	filenames := []string{
		"/a",
		"/bar/baz",
		"/foo/0",
		"/foo/1",
		"/foo/2/a",
		"/foo/2/b",
		"/foo/3",
		"/foot",
	}
	for _, filename := range filenames {
		f, err := fs.Create(normalize(filename))
		if err != nil {
			t.Fatalf("Create %q: %v", filename, err)
		}
		if err := f.Close(); err != nil {
			t.Fatalf("Close %q: %v", filename, err)
		}
	}

	{
		got := fs.(*fileSystem).String()
		want := normalize(`          /
       0    a
            bar/
       0      baz
            foo/
       0      0
       0      1
              2/
       0        a
       0        b
       0      3
       0    foot
`)
		if got != want {
			t.Fatalf("String:\n----got----\n%s----want----\n%s", got, want)
		}
	}

	testCases := []string{
		"/:a bar foo foot",
		"/bar:baz",
		"/bar/:baz",
		"/baz:",
		"/baz/:",
		"/foo:0 1 2 3",
		"/foo/:0 1 2 3",
		"/foo/1:",
		"/foo/1/:",
		"/foo/2:a b",
		"/foo/2/:a b",
		"/foot:",
		"/foot/:",
	}
	for _, tc := range testCases {
		s := strings.Split(tc, ":")
		list, _ := fs.List(normalize(s[0]))
		sort.Strings(list)
		got := strings.Join(list, " ")
		want := s[1]
		if got != want {
			t.Errorf("List %q: got %q, want %q", s[0], got, want)
		}
	}
}
<commit_msg>leveldb: rewrite memfs_test's TestBasics as a table-driven test.<commit_after>// Copyright 2012 The LevelDB-Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package memfs

import (
	"io"
	"os"
	"sort"
	"strconv"
	"strings"
	"testing"

	"code.google.com/p/leveldb-go/leveldb/db"
)

func normalize(name string) string {
	if os.PathSeparator == '/' {
		return name
	}
	return strings.Replace(name, "/", string(os.PathSeparator), -1)
}

func TestBasics(t *testing.T) {
	fs := New()
	testCases := []string{
		// Create a top-level file.
		"1a: create /foo",
		// Create a child of that file. It should fail, since /foo is not a directory.
		"2a: create /foo/x fails",
		// Create a third-level file. It should fail, since /bar has not been created.
		// Similarly, opening that file should fail.
		"3a: create /bar/baz/y fails",
		"3b: open /bar/baz/y fails",
		// Make the /bar/baz directory; create a third-level file. Creation should now succeed.
		"4a: mkdirall /bar/baz",
		"4b: f = create /bar/baz/y",
		"4c: f.stat.name == y",
		// Write some data; read it back.
		"5a: f.write abcde",
		"5b: f.close",
		"5c: f = open /bar/baz/y",
		"5d: f.read 5 == abcde",
		"5e: f.readat 2 1 == bc",
		"5f: f.close",
		// Remove the file twice. The first should succeed, the second should fail.
		"6a: remove /bar/baz/y",
		"6b: remove /bar/baz/y fails",
		"6c: open /bar/baz/y fails",
		// Rename /foo to /goo. Trying to open /foo should succeed before the rename and
		// fail afterwards, and vice versa for /goo.
		"7a: open /foo",
		"7b: open /goo fails",
		"7c: rename /foo /goo",
		"7d: open /foo fails",
		"7e: open /goo",
		// Create /bar/baz/z and rename /bar/baz to /bar/caz.
		"8a: create /bar/baz/z",
		"8b: open /bar/baz/z",
		"8c: open /bar/caz/z fails",
		"8d: rename /bar/baz /bar/caz",
		"8e: open /bar/baz/z fails",
		"8f: open /bar/caz/z",
	}
	var f db.File
	for _, tc := range testCases {
		s := strings.Split(tc, " ")[1:]

		saveF := s[0] == "f" && s[1] == "="
		if saveF {
			s = s[2:]
		}

		fails := s[len(s)-1] == "fails"
		if fails {
			s = s[:len(s)-1]
		}

		var (
			fi  os.FileInfo
			g   db.File
			err error
		)
		switch s[0] {
		case "create":
			g, err = fs.Create(normalize(s[1]))
		case "open":
			g, err = fs.Open(normalize(s[1]))
		case "mkdirall":
			err = fs.MkdirAll(normalize(s[1]), 0755)
		case "remove":
			err = fs.Remove(normalize(s[1]))
		case "rename":
			err = fs.Rename(normalize(s[1]), normalize(s[2]))
		case "f.write":
			_, err = f.Write([]byte(s[1]))
		case "f.read":
			n, _ := strconv.Atoi(s[1])
			buf := make([]byte, n)
			_, err = io.ReadFull(f, buf)
			if err != nil {
				break
			}
			if got, want := string(buf), s[3]; got != want {
				t.Fatalf("%q: got %q, want %q", tc, got, want)
			}
		case "f.readat":
			n, _ := strconv.Atoi(s[1])
			off, _ := strconv.Atoi(s[2])
			buf := make([]byte, n)
			_, err = f.ReadAt(buf, int64(off))
			if err != nil {
				break
			}
			if got, want := string(buf), s[4]; got != want {
				t.Fatalf("%q: got %q, want %q", tc, got, want)
			}
		case "f.close":
			f, err = nil, f.Close()
		case "f.stat.name":
			fi, err = f.Stat()
			if err != nil {
				break
			}
			if got, want := fi.Name(), s[2]; got != want {
				t.Fatalf("%q: got %q, want %q", tc, got, want)
			}
		default:
			t.Fatalf("bad test case: %q", tc)
		}

		if saveF {
			f, g = g, nil
		} else if g != nil {
			g.Close()
		}

		if fails {
			if err == nil {
				t.Fatalf("%q: got nil error, want non-nil", tc)
			}
		} else {
			if err != nil {
				t.Fatalf("%q: %v", tc, err)
			}
		}
	}
}

func TestList(t *testing.T) {
	fs := New()

	dirnames := []string{
		"/bar",
		"/foo/2",
	}
	for _, dirname := range dirnames {
		err := fs.MkdirAll(dirname, 0755)
		if err != nil {
			t.Fatalf("MkdirAll %q: %v", dirname, err)
		}
	}

	filenames := []string{
		"/a",
		"/bar/baz",
		"/foo/0",
		"/foo/1",
		"/foo/2/a",
		"/foo/2/b",
		"/foo/3",
		"/foot",
	}
	for _, filename := range filenames {
		f, err := fs.Create(normalize(filename))
		if err != nil {
			t.Fatalf("Create %q: %v", filename, err)
		}
		if err := f.Close(); err != nil {
			t.Fatalf("Close %q: %v", filename, err)
		}
	}

	{
		got := fs.(*fileSystem).String()
		want := normalize(`          /
       0    a
            bar/
       0      baz
            foo/
       0      0
       0      1
              2/
       0        a
       0        b
       0      3
       0    foot
`)
		if got != want {
			t.Fatalf("String:\n----got----\n%s----want----\n%s", got, want)
		}
	}

	testCases := []string{
		"/:a bar foo foot",
		"/bar:baz",
		"/bar/:baz",
		"/baz:",
		"/baz/:",
		"/foo:0 1 2 3",
		"/foo/:0 1 2 3",
		"/foo/1:",
		"/foo/1/:",
		"/foo/2:a b",
		"/foo/2/:a b",
		"/foot:",
		"/foot/:",
	}
	for _, tc := range testCases {
		s := strings.Split(tc, ":")
		list, _ := fs.List(normalize(s[0]))
		sort.Strings(list)
		got := strings.Join(list, " ")
		want := s[1]
		if got != want {
			t.Errorf("List %q: got %q, want %q", s[0], got, want)
		}
	}
}
<|endoftext|>