<commit_before>package collector

import (
	"encoding/json"
	"fullerite/metric"

	"reflect"
	"testing"

	l "github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
	"github.com/stretchr/testify/assert"
)

func getSUT() *DockerStats {
	expectedChan := make(chan metric.Metric)
	var expectedLogger = defaultLog.WithFields(l.Fields{"collector": "fullerite"})

	return newDockerStats(expectedChan, 10, expectedLogger).(*DockerStats)
}

func TestDockerStatsNewDockerStats(t *testing.T) {
	expectedChan := make(chan metric.Metric)
	var expectedLogger = defaultLog.WithFields(l.Fields{"collector": "fullerite"})
	expectedType := make(map[string]*CPUValues)

	d := newDockerStats(expectedChan, 10, expectedLogger).(*DockerStats)

	assert.Equal(t, d.log, expectedLogger)
	assert.Equal(t, d.channel, expectedChan)
	assert.Equal(t, d.interval, 10)
	assert.Equal(t, d.name, "DockerStats")
	assert.Equal(t, reflect.TypeOf(d.previousCPUValues), reflect.TypeOf(expectedType))
	assert.Equal(t, len(d.previousCPUValues), 0)
	d.Configure(make(map[string]interface{}))
	assert.Equal(t, d.GetEndpoint(), endpoint)
}

func TestDockerStatsConfigureEmptyConfig(t *testing.T) {
	config := make(map[string]interface{})

	d := newDockerStats(nil, 123, nil).(*DockerStats)
	d.Configure(config)

	assert.Equal(t, 123, d.Interval())
}

func TestDockerStatsConfigure(t *testing.T) {
	config := make(map[string]interface{})
	config["interval"] = 9999

	d := newDockerStats(nil, 123, nil).(*DockerStats)
	d.Configure(config)

	assert.Equal(t, 9999, d.Interval())
}

func TestDockerStatsBuildMetrics(t *testing.T) {
	config := make(map[string]interface{})
	envVars := []byte(`
	{
		"service_name":  {
			"MESOS_TASK_ID": "[^\\.]*"
		},
		"instance_name": {
			"MESOS_TASK_ID": "\\.([^\\.]*)\\."}
	}`)
	var val map[string]interface{}

	err := json.Unmarshal(envVars, &val)
	assert.Equal(t, err, nil)
	config["generatedDimensions"] = val

	stats := new(docker.Stats)
	stats.Networks.eth0.RxBytes = 10
	stats.Networks.eth0.TxBytes = 20
	stats.MemoryStats.Usage = 50
	stats.MemoryStats.Limit = 70

	containerJSON := []byte(`
	{
		"ID": "test-id",
		"Name": "test-container",
		"Config": {
			"Env": [
				"MESOS_TASK_ID=my--service.main.blablagit6bdsadnoise"
			]
		}
	}`)
	var container *docker.Container
	err = json.Unmarshal(containerJSON, &container)
	assert.Equal(t, err, nil)

	expectedDims := map[string]string{
		"container_id":   "test-id",
		"container_name": "test-container",
		"service_name":   "my_service",
		"instance_name":  "main",
		"iface":          "eth0",
	}

	expectedDimsGen := map[string]string{
		"service_name":  "my_service",
		"instance_name": "main",
	}
	expectedMetrics := []metric.Metric{
		metric.Metric{"DockerRxBytes", "cumcounter", 10, expectedDims},
		metric.Metric{"DockerTxBytes", "cumcounter", 20, expectedDims},
		metric.Metric{"DockerMemoryUsed", "gauge", 50, expectedDims},
		metric.Metric{"DockerMemoryLimit", "gauge", 70, expectedDims},
		metric.Metric{"DockerCpuPercentage", "gauge", 0.5, expectedDims},
		metric.Metric{"DockerContainerCount", "counter", 1, expectedDimsGen},
	}

	d := getSUT()
	d.Configure(config)
	ret := d.buildMetrics(container, stats, 0.5)

	assert.Equal(t, ret, expectedMetrics)
}

func TestDockerStatsBuildMetricsWithNameAsEnvVariable(t *testing.T) {
	config := make(map[string]interface{})
	envVars := []byte(`
	{
		"service_name": {
			"SERVICE_NAME": ".*"
		}
	}`)
	var val map[string]interface{}

	err := json.Unmarshal(envVars, &val)
	assert.Equal(t, err, nil)
	config["generatedDimensions"] = val

	stats := new(docker.Stats)
	stats.Network.RxBytes = 10
	stats.Network.TxBytes = 20
	stats.MemoryStats.Usage = 50
	stats.MemoryStats.Limit = 70

	containerJSON := []byte(`
	{
		"ID": "test-id",
		"Name": "test-container",
		"Config": {
			"Env": [
				"SERVICE_NAME=my_service"
			]
		}
	}`)
	var container *docker.Container
	err = json.Unmarshal(containerJSON, &container)
	assert.Equal(t, err, nil)

	expectedDims := map[string]string{
		"container_id":   "test-id",
		"container_name": "test-container",
		"service_name":   "my_service",
	}
	expectedDimsGen := map[string]string{
		"service_name": "my_service",
	}
	expectedMetrics := []metric.Metric{
		metric.Metric{"DockerRxBytes", "cumcounter", 10, expectedDims},
		metric.Metric{"DockerTxBytes", "cumcounter", 20, expectedDims},
		metric.Metric{"DockerMemoryUsed", "gauge", 50, expectedDims},
		metric.Metric{"DockerMemoryLimit", "gauge", 70, expectedDims},
		metric.Metric{"DockerCpuPercentage", "gauge", 0.5, expectedDims},
		metric.Metric{"DockerContainerCount", "counter", 1, expectedDimsGen},
	}

	d := getSUT()
	d.Configure(config)
	ret := d.buildMetrics(container, stats, 0.5)

	assert.Equal(t, ret, expectedMetrics)
}

func TestDockerStatsCalculateCPUPercent(t *testing.T) {
	var previousTotalUsage = uint64(0)
	var previousSystem = uint64(0)

	stats := new(docker.Stats)
	stats.CPUStats.CPUUsage.PercpuUsage = make([]uint64, 24)
	stats.CPUStats.CPUUsage.TotalUsage = 1261158030354
	stats.CPUStats.SystemCPUUsage = 108086414700000000

	assert.Equal(t, 0.02800332753427522, calculateCPUPercent(previousTotalUsage, previousSystem, stats))

	previousTotalUsage = stats.CPUStats.CPUUsage.TotalUsage
	previousSystem = stats.CPUStats.SystemCPUUsage
	stats.CPUStats.CPUUsage.TotalUsage = 1261164064229
	stats.CPUStats.SystemCPUUsage = 108086652820000000

	assert.Equal(t, 0.060815135225936505, calculateCPUPercent(previousTotalUsage, previousSystem, stats))
}
<commit_msg>differentiate dimensions, correct order of expected metrics<commit_after>package collector

import (
	"encoding/json"
	"fullerite/metric"

	"reflect"
	"testing"

	l "github.com/Sirupsen/logrus"
	"github.com/fsouza/go-dockerclient"
	"github.com/stretchr/testify/assert"
)

func getSUT() *DockerStats {
	expectedChan := make(chan metric.Metric)
	var expectedLogger = defaultLog.WithFields(l.Fields{"collector": "fullerite"})

	return newDockerStats(expectedChan, 10, expectedLogger).(*DockerStats)
}

func TestDockerStatsNewDockerStats(t *testing.T) {
	expectedChan := make(chan metric.Metric)
	var expectedLogger = defaultLog.WithFields(l.Fields{"collector": "fullerite"})
	expectedType := make(map[string]*CPUValues)

	d := newDockerStats(expectedChan, 10, expectedLogger).(*DockerStats)

	assert.Equal(t, d.log, expectedLogger)
	assert.Equal(t, d.channel, expectedChan)
	assert.Equal(t, d.interval, 10)
	assert.Equal(t, d.name, "DockerStats")
	assert.Equal(t, reflect.TypeOf(d.previousCPUValues), reflect.TypeOf(expectedType))
	assert.Equal(t, len(d.previousCPUValues), 0)
	d.Configure(make(map[string]interface{}))
	assert.Equal(t, d.GetEndpoint(), endpoint)
}

func TestDockerStatsConfigureEmptyConfig(t *testing.T) {
	config := make(map[string]interface{})

	d := newDockerStats(nil, 123, nil).(*DockerStats)
	d.Configure(config)

	assert.Equal(t, 123, d.Interval())
}

func TestDockerStatsConfigure(t *testing.T) {
	config := make(map[string]interface{})
	config["interval"] = 9999

	d := newDockerStats(nil, 123, nil).(*DockerStats)
	d.Configure(config)

	assert.Equal(t, 9999, d.Interval())
}

func TestDockerStatsBuildMetrics(t *testing.T) {
	config := make(map[string]interface{})
	envVars := []byte(`
	{
		"service_name":  {
			"MESOS_TASK_ID": "[^\\.]*"
		},
		"instance_name": {
			"MESOS_TASK_ID": "\\.([^\\.]*)\\."}
	}`)
	var val map[string]interface{}

	err := json.Unmarshal(envVars, &val)
	assert.Equal(t, err, nil)
	config["generatedDimensions"] = val

	stats := new(docker.Stats)
	stats.Networks = make(map[string]docker.NetworkStats)
	stats.Networks["eth0"] = docker.NetworkStats{RxBytes: 10, TxBytes: 20}
	stats.MemoryStats.Usage = 50
	stats.MemoryStats.Limit = 70

	containerJSON := []byte(`
	{
		"ID": "test-id",
		"Name": "test-container",
		"Config": {
			"Env": [
				"MESOS_TASK_ID=my--service.main.blablagit6bdsadnoise"
			]
		}
	}`)
	var container *docker.Container
	err = json.Unmarshal(containerJSON, &container)
	assert.Equal(t, err, nil)

	baseDims := map[string]string{
		"container_id":   "test-id",
		"container_name": "test-container",
		"service_name":   "my_service",
		"instance_name":  "main",
	}
	netDims := map[string]string{
		"container_id":   "test-id",
		"container_name": "test-container",
		"service_name":   "my_service",
		"instance_name":  "main",
		"iface":          "eth0",
	}

	expectedDimsGen := map[string]string{
		"service_name":  "my_service",
		"instance_name": "main",
	}
	expectedMetrics := []metric.Metric{
		metric.Metric{"DockerMemoryUsed", "gauge", 50, baseDims},
		metric.Metric{"DockerMemoryLimit", "gauge", 70, baseDims},
		metric.Metric{"DockerCpuPercentage", "gauge", 0.5, baseDims},
		metric.Metric{"DockerTxBytes", "cumcounter", 20, netDims},
		metric.Metric{"DockerRxBytes", "cumcounter", 10, netDims},
		metric.Metric{"DockerContainerCount", "counter", 1, expectedDimsGen},
	}

	d := getSUT()
	d.Configure(config)
	ret := d.buildMetrics(container, stats, 0.5)
	assert.Equal(t, ret, expectedMetrics)
}

func TestDockerStatsBuildMetricsWithNameAsEnvVariable(t *testing.T) {
	config := make(map[string]interface{})
	envVars := []byte(`
	{
		"service_name": {
			"SERVICE_NAME": ".*"
		}
	}`)
	var val map[string]interface{}

	err := json.Unmarshal(envVars, &val)
	assert.Equal(t, err, nil)
	config["generatedDimensions"] = val

	stats := new(docker.Stats)
	stats.MemoryStats.Usage = 50
	stats.MemoryStats.Limit = 70

	containerJSON := []byte(`
	{
		"ID": "test-id",
		"Name": "test-container",
		"Config": {
			"Env": [
				"SERVICE_NAME=my_service"
			]
		}
	}`)
	var container *docker.Container
	err = json.Unmarshal(containerJSON, &container)
	assert.Equal(t, err, nil)

	expectedDims := map[string]string{
		"container_id":   "test-id",
		"container_name": "test-container",
		"service_name":   "my_service",
	}
	expectedDimsGen := map[string]string{
		"service_name": "my_service",
	}
	expectedMetrics := []metric.Metric{
		metric.Metric{"DockerMemoryUsed", "gauge", 50, expectedDims},
		metric.Metric{"DockerMemoryLimit", "gauge", 70, expectedDims},
		metric.Metric{"DockerCpuPercentage", "gauge", 0.5, expectedDims},
		metric.Metric{"DockerContainerCount", "counter", 1, expectedDimsGen},
	}

	d := getSUT()
	d.Configure(config)
	ret := d.buildMetrics(container, stats, 0.5)

	assert.Equal(t, ret, expectedMetrics)
}

func TestDockerStatsCalculateCPUPercent(t *testing.T) {
	var previousTotalUsage = uint64(0)
	var previousSystem = uint64(0)

	stats := new(docker.Stats)
	stats.CPUStats.CPUUsage.PercpuUsage = make([]uint64, 24)
	stats.CPUStats.CPUUsage.TotalUsage = 1261158030354
	stats.CPUStats.SystemCPUUsage = 108086414700000000

	assert.Equal(t, 0.02800332753427522, calculateCPUPercent(previousTotalUsage, previousSystem, stats))

	previousTotalUsage = stats.CPUStats.CPUUsage.TotalUsage
	previousSystem = stats.CPUStats.SystemCPUUsage
	stats.CPUStats.CPUUsage.TotalUsage = 1261164064229
	stats.CPUStats.SystemCPUUsage = 108086652820000000

	assert.Equal(t, 0.060815135225936505, calculateCPUPercent(previousTotalUsage, previousSystem, stats))
}
<|endoftext|>


<commit_before>package storage

import (
	"bytes"
	"encoding/binary"
	"os"
	"path/filepath"
)

/*
Create storage
*/
func (m *ManagerStruct) Create(ID string) error {
	f, err := os.OpenFile(filepath.Join(dataPath, ID), os.O_CREATE|os.O_RDWR, 0644)
	if err != nil {
		return err
	}
	m.cache.Add(ID, f)
	return nil
}

/*
SaveData ...
*/
func (m *ManagerStruct) SaveData(ID string, data []byte, offset int64) error {
	f, err := m.getFileFromCache(ID)
	if err != nil {
		return err
	}
	buf := new(bytes.Buffer)
	if err = binary.Write(buf, binary.BigEndian, data); err != nil {
		return err
	}
	_, err = f.WriteAt(buf.Bytes(), offset)
	return err
}

/*
DeleteData ...
*/
func (m *ManagerStruct) DeleteData(ID string) error {
	v, ok := m.cache.Peek(ID)
	if ok {
		v.(*os.File).Close()
	}
	path := filepath.Join(dataPath, ID)
	/*
		if _, err := os.Stat(path); err != nil {
			return nil
		}
	*/
	return os.Remove(path)
}

/*
LoadData ...
*/
func (m *ManagerStruct) LoadData(ID string, offset int64, length int64) ([]byte, error) {
	f, err := m.getFileFromCache(ID)
	if err != nil {
		return nil, err
	}
	if length == 0 {
		if info, err := f.Stat(); err == nil {
			length = info.Size()
			length -= offset
		} else {
			return nil, err
		}
	}
	data := make([]byte, length)
	_, err = f.ReadAt(data, offset)
	return data, err
}

func (m *ManagerStruct) getFileFromCache(ID string) (*os.File, error) {
	v, ok := m.cache.Get(ID)
	var f *os.File
	var err error
	if !ok {
		f, err = os.OpenFile(filepath.Join(dataPath, ID), os.O_CREATE|os.O_RDWR, 0644)
		if err != nil {
			return nil, err
		}
	} else {
		f = v.(*os.File)
	}
	return f, nil
}
<commit_msg>Remove commented out code<commit_after>package storage

import (
	"bytes"
	"encoding/binary"
	"os"
	"path/filepath"
)

/*
Create storage
*/
func (m *ManagerStruct) Create(ID string) error {
	f, err := os.OpenFile(filepath.Join(dataPath, ID), os.O_CREATE|os.O_RDWR, 0644)
	if err != nil {
		return err
	}
	m.cache.Add(ID, f)
	return nil
}

/*
SaveData ...
*/
func (m *ManagerStruct) SaveData(ID string, data []byte, offset int64) error {
	f, err := m.getFileFromCache(ID)
	if err != nil {
		return err
	}
	buf := new(bytes.Buffer)
	if err = binary.Write(buf, binary.BigEndian, data); err != nil {
		return err
	}
	_, err = f.WriteAt(buf.Bytes(), offset)
	return err
}

/*
DeleteData ...
*/
func (m *ManagerStruct) DeleteData(ID string) error {
	v, ok := m.cache.Peek(ID)
	if ok {
		v.(*os.File).Close()
	}
	path := filepath.Join(dataPath, ID)
	return os.Remove(path)
}

/*
LoadData ...
*/
func (m *ManagerStruct) LoadData(ID string, offset int64, length int64) ([]byte, error) {
	f, err := m.getFileFromCache(ID)
	if err != nil {
		return nil, err
	}
	if length == 0 {
		if info, err := f.Stat(); err == nil {
			length = info.Size()
			length -= offset
		} else {
			return nil, err
		}
	}
	data := make([]byte, length)
	_, err = f.ReadAt(data, offset)
	return data, err
}

func (m *ManagerStruct) getFileFromCache(ID string) (*os.File, error) {
	v, ok := m.cache.Get(ID)
	var f *os.File
	var err error
	if !ok {
		f, err = os.OpenFile(filepath.Join(dataPath, ID), os.O_CREATE|os.O_RDWR, 0644)
		if err != nil {
			return nil, err
		}
	} else {
		f = v.(*os.File)
	}
	return f, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package storage

import (
	"sync"
	"testing"
	"time"

	"github.com/cockroachdb/cockroach/proto"
	"github.com/cockroachdb/cockroach/storage/engine"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/hlc"
	"github.com/cockroachdb/cockroach/util/log"
)

// Test implementation of a range iterator which cycles through
// a slice of ranges.
type testIterator struct {
	index  int
	ranges []Range
	count  int
	start  time.Time
	total  time.Duration
	sync.Mutex
}

func newTestIterator(count int) *testIterator {
	ti := &testIterator{
		start: time.Now(),
	}
	ti.ranges = make([]Range, count)
	// Initialize range stats for each range so the scanner can use them.
	for i := range ti.ranges {
		ti.ranges[i].stats = &rangeStats{
			raftID: int64(i),
			MVCCStats: engine.MVCCStats{
				KeyBytes:  1,
				ValBytes:  2,
				KeyCount:  1,
				LiveCount: 1,
			},
		}
	}
	return ti
}

func (ti *testIterator) Next() *Range {
	ti.Lock()
	defer ti.Unlock()
	if ti.index >= len(ti.ranges) {
		return nil
	}
	oldIndex := ti.index
	ti.index++
	return &ti.ranges[oldIndex]
}

func (ti *testIterator) EstimatedCount() int {
	ti.Lock()
	defer ti.Unlock()
	return len(ti.ranges) - ti.index
}

func (ti *testIterator) Reset() {
	ti.Lock()
	defer ti.Unlock()
	ti.index = 0
	now := time.Now()
	ti.total += now.Sub(ti.start)
	ti.start = now
	ti.count++
}

func (ti *testIterator) remove(index int) *Range {
	ti.Lock()
	defer ti.Unlock()
	var rng *Range
	if index < len(ti.ranges) {
		rng = &ti.ranges[index]
		ti.ranges = append(ti.ranges[:index], ti.ranges[index+1:]...)
	}
	return rng
}

func (ti *testIterator) avgScan() time.Duration {
	ti.Lock()
	defer ti.Unlock()
	return time.Duration(ti.total.Nanoseconds() / int64(ti.count))
}

// Test implementation of a range queue which adds range to an
// internal slice.
type testQueue struct {
	sync.Mutex // Protects ranges, done & processed count
	ranges     []*Range
	done       bool
	processed  int
}

func (tq *testQueue) Start(clock *hlc.Clock, stopper *util.Stopper) {
	stopper.Add(1)
	go func() {
		for {
			select {
			case <-time.After(1 * time.Millisecond):
				tq.Lock()
				if len(tq.ranges) > 0 {
					tq.ranges = tq.ranges[1:]
					tq.processed++
				}
				tq.Unlock()
			case <-stopper.ShouldStop():
				tq.Lock()
				tq.done = true
				tq.Unlock()
				stopper.SetStopped()
				return
			}
		}
	}()
}

func (tq *testQueue) MaybeAdd(rng *Range, now proto.Timestamp) {
	tq.Lock()
	defer tq.Unlock()
	if index := tq.indexOf(rng); index == -1 {
		tq.ranges = append(tq.ranges, rng)
	}
}

func (tq *testQueue) MaybeRemove(rng *Range) {
	tq.Lock()
	defer tq.Unlock()
	if index := tq.indexOf(rng); index != -1 {
		tq.ranges = append(tq.ranges[:index], tq.ranges[index+1:]...)
	}
}

func (tq *testQueue) count() int {
	tq.Lock()
	defer tq.Unlock()
	return len(tq.ranges)
}

func (tq *testQueue) indexOf(rng *Range) int {
	for i, r := range tq.ranges {
		if r == rng {
			return i
		}
	}
	return -1
}

func (tq *testQueue) isDone() bool {
	tq.Lock()
	defer tq.Unlock()
	return tq.done
}

// TestScannerAddToQueues verifies that ranges are added to and
// removed from multiple queues.
func TestScannerAddToQueues(t *testing.T) {
	const count = 3
	iter := newTestIterator(count)
	q1, q2 := &testQueue{}, &testQueue{}
	s := newRangeScanner(1*time.Millisecond, iter)
	s.AddQueues(q1, q2)
	mc := hlc.NewManualClock(0)
	clock := hlc.NewClock(mc.UnixNano)
	stopper := util.NewStopper(0)

	// Start queue and verify that all ranges are added to both queues.
	s.Start(clock, stopper)
	if err := util.IsTrueWithin(func() bool {
		return q1.count() == count && q2.count() == count
	}, 50*time.Millisecond); err != nil {
		t.Error(err)
	}

	// Remove first range and verify it does not exist in either range.
	rng := iter.remove(0)
	s.RemoveRange(rng)
	if err := util.IsTrueWithin(func() bool {
		return q1.count() == count-1 && q2.count() == count-1
	}, 10*time.Millisecond); err != nil {
		t.Error(err)
	}

	// Stop scanner and verify both queues are stopped.
	stopper.Stop()
	if !q1.isDone() || !q2.isDone() {
		t.Errorf("expected all queues to stop; got %t, %t", q1.isDone(), q2.isDone())
	}
}

// TestScannerTiming verifies that ranges are scanned, regardless
// of how many, to match scanInterval.
//
// TODO(spencer): in order to make this test not take too much time,
// we're running these loops at speeds where clock ticks may be
// an issue on virtual machines used for continuous integration.
func TestScannerTiming(t *testing.T) {
	const count = 3
	const runTime = 50 * time.Millisecond
	const maxError = 7500 * time.Microsecond
	durations := []time.Duration{
		5 * time.Millisecond,
		12500 * time.Microsecond,
	}
	for i, duration := range durations {
		iter := newTestIterator(count)
		q := &testQueue{}
		s := newRangeScanner(duration, iter)
		s.AddQueues(q)
		mc := hlc.NewManualClock(0)
		clock := hlc.NewClock(mc.UnixNano)
		stopper := util.NewStopper(0)
		defer stopper.Stop()
		s.Start(clock, stopper)
		time.Sleep(runTime)

		avg := iter.avgScan()
		log.Infof("%d: average scan: %s\n", i, avg)
		if avg.Nanoseconds()-duration.Nanoseconds() > maxError.Nanoseconds() ||
			duration.Nanoseconds()-avg.Nanoseconds() > maxError.Nanoseconds() {
			t.Errorf("expected %s, got %s: exceeds max error of %s", duration, avg, maxError)
		}
	}
}

// TestScannerEmptyIterator verifies that an empty iterator doesn't busy loop.
func TestScannerEmptyIterator(t *testing.T) {
	iter := newTestIterator(0)
	q := &testQueue{}
	s := newRangeScanner(1*time.Millisecond, iter)
	s.AddQueues(q)
	mc := hlc.NewManualClock(0)
	clock := hlc.NewClock(mc.UnixNano)
	stopper := util.NewStopper(0)
	defer stopper.Stop()
	s.Start(clock, stopper)
	time.Sleep(3 * time.Millisecond)
	if count := s.Count(); count > 3 {
		t.Errorf("expected three loops; got %d", count)
	}
}

// TestScannerStats verifies that stats accumulate from all ranges.
func TestScannerStats(t *testing.T) {
	const count = 3
	iter := newTestIterator(count)
	q := &testQueue{}
	stopper := util.NewStopper(0)
	defer stopper.Stop()
	s := newRangeScanner(1*time.Millisecond, iter)
	s.AddQueues(q)
	mc := hlc.NewManualClock(0)
	clock := hlc.NewClock(mc.UnixNano)
	// At start, scanner stats should be blank for MVCC, but have accurate number of ranges.
	if rc := s.Stats().RangeCount; rc != count {
		t.Errorf("range count expected %d; got %d", count, rc)
	}
	if vb := s.Stats().MVCC.ValBytes; vb != 0 {
		t.Errorf("value bytes expected %d; got %d", 0, vb)
	}
	s.Start(clock, stopper)
	// We expect a full run to accumulate stats from all ranges.
	if err := util.IsTrueWithin(func() bool {
		if rc := s.Stats().RangeCount; rc != count {
			return false
		}
		if vb := s.Stats().MVCC.ValBytes; vb != count*2 {
			return false
		}
		return true
	}, 100*time.Millisecond); err != nil {
		t.Error(err)
	}
}
<commit_msg>Slow down TestScannerTiming so it can pass reliably on circleci in race mode.<commit_after>// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Spencer Kimball (spencer.kimball@gmail.com)

package storage

import (
	"sync"
	"testing"
	"time"

	"github.com/cockroachdb/cockroach/proto"
	"github.com/cockroachdb/cockroach/storage/engine"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/hlc"
	"github.com/cockroachdb/cockroach/util/log"
)

// Test implementation of a range iterator which cycles through
// a slice of ranges.
type testIterator struct {
	index  int
	ranges []Range
	count  int
	start  time.Time
	total  time.Duration
	sync.Mutex
}

func newTestIterator(count int) *testIterator {
	ti := &testIterator{
		start: time.Now(),
	}
	ti.ranges = make([]Range, count)
	// Initialize range stats for each range so the scanner can use them.
	for i := range ti.ranges {
		ti.ranges[i].stats = &rangeStats{
			raftID: int64(i),
			MVCCStats: engine.MVCCStats{
				KeyBytes:  1,
				ValBytes:  2,
				KeyCount:  1,
				LiveCount: 1,
			},
		}
	}
	return ti
}

func (ti *testIterator) Next() *Range {
	ti.Lock()
	defer ti.Unlock()
	if ti.index >= len(ti.ranges) {
		return nil
	}
	oldIndex := ti.index
	ti.index++
	return &ti.ranges[oldIndex]
}

func (ti *testIterator) EstimatedCount() int {
	ti.Lock()
	defer ti.Unlock()
	return len(ti.ranges) - ti.index
}

func (ti *testIterator) Reset() {
	ti.Lock()
	defer ti.Unlock()
	ti.index = 0
	now := time.Now()
	ti.total += now.Sub(ti.start)
	ti.start = now
	ti.count++
}

func (ti *testIterator) remove(index int) *Range {
	ti.Lock()
	defer ti.Unlock()
	var rng *Range
	if index < len(ti.ranges) {
		rng = &ti.ranges[index]
		ti.ranges = append(ti.ranges[:index], ti.ranges[index+1:]...)
	}
	return rng
}

func (ti *testIterator) avgScan() time.Duration {
	ti.Lock()
	defer ti.Unlock()
	return time.Duration(ti.total.Nanoseconds() / int64(ti.count))
}

// Test implementation of a range queue which adds range to an
// internal slice.
type testQueue struct {
	sync.Mutex // Protects ranges, done & processed count
	ranges     []*Range
	done       bool
	processed  int
}

func (tq *testQueue) Start(clock *hlc.Clock, stopper *util.Stopper) {
	stopper.Add(1)
	go func() {
		for {
			select {
			case <-time.After(1 * time.Millisecond):
				tq.Lock()
				if len(tq.ranges) > 0 {
					tq.ranges = tq.ranges[1:]
					tq.processed++
				}
				tq.Unlock()
			case <-stopper.ShouldStop():
				tq.Lock()
				tq.done = true
				tq.Unlock()
				stopper.SetStopped()
				return
			}
		}
	}()
}

func (tq *testQueue) MaybeAdd(rng *Range, now proto.Timestamp) {
	tq.Lock()
	defer tq.Unlock()
	if index := tq.indexOf(rng); index == -1 {
		tq.ranges = append(tq.ranges, rng)
	}
}

func (tq *testQueue) MaybeRemove(rng *Range) {
	tq.Lock()
	defer tq.Unlock()
	if index := tq.indexOf(rng); index != -1 {
		tq.ranges = append(tq.ranges[:index], tq.ranges[index+1:]...)
	}
}

func (tq *testQueue) count() int {
	tq.Lock()
	defer tq.Unlock()
	return len(tq.ranges)
}

func (tq *testQueue) indexOf(rng *Range) int {
	for i, r := range tq.ranges {
		if r == rng {
			return i
		}
	}
	return -1
}

func (tq *testQueue) isDone() bool {
	tq.Lock()
	defer tq.Unlock()
	return tq.done
}

// TestScannerAddToQueues verifies that ranges are added to and
// removed from multiple queues.
func TestScannerAddToQueues(t *testing.T) {
	const count = 3
	iter := newTestIterator(count)
	q1, q2 := &testQueue{}, &testQueue{}
	s := newRangeScanner(1*time.Millisecond, iter)
	s.AddQueues(q1, q2)
	mc := hlc.NewManualClock(0)
	clock := hlc.NewClock(mc.UnixNano)
	stopper := util.NewStopper(0)

	// Start queue and verify that all ranges are added to both queues.
	s.Start(clock, stopper)
	if err := util.IsTrueWithin(func() bool {
		return q1.count() == count && q2.count() == count
	}, 50*time.Millisecond); err != nil {
		t.Error(err)
	}

	// Remove first range and verify it does not exist in either range.
	rng := iter.remove(0)
	s.RemoveRange(rng)
	if err := util.IsTrueWithin(func() bool {
		return q1.count() == count-1 && q2.count() == count-1
	}, 10*time.Millisecond); err != nil {
		t.Error(err)
	}

	// Stop scanner and verify both queues are stopped.
	stopper.Stop()
	if !q1.isDone() || !q2.isDone() {
		t.Errorf("expected all queues to stop; got %t, %t", q1.isDone(), q2.isDone())
	}
}

// TestScannerTiming verifies that ranges are scanned, regardless
// of how many, to match scanInterval.
func TestScannerTiming(t *testing.T) {
	const count = 3
	const runTime = 100 * time.Millisecond
	const maxError = 7500 * time.Microsecond
	durations := []time.Duration{
		10 * time.Millisecond,
		25 * time.Millisecond,
	}
	for i, duration := range durations {
		iter := newTestIterator(count)
		q := &testQueue{}
		s := newRangeScanner(duration, iter)
		s.AddQueues(q)
		mc := hlc.NewManualClock(0)
		clock := hlc.NewClock(mc.UnixNano)
		stopper := util.NewStopper(0)
		defer stopper.Stop()
		s.Start(clock, stopper)
		time.Sleep(runTime)

		avg := iter.avgScan()
		log.Infof("%d: average scan: %s\n", i, avg)
		if avg.Nanoseconds()-duration.Nanoseconds() > maxError.Nanoseconds() ||
			duration.Nanoseconds()-avg.Nanoseconds() > maxError.Nanoseconds() {
			t.Errorf("expected %s, got %s: exceeds max error of %s", duration, avg, maxError)
		}
	}
}

// TestScannerEmptyIterator verifies that an empty iterator doesn't busy loop.
func TestScannerEmptyIterator(t *testing.T) {
	iter := newTestIterator(0)
	q := &testQueue{}
	s := newRangeScanner(1*time.Millisecond, iter)
	s.AddQueues(q)
	mc := hlc.NewManualClock(0)
	clock := hlc.NewClock(mc.UnixNano)
	stopper := util.NewStopper(0)
	defer stopper.Stop()
	s.Start(clock, stopper)
	time.Sleep(3 * time.Millisecond)
	if count := s.Count(); count > 3 {
		t.Errorf("expected three loops; got %d", count)
	}
}

// TestScannerStats verifies that stats accumulate from all ranges.
func TestScannerStats(t *testing.T) {
	const count = 3
	iter := newTestIterator(count)
	q := &testQueue{}
	stopper := util.NewStopper(0)
	defer stopper.Stop()
	s := newRangeScanner(1*time.Millisecond, iter)
	s.AddQueues(q)
	mc := hlc.NewManualClock(0)
	clock := hlc.NewClock(mc.UnixNano)
	// At start, scanner stats should be blank for MVCC, but have accurate number of ranges.
	if rc := s.Stats().RangeCount; rc != count {
		t.Errorf("range count expected %d; got %d", count, rc)
	}
	if vb := s.Stats().MVCC.ValBytes; vb != 0 {
		t.Errorf("value bytes expected %d; got %d", 0, vb)
	}
	s.Start(clock, stopper)
	// We expect a full run to accumulate stats from all ranges.
	if err := util.IsTrueWithin(func() bool {
		if rc := s.Stats().RangeCount; rc != count {
			return false
		}
		if vb := s.Stats().MVCC.ValBytes; vb != count*2 {
			return false
		}
		return true
	}, 100*time.Millisecond); err != nil {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"log"

	"github.com/coopernurse/gorp"
	_ "github.com/lib/pq"
)

func populateLocations(locationsMap *gorp.DbMap) {
	tuftsResQuad := initLocation("Tufts Res Quad", 42.408565, -71.121765)
	hodgkinsPark := initLocation("Hodgkins Park", 42.399566,-71.124595)
	danehyPark   := initLocation("Danehy Park", 42.388306,-71.137507)
	ledermanPark := initLocation("Lederman Park", 42.363649,-71.071774)

	err := locationsMap.Insert(tuftsResQuad, hodgkinsPark,
							   danehyPark, ledermanPark)

	if err != nil {
		log.Fatal(err)
	}
}

func initDB() *gorp.DbMap{
	db, err := sql.Open("postgres", "postgres://yourPostgresDatabaseLogin")

	if err != nil {
		log.Fatal(err)
	}

	dbMap := &gorp.DbMap{Db: db, Dialect: gorp.PostgresDialect{}}

	locationsTable :=
	  dbMap.AddTableWithName(Location{}, "locations").SetKeys(true, "Id")

	locationsTable.ColMap("Name").SetNotNull(true).SetUnique(true)
	locationsTable.ColMap("Lat").SetNotNull(true)
	locationsTable.ColMap("Lng").SetNotNull(true)

	err = dbMap.DropTablesIfExists()
	if err != nil {
		log.Fatal(err)
	}

	err = dbMap.CreateTablesIfNotExists()
	if err != nil {
		log.Fatal(err)
	}

	populateLocations(dbMap)

	return dbMap
}
<commit_msg>Added db.Ping() call in initDB.go<commit_after>package main

import (
	"database/sql"
	"log"

	"github.com/coopernurse/gorp"
	_ "github.com/lib/pq"
)

func populateLocations(locationsMap *gorp.DbMap) {
	tuftsResQuad := initLocation("Tufts Res Quad", 42.408565, -71.121765)
	hodgkinsPark := initLocation("Hodgkins Park", 42.399566,-71.124595)
	danehyPark   := initLocation("Danehy Park", 42.388306,-71.137507)
	ledermanPark := initLocation("Lederman Park", 42.363649,-71.071774)

	err := locationsMap.Insert(tuftsResQuad, hodgkinsPark,
							   danehyPark, ledermanPark)

	if err != nil {
		log.Fatal(err)
	}
}

func initDB() *gorp.DbMap{
	db, err := sql.Open("postgres", "postgres://yourPostgresDatabaseLogin")

	if err != nil {
		log.Fatal(err)
	}
	err = db.Ping()
	if err != nil {
		log.Fatal(err)
	}

	dbMap := &gorp.DbMap{Db: db, Dialect: gorp.PostgresDialect{}}

	locationsTable :=
	  dbMap.AddTableWithName(Location{}, "locations").SetKeys(true, "Id")

	locationsTable.ColMap("Name").SetNotNull(true).SetUnique(true)
	locationsTable.ColMap("Lat").SetNotNull(true)
	locationsTable.ColMap("Lng").SetNotNull(true)

	err = dbMap.DropTablesIfExists()
	if err != nil {
		log.Fatal(err)
	}

	err = dbMap.CreateTablesIfNotExists()
	if err != nil {
		log.Fatal(err)
	}

	populateLocations(dbMap)

	return dbMap
}
<|endoftext|>


<commit_before>// Package inject provides utilities for mapping and injecting dependencies in various ways.
package inject

import (
	"fmt"
	"reflect"
)

// Injector represents an interface for mapping and injecting dependencies into structs
// and function arguments.
type Injector interface {
	Applicator
	Invoker
	TypeMapper
	// SetParent sets the parent of the injector. If the injector cannot find a
	// dependency in its Type map it will check it's parent before returning an
	// error.
	SetParent(Injector)
}

// Applicator represents an interface for mapping dependencies to a struct.
type Applicator interface {
	// Maps dependencies in the Type map to each field in the struct
	// that is tagged with 'inject'. Returns an error if the injection
	// fails.
	Apply(interface{}) error
}

// Invoker represents an interface for calling functions via reflection.
type Invoker interface {
	// Invoke attempts to call the interface{} provided as a function,
	// providing dependencies for function arguments based on Type. Returns
	// a slice of reflect.Value representing the returned values of the function.
	// Returns an error if the injection fails.
	Invoke(interface{}) ([]reflect.Value, error)
}

// TypeMapper represents an interface for mapping interface{} values based on type.
type TypeMapper interface {
	// Maps the interface{} value based on it's immediate type from reflect.TypeOf.
	Map(interface{}) TypeMapper
	// Maps the interface{} value based on the pointer of an Interface provided.
	// This is really only useful for mapping a value as an interface, as interfaces
	// cannot at this time be referenced directly without a pointer.
	MapTo(interface{}, interface{}) TypeMapper
	// Returns the Value that is mapped to the current type. Returns a zeroed Value if
	// the Type has not been mapped.
	Get(reflect.Type) reflect.Value
}

type injector struct {
	values map[reflect.Type]reflect.Value
	parent Injector
}

// InterfaceOf dereferences a pointer to an Interface type.
// It panics if value is not an pointer to an interface.
func InterfaceOf(value interface{}) reflect.Type {
	t := reflect.TypeOf(value)

	for t.Kind() == reflect.Ptr {
		t = t.Elem()
	}

	if t.Kind() != reflect.Interface {
		panic("Called inject.InterfaceOf with a value that is not a pointer to an interface. (*MyInterface)(nil)")
	}

	return t
}

// New returns a new Injector.
func New() Injector {
	return &injector{
		values: make(map[reflect.Type]reflect.Value),
	}
}

// Invoke attempts to call the interface{} provided as a function,
// providing dependencies for function arguments based on Type.
// Returns a slice of reflect.Value representing the returned values of the function.
// Returns an error if the injection fails.
// It panics if f is not a function
func (inj *injector) Invoke(f interface{}) ([]reflect.Value, error) {
	t := reflect.TypeOf(f)

	var in = make([]reflect.Value, t.NumIn()) //Panic if t is not kind of Func
	for i := 0; i < t.NumIn(); i++ {
		argType := t.In(i)
		val := inj.Get(argType)
		if !val.IsValid() {
			return nil, fmt.Errorf("Value not found for type %v", argType)
		}

		in[i] = val
	}

	return reflect.ValueOf(f).Call(in), nil
}

// Maps dependencies in the Type map to each field in the struct
// that is tagged with 'inject'.
// Returns an error if the injection fails.
func (inj *injector) Apply(val interface{}) error {
	v := reflect.ValueOf(val)

	for v.Kind() == reflect.Ptr {
		v = v.Elem()
	}

	if v.Kind() != reflect.Struct {
		return nil // Should not panic here ?
	}

	t := v.Type()

	for i := 0; i < v.NumField(); i++ {
		f := v.Field(i)
		structField := t.Field(i)
		if f.CanSet() && structField.Tag == "inject" {
			ft := f.Type()
			v := inj.Get(ft)
			if !v.IsValid() {
				return fmt.Errorf("Value not found for type %v", ft)
			}

			f.Set(v)
		}

	}

	return nil
}

// Maps the concrete value of val to its dynamic type using reflect.TypeOf,
// It returns the TypeMapper registered in.
func (i *injector) Map(val interface{}) TypeMapper {
	i.values[reflect.TypeOf(val)] = reflect.ValueOf(val)
	return i
}

func (i *injector) MapTo(val interface{}, ifacePtr interface{}) TypeMapper {
	i.values[InterfaceOf(ifacePtr)] = reflect.ValueOf(val)
	return i
}

func (i *injector) Get(t reflect.Type) reflect.Value {
	val := i.values[t]

	if val.IsValid() {
		return val
	}

	// no concrete types found, try to find implementors
	// if t is an interface
	if t.Kind() == reflect.Interface {
		for k, v := range i.values {
			if k.Implements(t) {
				val = v
				break
			}
		}
	}

  // Still no type found, try to look it up on the parent
	if !val.IsValid() && i.parent != nil {
		val = i.parent.Get(t)
	}

	return val

}

func (i *injector) SetParent(parent Injector) {
	i.parent = parent
}
<commit_msg>Formatting<commit_after>// Package inject provides utilities for mapping and injecting dependencies in various ways.
package inject

import (
	"fmt"
	"reflect"
)

// Injector represents an interface for mapping and injecting dependencies into structs
// and function arguments.
type Injector interface {
	Applicator
	Invoker
	TypeMapper
	// SetParent sets the parent of the injector. If the injector cannot find a
	// dependency in its Type map it will check it's parent before returning an
	// error.
	SetParent(Injector)
}

// Applicator represents an interface for mapping dependencies to a struct.
type Applicator interface {
	// Maps dependencies in the Type map to each field in the struct
	// that is tagged with 'inject'. Returns an error if the injection
	// fails.
	Apply(interface{}) error
}

// Invoker represents an interface for calling functions via reflection.
type Invoker interface {
	// Invoke attempts to call the interface{} provided as a function,
	// providing dependencies for function arguments based on Type. Returns
	// a slice of reflect.Value representing the returned values of the function.
	// Returns an error if the injection fails.
	Invoke(interface{}) ([]reflect.Value, error)
}

// TypeMapper represents an interface for mapping interface{} values based on type.
type TypeMapper interface {
	// Maps the interface{} value based on it's immediate type from reflect.TypeOf.
	Map(interface{}) TypeMapper
	// Maps the interface{} value based on the pointer of an Interface provided.
	// This is really only useful for mapping a value as an interface, as interfaces
	// cannot at this time be referenced directly without a pointer.
	MapTo(interface{}, interface{}) TypeMapper
	// Returns the Value that is mapped to the current type. Returns a zeroed Value if
	// the Type has not been mapped.
	Get(reflect.Type) reflect.Value
}

type injector struct {
	values map[reflect.Type]reflect.Value
	parent Injector
}

// InterfaceOf dereferences a pointer to an Interface type.
// It panics if value is not an pointer to an interface.
func InterfaceOf(value interface{}) reflect.Type {
	t := reflect.TypeOf(value)

	for t.Kind() == reflect.Ptr {
		t = t.Elem()
	}

	if t.Kind() != reflect.Interface {
		panic("Called inject.InterfaceOf with a value that is not a pointer to an interface. (*MyInterface)(nil)")
	}

	return t
}

// New returns a new Injector.
func New() Injector {
	return &injector{
		values: make(map[reflect.Type]reflect.Value),
	}
}

// Invoke attempts to call the interface{} provided as a function,
// providing dependencies for function arguments based on Type.
// Returns a slice of reflect.Value representing the returned values of the function.
// Returns an error if the injection fails.
// It panics if f is not a function
func (inj *injector) Invoke(f interface{}) ([]reflect.Value, error) {
	t := reflect.TypeOf(f)

	var in = make([]reflect.Value, t.NumIn()) //Panic if t is not kind of Func
	for i := 0; i < t.NumIn(); i++ {
		argType := t.In(i)
		val := inj.Get(argType)
		if !val.IsValid() {
			return nil, fmt.Errorf("Value not found for type %v", argType)
		}

		in[i] = val
	}

	return reflect.ValueOf(f).Call(in), nil
}

// Maps dependencies in the Type map to each field in the struct
// that is tagged with 'inject'.
// Returns an error if the injection fails.
func (inj *injector) Apply(val interface{}) error {
	v := reflect.ValueOf(val)

	for v.Kind() == reflect.Ptr {
		v = v.Elem()
	}

	if v.Kind() != reflect.Struct {
		return nil // Should not panic here ?
	}

	t := v.Type()

	for i := 0; i < v.NumField(); i++ {
		f := v.Field(i)
		structField := t.Field(i)
		if f.CanSet() && structField.Tag == "inject" {
			ft := f.Type()
			v := inj.Get(ft)
			if !v.IsValid() {
				return fmt.Errorf("Value not found for type %v", ft)
			}

			f.Set(v)
		}

	}

	return nil
}

// Maps the concrete value of val to its dynamic type using reflect.TypeOf,
// It returns the TypeMapper registered in.
func (i *injector) Map(val interface{}) TypeMapper {
	i.values[reflect.TypeOf(val)] = reflect.ValueOf(val)
	return i
}

func (i *injector) MapTo(val interface{}, ifacePtr interface{}) TypeMapper {
	i.values[InterfaceOf(ifacePtr)] = reflect.ValueOf(val)
	return i
}

func (i *injector) Get(t reflect.Type) reflect.Value {
	val := i.values[t]

	if val.IsValid() {
		return val
	}

	// no concrete types found, try to find implementors
	// if t is an interface
	if t.Kind() == reflect.Interface {
		for k, v := range i.values {
			if k.Implements(t) {
				val = v
				break
			}
		}
	}

	// Still no type found, try to look it up on the parent
	if !val.IsValid() && i.parent != nil {
		val = i.parent.Get(t)
	}

	return val

}

func (i *injector) SetParent(parent Injector) {
	i.parent = parent
}
<|endoftext|>


<commit_before>package byteio

import "io"

type EndianReader interface {
	io.Reader
	ReadUint8() (uint8, int, error)
	ReadInt8() (int8, int, error)
	ReadUint16() (uint16, int, error)
	ReadInt16() (int16, int, error)
	ReadUint32() (uint32, int, error)
	ReadInt32() (int32, int, error)
	ReadUint64() (uint64, int, error)
	ReadInt64() (int64, int, error)
}

type EndianWriter interface {
	io.Writer
	WriteUint8(uint8) (int, error)
	WriteInt8(int8) (int, error)
	WriteUint16(uint16) (int, error)
	WriteInt16(int16) (int, error)
	WriteUint32(uint32) (int, error)
	WriteInt32(int32) (int, error)
	WriteUint64(uint64) (int, error)
	WriteInt64(int64) (int, error)
}
<commit_msg>added missing methods to interfaces<commit_after>package byteio

import "io"

type EndianReader interface {
	io.Reader
	ReadUint8() (uint8, int, error)
	ReadInt8() (int8, int, error)
	ReadUint16() (uint16, int, error)
	ReadInt16() (int16, int, error)
	ReadUint32() (uint32, int, error)
	ReadInt32() (int32, int, error)
	ReadUint64() (uint64, int, error)
	ReadInt64() (int64, int, error)
	ReadFloat32() (float32, int, error)
	ReadFloat64() (float64, int, error)
}

type EndianWriter interface {
	io.Writer
	WriteUint8(uint8) (int, error)
	WriteInt8(int8) (int, error)
	WriteUint16(uint16) (int, error)
	WriteInt16(int16) (int, error)
	WriteUint32(uint32) (int, error)
	WriteInt32(int32) (int, error)
	WriteUint64(uint64) (int, error)
	WriteInt64(int64) (int, error)
	WriteFloat32(float32) (int, error)
	WriteFloat64(float64) (int, error)
}
<|endoftext|>


<commit_before>package ekanite

import (
	"expvar"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/ekanite/ekanite/input"
)

const (
	DefaultNumShards       = 16
	DefaultIndexDuration   = 24 * time.Hour
	DefaultRetentionPeriod = 24 * time.Hour

	RetentionCheckInterval = time.Hour
)

var (
	stats = expvar.NewMap("engine")
)

// EventIndex is the interface a system than can index events must implement.
type EventIndexer interface {
	Index(events []*Event) error
}

// Batcher accepts "input events", and once it has a certain number, or a certain amount
// of time has passed, sends those as indexable Events to an Indexer. It also supports a
// maximum number of unprocessed Events it will keep pending. Once this limit is reached,
// it will not accept anymore until outstanding Events are processed.
type Batcher struct {
	indexer  EventIndexer
	size     int
	duration time.Duration

	c chan *input.Event
}

// NewBatcher returns a Batcher for EventIndexer e, a batching size of sz, a maximum duration
// of dur, and a maximum outstanding count of max.
func NewBatcher(e EventIndexer, sz int, dur time.Duration, max int) *Batcher {
	return &Batcher{
		indexer:  e,
		size:     sz,
		duration: dur,
		c:        make(chan *input.Event, max),
	}
}

// Start starts the batching process.
func (b *Batcher) Start(errChan chan<- error) error {
	go func() {
		batch := make([]*Event, 0, b.size)
		timer := time.NewTimer(b.duration)
		timer.Stop() // Stop any first firing.

		send := func() {
			err := b.indexer.Index(batch)
			if err != nil {
				stats.Add("batchIndexedError", 1)
				return
			}
			stats.Add("batchIndexed", 1)
			stats.Add("eventsIndexed", int64(len(batch)))
			if errChan != nil {
				errChan <- err
			}
			batch = make([]*Event, 0, b.size)
		}

		for {
			select {
			case event := <-b.c:
				idxEvent := &Event{
					event,
				}
				batch = append(batch, idxEvent)
				if len(batch) == 1 {
					timer.Reset(b.duration)
				}
				if len(batch) == b.size {
					timer.Stop()
					send()
				}
			case <-timer.C:
				stats.Add("batchTimeout", 1)
				send()
			}
		}
	}()

	return nil
}

// C returns the channel on the batcher to which events should be sent.
func (b *Batcher) C() chan<- *input.Event {
	return b.c
}

// Engine is the component that performs all indexing.
type Engine struct {
	path            string        // Path to all indexed data
	NumShards       int           // Number of shards to use when creating an index.
	IndexDuration   time.Duration // Duration of created indexes.
	RetentionPeriod time.Duration // How long after Index end-time to hang onto data.

	mu      sync.RWMutex
	indexes Indexes

	open bool
	done chan struct{}
	wg   sync.WaitGroup

	Logger *log.Logger
}

// NewEngine returns a new indexing engine, which will use any data located at path.
func NewEngine(path string) *Engine {
	return &Engine{
		path:            path,
		NumShards:       DefaultNumShards,
		IndexDuration:   DefaultIndexDuration,
		RetentionPeriod: DefaultRetentionPeriod,
		done:            make(chan struct{}),
		Logger:          log.New(os.Stderr, "[engine] ", log.LstdFlags),
	}
}

// Open opens the engine.
func (e *Engine) Open() error {
	if err := os.MkdirAll(e.path, 0755); err != nil {
		return err
	}
	d, err := os.Open(e.path)
	if err != nil {
		return fmt.Errorf("failed to open engine: %s", err.Error())
	}

	fis, err := d.Readdir(0)
	if err != nil {
		return err
	}

	// Open all indexes.
	for _, fi := range fis {
		if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
			continue
		}
		indexPath := filepath.Join(e.path, fi.Name())
		i, err := OpenIndex(indexPath)
		if err != nil {
			return fmt.Errorf("engine failed to open at index %s: %s", indexPath, err.Error())
		}
		log.Printf("engine opened index with %d shard(s) at %s", len(i.Shards), indexPath)
		e.indexes = append(e.indexes, i)
		sort.Sort(e.indexes)
	}

	e.wg.Add(1)
	go e.runRetentionEnforcement()

	e.open = true
	return nil
}

// Close closes the engine.
func (e *Engine) Close() error {
	if !e.open {
		return nil
	}

	for _, i := range e.indexes {
		if err := i.Close(); err != nil {
			return err
		}
	}

	close(e.done)
	e.wg.Wait()

	e.open = false
	return nil
}

// Total returns the total number of documents indexed.
func (e *Engine) Total() (uint64, error) {
	e.mu.RLock()
	defer e.mu.RUnlock()

	var total uint64
	for _, i := range e.indexes {
		t, err := i.Total()
		if err != nil {
			return 0, err
		}
		total += t
	}
	return total, nil
}

// runRetentionEnforcement periodically run retention enforcement.
func (e *Engine) runRetentionEnforcement() {
	defer e.wg.Done()
	for {
		select {
		case <-e.done:
			return

		case <-time.After(RetentionCheckInterval):
			e.Logger.Print("retention enforcement commencing")
			stats.Add("retentionEnforcementRun", 1)
			e.enforceRetention()
		}
	}
}

// enforceRetention removes indexes which have aged out.
func (e *Engine) enforceRetention() {
	e.mu.Lock()
	defer e.mu.Unlock()

	filtered := e.indexes[:0]
	for _, i := range e.indexes {
		if i.Expired(time.Now().UTC(), e.RetentionPeriod) {
			if err := DeleteIndex(i); err != nil {
				e.Logger.Printf("retention enforcement failed to delete index %s: %s", i.path, err.Error())
			} else {
				e.Logger.Printf("retention enforcement deleted index %s", i.path)
				stats.Add("retentionEnforcementDeletions", 1)
			}
		} else {
			filtered = append(filtered, i)
		}
	}
	e.indexes = filtered
	return
}

// indexForReferenceTime returns an index suitable for indexing an event
// for the given reference time. Must be called under RLock.
func (e *Engine) indexForReferenceTime(t time.Time) *Index {
	for _, i := range e.indexes {
		if i.Contains(t) {
			return i
		}
	}
	return nil
}

// createIndex creates an index with a given start and end time and adds the
// created index to the Engine's store. It must be called under lock.
func (e *Engine) createIndex(startTime, endTime time.Time) (*Index, error) {
	// There cannot be two indexes with the same start time, since this would mean
	// two indexes with the same path. So if an index already exists with the requested
	// start time, use that index's end time as the start time.
	var idx *Index
	for _, i := range e.indexes {
		if i.startTime == startTime {
			idx = i
			break
		}
	}
	if idx != nil {
		startTime = idx.endTime // XXX This could still align with another start time! Needs some sort of loop.
		assert(!startTime.After(endTime), "new start time after end time")
	}

	i, err := NewIndex(e.path, startTime, endTime, e.NumShards)
	if err != nil {
		return nil, err
	}
	e.indexes = append(e.indexes, i)
	sort.Sort(e.indexes)

	e.Logger.Printf("index %s created with %d shards, start time: %s, end time: %s",
		i.Path(), e.NumShards, i.StartTime(), i.EndTime())
	return i, nil
}

// createIndexForReferenceTime creates an index suitable for indexing an event at the given
// reference time.
func (e *Engine) createIndexForReferenceTime(rt time.Time) (*Index, error) {
	start := rt.Truncate(e.IndexDuration).UTC()
	end := start.Add(e.IndexDuration).UTC()
	return e.createIndex(start, end)
}

// Index indexes a batch of Events. It blocks until all processing has completed.
func (e *Engine) Index(events []*Event) error {
	e.mu.RLock()
	defer e.mu.RUnlock()

	var wg sync.WaitGroup

	// De-multiplex the batch into sub-batches, one sub-batch for each Index.
	subBatches := make(map[*Index][]Document, 0)

	for _, ev := range events {
		index := e.indexForReferenceTime(ev.ReferenceTime())
		if index == nil {
			func() {
				// Take a RWLock, check again, and create a new index if necessary.
				// Doing this in a function makes lock management foolproof.
				e.mu.RUnlock()
				defer e.mu.RLock()
				e.mu.Lock()
				defer e.mu.Unlock()

				index = e.indexForReferenceTime(ev.ReferenceTime())
				if index == nil {
					var err error
					index, err = e.createIndexForReferenceTime(ev.ReferenceTime())
					if err != nil || index == nil {
						panic(fmt.Sprintf("failed to create index for %s: %s", ev.ReferenceTime(), err))
					}
				}
			}()
		}

		if _, ok := subBatches[index]; !ok {
			subBatches[index] = make([]Document, 0)
		}
		subBatches[index] = append(subBatches[index], ev)
	}

	// Index each batch in parallel.
	for index, subBatch := range subBatches {
		wg.Add(1)
		go func(i *Index, b []Document) {
			defer wg.Done()
			i.Index(b)
		}(index, subBatch)
	}
	wg.Wait()
	return nil
}

// Search performs a search.
func (e *Engine) Search(query string) (<-chan string, error) {
	e.mu.RLock()
	defer e.mu.RUnlock()
	stats.Add("queriesRx", 1)

	// Buffer channel to control how many docs are sent back. XXX Will this allow
	// the client to control? Possibly.
	c := make(chan string, 1)

	go func() {
		// Sequentially search each index, starting with the earliest in time.
		// This could be done in parallel but more sorting would be required.

		for i := len(e.indexes) - 1; i >= 0; i-- {
			e.Logger.Printf("searching index %s", e.indexes[i].Path())
			ids, err := e.indexes[i].Search(query)
			if err != nil {
				e.Logger.Println("error performing search:", err.Error())
				break
			}
			for _, id := range ids {
				b, err := e.indexes[i].Document(id)
				if err != nil {
					e.Logger.Println("error getting document:", err.Error())
					break
				}
				stats.Add("docsIDsRetrived", 1)
				c <- string(b) // There is excessive byte-slice-to-strings here.
			}
		}
		close(c)
	}()

	return c, nil
}

// Path returns the path to the indexed data directory.
func (e *Engine) Path() string {
	return e.path
}

// assert will panic with a given formatted message if the given condition is false.
func assert(condition bool, msg string, v ...interface{}) {
	if !condition {
		panic(fmt.Sprintf("assert failed: "+msg, v...))
	}
}
<commit_msg>Some minor comment fixes<commit_after>package ekanite

import (
	"expvar"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/ekanite/ekanite/input"
)

// Engine defaults
const (
	DefaultNumShards       = 16
	DefaultIndexDuration   = 24 * time.Hour
	DefaultRetentionPeriod = 24 * time.Hour

	RetentionCheckInterval = time.Hour
)

// Engine stats
var (
	stats = expvar.NewMap("engine")
)

// EventIndex is the interface a system than can index events must implement.
type EventIndexer interface {
	Index(events []*Event) error
}

// Batcher accepts "input events", and once it has a certain number, or a certain amount
// of time has passed, sends those as indexable Events to an Indexer. It also supports a
// maximum number of unprocessed Events it will keep pending. Once this limit is reached,
// it will not accept anymore until outstanding Events are processed.
type Batcher struct {
	indexer  EventIndexer
	size     int
	duration time.Duration

	c chan *input.Event
}

// NewBatcher returns a Batcher for EventIndexer e, a batching size of sz, a maximum duration
// of dur, and a maximum outstanding count of max.
func NewBatcher(e EventIndexer, sz int, dur time.Duration, max int) *Batcher {
	return &Batcher{
		indexer:  e,
		size:     sz,
		duration: dur,
		c:        make(chan *input.Event, max),
	}
}

// Start starts the batching process.
func (b *Batcher) Start(errChan chan<- error) error {
	go func() {
		batch := make([]*Event, 0, b.size)
		timer := time.NewTimer(b.duration)
		timer.Stop() // Stop any first firing.

		send := func() {
			err := b.indexer.Index(batch)
			if err != nil {
				stats.Add("batchIndexedError", 1)
				return
			}
			stats.Add("batchIndexed", 1)
			stats.Add("eventsIndexed", int64(len(batch)))
			if errChan != nil {
				errChan <- err
			}
			batch = make([]*Event, 0, b.size)
		}

		for {
			select {
			case event := <-b.c:
				idxEvent := &Event{
					event,
				}
				batch = append(batch, idxEvent)
				if len(batch) == 1 {
					timer.Reset(b.duration)
				}
				if len(batch) == b.size {
					timer.Stop()
					send()
				}
			case <-timer.C:
				stats.Add("batchTimeout", 1)
				send()
			}
		}
	}()

	return nil
}

// C returns the channel on the batcher to which events should be sent.
func (b *Batcher) C() chan<- *input.Event {
	return b.c
}

// Engine is the component that performs all indexing.
type Engine struct {
	path            string        // Path to all indexed data
	NumShards       int           // Number of shards to use when creating an index.
	IndexDuration   time.Duration // Duration of created indexes.
	RetentionPeriod time.Duration // How long after Index end-time to hang onto data.

	mu      sync.RWMutex
	indexes Indexes

	open bool
	done chan struct{}
	wg   sync.WaitGroup

	Logger *log.Logger
}

// NewEngine returns a new indexing engine, which will use any data located at path.
func NewEngine(path string) *Engine {
	return &Engine{
		path:            path,
		NumShards:       DefaultNumShards,
		IndexDuration:   DefaultIndexDuration,
		RetentionPeriod: DefaultRetentionPeriod,
		done:            make(chan struct{}),
		Logger:          log.New(os.Stderr, "[engine] ", log.LstdFlags),
	}
}

// Open opens the engine.
func (e *Engine) Open() error {
	if err := os.MkdirAll(e.path, 0755); err != nil {
		return err
	}
	d, err := os.Open(e.path)
	if err != nil {
		return fmt.Errorf("failed to open engine: %s", err.Error())
	}

	fis, err := d.Readdir(0)
	if err != nil {
		return err
	}

	// Open all indexes.
	for _, fi := range fis {
		if !fi.IsDir() || strings.HasPrefix(fi.Name(), ".") {
			continue
		}
		indexPath := filepath.Join(e.path, fi.Name())
		i, err := OpenIndex(indexPath)
		if err != nil {
			return fmt.Errorf("engine failed to open at index %s: %s", indexPath, err.Error())
		}
		log.Printf("engine opened index with %d shard(s) at %s", len(i.Shards), indexPath)
		e.indexes = append(e.indexes, i)
		sort.Sort(e.indexes)
	}

	e.wg.Add(1)
	go e.runRetentionEnforcement()

	e.open = true
	return nil
}

// Close closes the engine.
func (e *Engine) Close() error {
	if !e.open {
		return nil
	}

	for _, i := range e.indexes {
		if err := i.Close(); err != nil {
			return err
		}
	}

	close(e.done)
	e.wg.Wait()

	e.open = false
	return nil
}

// Total returns the total number of documents indexed.
func (e *Engine) Total() (uint64, error) {
	e.mu.RLock()
	defer e.mu.RUnlock()

	var total uint64
	for _, i := range e.indexes {
		t, err := i.Total()
		if err != nil {
			return 0, err
		}
		total += t
	}
	return total, nil
}

// runRetentionEnforcement periodically run retention enforcement.
func (e *Engine) runRetentionEnforcement() {
	defer e.wg.Done()
	for {
		select {
		case <-e.done:
			return

		case <-time.After(RetentionCheckInterval):
			e.Logger.Print("retention enforcement commencing")
			stats.Add("retentionEnforcementRun", 1)
			e.enforceRetention()
		}
	}
}

// enforceRetention removes indexes which have aged out.
func (e *Engine) enforceRetention() {
	e.mu.Lock()
	defer e.mu.Unlock()

	filtered := e.indexes[:0]
	for _, i := range e.indexes {
		if i.Expired(time.Now().UTC(), e.RetentionPeriod) {
			if err := DeleteIndex(i); err != nil {
				e.Logger.Printf("retention enforcement failed to delete index %s: %s", i.path, err.Error())
			} else {
				e.Logger.Printf("retention enforcement deleted index %s", i.path)
				stats.Add("retentionEnforcementDeletions", 1)
			}
		} else {
			filtered = append(filtered, i)
		}
	}
	e.indexes = filtered
	return
}

// indexForReferenceTime returns an index suitable for indexing an event
// for the given reference time. Must be called under RLock.
func (e *Engine) indexForReferenceTime(t time.Time) *Index {
	for _, i := range e.indexes {
		if i.Contains(t) {
			return i
		}
	}
	return nil
}

// createIndex creates an index with a given start and end time and adds the
// created index to the Engine's store. It must be called under lock.
func (e *Engine) createIndex(startTime, endTime time.Time) (*Index, error) {
	// There cannot be two indexes with the same start time, since this would mean
	// two indexes with the same path. So if an index already exists with the requested
	// start time, use that index's end time as the start time.
	var idx *Index
	for _, i := range e.indexes {
		if i.startTime == startTime {
			idx = i
			break
		}
	}
	if idx != nil {
		startTime = idx.endTime // XXX This could still align with another start time! Needs some sort of loop.
		assert(!startTime.After(endTime), "new start time after end time")
	}

	i, err := NewIndex(e.path, startTime, endTime, e.NumShards)
	if err != nil {
		return nil, err
	}
	e.indexes = append(e.indexes, i)
	sort.Sort(e.indexes)

	e.Logger.Printf("index %s created with %d shards, start time: %s, end time: %s",
		i.Path(), e.NumShards, i.StartTime(), i.EndTime())
	return i, nil
}

// createIndexForReferenceTime creates an index suitable for indexing an event at the given
// reference time.
func (e *Engine) createIndexForReferenceTime(rt time.Time) (*Index, error) {
	start := rt.Truncate(e.IndexDuration).UTC()
	end := start.Add(e.IndexDuration).UTC()
	return e.createIndex(start, end)
}

// Index indexes a batch of Events. It blocks until all processing has completed.
func (e *Engine) Index(events []*Event) error {
	e.mu.RLock()
	defer e.mu.RUnlock()

	var wg sync.WaitGroup

	// De-multiplex the batch into sub-batches, one sub-batch for each Index.
	subBatches := make(map[*Index][]Document, 0)

	for _, ev := range events {
		index := e.indexForReferenceTime(ev.ReferenceTime())
		if index == nil {
			func() {
				// Take a RWLock, check again, and create a new index if necessary.
				// Doing this in a function makes lock management foolproof.
				e.mu.RUnlock()
				defer e.mu.RLock()
				e.mu.Lock()
				defer e.mu.Unlock()

				index = e.indexForReferenceTime(ev.ReferenceTime())
				if index == nil {
					var err error
					index, err = e.createIndexForReferenceTime(ev.ReferenceTime())
					if err != nil || index == nil {
						panic(fmt.Sprintf("failed to create index for %s: %s", ev.ReferenceTime(), err))
					}
				}
			}()
		}

		if _, ok := subBatches[index]; !ok {
			subBatches[index] = make([]Document, 0)
		}
		subBatches[index] = append(subBatches[index], ev)
	}

	// Index each batch in parallel.
	for index, subBatch := range subBatches {
		wg.Add(1)
		go func(i *Index, b []Document) {
			defer wg.Done()
			i.Index(b)
		}(index, subBatch)
	}
	wg.Wait()
	return nil
}

// Search performs a search.
func (e *Engine) Search(query string) (<-chan string, error) {
	e.mu.RLock()
	defer e.mu.RUnlock()
	stats.Add("queriesRx", 1)

	// Buffer channel to control how many docs are sent back. XXX Will this allow
	// the client to control? Possibly.
	c := make(chan string, 1)

	go func() {
		// Sequentially search each index, starting with the earliest in time.
		// This could be done in parallel but more sorting would be required.
		for i := len(e.indexes) - 1; i >= 0; i-- {
			e.Logger.Printf("searching index %s", e.indexes[i].Path())
			ids, err := e.indexes[i].Search(query)
			if err != nil {
				e.Logger.Println("error performing search:", err.Error())
				break
			}
			for _, id := range ids {
				b, err := e.indexes[i].Document(id)
				if err != nil {
					e.Logger.Println("error getting document:", err.Error())
					break
				}
				stats.Add("docsIDsRetrived", 1)
				c <- string(b) // There is excessive byte-slice-to-strings here.
			}
		}
		close(c)
	}()

	return c, nil
}

// Path returns the path to the indexed data directory.
func (e *Engine) Path() string {
	return e.path
}

// assert will panic with a given formatted message if the given condition is false.
func assert(condition bool, msg string, v ...interface{}) {
	if !condition {
		panic(fmt.Sprintf("assert failed: "+msg, v...))
	}
}
<|endoftext|>


<commit_before>package notification

import (
	"encoding/json"
	"strings"
)

// StatusRule includes parametes of status rules.
type StatusRule struct {
	CurrentLevel  CheckLevel  `json:"currentLevel"`
	PreviousLevel *CheckLevel `json:"previousLevel"`
}

// CheckLevel is the enum value of status levels.
type CheckLevel int

// consts of CheckStatusLevel
const (
	Unknown CheckLevel = iota
	Ok
	Info
	Warn
	Critical
	Any
)

var checkLevels = []string{
	"UNKNOWN",
	"OK",
	"INFO",
	"WARN",
	"CRIT",
	"ANY",
}

var checkLevelMaps = map[string]CheckLevel{
	"UNKNOWN": Unknown,
	"OK":      Ok,
	"INFO":    Info,
	"WARN":    Warn,
	"CRIT":    Critical,
	"ANY":     Any,
}

// MarshalJSON implements json.Marshaller.
func (cl CheckLevel) MarshalJSON() ([]byte, error) {
	return json.Marshal(cl.String())
}

// UnmarshalJSON implements json.Unmarshaller.
func (cl *CheckLevel) UnmarshalJSON(b []byte) error {
	var ss string
	if err := json.Unmarshal(b, &ss); err != nil {
		return err
	}
	*cl = ParseCheckLevel(strings.ToUpper(ss))
	return nil
}

// String returns the string value, invalid CheckLevel will return Unknown.
func (cl CheckLevel) String() string {
	if cl < Unknown || cl > Critical {
		cl = Unknown
	}
	return checkLevels[cl]
}

// ParseCheckLevel will parse the string to checkLevel
func ParseCheckLevel(s string) CheckLevel {
	if cl, ok := checkLevelMaps[s]; ok {
		return cl
	}
	return Unknown
}
<commit_msg>fix(notification): fix the any checkLevel  string<commit_after>package notification

import (
	"encoding/json"
	"strings"
)

// StatusRule includes parametes of status rules.
type StatusRule struct {
	CurrentLevel  CheckLevel  `json:"currentLevel"`
	PreviousLevel *CheckLevel `json:"previousLevel"`
}

// CheckLevel is the enum value of status levels.
type CheckLevel int

// consts of CheckStatusLevel
const (
	Unknown CheckLevel = iota
	Ok
	Info
	Warn
	Critical
	Any
)

var checkLevels = []string{
	"UNKNOWN",
	"OK",
	"INFO",
	"WARN",
	"CRIT",
	"ANY",
}

var checkLevelMaps = map[string]CheckLevel{
	"UNKNOWN": Unknown,
	"OK":      Ok,
	"INFO":    Info,
	"WARN":    Warn,
	"CRIT":    Critical,
	"ANY":     Any,
}

// MarshalJSON implements json.Marshaller.
func (cl CheckLevel) MarshalJSON() ([]byte, error) {
	return json.Marshal(cl.String())
}

// UnmarshalJSON implements json.Unmarshaller.
func (cl *CheckLevel) UnmarshalJSON(b []byte) error {
	var ss string
	if err := json.Unmarshal(b, &ss); err != nil {
		return err
	}
	*cl = ParseCheckLevel(strings.ToUpper(ss))
	return nil
}

// String returns the string value, invalid CheckLevel will return Unknown.
func (cl CheckLevel) String() string {
	if cl < Unknown || cl > Any {
		cl = Unknown
	}
	return checkLevels[cl]
}

// ParseCheckLevel will parse the string to checkLevel
func ParseCheckLevel(s string) CheckLevel {
	if cl, ok := checkLevelMaps[s]; ok {
		return cl
	}
	return Unknown
}
<|endoftext|>


<commit_before>package sweetiebot

import (
  "sync/atomic"
  "time"
)

type AtomicFlag struct {
  flag uint32
}

type SaturationLimit struct {
  times []int64
  index int
  lock AtomicFlag
}

func realmod(x int, m int) int {
    x %= m
    if x < 0 { x += m }
    return x
}

func (f *AtomicFlag) test_and_set() bool {
  return atomic.SwapUint32(&f.flag, 1) != 0
}

func (f *AtomicFlag) clear() {
   atomic.SwapUint32(&f.flag, 0)
}

func (s *SaturationLimit) append(time int64) {
  for s.lock.test_and_set() {}
  s.index = realmod(s.index+1, len(s.times))
  s.times[s.index] = time
  s.lock.clear()
}

// Used for our own saturation limits, where we check to see if sending the message would violate our limit BEFORE we actually send it.
func (s *SaturationLimit) check(num int, period int64, curtime int64) bool {
  for s.lock.test_and_set() {}
  i := realmod(s.index-(num-1), len(s.times))
  b := (curtime - s.times[i]) <= period
  s.lock.clear()
  return b
}

// Used for spam detection, where we always insert the message first (because it's already happened) and THEN check to see if it violated the limit.
func (s *SaturationLimit) checkafter(num int, period int64) bool {
  for s.lock.test_and_set() {}
  i := realmod(s.index-num, len(s.times))
  b := (s.times[s.index] - s.times[i]) <= period
  s.lock.clear()
  return b
}

func CheckRateLimit(prevtime *int64, interval int64) bool {
    return time.Now().UTC().Unix() - *prevtime > interval
}

func RateLimit(prevtime *int64, interval int64) bool {
  t := time.Now().UTC().Unix()
  d := *prevtime // perform a read so it doesn't change on us
  if t - d > interval {
    *prevtime = t // CompareAndSwapInt64 doesn't work on x86, temporarily removing this
    return true
    //return atomic.CompareAndSwapInt64(prevtime, d, t) // If the swapped failed, it means another thread already sent a message and swapped it out, so don't send a message.
  }
  return false
}

func CheckShutup(channel string) bool {
  return channel != sb.DebugChannelID || CheckRateLimit(&sb.lastshutup, sb.config.Maxshutup);
}<commit_msg>Remove debug code<commit_after>package sweetiebot

import (
  "sync/atomic"
  "time"
)

type AtomicFlag struct {
  flag uint32
}

type SaturationLimit struct {
  times []int64
  index int
  lock AtomicFlag
}

func realmod(x int, m int) int {
    x %= m
    if x < 0 { x += m }
    return x
}

func (f *AtomicFlag) test_and_set() bool {
  return atomic.SwapUint32(&f.flag, 1) != 0
}

func (f *AtomicFlag) clear() {
   atomic.SwapUint32(&f.flag, 0)
}

func (s *SaturationLimit) append(time int64) {
  for s.lock.test_and_set() {}
  s.index = realmod(s.index+1, len(s.times))
  s.times[s.index] = time
  s.lock.clear()
}

// Used for our own saturation limits, where we check to see if sending the message would violate our limit BEFORE we actually send it.
func (s *SaturationLimit) check(num int, period int64, curtime int64) bool {
  for s.lock.test_and_set() {}
  i := realmod(s.index-(num-1), len(s.times))
  b := (curtime - s.times[i]) <= period
  s.lock.clear()
  return b
}

// Used for spam detection, where we always insert the message first (because it's already happened) and THEN check to see if it violated the limit.
func (s *SaturationLimit) checkafter(num int, period int64) bool {
  for s.lock.test_and_set() {}
  i := realmod(s.index-num, len(s.times))
  b := (s.times[s.index] - s.times[i]) <= period
  s.lock.clear()
  return b
}

func CheckRateLimit(prevtime *int64, interval int64) bool {
    return time.Now().UTC().Unix() - *prevtime > interval
}

func RateLimit(prevtime *int64, interval int64) bool {
  t := time.Now().UTC().Unix()
  d := *prevtime // perform a read so it doesn't change on us
  if t - d > interval {
    *prevtime = t // CompareAndSwapInt64 doesn't work on x86, temporarily removing this
    return true
    //return atomic.CompareAndSwapInt64(prevtime, d, t) // If the swapped failed, it means another thread already sent a message and swapped it out, so don't send a message.
  }
  return false
}

func CheckShutup(channel string) bool {
  return channel != sb.ManeChannelID || CheckRateLimit(&sb.lastshutup, sb.config.Maxshutup);
}<|endoftext|>


<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dircache

// This file defines structures that keep track of individual target directories.
// It particular it keeps a count of entries from the directory still in the LRU
// and handles refreshing of directory entries.

import (
	"strings"
	"sync"
	"time"

	"upspin.io/bind"
	"upspin.io/errors"
	"upspin.io/log"
	"upspin.io/path"
	"upspin.io/upspin"
)

// proxiedDir contains information about a proxied user directoies.
type proxiedDir struct {
	l     *clog
	ep    *upspin.Endpoint // endpoint for directory server
	order int64            // last order seen in watch
	atime time.Time        // time of last access
	user  upspin.UserName

	die   chan bool // channel used to tell watcher to die
	dying chan bool // channel used to confirm watcher is dying
}

// proxiedDirs is used to translate between a user name and the relevant cached directory.
type proxiedDirs struct {
	sync.Mutex

	closing bool // when this is true do not allocate any new watchers
	l       *clog
	m       map[upspin.UserName]*proxiedDir
}

func newProxiedDirs(l *clog) *proxiedDirs {
	return &proxiedDirs{m: make(map[upspin.UserName]*proxiedDir), l: l}
}

// close terminates all watchers.
func (p *proxiedDirs) close() {
	p.Lock()
	defer p.Unlock()
	if p.closing {
		return
	}
	p.closing = true
	for _, d := range p.m {
		d.close()
	}
}

// proxyFor saves the endpoint and makes sure it is being watched.
func (p *proxiedDirs) proxyFor(name upspin.PathName, ep *upspin.Endpoint) {
	p.Lock()
	defer p.Unlock()
	if p.closing {
		return
	}

	parsed, err := path.Parse(name)
	if err != nil {
		log.Info.Printf("parse error on a cleaned name: %s", name)
		return
	}
	u := parsed.User()
	d := p.m[u]
	if d == nil {
		d = &proxiedDir{l: p.l, ep: ep, user: u}
		p.m[u] = d
	}

	// Remember when we last accessed this proxied directory.
	// TODO: Use this time to stop listening to directories we
	// haven't looked at in a long time. We will also have to
	// forget about cached information for them if we stop
	// watching.
	d.atime = time.Now()

	// If the endpoint changed, kill off the current watcher.
	if d.ep != ep {
		d.close()
	}

	// Start a watcher if none is running.
	d.ep = ep
	if d.die == nil {
		d.die = make(chan bool)
		d.dying = make(chan bool)
		go d.watcher()
	}
}

// setOrder remembers an order read from the logfile.
func (p *proxiedDirs) setOrder(name upspin.PathName, order int64) {
	p.Lock()
	defer p.Unlock()
	if p.closing {
		return
	}

	parsed, err := path.Parse(name)
	if err != nil {
		log.Info.Printf("parse error on a cleaned name: %s", name)
		return
	}
	u := parsed.User()
	d := p.m[u]
	if d == nil {
		d = &proxiedDir{l: p.l, user: u}
		p.m[u] = d
	}
	d.order = order
}

// close terminates the goroutines associated with a proxied dir.
func (d *proxiedDir) close() {
	if d.die != nil {
		close(d.die)
		<-d.dying
		d.die = nil
	}
}

// watcher watches a directory and caches any changes to something already in the LRU.
func (d *proxiedDir) watcher() {
	defer close(d.dying)
	nextLogTime := time.Now()
	for {
		err := d.watch()
		if err == nil {
			// watch() only returns if the watcher has been told to die
			// or if there is an error requiring a new Watch.
			return
		}
		if err == upspin.ErrNotSupported {
			// Can't survive this.
			log.Info.Printf("rpc/dircache.watcher: %s: %s", d.user, err)
			return
		}
		if strings.Contains(err.Error(), "log misaligned") {
			// For now we have a problem with unreadable log entries
			// blocking the log reading. If we hit that, just
			// continue with the current state.
			d.order = -1
		}
		if time.Now().After(nextLogTime) {
			log.Info.Printf("rpc/dircache.watcher: %s: %s", d.user, err)
			nextLogTime = time.Now().Add(time.Hour)
		}
		time.Sleep(10 * time.Second)
	}
}

// watch loops receiving watch events. It returns nil if told to die.
// Otherwise it returns whatever error was encountered.
func (d *proxiedDir) watch() error {
	dir, err := bind.DirServer(d.l.cfg, *d.ep)
	if err != nil {
		return err
	}
	done := make(chan struct{})
	defer close(done)
	event, err := dir.Watch(upspin.PathName(string(d.user)+"/"), d.order, done)
	if err != nil {
		return err
	}

	// Loop receiving events until we are told to stop or the event stream is closed.
	for {
		select {
		case <-d.die:
			return nil
		case e, ok := <-event:
			if !ok {
				return errors.E("Watch event stream closed")
			}
			if err := d.handleEvent(&e); err != nil {
				return err
			}
		}
	}
}

func (d *proxiedDir) handleEvent(e *upspin.Event) error {
	// Something odd happened?
	if e.Error != nil {
		log.Info.Printf("rpc/dircache.handleEvent: %s", e.Error)
		return e.Error
	}

	// Is this a file we are watching?
	log.Debug.Printf("watch entry %s %v", e.Entry.Name, e)
	_, ok := d.l.lru.Get(lruKey{name: e.Entry.Name, glob: false})
	if !ok {
		// Not a file we are watching, how about in a directory we are watching?
		dirName := path.DropPath(e.Entry.Name, 1)
		if dirName == e.Entry.Name {
			return nil
		}
		_, ok := d.l.lru.Get(lruKey{name: dirName, glob: true})
		if !ok {
			return nil
		}
	}

	// This is an event we care about.
	d.order = e.Order
	op := lookupReq
	if e.Delete {
		op = deleteReq
	}
	d.l.logRequestWithOrder(op, e.Entry.Name, nil, e.Entry, e.Order)
	d.l.flush()
	return nil
}
<commit_msg>dir/dircache: I was treating each call as a new session.<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dircache

// This file defines structures that keep track of individual target directories.
// It particular it keeps a count of entries from the directory still in the LRU
// and handles refreshing of directory entries.

import (
	"strings"
	"sync"
	"time"

	"upspin.io/bind"
	"upspin.io/errors"
	"upspin.io/log"
	"upspin.io/path"
	"upspin.io/upspin"
)

// proxiedDir contains information about a proxied user directoies.
type proxiedDir struct {
	l     *clog
	ep    *upspin.Endpoint // endpoint for directory server
	order int64            // last order seen in watch
	atime time.Time        // time of last access
	user  upspin.UserName

	die   chan bool // channel used to tell watcher to die
	dying chan bool // channel used to confirm watcher is dying
}

// proxiedDirs is used to translate between a user name and the relevant cached directory.
type proxiedDirs struct {
	sync.Mutex

	closing bool // when this is true do not allocate any new watchers
	l       *clog
	m       map[upspin.UserName]*proxiedDir
}

func newProxiedDirs(l *clog) *proxiedDirs {
	return &proxiedDirs{m: make(map[upspin.UserName]*proxiedDir), l: l}
}

// close terminates all watchers.
func (p *proxiedDirs) close() {
	p.Lock()
	defer p.Unlock()
	if p.closing {
		return
	}
	p.closing = true
	for _, d := range p.m {
		d.close()
	}
}

// proxyFor saves the endpoint and makes sure it is being watched.
func (p *proxiedDirs) proxyFor(name upspin.PathName, ep *upspin.Endpoint) {
	p.Lock()
	defer p.Unlock()
	if p.closing {
		return
	}

	parsed, err := path.Parse(name)
	if err != nil {
		log.Info.Printf("parse error on a cleaned name: %s", name)
		return
	}
	u := parsed.User()
	d := p.m[u]
	if d == nil {
		d = &proxiedDir{l: p.l, ep: ep, user: u}
		p.m[u] = d
	}

	// Remember when we last accessed this proxied directory.
	// TODO: Use this time to stop listening to directories we
	// haven't looked at in a long time. We will also have to
	// forget about cached information for them if we stop
	// watching.
	d.atime = time.Now()

	// If the endpoint changed, kill off the current watcher.
	if d.ep != nil && *d.ep != *ep {
		d.close()
	}

	// Start a watcher if none is running.
	d.ep = ep
	if d.die == nil {
		d.die = make(chan bool)
		d.dying = make(chan bool)
		go d.watcher()
	}
}

// setOrder remembers an order read from the logfile.
func (p *proxiedDirs) setOrder(name upspin.PathName, order int64) {
	p.Lock()
	defer p.Unlock()
	if p.closing {
		return
	}

	parsed, err := path.Parse(name)
	if err != nil {
		log.Info.Printf("parse error on a cleaned name: %s", name)
		return
	}
	u := parsed.User()
	d := p.m[u]
	if d == nil {
		d = &proxiedDir{l: p.l, user: u}
		p.m[u] = d
	}
	d.order = order
}

// close terminates the goroutines associated with a proxied dir.
func (d *proxiedDir) close() {
	if d.die != nil {
		close(d.die)
		<-d.dying
		d.die = nil
	}
}

// watcher watches a directory and caches any changes to something already in the LRU.
func (d *proxiedDir) watcher() {
	log.Debug.Printf("dircache.Watcher %s %s", d.user, d.ep)
	defer close(d.dying)
	nextLogTime := time.Now()
	for {
		err := d.watch()
		if err == nil {
			log.Debug.Printf("dircache.Watcher %s %s exiting", d.user, d.ep)
			// watch() only returns if the watcher has been told to die
			// or if there is an error requiring a new Watch.
			return
		}
		if err == upspin.ErrNotSupported {
			// Can't survive this.
			log.Debug.Printf("rpc/dircache.watcher: %s: %s", d.user, err)
			return
		}
		if strings.Contains(err.Error(), "log misaligned") {
			// For now we have a problem with unreadable log entries
			// blocking the log reading. If we hit that, just
			// continue with the current state.
			d.order = -1
		}
		if time.Now().After(nextLogTime) {
			log.Info.Printf("rpc/dircache.watcher: %s: %s", d.user, err)
			nextLogTime = time.Now().Add(time.Hour)
		}
		time.Sleep(10 * time.Second)
	}
}

// watch loops receiving watch events. It returns nil if told to die.
// Otherwise it returns whatever error was encountered.
func (d *proxiedDir) watch() error {
	dir, err := bind.DirServer(d.l.cfg, *d.ep)
	if err != nil {
		return err
	}
	done := make(chan struct{})
	defer close(done)
	event, err := dir.Watch(upspin.PathName(string(d.user)+"/"), d.order, done)
	if err != nil {
		return err
	}

	// Loop receiving events until we are told to stop or the event stream is closed.
	for {
		select {
		case <-d.die:
			return nil
		case e, ok := <-event:
			if !ok {
				return errors.E("Watch event stream closed")
			}
			if err := d.handleEvent(&e); err != nil {
				return err
			}
		}
	}
}

func (d *proxiedDir) handleEvent(e *upspin.Event) error {
	// Something odd happened?
	if e.Error != nil {
		log.Info.Printf("rpc/dircache.handleEvent: %s", e.Error)
		return e.Error
	}

	// Is this a file we are watching?
	log.Debug.Printf("watch entry %s %v", e.Entry.Name, e)
	_, ok := d.l.lru.Get(lruKey{name: e.Entry.Name, glob: false})
	if !ok {
		// Not a file we are watching, how about in a directory we are watching?
		dirName := path.DropPath(e.Entry.Name, 1)
		if dirName == e.Entry.Name {
			return nil
		}
		_, ok := d.l.lru.Get(lruKey{name: dirName, glob: true})
		if !ok {
			return nil
		}
	}

	// This is an event we care about.
	d.order = e.Order
	op := lookupReq
	if e.Delete {
		op = deleteReq
	}
	d.l.logRequestWithOrder(op, e.Entry.Name, nil, e.Entry, e.Order)
	d.l.flush()
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"path"
	"path/filepath"
	"strings"

	"github.com/lxc/lxd/lxc/config"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/i18n"
)

// defaultAliases contains LXC's built-in command line aliases.  The built-in
// aliases are checked only if no user-defined alias was found.
var defaultAliases = map[string]string{
	"shell": "exec @ARGS@ -- su -l",
}

func findAlias(aliases map[string]string, origArgs []string) ([]string, []string, bool) {
	foundAlias := false
	aliasKey := []string{}
	aliasValue := []string{}

	for k, v := range aliases {
		foundAlias = true
		for i, key := range strings.Split(k, " ") {
			if len(origArgs) <= i+1 || origArgs[i+1] != key {
				foundAlias = false
				break
			}
		}

		if foundAlias {
			aliasKey = strings.Split(k, " ")
			aliasValue = strings.Split(v, " ")
			break
		}
	}

	return aliasKey, aliasValue, foundAlias
}

func expandAlias(conf *config.Config, args []string) ([]string, bool) {
	var newArgs []string
	var origArgs []string

	for _, arg := range args[1:] {
		if arg[0] != '-' {
			break
		}

		newArgs = append(newArgs, arg)
	}

	origArgs = append([]string{args[0]}, args[len(newArgs)+1:]...)

	aliasKey, aliasValue, foundAlias := findAlias(conf.Aliases, origArgs)
	if !foundAlias {
		aliasKey, aliasValue, foundAlias = findAlias(defaultAliases, origArgs)
		if !foundAlias {
			return []string{}, false
		}
	}

	if !strings.HasPrefix(aliasValue[0], "/") {
		newArgs = append([]string{origArgs[0]}, newArgs...)
	}
	hasReplacedArgsVar := false

	for i, aliasArg := range aliasValue {
		if aliasArg == "@ARGS@" && len(origArgs) > i {
			newArgs = append(newArgs, origArgs[i+1:]...)
			hasReplacedArgsVar = true
		} else {
			newArgs = append(newArgs, aliasArg)
		}
	}

	if !hasReplacedArgsVar {
		// Add the rest of the arguments
		newArgs = append(newArgs, origArgs[len(aliasKey)+1:]...)
	}

	return newArgs, true
}

func execIfAliases() error {
	args := os.Args

	// Avoid loops
	if os.Getenv("LXC_ALIASES") == "1" {
		return nil
	}

	// Figure out the config directory and config path
	var configDir string
	if os.Getenv("LXD_CONF") != "" {
		configDir = os.Getenv("LXD_CONF")
	} else if os.Getenv("HOME") != "" {
		configDir = path.Join(os.Getenv("HOME"), ".config", "lxc")
	} else {
		user, err := user.Current()
		if err != nil {
			return nil
		}

		configDir = path.Join(user.HomeDir, ".config", "lxc")
	}

	confPath := os.ExpandEnv(path.Join(configDir, "config.yml"))

	// Load the configuration
	var conf *config.Config
	var err error
	if shared.PathExists(confPath) {
		conf, err = config.LoadConfig(confPath)
		if err != nil {
			return nil
		}
	} else {
		conf = config.NewConfig(filepath.Dir(confPath), true)
	}

	// Expand the aliases
	newArgs, expanded := expandAlias(conf, args)
	if !expanded {
		return nil
	}

	// Look for the executable
	path, err := exec.LookPath(newArgs[0])
	if err != nil {
		return fmt.Errorf(i18n.G("Processing aliases failed: %s"), err)
	}

	// Re-exec
	environ := getEnviron()
	environ = append(environ, "LXC_ALIASES=1")
	ret := doExec(path, newArgs, environ)
	return fmt.Errorf(i18n.G("Processing aliases failed: %s"), ret)
}
<commit_msg>lxc/main/aliases: Fix panic when empty argument passed to lxc command<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"path"
	"path/filepath"
	"strings"

	"github.com/lxc/lxd/lxc/config"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/i18n"
)

// defaultAliases contains LXC's built-in command line aliases.  The built-in
// aliases are checked only if no user-defined alias was found.
var defaultAliases = map[string]string{
	"shell": "exec @ARGS@ -- su -l",
}

func findAlias(aliases map[string]string, origArgs []string) ([]string, []string, bool) {
	foundAlias := false
	aliasKey := []string{}
	aliasValue := []string{}

	for k, v := range aliases {
		foundAlias = true
		for i, key := range strings.Split(k, " ") {
			if len(origArgs) <= i+1 || origArgs[i+1] != key {
				foundAlias = false
				break
			}
		}

		if foundAlias {
			aliasKey = strings.Split(k, " ")
			aliasValue = strings.Split(v, " ")
			break
		}
	}

	return aliasKey, aliasValue, foundAlias
}

func expandAlias(conf *config.Config, args []string) ([]string, bool) {
	var newArgs []string
	var origArgs []string

	for _, arg := range args[1:] {
		if !strings.HasPrefix(arg, "-") {
			break
		}

		newArgs = append(newArgs, arg)
	}

	origArgs = append([]string{args[0]}, args[len(newArgs)+1:]...)

	aliasKey, aliasValue, foundAlias := findAlias(conf.Aliases, origArgs)
	if !foundAlias {
		aliasKey, aliasValue, foundAlias = findAlias(defaultAliases, origArgs)
		if !foundAlias {
			return []string{}, false
		}
	}

	if !strings.HasPrefix(aliasValue[0], "/") {
		newArgs = append([]string{origArgs[0]}, newArgs...)
	}
	hasReplacedArgsVar := false

	for i, aliasArg := range aliasValue {
		if aliasArg == "@ARGS@" && len(origArgs) > i {
			newArgs = append(newArgs, origArgs[i+1:]...)
			hasReplacedArgsVar = true
		} else {
			newArgs = append(newArgs, aliasArg)
		}
	}

	if !hasReplacedArgsVar {
		// Add the rest of the arguments
		newArgs = append(newArgs, origArgs[len(aliasKey)+1:]...)
	}

	return newArgs, true
}

func execIfAliases() error {
	args := os.Args

	// Avoid loops
	if os.Getenv("LXC_ALIASES") == "1" {
		return nil
	}

	// Figure out the config directory and config path
	var configDir string
	if os.Getenv("LXD_CONF") != "" {
		configDir = os.Getenv("LXD_CONF")
	} else if os.Getenv("HOME") != "" {
		configDir = path.Join(os.Getenv("HOME"), ".config", "lxc")
	} else {
		user, err := user.Current()
		if err != nil {
			return nil
		}

		configDir = path.Join(user.HomeDir, ".config", "lxc")
	}

	confPath := os.ExpandEnv(path.Join(configDir, "config.yml"))

	// Load the configuration
	var conf *config.Config
	var err error
	if shared.PathExists(confPath) {
		conf, err = config.LoadConfig(confPath)
		if err != nil {
			return nil
		}
	} else {
		conf = config.NewConfig(filepath.Dir(confPath), true)
	}

	// Expand the aliases
	newArgs, expanded := expandAlias(conf, args)
	if !expanded {
		return nil
	}

	// Look for the executable
	path, err := exec.LookPath(newArgs[0])
	if err != nil {
		return fmt.Errorf(i18n.G("Processing aliases failed: %s"), err)
	}

	// Re-exec
	environ := getEnviron()
	environ = append(environ, "LXC_ALIASES=1")
	ret := doExec(path, newArgs, environ)
	return fmt.Errorf(i18n.G("Processing aliases failed: %s"), ret)
}
<|endoftext|>


<commit_before>package main

import (
	// "bytes"
	"encoding/json"
	"fmt"
	// "html/template"
	// "io/ioutil"
	// "math"
	// "os"
	// "path/filepath"
	"runtime"
	"strconv"
	"strings"
	// "time"
	"sync"
	// "github.com/wgliang/goreporter/linters/aligncheck"
	"github.com/wgliang/goreporter/linters/copycheck"
	"github.com/wgliang/goreporter/linters/cyclo"
	// "github.com/wgliang/goreporter/linters/deadcode"
	// "github.com/wgliang/goreporter/linters/errorcheck"
	"github.com/wgliang/goreporter/linters/simplecode"
	"github.com/wgliang/goreporter/linters/staticscan"
	// "github.com/wgliang/goreporter/linters/structcheck"
	"github.com/wgliang/goreporter/linters/unittest"
	// "github.com/wgliang/goreporter/linters/varcheck"
)

var system string

func init() {
	if runtime.GOOS == `windows` {
		system = `\`
	} else {
		system = `/`
	}
}

func NewReporter() *Reporter {
	return &Reporter{}
}

func (r *Reporter) Engine(projectPath string, exceptPackages string) []byte {
	fmt.Println("start code quality assessment...")

	dirsUnitTest, err := DirList(projectPath, "_test.go", exceptPackages)
	if err != nil {
		fmt.Println(err)
	}
	r.Project = projectName(projectPath)
	var wg sync.WaitGroup

	// run linter:unit test
	wg.Add(1)
	go func() {
		packagesTestDetail := make(map[string]PackageTest, 0)
		packagesRaceDetail := make(map[string][]string, 0)
		fmt.Println("running unit test...")
		sumCover := 0.0
		countCover := 0
		for pkgName, pkgPath := range dirsUnitTest {
			unitTestRes, unitRaceRes := unittest.UnitTest(pkgPath)
			var packageTest PackageTest
			if len(unitTestRes) >= 1 {
				testres := unitTestRes[pkgName]
				if len(testres) > 5 {
					if testres[0] == "ok" {
						packageTest.IsPass = true
					} else {
						packageTest.IsPass = false
					}
					timeLen := len(testres[2])
					if timeLen > 1 {
						time, err := strconv.ParseFloat(testres[2][:(timeLen-1)], 64)
						if err == nil {
							packageTest.Time = time
						} else {
							fmt.Println(err)
						}
					}
					packageTest.Coverage = testres[4]

					coverLen := len(testres[4])
					if coverLen > 1 {
						coverFloat, _ := strconv.ParseFloat(testres[4][:(coverLen-1)], 64)
						sumCover = sumCover + coverFloat
						countCover = countCover + 1
					} else {
						countCover = countCover + 1
					}
				} else {
					packageTest.Coverage = "0%"
					countCover = countCover + 1
				}
			} else {
				packageTest.Coverage = "0%"
				countCover = countCover + 1
			}

			packagesTestDetail[pkgName] = packageTest
			if len(unitRaceRes[pkgName]) > 0 {
				packagesRaceDetail[pkgName] = unitRaceRes[pkgName]
			}
		}
		r.UnitTestx.PackagesTestDetail = packagesTestDetail
		r.UnitTestx.AvgCover = fmt.Sprintf("%.1f", sumCover/float64(countCover)) + "%"
		r.UnitTestx.PackagesRaceDetail = packagesRaceDetail
		wg.Done()
		fmt.Println("unit test over!")
	}()

	fmt.Println("computing cyclo...")
	dirsAll, err := DirList(projectPath, ".go", exceptPackages)
	if err != nil {
		fmt.Println(err)
	}
	wg.Add(1)
	go func() {
		cycloRes := make(map[string]Cycloi, 0)
		for pkgName, pkgPath := range dirsAll {
			cyclo, avg := cyclo.Cyclo(pkgPath)
			cycloRes[pkgName] = Cycloi{
				Average: avg,
				Result:  cyclo,
			}
		}
		r.Cyclox = cycloRes
		wg.Done()
		fmt.Println("cyclo over!")
	}()

	fmt.Println("simpling code...")
	wg.Add(1)
	go func() {
		simples := simplecode.SimpleCode(projectPath)
		simpleTips := make(map[string][]string, 0)
		for _, tips := range simples {
			index := strings.Index(tips, ":")
			simpleTips[PackageAbsPathExceptSuffix(tips[0:index])] = append(simpleTips[PackageAbsPathExceptSuffix(tips[0:index])], tips)
		}
		r.SimpleTips = simpleTips
		wg.Done()
	}()
	fmt.Println("simpled code!")

	fmt.Println("checking copy code...")
	wg.Add(1)
	go func() {
		x := copycheck.CopyCheck(projectPath, "_test.go")
		r.CopyTips = x
		wg.Done()
		fmt.Println("checked copy code!")
	}()

	fmt.Println("running staticscan...")
	wg.Add(1)
	go func() {
		staticscan.StaticScan(projectPath)
		scanTips := make(map[string][]string, 0)
		tips := staticscan.StaticScan(projectPath)
		for _, tip := range tips {
			index := strings.Index(tip, ":")
			scanTips[PackageAbsPathExceptSuffix(tip[0:index])] = append(scanTips[PackageAbsPathExceptSuffix(tip[0:index])], tip)
		}
		r.ScanTips = scanTips
		wg.Done()
		fmt.Println("staticscan over!")
	}()

	wg.Wait()
	fmt.Println("finished code quality assessment...")
	return r.formateReport2Json()
}

func (r *Reporter) formateReport2Json() []byte {
	report, err := json.Marshal(r)
	if err != nil {
		fmt.Println("json err:", err)
	}
	fmt.Println(string(report))
	return report
}
<commit_msg>fix som bug rewrite it<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"runtime"
	"strconv"
	"strings"
	"sync"

	// "github.com/wgliang/goreporter/linters/aligncheck"
	"github.com/wgliang/goreporter/linters/copycheck"
	"github.com/wgliang/goreporter/linters/cyclo"
	"github.com/wgliang/goreporter/linters/deadcode"
	"github.com/wgliang/goreporter/linters/depend"
	// "github.com/wgliang/goreporter/linters/errorcheck"
	"github.com/wgliang/goreporter/linters/simplecode"
	"github.com/wgliang/goreporter/linters/staticscan"
	// "github.com/wgliang/goreporter/linters/structcheck"
	"github.com/wgliang/goreporter/linters/unittest"
	// "github.com/wgliang/goreporter/linters/varcheck"
)

var system string

func init() {
	if runtime.GOOS == `windows` {
		system = `\`
	} else {
		system = `/`
	}
}

func NewReporter() *Reporter {
	return &Reporter{}
}

func (r *Reporter) Engine(projectPath string, exceptPackages string) {
	fmt.Println("start code quality assessment...")

	dirsUnitTest, err := DirList(projectPath, "_test.go", exceptPackages)
	if err != nil {
		fmt.Println(err)
	}
	r.Project = projectName(projectPath)
	var wg sync.WaitGroup

	// run linter:unit test
	wg.Add(1)
	go func() {
		fmt.Println("running unit test...")
		packagesTestDetail := struct {
			Values map[string]PackageTest
			mux    *sync.RWMutex
		}{make(map[string]PackageTest, 0), new(sync.RWMutex)}
		packagesRaceDetail := struct {
			Values map[string][]string
			mux    *sync.RWMutex
		}{make(map[string][]string, 0), new(sync.RWMutex)}

		sumCover := 0.0
		countCover := 0
		var pkg sync.WaitGroup
		for pkgName, pkgPath := range dirsUnitTest {
			pkg.Add(1)
			go func(pkgName, pkgPath string) {
				unitTestRes, unitRaceRes := unittest.UnitTest(pkgPath)
				var packageTest PackageTest
				if len(unitTestRes) >= 1 {
					testres := unitTestRes[pkgName]
					if len(testres) > 5 {
						if testres[0] == "ok" {
							packageTest.IsPass = true
						} else {
							packageTest.IsPass = false
						}
						timeLen := len(testres[2])
						if timeLen > 1 {
							time, err := strconv.ParseFloat(testres[2][:(timeLen-1)], 64)
							if err == nil {
								packageTest.Time = time
							} else {
								fmt.Println(err)
							}
						}
						packageTest.Coverage = testres[4]

						coverLen := len(testres[4])
						if coverLen > 1 {
							coverFloat, _ := strconv.ParseFloat(testres[4][:(coverLen-1)], 64)
							sumCover = sumCover + coverFloat
							countCover = countCover + 1
						} else {
							countCover = countCover + 1
						}
					} else {
						packageTest.Coverage = "0%"
						countCover = countCover + 1
					}
				} else {
					packageTest.Coverage = "0%"
					countCover = countCover + 1
				}
				packagesTestDetail.mux.Lock()
				packagesTestDetail.Values[pkgName] = packageTest
				packagesTestDetail.mux.Unlock()

				if len(unitRaceRes[pkgName]) > 0 {
					packagesRaceDetail.mux.Lock()
					packagesRaceDetail.Values[pkgName] = unitRaceRes[pkgName]
					packagesRaceDetail.mux.Unlock()
				}
				pkg.Done()
			}(pkgName, pkgPath)
		}

		pkg.Wait()
		packagesTestDetail.mux.Lock()
		r.UnitTestx.PackagesTestDetail = packagesTestDetail.Values
		packagesTestDetail.mux.Unlock()
		r.UnitTestx.AvgCover = fmt.Sprintf("%.1f", sumCover/float64(countCover)) + "%"
		packagesRaceDetail.mux.Lock()
		r.UnitTestx.PackagesRaceDetail = packagesRaceDetail.Values
		packagesRaceDetail.mux.Unlock()

		wg.Done()
		fmt.Println("unit test over!")
	}()

	fmt.Println("computing cyclo...")
	dirsAll, err := DirList(projectPath, ".go", exceptPackages)
	if err != nil {
		fmt.Println(err)
	}
	wg.Add(1)
	go func() {
		cycloRes := make(map[string]Cycloi, 0)
		for pkgName, pkgPath := range dirsAll {
			cyclo, avg := cyclo.Cyclo(pkgPath)
			cycloRes[pkgName] = Cycloi{
				Average: avg,
				Result:  cyclo,
			}
		}
		r.Cyclox = cycloRes
		wg.Done()
		fmt.Println("cyclo over!")
	}()

	fmt.Println("simpling code...")
	wg.Add(1)
	go func() {
		simples := simplecode.SimpleCode(projectPath)
		simpleTips := make(map[string][]string, 0)
		for _, tips := range simples {
			index := strings.Index(tips, ":")
			simpleTips[PackageAbsPathExceptSuffix(tips[0:index])] = append(simpleTips[PackageAbsPathExceptSuffix(tips[0:index])], tips)
		}
		r.SimpleTips = simpleTips
		wg.Done()
	}()
	fmt.Println("simpled code!")

	fmt.Println("checking copy code...")
	wg.Add(1)
	go func() {
		x := copycheck.CopyCheck(projectPath, "_test.go")
		r.CopyTips = x
		wg.Done()
		fmt.Println("checked copy code!")
	}()

	fmt.Println("running staticscan...")
	wg.Add(1)
	go func() {
		staticscan.StaticScan(projectPath)
		scanTips := make(map[string][]string, 0)
		tips := staticscan.StaticScan(projectPath)
		for _, tip := range tips {
			index := strings.Index(tip, ":")
			scanTips[PackageAbsPathExceptSuffix(tip[0:index])] = append(scanTips[PackageAbsPathExceptSuffix(tip[0:index])], tip)
		}
		r.ScanTips = scanTips
		wg.Done()
		fmt.Println("staticscan over!")
	}()

	fmt.Println("creating depend graph...")
	wg.Add(1)
	go func() {
		r.DependGraph = depend.Depend(projectPath, exceptPackages)
		wg.Done()
		fmt.Println("created depend graph")
	}()

	fmt.Println("checking dead code...")
	wg.Add(1)
	go func() {
		r.DeadCode = deadcode.DeadCode(projectPath)
		wg.Done()
		fmt.Println("checked dead code")
	}()

	fmt.Println("getting import packages...")
	var importPkgs []string
	wg.Add(1)
	go func() {
		importPkgs = unittest.GoListWithImportPackages(projectPath)
		wg.Done()
	}()

	wg.Wait()

	// get all no unit test packages
	noTestPackage := make([]string, 0)
	for i := 0; i < len(importPkgs); i++ {
		if _, ok := r.UnitTestx.PackagesTestDetail[importPkgs[i]]; !ok {
			noTestPackage = append(noTestPackage, importPkgs[i])
		}
	}
	r.NoTestPkg = noTestPackage

	fmt.Println("finished code quality assessment...")
}

func (r *Reporter) formateReport2Json() []byte {
	report, err := json.Marshal(r)
	if err != nil {
		fmt.Println("json err:", err)
	}

	return report
}
<|endoftext|>