<commit_before>package geo

import (
	"encoding/json"
	"fmt"
	"log"
	"testing"
)

// Tests that a call to NewPoint should return a pointer to a Point with the specified values assigned correctly.
func TestNewPoint(t *testing.T) {
	p := NewPoint(40.5, 120.5)

	if p == nil {
		t.Error("Expected to get a pointer to a new point, but got nil instead.")
	}

	if p.lat != 40.5 {
		t.Errorf("Expected to be able to specify 40.5 as the lat value of a new point, but got %f instead", p.lat)
	}

	if p.lng != 120.5 {
		t.Errorf("Expected to be able to specify 120.5 as the lng value of a new point, but got %f instead", p.lng)
	}
}

// Tests that calling GetLat() after creating a new point returns the expected lat value.
func TestLat(t *testing.T) {
	p := NewPoint(40.5, 120.5)

	lat := p.Lat()

	if lat != 40.5 {
		t.Error("Expected a call to GetLat() to return the same lat value as was set before, but got %f instead", lat)
	}
}

// Tests that calling GetLng() after creating a new point returns the expected lng value.
func TestLng(t *testing.T) {
	p := NewPoint(40.5, 120.5)

	lng := p.Lng()

	if lng != 120.5 {
		t.Error("Expected a call to GetLng() to return the same lat value as was set before, but got %f instead", lng)
	}
}

// Seems brittle :\
func TestGreatCircleDistance(t *testing.T) {
	// Test that SEA and SFO are ~ 1091km apart, accurate to 100 meters.
	sea := &Point{lat: 47.4489, lng: -122.3094}
	sfo := &Point{lat: 37.6160933, lng: -122.3924223}
	sfoToSea := 1093.379199082169

	dist := sea.GreatCircleDistance(sfo)

	if !(dist < (sfoToSea+0.1) && dist > (sfoToSea-0.1)) {
		t.Error("Unnacceptable result.", dist)
	}
}

func TestPointAtDistanceAndBearing(t *testing.T) {
	sea := &Point{lat: 47.44745785, lng: -122.308065668024}
	p := sea.PointAtDistanceAndBearing(1090.7, 180)

	// Expected results of transposing point
	// ~1091km at bearing of 180 degrees
	resultLat := 37.638557
	resultLng := -122.308066

	withinLatBounds := p.lat < resultLat+0.001 && p.lat > resultLat-0.001
	withinLngBounds := p.lng < resultLng+0.001 && p.lng > resultLng-0.001
	if !(withinLatBounds && withinLngBounds) {
		t.Error("Unnacceptable result.", fmt.Sprintf("[%f, %f]", p.lat, p.lng))
	}
}

func TestBearingTo(t *testing.T) {
	p1 := &Point{lat: 40.7486, lng: -73.9864}
	p2 := &Point{lat: 0.0, lng: 0.0}
	bearing := p1.BearingTo(p2)

	// Expected bearing 60 degrees
	resultBearing := 100.610833

	withinBearingBounds := bearing < resultBearing+0.001 && bearing > resultBearing-0.001
	if !withinBearingBounds {
		t.Error("Unnacceptable result.", fmt.Sprintf("%f", bearing))
	}
}

func TestMidpointTo(t *testing.T) {
	p1 := &Point{lat: 52.205, lng: 0.119}
	p2 := &Point{lat: 48.857, lng: 2.351}

	p := p1.MidpointTo(p2)

	// Expected midpoint 50.5363°N, 001.2746°E
	resultLat := 50.53632
	resultLng := 1.274614

	withinLatBounds := p.lat < resultLat+0.001 && p.lat > resultLat-0.001
	withinLngBounds := p.lng < resultLng+0.001 && p.lng > resultLng-0.001
	if !(withinLatBounds && withinLngBounds) {
		t.Error("Unnacceptable result.", fmt.Sprintf("[%f, %f]", p.lat, p.lng))
	}
}

// Enures that a point can be marhalled into JSON
func TestMarshalJSON(t *testing.T) {
	p := NewPoint(40.7486, -73.9864)
	res, err := json.Marshal(p)

	if err != nil {
		log.Print(err)
		t.Error("Should not encounter an error when attempting to Marshal a Point to JSON")
	}

	if string(res) != `{"lat":40.7486,"lng":-73.9864}` {
		t.Error("Point should correctly Marshal to JSON")
	}
}

// Enures that a point can be unmarhalled from JSON
func TestUnmarshalJSON(t *testing.T) {
	data := []byte(`{"lat":40.7486,"lng":-73.9864}`)
	p := &Point{}
	err := p.UnmarshalJSON(data)

	if err != nil {
		t.Errorf("Should not encounter an error when attempting to Unmarshal a Point from JSON")
	}

	if p.lat != 40.7486 || p.lng != -73.9864 {
		t.Errorf("Point has mismatched data after Unmarshalling from JSON")
	}
}
<commit_msg>Unit tests for Point.MarshalBinary() and Point.UnmarshalBinary().<commit_after>package geo

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"log"
	"testing"
)

// Tests that a call to NewPoint should return a pointer to a Point with the specified values assigned correctly.
func TestNewPoint(t *testing.T) {
	p := NewPoint(40.5, 120.5)

	if p == nil {
		t.Error("Expected to get a pointer to a new point, but got nil instead.")
	}

	if p.lat != 40.5 {
		t.Errorf("Expected to be able to specify 40.5 as the lat value of a new point, but got %f instead", p.lat)
	}

	if p.lng != 120.5 {
		t.Errorf("Expected to be able to specify 120.5 as the lng value of a new point, but got %f instead", p.lng)
	}
}

// Tests that calling GetLat() after creating a new point returns the expected lat value.
func TestLat(t *testing.T) {
	p := NewPoint(40.5, 120.5)

	lat := p.Lat()

	if lat != 40.5 {
		t.Error("Expected a call to GetLat() to return the same lat value as was set before, but got %f instead", lat)
	}
}

// Tests that calling GetLng() after creating a new point returns the expected lng value.
func TestLng(t *testing.T) {
	p := NewPoint(40.5, 120.5)

	lng := p.Lng()

	if lng != 120.5 {
		t.Error("Expected a call to GetLng() to return the same lat value as was set before, but got %f instead", lng)
	}
}

// Seems brittle :\
func TestGreatCircleDistance(t *testing.T) {
	// Test that SEA and SFO are ~ 1091km apart, accurate to 100 meters.
	sea := &Point{lat: 47.4489, lng: -122.3094}
	sfo := &Point{lat: 37.6160933, lng: -122.3924223}
	sfoToSea := 1093.379199082169

	dist := sea.GreatCircleDistance(sfo)

	if !(dist < (sfoToSea+0.1) && dist > (sfoToSea-0.1)) {
		t.Error("Unnacceptable result.", dist)
	}
}

func TestPointAtDistanceAndBearing(t *testing.T) {
	sea := &Point{lat: 47.44745785, lng: -122.308065668024}
	p := sea.PointAtDistanceAndBearing(1090.7, 180)

	// Expected results of transposing point
	// ~1091km at bearing of 180 degrees
	resultLat := 37.638557
	resultLng := -122.308066

	withinLatBounds := p.lat < resultLat+0.001 && p.lat > resultLat-0.001
	withinLngBounds := p.lng < resultLng+0.001 && p.lng > resultLng-0.001
	if !(withinLatBounds && withinLngBounds) {
		t.Error("Unnacceptable result.", fmt.Sprintf("[%f, %f]", p.lat, p.lng))
	}
}

func TestBearingTo(t *testing.T) {
	p1 := &Point{lat: 40.7486, lng: -73.9864}
	p2 := &Point{lat: 0.0, lng: 0.0}
	bearing := p1.BearingTo(p2)

	// Expected bearing 60 degrees
	resultBearing := 100.610833

	withinBearingBounds := bearing < resultBearing+0.001 && bearing > resultBearing-0.001
	if !withinBearingBounds {
		t.Error("Unnacceptable result.", fmt.Sprintf("%f", bearing))
	}
}

func TestMidpointTo(t *testing.T) {
	p1 := &Point{lat: 52.205, lng: 0.119}
	p2 := &Point{lat: 48.857, lng: 2.351}

	p := p1.MidpointTo(p2)

	// Expected midpoint 50.5363°N, 001.2746°E
	resultLat := 50.53632
	resultLng := 1.274614

	withinLatBounds := p.lat < resultLat+0.001 && p.lat > resultLat-0.001
	withinLngBounds := p.lng < resultLng+0.001 && p.lng > resultLng-0.001
	if !(withinLatBounds && withinLngBounds) {
		t.Error("Unnacceptable result.", fmt.Sprintf("[%f, %f]", p.lat, p.lng))
	}
}

// Ensures that a point can be marhalled into JSON
func TestMarshalJSON(t *testing.T) {
	p := NewPoint(40.7486, -73.9864)
	res, err := json.Marshal(p)

	if err != nil {
		log.Print(err)
		t.Error("Should not encounter an error when attempting to Marshal a Point to JSON")
	}

	if string(res) != `{"lat":40.7486,"lng":-73.9864}` {
		t.Error("Point should correctly Marshal to JSON")
	}
}

// Ensures that a point can be unmarhalled from JSON
func TestUnmarshalJSON(t *testing.T) {
	data := []byte(`{"lat":40.7486,"lng":-73.9864}`)
	p := &Point{}
	err := p.UnmarshalJSON(data)

	if err != nil {
		t.Errorf("Should not encounter an error when attempting to Unmarshal a Point from JSON")
	}

	if p.lat != 40.7486 || p.lng != -73.9864 {
		t.Errorf("Point has mismatched data after Unmarshalling from JSON")
	}
}

// Ensure that a point can be marshalled into slice of binaries
func TestMarshalBinary(t *testing.T) {
	lat, long := 40.7486, -73.9864
	p := NewPoint(lat, long)
	actual, err := p.MarshalBinary()
	if err != nil {
		t.Error("Should not encounter an error when attempting to Marshal a Point to binary", err)
	}

	expected, err := coordinatesToBytes(lat, long)
	if err != nil {
		t.Error("Unable to convert coordinates to bytes slice.", err)
	}

	if !bytes.Equal(actual, expected) {
		t.Errorf("Point should correctly Marshal to Binary.\nExpected %v\nBut got %v", expected, actual)
	}
}

// Ensure that a point can be unmarshalled from a slice of binaries
func TestUnmarshalBinary(t *testing.T) {
	lat, long := 40.7486, -73.9864
	coordinates, err := coordinatesToBytes(lat, long)
	if err != nil {
		t.Error("Unable to convert coordinates to bytes slice.", err)
	}

	actual := &Point{}
	err = actual.UnmarshalBinary(coordinates)
	if err != nil {
		t.Error("Should not encounter an error when attempting to Unmarshal a Point from binary", err)
	}

	expected := NewPoint(lat, long)
	if !assertPointsEqual(actual, expected, 4) {
		t.Errorf("Point should correctly Marshal to Binary.\nExpected %+v\nBut got %+v", expected, actual)
	}
}

func coordinatesToBytes(lat, long float64) ([]byte, error) {
	var buf bytes.Buffer
	if err := binary.Write(&buf, binary.LittleEndian, lat); err != nil {
		return nil, err
	}
	if err := binary.Write(&buf, binary.LittleEndian, long); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

// Asserts true when the latitude and longtitude of p1 and p2 are equal up to a certain number of decimal places.
// Precision is used to define that number of decimal places.
func assertPointsEqual(p1, p2 *Point, precision int) bool {
	roundedLat1, roundedLng1 := int(p1.lat*float64(precision))/precision, int(p1.lng*float64(precision))/precision
	roundedLat2, roundedLng2 := int(p2.lat*float64(precision))/precision, int(p2.lng*float64(precision))/precision
	return roundedLat1 == roundedLat2 && roundedLng1 == roundedLng2
}
<|endoftext|>


<commit_before>package rpcd

import (
	"errors"
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/format"
	"github.com/Symantec/Dominator/lib/fsutil"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/objectcache"
	"github.com/Symantec/Dominator/lib/objectclient"
	"github.com/Symantec/Dominator/proto/sub"
	"io"
	"os"
	"path"
	"syscall"
	"time"
)

var (
	exitOnFetchFailure = flag.Bool("exitOnFetchFailure", false,
		"If true, exit if there are fetch failures. For debugging only")
	// TODO(rgooch): Remove this flag once data corruption is fixed, so that
	//               scanning always continues during a fetch.
	stopScanDuringFetch = flag.Bool("stopScanDuringFetch", true,
		"If true, stop scan during fetching. This reduces the chance of fetch problems")
)

func (t *rpcType) Fetch(request sub.FetchRequest,
	reply *sub.FetchResponse) error {
	if *readOnly {
		txt := "Fetch() rejected due to read-only mode"
		logger.Println(txt)
		return errors.New(txt)
	}
	rwLock.Lock()
	defer rwLock.Unlock()
	logger.Printf("Fetch() %d objects\n", len(request.Hashes))
	if fetchInProgress {
		logger.Println("Error: fetch already in progress")
		return errors.New("fetch already in progress")
	}
	if updateInProgress {
		logger.Println("Error: update in progress")
		return errors.New("update in progress")
	}
	fetchInProgress = true
	go doFetch(request)
	return nil
}

func doFetch(request sub.FetchRequest) {
	defer clearFetchInProgress()
	if *stopScanDuringFetch {
		disableScannerFunc(true)
		defer disableScannerFunc(false)
	}
	objectServer := objectclient.NewObjectClient(request.ServerAddress)
	benchmark := false
	if networkReaderContext.MaximumSpeed() < 1 {
		benchmark = enoughBytesForBenchmark(objectServer, request)
		if benchmark {
			objectServer.SetExclusiveGetObjects(true)
			logger.Println("Benchmarking network speed")
		}
	}
	objectsReader, err := objectServer.GetObjects(request.Hashes)
	if err != nil {
		logger.Printf("Error getting object reader:\t%s\n", err.Error())
		if *exitOnFetchFailure {
			os.Exit(1)
		}
		return
	}
	var totalLength uint64
	timeStart := time.Now()
	for _, hash := range request.Hashes {
		length, reader, err := objectsReader.NextObject()
		if err != nil {
			logger.Println(err)
			if *exitOnFetchFailure {
				os.Exit(1)
			}
			return
		}
		err = readOne(hash, length, networkReaderContext.NewReader(reader))
		reader.Close()
		if err != nil {
			logger.Println(err)
			if *exitOnFetchFailure {
				os.Exit(1)
			}
			return
		}
		totalLength += length
	}
	duration := time.Since(timeStart)
	speed := uint64(float64(totalLength) / duration.Seconds())
	if benchmark {
		file, err := os.Create(netbenchFilename)
		if err == nil {
			fmt.Fprintf(file, "%d\n", speed)
			file.Close()
		}
		networkReaderContext.InitialiseMaximumSpeed(speed)
	}
	logger.Printf("Fetch() complete. Read: %s in %s (%s/s)\n",
		format.FormatBytes(totalLength), duration, format.FormatBytes(speed))
	rescanObjectCacheChannel <- true
}

func enoughBytesForBenchmark(objectServer *objectclient.ObjectClient,
	request sub.FetchRequest) bool {
	lengths, err := objectServer.CheckObjects(request.Hashes)
	if err != nil {
		return false
	}
	var totalLength uint64
	for _, length := range lengths {
		totalLength += length
	}
	if totalLength > 1024*1024*64 {
		return true
	}
	return false
}

func readOne(hash hash.Hash, length uint64, reader io.Reader) error {
	filename := path.Join(objectsDir, objectcache.HashToFilename(hash))
	dirname := path.Dir(filename)
	if err := os.MkdirAll(dirname, syscall.S_IRWXU); err != nil {
		return err
	}
	return fsutil.CopyToFile(filename, reader, int64(length))
}

func clearFetchInProgress() {
	rwLock.Lock()
	defer rwLock.Unlock()
	fetchInProgress = false
}
<commit_msg>Call new ObjectsReader.Close() method in subd.Fetch() handler.<commit_after>package rpcd

import (
	"errors"
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/format"
	"github.com/Symantec/Dominator/lib/fsutil"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/objectcache"
	"github.com/Symantec/Dominator/lib/objectclient"
	"github.com/Symantec/Dominator/proto/sub"
	"io"
	"os"
	"path"
	"syscall"
	"time"
)

var (
	exitOnFetchFailure = flag.Bool("exitOnFetchFailure", false,
		"If true, exit if there are fetch failures. For debugging only")
	// TODO(rgooch): Remove this flag once data corruption is fixed, so that
	//               scanning always continues during a fetch.
	stopScanDuringFetch = flag.Bool("stopScanDuringFetch", true,
		"If true, stop scan during fetching. This reduces the chance of fetch problems")
)

func (t *rpcType) Fetch(request sub.FetchRequest,
	reply *sub.FetchResponse) error {
	if *readOnly {
		txt := "Fetch() rejected due to read-only mode"
		logger.Println(txt)
		return errors.New(txt)
	}
	rwLock.Lock()
	defer rwLock.Unlock()
	logger.Printf("Fetch() %d objects\n", len(request.Hashes))
	if fetchInProgress {
		logger.Println("Error: fetch already in progress")
		return errors.New("fetch already in progress")
	}
	if updateInProgress {
		logger.Println("Error: update in progress")
		return errors.New("update in progress")
	}
	fetchInProgress = true
	go doFetch(request)
	return nil
}

func doFetch(request sub.FetchRequest) {
	defer clearFetchInProgress()
	if *stopScanDuringFetch {
		disableScannerFunc(true)
		defer disableScannerFunc(false)
	}
	objectServer := objectclient.NewObjectClient(request.ServerAddress)
	benchmark := false
	if networkReaderContext.MaximumSpeed() < 1 {
		benchmark = enoughBytesForBenchmark(objectServer, request)
		if benchmark {
			objectServer.SetExclusiveGetObjects(true)
			logger.Println("Benchmarking network speed")
		}
	}
	objectsReader, err := objectServer.GetObjects(request.Hashes)
	if err != nil {
		logger.Printf("Error getting object reader:\t%s\n", err.Error())
		if *exitOnFetchFailure {
			os.Exit(1)
		}
		return
	}
	defer objectsReader.Close()
	var totalLength uint64
	timeStart := time.Now()
	for _, hash := range request.Hashes {
		length, reader, err := objectsReader.NextObject()
		if err != nil {
			logger.Println(err)
			if *exitOnFetchFailure {
				os.Exit(1)
			}
			return
		}
		err = readOne(hash, length, networkReaderContext.NewReader(reader))
		reader.Close()
		if err != nil {
			logger.Println(err)
			if *exitOnFetchFailure {
				os.Exit(1)
			}
			return
		}
		totalLength += length
	}
	duration := time.Since(timeStart)
	speed := uint64(float64(totalLength) / duration.Seconds())
	if benchmark {
		file, err := os.Create(netbenchFilename)
		if err == nil {
			fmt.Fprintf(file, "%d\n", speed)
			file.Close()
		}
		networkReaderContext.InitialiseMaximumSpeed(speed)
	}
	logger.Printf("Fetch() complete. Read: %s in %s (%s/s)\n",
		format.FormatBytes(totalLength), duration, format.FormatBytes(speed))
	rescanObjectCacheChannel <- true
}

func enoughBytesForBenchmark(objectServer *objectclient.ObjectClient,
	request sub.FetchRequest) bool {
	lengths, err := objectServer.CheckObjects(request.Hashes)
	if err != nil {
		return false
	}
	var totalLength uint64
	for _, length := range lengths {
		totalLength += length
	}
	if totalLength > 1024*1024*64 {
		return true
	}
	return false
}

func readOne(hash hash.Hash, length uint64, reader io.Reader) error {
	filename := path.Join(objectsDir, objectcache.HashToFilename(hash))
	dirname := path.Dir(filename)
	if err := os.MkdirAll(dirname, syscall.S_IRWXU); err != nil {
		return err
	}
	return fsutil.CopyToFile(filename, reader, int64(length))
}

func clearFetchInProgress() {
	rwLock.Lock()
	defer rwLock.Unlock()
	fetchInProgress = false
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/anchor/vaultaire-collector-nagios/perfdata"
	"time"
)

// PerfDataWriteResult holds write statistics for reporting purposes.
type PerfDataWriteResult struct {
	RecordsWritten int
	Times          []int64
	Failed         bool
}

// processPerfDataRecord takes a slice of 'KEY::VALUE' pairs from one
// line of perfdata, and does the following:
//
// - parses them as either a service check result or a host check
//   result
// - extracts the individual metrics from the result
// - writes each metric to the supplied PerfDatumWriter
// - writes throughput statistics to the `written` channel
//
// This function is designed to be called in parallel, so it blocks on
// reading from a semaphore channel before proceeding.
func processPerfDataRecord(written chan PerfDataWriteResult, semaphore chan int, writer PerfDatumWriter, line []string) {
	<-semaphore
	writeResult := new(PerfDataWriteResult)
	datum, err := perfdata.NewPerfDatum(line)
	if err != nil {
		Log.Errorf("Error parsing record: %v", err)
		writeResult.Failed = true
		written <- *writeResult
		semaphore <- 1
		return
	}
	// Record parsed, extract the individual perfdata
	metrics, err := datum.RenderMetrics()
	if err != nil {
		Log.Errorf("Could not extract perfdata: %v", err)
		writeResult.Failed = true
		written <- *writeResult
		semaphore <- 1
		return
	}
	// Got everything we need, now write it to our storage backend
	for _, metricRecord := range metrics {
		preTime := time.Now()
		err := writer.Write(metricRecord)
		postTime := time.Now()
		writeTime := postTime.UnixNano() - preTime.UnixNano()
		writeSeconds := float64(writeTime) / 1000000000.0
		Log.Debugf("Write took %v seconds.", writeSeconds)
		writeResult.Times = append(writeResult.Times, writeTime)
		if err != nil {
			Log.Errorf("Failed to write %v: %v", metricRecord.GetKey(), err)
		} else {
			writeResult.RecordsWritten += 1
		}
	}
	written <- *writeResult
	semaphore <- 1
}
<commit_msg>Remove useless debugging output<commit_after>package main

import (
	"github.com/anchor/vaultaire-collector-nagios/perfdata"
	"time"
)

// PerfDataWriteResult holds write statistics for reporting purposes.
type PerfDataWriteResult struct {
	RecordsWritten int
	Times          []int64
	Failed         bool
}

// processPerfDataRecord takes a slice of 'KEY::VALUE' pairs from one
// line of perfdata, and does the following:
//
// - parses them as either a service check result or a host check
//   result
// - extracts the individual metrics from the result
// - writes each metric to the supplied PerfDatumWriter
// - writes throughput statistics to the `written` channel
//
// This function is designed to be called in parallel, so it blocks on
// reading from a semaphore channel before proceeding.
func processPerfDataRecord(written chan PerfDataWriteResult, semaphore chan int, writer PerfDatumWriter, line []string) {
	<-semaphore
	writeResult := new(PerfDataWriteResult)
	datum, err := perfdata.NewPerfDatum(line)
	if err != nil {
		Log.Errorf("Error parsing record: %v", err)
		writeResult.Failed = true
		written <- *writeResult
		semaphore <- 1
		return
	}
	// Record parsed, extract the individual perfdata
	metrics, err := datum.RenderMetrics()
	if err != nil {
		Log.Errorf("Could not extract perfdata: %v", err)
		writeResult.Failed = true
		written <- *writeResult
		semaphore <- 1
		return
	}
	// Got everything we need, now write it to our storage backend
	for _, metricRecord := range metrics {
		preTime := time.Now()
		err := writer.Write(metricRecord)
		postTime := time.Now()
		writeTime := postTime.UnixNano() - preTime.UnixNano()
		writeSeconds := float64(writeTime) / 1000000000.0
		writeResult.Times = append(writeResult.Times, writeTime)
		if err != nil {
			Log.Errorf("Failed to write %v: %v", metricRecord.GetKey(), err)
		} else {
			writeResult.RecordsWritten += 1
		}
	}
	written <- *writeResult
	semaphore <- 1
}
<|endoftext|>


<commit_before>package charon

import (
	"database/sql"
	"os"
	"testing"
	"time"

	"github.com/coreos/etcd/Godeps/_workspace/src/golang.org/x/crypto/bcrypt"
	klog "github.com/go-kit/kit/log"
	_ "github.com/lib/pq"
	"github.com/piotrkowalczuk/mnemosyne"
	"github.com/piotrkowalczuk/sklog"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

type endToEndSuite struct {
	db             *sql.DB
	hasher         PasswordHasher
	userRepository userProvider

	charon       RPCClient
	charonConn   *grpc.ClientConn
	charonDaemon *Daemon

	mnemosyne       mnemosyne.RPCClient
	mnemosyneConn   *grpc.ClientConn
	mnemosyneDaemon *mnemosyne.Daemon
}

func (etes *endToEndSuite) setup(t *testing.T) {
	if testing.Short() {
		t.Skip("e2e suite ignored in short mode")
	}

	var err error

	mnemosyneTCP := listenTCP(t)
	charonTCP := listenTCP(t)
	logger := sklog.NewTestLogger(t)
	_ = klog.NewJSONLogger(os.Stdout)
	grpclog.SetLogger(sklog.NewGRPCLogger(logger))

	etes.mnemosyneDaemon = mnemosyne.NewDaemon(&mnemosyne.DaemonOpts{
		Namespace:              "mnemosyne",
		MonitoringEngine:       mnemosyne.MonitoringEnginePrometheus,
		StoragePostgresAddress: testPostgresAddress,
		Logger:                 logger,
		RPCListener:            mnemosyneTCP,
	})
	if err = etes.mnemosyneDaemon.Run(); err != nil {
		t.Fatalf("mnemosyne daemon start error: %s", err.Error())
	}
	t.Logf("mnemosyne deamon running on: %s", etes.mnemosyneDaemon.Addr().String())

	etes.charonDaemon = NewDaemon(&DaemonOpts{
		Namespace:          "charon",
		MonitoringEngine:   MonitoringEnginePrometheus,
		MnemosyneAddress:   etes.mnemosyneDaemon.Addr().String(),
		Logger:             logger,
		PostgresAddress:    testPostgresAddress,
		RPCListener:        charonTCP,
		PasswordBCryptCost: bcrypt.MinCost,
	})
	if err = etes.charonDaemon.Run(); err != nil {
		t.Fatalf("charon daemon start error: %s", err.Error())
	}
	t.Logf("charon deamon running on: %s", etes.charonDaemon.Addr().String())

	if etes.mnemosyneConn, err = grpc.Dial(
		mnemosyneTCP.Addr().String(),
		grpc.WithInsecure(),
		grpc.WithBlock(),
		grpc.WithTimeout(2*time.Second),
	); err != nil {
		t.Fatalf("mnemosyne grpc connection error: %s", err.Error())
	}
	if etes.charonConn, err = grpc.Dial(
		charonTCP.Addr().String(),
		grpc.WithInsecure(),
		grpc.WithBlock(),
		grpc.WithTimeout(2*time.Second),
	); err != nil {
		t.Fatalf("charon grpc connection error: %s", err.Error())
	}
	if etes.db, err = sql.Open("postgres", testPostgresAddress); err != nil {
		t.Fatalf("postgres connection error: %s", err.Error())
	}
	if err := setupDatabase(etes.db); err != nil {
		t.Fatalf("database setup error: %s", err.Error())
	}
	if etes.hasher, err = NewBCryptPasswordHasher(bcrypt.MinCost); err != nil {
		t.Fatalf("password hasher error: %s", err.Error())
	}

	etes.charon = NewRPCClient(etes.charonConn)
	etes.mnemosyne = mnemosyne.NewRPCClient(etes.mnemosyneConn)
	etes.userRepository = newUserRepository(etes.db)

	if _, err = createDumyTestUser(etes.userRepository, etes.hasher); err != nil {
		t.Fatalf("dummy user error: %s", err.Error())
	}
}

func (etes *endToEndSuite) teardown(t *testing.T) {
	grpcClose := func(conn *grpc.ClientConn) error {
		state, err := conn.State()
		if err != nil {
			return err
		}
		if state != grpc.Shutdown {
			if err = conn.Close(); err != nil {
				return err
			}
		}
		return nil
	}

	if err := teardownDatabase(etes.db); err != nil {
		t.Errorf("e2e suite database teardown error: %s", err.Error())
	}
	if err := grpcClose(etes.mnemosyneConn); err != nil {
		t.Errorf("e2e suite mnemosyne conn close error: %s", err.Error())
	}
	if err := grpcClose(etes.charonConn); err != nil {
		t.Errorf("e2e suite charon conn close error: %s", err.Error())
	}

	if err := etes.mnemosyneDaemon.Close(); err != nil {
		t.Errorf("e2e suite mnemosyne daemon close error: %s", err.Error())
	}
	if err := etes.charonDaemon.Close(); err != nil {
		t.Errorf("e2e suite charon daemon close error: %s", err.Error())
	}

	if err := etes.db.Close(); err != nil {
		t.Errorf("e2e suite database conn close error: %s", err.Error())
	}
}
<commit_msg>bcrypt import fix<commit_after>package charon

import (
	"database/sql"
	"os"
	"testing"
	"time"

	klog "github.com/go-kit/kit/log"
	_ "github.com/lib/pq"
	"github.com/piotrkowalczuk/mnemosyne"
	"github.com/piotrkowalczuk/sklog"
	"golang.org/x/crypto/bcrypt"
	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

type endToEndSuite struct {
	db             *sql.DB
	hasher         PasswordHasher
	userRepository userProvider

	charon       RPCClient
	charonConn   *grpc.ClientConn
	charonDaemon *Daemon

	mnemosyne       mnemosyne.RPCClient
	mnemosyneConn   *grpc.ClientConn
	mnemosyneDaemon *mnemosyne.Daemon
}

func (etes *endToEndSuite) setup(t *testing.T) {
	if testing.Short() {
		t.Skip("e2e suite ignored in short mode")
	}

	var err error

	mnemosyneTCP := listenTCP(t)
	charonTCP := listenTCP(t)
	logger := sklog.NewTestLogger(t)
	_ = klog.NewJSONLogger(os.Stdout)
	grpclog.SetLogger(sklog.NewGRPCLogger(logger))

	etes.mnemosyneDaemon = mnemosyne.NewDaemon(&mnemosyne.DaemonOpts{
		Namespace:              "mnemosyne",
		MonitoringEngine:       mnemosyne.MonitoringEnginePrometheus,
		StoragePostgresAddress: testPostgresAddress,
		Logger:                 logger,
		RPCListener:            mnemosyneTCP,
	})
	if err = etes.mnemosyneDaemon.Run(); err != nil {
		t.Fatalf("mnemosyne daemon start error: %s", err.Error())
	}
	t.Logf("mnemosyne deamon running on: %s", etes.mnemosyneDaemon.Addr().String())

	etes.charonDaemon = NewDaemon(&DaemonOpts{
		Namespace:          "charon",
		MonitoringEngine:   MonitoringEnginePrometheus,
		MnemosyneAddress:   etes.mnemosyneDaemon.Addr().String(),
		Logger:             logger,
		PostgresAddress:    testPostgresAddress,
		RPCListener:        charonTCP,
		PasswordBCryptCost: bcrypt.MinCost,
	})
	if err = etes.charonDaemon.Run(); err != nil {
		t.Fatalf("charon daemon start error: %s", err.Error())
	}
	t.Logf("charon deamon running on: %s", etes.charonDaemon.Addr().String())

	if etes.mnemosyneConn, err = grpc.Dial(
		mnemosyneTCP.Addr().String(),
		grpc.WithInsecure(),
		grpc.WithBlock(),
		grpc.WithTimeout(2*time.Second),
	); err != nil {
		t.Fatalf("mnemosyne grpc connection error: %s", err.Error())
	}
	if etes.charonConn, err = grpc.Dial(
		charonTCP.Addr().String(),
		grpc.WithInsecure(),
		grpc.WithBlock(),
		grpc.WithTimeout(2*time.Second),
	); err != nil {
		t.Fatalf("charon grpc connection error: %s", err.Error())
	}
	if etes.db, err = sql.Open("postgres", testPostgresAddress); err != nil {
		t.Fatalf("postgres connection error: %s", err.Error())
	}
	if err := setupDatabase(etes.db); err != nil {
		t.Fatalf("database setup error: %s", err.Error())
	}
	if etes.hasher, err = NewBCryptPasswordHasher(bcrypt.MinCost); err != nil {
		t.Fatalf("password hasher error: %s", err.Error())
	}

	etes.charon = NewRPCClient(etes.charonConn)
	etes.mnemosyne = mnemosyne.NewRPCClient(etes.mnemosyneConn)
	etes.userRepository = newUserRepository(etes.db)

	if _, err = createDumyTestUser(etes.userRepository, etes.hasher); err != nil {
		t.Fatalf("dummy user error: %s", err.Error())
	}
}

func (etes *endToEndSuite) teardown(t *testing.T) {
	grpcClose := func(conn *grpc.ClientConn) error {
		state, err := conn.State()
		if err != nil {
			return err
		}
		if state != grpc.Shutdown {
			if err = conn.Close(); err != nil {
				return err
			}
		}
		return nil
	}

	if err := teardownDatabase(etes.db); err != nil {
		t.Errorf("e2e suite database teardown error: %s", err.Error())
	}
	if err := grpcClose(etes.mnemosyneConn); err != nil {
		t.Errorf("e2e suite mnemosyne conn close error: %s", err.Error())
	}
	if err := grpcClose(etes.charonConn); err != nil {
		t.Errorf("e2e suite charon conn close error: %s", err.Error())
	}

	if err := etes.mnemosyneDaemon.Close(); err != nil {
		t.Errorf("e2e suite mnemosyne daemon close error: %s", err.Error())
	}
	if err := etes.charonDaemon.Close(); err != nil {
		t.Errorf("e2e suite charon daemon close error: %s", err.Error())
	}

	if err := etes.db.Close(); err != nil {
		t.Errorf("e2e suite database conn close error: %s", err.Error())
	}
}
<|endoftext|>


<commit_before>package swagger2

import (
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"github.com/grokify/gocharts/data/histogram"
	"github.com/grokify/gotilla/encoding/csvutil"
	"github.com/grokify/gotilla/type/stringsutil"
)

func CountEndpointsByTag(spec Specification, tags []string) histogram.HistogramSet {
	tags = stringsutil.SliceLinesTrimSpace(tags, true)
	hist := histogram.NewHistogramSet()
	for url, path := range spec.Paths {
		hist = countEndpointByTag(hist, tags, url, http.MethodGet, path.Get)
		hist = countEndpointByTag(hist, tags, url, http.MethodPatch, path.Patch)
		hist = countEndpointByTag(hist, tags, url, http.MethodPut, path.Put)
		hist = countEndpointByTag(hist, tags, url, http.MethodPost, path.Post)
		hist = countEndpointByTag(hist, tags, url, http.MethodDelete, path.Delete)
	}
	return hist
}

func countEndpointByTag(hist histogram.HistogramSet, tags []string, url string, method string, ep *Endpoint) histogram.HistogramSet {
	if ep == nil {
		return hist
	}
	method = strings.ToUpper(strings.TrimSpace(method))
	url = strings.TrimSpace(url)
	endpoint := method + " " + url
	for _, tag := range ep.Tags {
		tag = strings.TrimSpace(tag)
		add := true
		if len(tags) > 0 {
			add = false
			for _, try := range tags {
				if tag == try {
					add = true
				}
			}
		}
		if !add {
			continue
		}
		if len(tag) > 0 {
			hist.Add(tag, endpoint, 1)
		}
	}
	hist.Inflate()
	return hist
}

func WriteEndpointCountCSV(filename string, hset histogram.HistogramSet) error {
	writer, file, err := csvutil.NewWriterFile(filename)
	if err != nil {
		return err
	}
	//defer file.Close()
	//defer writer.Close()
	header := []string{"Tag", "Tag Endpoint Count", "Method", "Path"}
	err = writer.Write(header)
	if err != nil {
		return err
	}
	for tagName, hist := range hset.HistogramMap {
		hist.Inflate()
		for endpoint := range hist.BinsFrequency {
			parts := strings.Split(endpoint, " ")
			if len(parts) >= 2 {
				row := []string{
					tagName,
					strconv.Itoa(hist.BinCount),
					strings.ToUpper(parts[0]),
					strings.Join(parts[1:], " ")}
				err := writer.Write(row)
				if err != nil {
					return err
				}
			}
		}
	}
	writer.Flush()
	err = writer.Error()
	if err != nil {
		return err
	}
	return file.Close()
}

// EndpointCount returns a count of the endpoints for a specification.
func EndpointCount(spec Specification) int {
	endpoints := map[string]int{}
	for url, path := range spec.Paths {
		url = strings.TrimSpace(url)
		if path.Get != nil && !path.Get.IsEmpty() {
			endpoints[fmt.Sprintf("%s %s", http.MethodGet, url)] = 1
		}
		if path.Patch != nil && !path.Patch.IsEmpty() {
			endpoints[fmt.Sprintf("%s %s", http.MethodPatch, url)] = 1
		}
		if path.Post != nil && !path.Post.IsEmpty() {
			endpoints[fmt.Sprintf("%s %s", http.MethodPost, url)] = 1
		}
		if path.Put != nil && !path.Put.IsEmpty() {
			endpoints[fmt.Sprintf("%s %s", http.MethodPut, url)] = 1
		}
		if path.Delete != nil && !path.Delete.IsEmpty() {
			endpoints[fmt.Sprintf("%s %s", http.MethodDelete, url)] = 1
		}
	}
	return len(endpoints)
}
<commit_msg>update for dependency<commit_after>package swagger2

import (
	"fmt"
	"net/http"
	"strconv"
	"strings"

	"github.com/grokify/gocharts/data/histogram"
	"github.com/grokify/gotilla/encoding/csvutil"
	"github.com/grokify/gotilla/type/stringsutil"
)

func CountEndpointsByTag(spec Specification, tags []string) histogram.HistogramSet {
	tags = stringsutil.SliceTrimSpace(tags, true)
	hist := histogram.NewHistogramSet()
	for url, path := range spec.Paths {
		hist = countEndpointByTag(hist, tags, url, http.MethodGet, path.Get)
		hist = countEndpointByTag(hist, tags, url, http.MethodPatch, path.Patch)
		hist = countEndpointByTag(hist, tags, url, http.MethodPut, path.Put)
		hist = countEndpointByTag(hist, tags, url, http.MethodPost, path.Post)
		hist = countEndpointByTag(hist, tags, url, http.MethodDelete, path.Delete)
	}
	return hist
}

func countEndpointByTag(hist histogram.HistogramSet, tags []string, url string, method string, ep *Endpoint) histogram.HistogramSet {
	if ep == nil {
		return hist
	}
	method = strings.ToUpper(strings.TrimSpace(method))
	url = strings.TrimSpace(url)
	endpoint := method + " " + url
	for _, tag := range ep.Tags {
		tag = strings.TrimSpace(tag)
		add := true
		if len(tags) > 0 {
			add = false
			for _, try := range tags {
				if tag == try {
					add = true
				}
			}
		}
		if !add {
			continue
		}
		if len(tag) > 0 {
			hist.Add(tag, endpoint, 1)
		}
	}
	hist.Inflate()
	return hist
}

func WriteEndpointCountCSV(filename string, hset histogram.HistogramSet) error {
	writer, file, err := csvutil.NewWriterFile(filename)
	if err != nil {
		return err
	}
	//defer file.Close()
	//defer writer.Close()
	header := []string{"Tag", "Tag Endpoint Count", "Method", "Path"}
	err = writer.Write(header)
	if err != nil {
		return err
	}
	for tagName, hist := range hset.HistogramMap {
		hist.Inflate()
		for endpoint := range hist.BinsFrequency {
			parts := strings.Split(endpoint, " ")
			if len(parts) >= 2 {
				row := []string{
					tagName,
					strconv.Itoa(hist.BinCount),
					strings.ToUpper(parts[0]),
					strings.Join(parts[1:], " ")}
				err := writer.Write(row)
				if err != nil {
					return err
				}
			}
		}
	}
	writer.Flush()
	err = writer.Error()
	if err != nil {
		return err
	}
	return file.Close()
}

// EndpointCount returns a count of the endpoints for a specification.
func EndpointCount(spec Specification) int {
	endpoints := map[string]int{}
	for url, path := range spec.Paths {
		url = strings.TrimSpace(url)
		if path.Get != nil && !path.Get.IsEmpty() {
			endpoints[fmt.Sprintf("%s %s", http.MethodGet, url)] = 1
		}
		if path.Patch != nil && !path.Patch.IsEmpty() {
			endpoints[fmt.Sprintf("%s %s", http.MethodPatch, url)] = 1
		}
		if path.Post != nil && !path.Post.IsEmpty() {
			endpoints[fmt.Sprintf("%s %s", http.MethodPost, url)] = 1
		}
		if path.Put != nil && !path.Put.IsEmpty() {
			endpoints[fmt.Sprintf("%s %s", http.MethodPut, url)] = 1
		}
		if path.Delete != nil && !path.Delete.IsEmpty() {
			endpoints[fmt.Sprintf("%s %s", http.MethodDelete, url)] = 1
		}
	}
	return len(endpoints)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"runtime"
	"time"
)

func startOomKiller(maxMb int) {
	go func() {
		const M = uint64(1024 * 1024)
		runtime.LockOSThread()

		var mst runtime.MemStats
		buf := make([]byte, 1*M)
		f := "MarGo: OOM.\n" +
			"Memory limit: %vm\n" +
			"Memory usage: %vm\n" +
			"Number goroutines: %v\n" +
			"------- begin stack trace ----\n" +
			"\n%s\n\n" +
			"-------  end stack trace  ----\n"

		for {
			runtime.ReadMemStats(&mst)
			alloc := int(mst.Sys / M)
			if alloc >= maxMb {
				n := runtime.Stack(buf, true)
				log.Fatalf(f, maxMb, alloc, runtime.NumGoroutine(), buf[:n])
			}
			time.Sleep(time.Millisecond * 50)
		}
	}()
}
<commit_msg>* reduce oom poll rate to waste less CPU<commit_after>package main

import (
	"log"
	"runtime"
	"time"
)

func startOomKiller(maxMb int) {
	go func() {
		const M = uint64(1024 * 1024)
		runtime.LockOSThread()

		var mst runtime.MemStats
		buf := make([]byte, 1*M)
		f := "MarGo: OOM.\n" +
			"Memory limit: %vm\n" +
			"Memory usage: %vm\n" +
			"Number goroutines: %v\n" +
			"------- begin stack trace ----\n" +
			"\n%s\n\n" +
			"-------  end stack trace  ----\n"

		for {
			runtime.ReadMemStats(&mst)
			alloc := int(mst.Sys / M)
			if alloc >= maxMb {
				n := runtime.Stack(buf, true)
				log.Fatalf(f, maxMb, alloc, runtime.NumGoroutine(), buf[:n])
			}
			time.Sleep(time.Millisecond * 100)
		}
	}()
}
<|endoftext|>


<commit_before>package gubled

import (
	"testing"

	"github.com/stretchr/testify/assert"

	"fmt"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/smancke/guble/client"
	"github.com/smancke/guble/gubled/config"
	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/testutil"
)

type testgroup struct {
	t                *testing.T
	groupID          int
	addr             string
	done             chan bool
	messagesToSend   int
	client1, client2 client.Client
	topic            string
}

func newTestgroup(t *testing.T, groupID int, addr string, messagesToSend int) *testgroup {
	return &testgroup{
		t:              t,
		groupID:        groupID,
		addr:           addr,
		done:           make(chan bool),
		messagesToSend: messagesToSend,
	}
}

func TestThroughput(t *testing.T) {
	//testutil.EnableDebugForMethod()
	defer testutil.ResetDefaultRegistryHealthCheck()

	dir, _ := ioutil.TempDir("", "guble_benchmarking_test")
	defer os.RemoveAll(dir)

	*config.HttpListen = "localhost:0"
	*config.KVS = "memory"
	*config.MS = "file"
	*config.StoragePath = dir

	service := StartService()
	defer func() {
		service.Stop()
	}()
	time.Sleep(time.Millisecond * 10)

	testgroupCount := 4
	messagesPerGroup := 100
	log.Printf("init the %v testgroups", testgroupCount)
	testgroups := make([]*testgroup, testgroupCount, testgroupCount)
	for i := range testgroups {
		testgroups[i] = newTestgroup(t, i, service.WebServer().GetAddr(), messagesPerGroup)
	}

	// init test
	log.Print("init the testgroups")
	for i := range testgroups {
		testgroups[i].Init()
	}

	defer func() {
		// cleanup tests
		log.Print("cleanup the testgroups")
		for i := range testgroups {
			testgroups[i].Clean()
		}
	}()

	// start test
	log.Print("start the testgroups")
	start := time.Now()
	for i := range testgroups {
		go testgroups[i].Start()
	}

	log.Print("wait for finishing")
	timeout := time.After(time.Second * 60)
	for i, test := range testgroups {
		//fmt.Printf("wating for test %v\n", i)
		select {
		case successFlag := <-test.done:
			if !successFlag {
				t.Logf("testgroup %v returned with error", i)
				t.FailNow()
				return
			}
		case <-timeout:
			t.Log("timeout. testgroups not ready before timeout")
			t.Fail()
			return
		}
	}

	end := time.Now()
	totalMessages := testgroupCount * messagesPerGroup
	throughput := float64(totalMessages) / end.Sub(start).Seconds()
	log.Printf("finished! Throughput: %v/sec (%v message in %v)", int(throughput), totalMessages, end.Sub(start))
}

func (tg *testgroup) Init() {
	tg.topic = fmt.Sprintf("/%v-foo", tg.groupID)
	var err error
	location := "ws://" + tg.addr + "/stream/user/xy"
	//location := "ws://gathermon.mancke.net:8080/stream/"
	//location := "ws://127.0.0.1:8080/stream/"
	tg.client1, err = client.Open(location, "http://localhost/", 10, false)
	if err != nil {
		panic(err)
	}
	tg.client2, err = client.Open(location, "http://localhost/", 10, false)
	if err != nil {
		panic(err)
	}

	tg.expectStatusMessage(protocol.SUCCESS_CONNECTED, "You are connected to the server.")

	tg.client1.Subscribe(tg.topic)
	time.Sleep(time.Millisecond * 1)
	//test.expectStatusMessage(protocol.SUCCESS_SUBSCRIBED_TO, test.topic)
}

func (tg *testgroup) expectStatusMessage(name string, arg string) {
	select {
	case notify := <-tg.client1.StatusMessages():
		assert.Equal(tg.t, name, notify.Name)
		assert.Equal(tg.t, arg, notify.Arg)
	case <-time.After(time.Second * 1):
		tg.t.Logf("[%v] no notification of type %s until timeout", tg.groupID, name)
		tg.done <- false
		tg.t.Fail()
		return
	}
}

func (tg *testgroup) Start() {
	go func() {
		for i := 0; i < tg.messagesToSend; i++ {
			body := fmt.Sprintf("Hallo-%v", i)
			tg.client2.Send(tg.topic, body, "")
		}
	}()

	for i := 0; i < tg.messagesToSend; i++ {
		body := fmt.Sprintf("Hallo-%v", i)

		select {
		case msg := <-tg.client1.Messages():
			assert.Equal(tg.t, body, msg.BodyAsString())
			assert.Equal(tg.t, tg.topic, string(msg.Path))
		case msg := <-tg.client1.Errors():
			tg.t.Logf("[%v] received error: %v", tg.groupID, msg)
			tg.done <- false
			tg.t.Fail()
			return
		case <-time.After(time.Second * 5):
			tg.t.Logf("[%v] no message received until timeout, expected message %v", tg.groupID, i)
			tg.done <- false
			tg.t.Fail()
			return
		}
	}
	tg.done <- true
}

func (tg *testgroup) Clean() {
	tg.client1.Close()
	tg.client2.Close()
}
<commit_msg>-short for test<commit_after>package gubled

import (
	"testing"

	"github.com/stretchr/testify/assert"

	"fmt"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/smancke/guble/client"
	"github.com/smancke/guble/gubled/config"
	"github.com/smancke/guble/protocol"
	"github.com/smancke/guble/testutil"
)

type testgroup struct {
	t                *testing.T
	groupID          int
	addr             string
	done             chan bool
	messagesToSend   int
	client1, client2 client.Client
	topic            string
}

func newTestgroup(t *testing.T, groupID int, addr string, messagesToSend int) *testgroup {
	return &testgroup{
		t:              t,
		groupID:        groupID,
		addr:           addr,
		done:           make(chan bool),
		messagesToSend: messagesToSend,
	}
}

func TestThroughput(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode.")
	}

	//testutil.EnableDebugForMethod()()
	defer testutil.ResetDefaultRegistryHealthCheck()

	dir, _ := ioutil.TempDir("", "guble_benchmarking_test")
	defer os.RemoveAll(dir)

	*config.HttpListen = "localhost:0"
	*config.KVS = "memory"
	*config.MS = "file"
	*config.StoragePath = dir

	service := StartService()
	defer func() {
		service.Stop()
	}()
	time.Sleep(time.Millisecond * 10)

	testgroupCount := 4
	messagesPerGroup := 100
	log.Printf("init the %v testgroups", testgroupCount)
	testgroups := make([]*testgroup, testgroupCount, testgroupCount)
	for i := range testgroups {
		testgroups[i] = newTestgroup(t, i, service.WebServer().GetAddr(), messagesPerGroup)
	}

	// init test
	log.Print("init the testgroups")
	for i := range testgroups {
		testgroups[i].Init()
	}

	defer func() {
		// cleanup tests
		log.Print("cleanup the testgroups")
		for i := range testgroups {
			testgroups[i].Clean()
		}
	}()

	// start test
	log.Print("start the testgroups")
	start := time.Now()
	for i := range testgroups {
		go testgroups[i].Start()
	}

	log.Print("wait for finishing")
	timeout := time.After(time.Second * 60)
	for i, test := range testgroups {
		//fmt.Printf("wating for test %v\n", i)
		select {
		case successFlag := <-test.done:
			if !successFlag {
				t.Logf("testgroup %v returned with error", i)
				t.FailNow()
				return
			}
		case <-timeout:
			t.Log("timeout. testgroups not ready before timeout")
			t.Fail()
			return
		}
	}

	end := time.Now()
	totalMessages := testgroupCount * messagesPerGroup
	throughput := float64(totalMessages) / end.Sub(start).Seconds()
	log.Printf("finished! Throughput: %v/sec (%v message in %v)", int(throughput), totalMessages, end.Sub(start))
}

func (tg *testgroup) Init() {
	tg.topic = fmt.Sprintf("/%v-foo", tg.groupID)
	var err error
	location := "ws://" + tg.addr + "/stream/user/xy"
	//location := "ws://gathermon.mancke.net:8080/stream/"
	//location := "ws://127.0.0.1:8080/stream/"
	tg.client1, err = client.Open(location, "http://localhost/", 10, false)
	if err != nil {
		panic(err)
	}
	tg.client2, err = client.Open(location, "http://localhost/", 10, false)
	if err != nil {
		panic(err)
	}

	tg.expectStatusMessage(protocol.SUCCESS_CONNECTED, "You are connected to the server.")

	tg.client1.Subscribe(tg.topic)
	time.Sleep(time.Millisecond * 1)
	//test.expectStatusMessage(protocol.SUCCESS_SUBSCRIBED_TO, test.topic)
}

func (tg *testgroup) expectStatusMessage(name string, arg string) {
	select {
	case notify := <-tg.client1.StatusMessages():
		assert.Equal(tg.t, name, notify.Name)
		assert.Equal(tg.t, arg, notify.Arg)
	case <-time.After(time.Second * 1):
		tg.t.Logf("[%v] no notification of type %s until timeout", tg.groupID, name)
		tg.done <- false
		tg.t.Fail()
		return
	}
}

func (tg *testgroup) Start() {
	go func() {
		for i := 0; i < tg.messagesToSend; i++ {
			body := fmt.Sprintf("Hallo-%v", i)
			tg.client2.Send(tg.topic, body, "")
		}
	}()

	for i := 0; i < tg.messagesToSend; i++ {
		body := fmt.Sprintf("Hallo-%v", i)

		select {
		case msg := <-tg.client1.Messages():
			assert.Equal(tg.t, body, msg.BodyAsString())
			assert.Equal(tg.t, tg.topic, string(msg.Path))
		case msg := <-tg.client1.Errors():
			tg.t.Logf("[%v] received error: %v", tg.groupID, msg)
			tg.done <- false
			tg.t.Fail()
			return
		case <-time.After(time.Second * 5):
			tg.t.Logf("[%v] no message received until timeout, expected message %v", tg.groupID, i)
			tg.done <- false
			tg.t.Fail()
			return
		}
	}
	tg.done <- true
}

func (tg *testgroup) Clean() {
	tg.client1.Close()
	tg.client2.Close()
}
<|endoftext|>


<commit_before>package pollster

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"time"
)

type DateEstimates []struct {
	Date      string `json:"date"`
	Estimates []struct {
		Choice string  `json:"choice"`
		Value  float32 `json:"value"`
	} `json:"estimates"`
}

type Estimates []struct {
	Choice         string  `json:"choice"`
	Value          float32 `json:"value"`
	LeadConfidence float32 `json:"lead_confidence"`
	FirstName      string  `json:"first_name"`
	LastName       string  `json:"last_name"`
	Party          string  `json:"party"`
	Incumbent      bool    `json:"incumbent"`
}

type Chart struct {
	Title         string        `json:"title"`
	Slug          string        `json:"slug"`
	Topic         string        `json:"topic"`
	State         string        `json:"state"`
	ShortTitle    string        `json:"short_title"`
	ElectionDate  string        `json:"election_date"`
	PollCount     int           `json:"poll_count"`
	LastUpdated   time.Time     `json:"last_updated"`
	Url           string        `json:"url"`
	Estimates     Estimates     `json:"estimates"`
	DateEstimates DateEstimates `json:"estimates_by_date"`
}

type Poll struct {
	Id           int    `json:"id"`
	Pollster     string `json:"pollster"`
	StartDate    string `json:"start_date"`
	EndDate      string `json:"end_date"`
	Method       string `json:"method"`
	Source       string `json:"source"`
	LastUpdated  string `json:"last_updated"`
	Partisan     string `json:"partisan"`
	Affiliation  string `json:"affiliation"`
	SurveyHouses []struct {
		Name  string `json:"name"`
		Party string `json:"party"`
	} `json:"survey_houses"`
	Sponsors []struct {
		Name  string `json:"name"`
		Party string `json:"party"`
	}
	Questions []struct {
		Name           string `json:"name"`
		Chart          string `json:"chart"`
		Topic          string `json:"topic"`
		State          string `json:"state"`
		Subpopulations []struct {
			Name          string `json:"name"`
			Observations  int    `json:"observations"`
			MarginOfError int    `json:"margin_of_error"`
			Responses     []struct {
				Choice    string `json:"choice"`
				Value     int    `json:"value"`
				FirstName string `json:"first_name"`
				LastName  string `json:"last_name"`
				Party     string `json:"party"`
				Incumbent bool   `json:"incumbent"`
			} `json:"responses"`
		} `json:"subpopulations"`
	} `json:"questions"`
}

var baseUrl = "http://elections.huffingtonpost.com/pollster/api/"

func handleError(err error) bool {
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	return false
}

func getJson(url string) []byte {
	res, err := http.Get(url)
	handleError(err)
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	handleError(err)
	return body
}

func buildUrl(url string, params map[string]string) string {
	for k, v := range params {
		url += fmt.Sprintf("%s=%s&", k, v)
	}
	return url
}

func Charts(params map[string]string) []Chart {
	url := buildUrl(baseUrl+"charts?", params)
	body := getJson(url)
	var charts []Chart
	json.Unmarshal(body, &charts)
	return charts
}

func (chart Chart) EstimatesByDate() DateEstimates {
	body := getJson(fmt.Sprintf(baseUrl+"charts/%s", chart.Slug))
	json.Unmarshal(body, &chart)
	return chart.DateEstimates
}

func Polls(params map[string]string) []Poll {
	url := buildUrl(baseUrl+"polls?", params)
	body := getJson(url)
	var polls []Poll
	json.Unmarshal(body, &polls)
	return polls
}
<commit_msg>Make base URL a constant<commit_after>package pollster

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"time"
)

type DateEstimates []struct {
	Date      string `json:"date"`
	Estimates []struct {
		Choice string  `json:"choice"`
		Value  float32 `json:"value"`
	} `json:"estimates"`
}

type Estimates []struct {
	Choice         string  `json:"choice"`
	Value          float32 `json:"value"`
	LeadConfidence float32 `json:"lead_confidence"`
	FirstName      string  `json:"first_name"`
	LastName       string  `json:"last_name"`
	Party          string  `json:"party"`
	Incumbent      bool    `json:"incumbent"`
}

type Chart struct {
	Title         string        `json:"title"`
	Slug          string        `json:"slug"`
	Topic         string        `json:"topic"`
	State         string        `json:"state"`
	ShortTitle    string        `json:"short_title"`
	ElectionDate  string        `json:"election_date"`
	PollCount     int           `json:"poll_count"`
	LastUpdated   time.Time     `json:"last_updated"`
	Url           string        `json:"url"`
	Estimates     Estimates     `json:"estimates"`
	DateEstimates DateEstimates `json:"estimates_by_date"`
}

type Poll struct {
	Id           int    `json:"id"`
	Pollster     string `json:"pollster"`
	StartDate    string `json:"start_date"`
	EndDate      string `json:"end_date"`
	Method       string `json:"method"`
	Source       string `json:"source"`
	LastUpdated  string `json:"last_updated"`
	Partisan     string `json:"partisan"`
	Affiliation  string `json:"affiliation"`
	SurveyHouses []struct {
		Name  string `json:"name"`
		Party string `json:"party"`
	} `json:"survey_houses"`
	Sponsors []struct {
		Name  string `json:"name"`
		Party string `json:"party"`
	}
	Questions []struct {
		Name           string `json:"name"`
		Chart          string `json:"chart"`
		Topic          string `json:"topic"`
		State          string `json:"state"`
		Subpopulations []struct {
			Name          string `json:"name"`
			Observations  int    `json:"observations"`
			MarginOfError int    `json:"margin_of_error"`
			Responses     []struct {
				Choice    string `json:"choice"`
				Value     int    `json:"value"`
				FirstName string `json:"first_name"`
				LastName  string `json:"last_name"`
				Party     string `json:"party"`
				Incumbent bool   `json:"incumbent"`
			} `json:"responses"`
		} `json:"subpopulations"`
	} `json:"questions"`
}

const baseUrl = "http://elections.huffingtonpost.com/pollster/api/"

func handleError(err error) bool {
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	return false
}

func getJson(url string) []byte {
	res, err := http.Get(url)
	handleError(err)
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	handleError(err)
	return body
}

func buildUrl(url string, params map[string]string) string {
	for k, v := range params {
		url += fmt.Sprintf("%s=%s&", k, v)
	}
	return url
}

func Charts(params map[string]string) []Chart {
	url := buildUrl(baseUrl+"charts?", params)
	body := getJson(url)
	var charts []Chart
	json.Unmarshal(body, &charts)
	return charts
}

func (chart Chart) EstimatesByDate() DateEstimates {
	body := getJson(fmt.Sprintf(baseUrl+"charts/%s", chart.Slug))
	json.Unmarshal(body, &chart)
	return chart.DateEstimates
}

func Polls(params map[string]string) []Poll {
	url := buildUrl(baseUrl+"polls?", params)
	body := getJson(url)
	var polls []Poll
	json.Unmarshal(body, &polls)
	return polls
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sync

import (
	"internal/race"
	"sync/atomic"
	"unsafe"
)

// A WaitGroup waits for a collection of goroutines to finish.
// The main goroutine calls Add to set the number of
// goroutines to wait for. Then each of the goroutines
// runs and calls Done when finished. At the same time,
// Wait can be used to block until all goroutines have finished.
//
// A WaitGroup must not be copied after first use.
//
// In the terminology of the Go memory model, a call to Done
// “synchronizes before” the return of any Wait call that it unblocks.
type WaitGroup struct {
	noCopy noCopy

	// 64-bit value: high 32 bits are counter, low 32 bits are waiter count.
	// 64-bit atomic operations require 64-bit alignment, but 32-bit
	// compilers only guarantee that 64-bit fields are 32-bit aligned.
	// For this reason on 32 bit architectures we need to check in state()
	// if state1 is aligned or not, and dynamically "swap" the field order if
	// needed.
	state1 uint64
	state2 uint32
}

// state returns pointers to the state and sema fields stored within wg.state*.
func (wg *WaitGroup) state() (statep *uint64, semap *uint32) {
	if unsafe.Alignof(wg.state1) == 8 || uintptr(unsafe.Pointer(&wg.state1))%8 == 0 {
		// state1 is 64-bit aligned: nothing to do.
		return &wg.state1, &wg.state2
	} else {
		// state1 is 32-bit aligned but not 64-bit aligned: this means that
		// (&state1)+4 is 64-bit aligned.
		state := (*[3]uint32)(unsafe.Pointer(&wg.state1))
		return (*uint64)(unsafe.Pointer(&state[1])), &state[0]
	}
}

// Add adds delta, which may be negative, to the WaitGroup counter.
// If the counter becomes zero, all goroutines blocked on Wait are released.
// If the counter goes negative, Add panics.
//
// Note that calls with a positive delta that occur when the counter is zero
// must happen before a Wait. Calls with a negative delta, or calls with a
// positive delta that start when the counter is greater than zero, may happen
// at any time.
// Typically this means the calls to Add should execute before the statement
// creating the goroutine or other event to be waited for.
// If a WaitGroup is reused to wait for several independent sets of events,
// new Add calls must happen after all previous Wait calls have returned.
// See the WaitGroup example.
func (wg *WaitGroup) Add(delta int) {
	statep, semap := wg.state()
	if race.Enabled {
		_ = *statep // trigger nil deref early
		if delta < 0 {
			// Synchronize decrements with Wait.
			race.ReleaseMerge(unsafe.Pointer(wg))
		}
		race.Disable()
		defer race.Enable()
	}
	state := atomic.AddUint64(statep, uint64(delta)<<32)
	v := int32(state >> 32)
	w := uint32(state)
	if race.Enabled && delta > 0 && v == int32(delta) {
		// The first increment must be synchronized with Wait.
		// Need to model this as a read, because there can be
		// several concurrent wg.counter transitions from 0.
		race.Read(unsafe.Pointer(semap))
	}
	if v < 0 {
		panic("sync: negative WaitGroup counter")
	}
	if w != 0 && delta > 0 && v == int32(delta) {
		panic("sync: WaitGroup misuse: Add called concurrently with Wait")
	}
	if v > 0 || w == 0 {
		return
	}
	// This goroutine has set counter to 0 when waiters > 0.
	// Now there can't be concurrent mutations of state:
	// - Adds must not happen concurrently with Wait,
	// - Wait does not increment waiters if it sees counter == 0.
	// Still do a cheap sanity check to detect WaitGroup misuse.
	if *statep != state {
		panic("sync: WaitGroup misuse: Add called concurrently with Wait")
	}
	// Reset waiters count to 0.
	*statep = 0
	for ; w != 0; w-- {
		runtime_Semrelease(semap, false, 0)
	}
}

// Done decrements the WaitGroup counter by one.
func (wg *WaitGroup) Done() {
	wg.Add(-1)
}

// Wait blocks until the WaitGroup counter is zero.
func (wg *WaitGroup) Wait() {
	statep, semap := wg.state()
	if race.Enabled {
		_ = *statep // trigger nil deref early
		race.Disable()
	}
	for {
		state := atomic.LoadUint64(statep)
		v := int32(state >> 32)
		w := uint32(state)
		if v == 0 {
			// Counter is 0, no need to wait.
			if race.Enabled {
				race.Enable()
				race.Acquire(unsafe.Pointer(wg))
			}
			return
		}
		// Increment waiters count.
		if atomic.CompareAndSwapUint64(statep, state, state+1) {
			if race.Enabled && w == 0 {
				// Wait must be synchronized with the first Add.
				// Need to model this is as a write to race with the read in Add.
				// As a consequence, can do the write only for the first waiter,
				// otherwise concurrent Waits will race with each other.
				race.Write(unsafe.Pointer(semap))
			}
			runtime_Semacquire(semap)
			if *statep != 0 {
				panic("sync: WaitGroup is reused before previous Wait has returned")
			}
			if race.Enabled {
				race.Enable()
				race.Acquire(unsafe.Pointer(wg))
			}
			return
		}
	}
}
<commit_msg>sync: use atomic.Uint64 for WaitGroup state<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sync

import (
	"internal/race"
	"sync/atomic"
	"unsafe"
)

// A WaitGroup waits for a collection of goroutines to finish.
// The main goroutine calls Add to set the number of
// goroutines to wait for. Then each of the goroutines
// runs and calls Done when finished. At the same time,
// Wait can be used to block until all goroutines have finished.
//
// A WaitGroup must not be copied after first use.
//
// In the terminology of the Go memory model, a call to Done
// “synchronizes before” the return of any Wait call that it unblocks.
type WaitGroup struct {
	noCopy noCopy

	state atomic.Uint64 // high 32 bits are counter, low 32 bits are waiter count.
	sema  uint32
}

// Add adds delta, which may be negative, to the WaitGroup counter.
// If the counter becomes zero, all goroutines blocked on Wait are released.
// If the counter goes negative, Add panics.
//
// Note that calls with a positive delta that occur when the counter is zero
// must happen before a Wait. Calls with a negative delta, or calls with a
// positive delta that start when the counter is greater than zero, may happen
// at any time.
// Typically this means the calls to Add should execute before the statement
// creating the goroutine or other event to be waited for.
// If a WaitGroup is reused to wait for several independent sets of events,
// new Add calls must happen after all previous Wait calls have returned.
// See the WaitGroup example.
func (wg *WaitGroup) Add(delta int) {
	if race.Enabled {
		if delta < 0 {
			// Synchronize decrements with Wait.
			race.ReleaseMerge(unsafe.Pointer(wg))
		}
		race.Disable()
		defer race.Enable()
	}
	state := wg.state.Add(uint64(delta) << 32)
	v := int32(state >> 32)
	w := uint32(state)
	if race.Enabled && delta > 0 && v == int32(delta) {
		// The first increment must be synchronized with Wait.
		// Need to model this as a read, because there can be
		// several concurrent wg.counter transitions from 0.
		race.Read(unsafe.Pointer(&wg.sema))
	}
	if v < 0 {
		panic("sync: negative WaitGroup counter")
	}
	if w != 0 && delta > 0 && v == int32(delta) {
		panic("sync: WaitGroup misuse: Add called concurrently with Wait")
	}
	if v > 0 || w == 0 {
		return
	}
	// This goroutine has set counter to 0 when waiters > 0.
	// Now there can't be concurrent mutations of state:
	// - Adds must not happen concurrently with Wait,
	// - Wait does not increment waiters if it sees counter == 0.
	// Still do a cheap sanity check to detect WaitGroup misuse.
	if wg.state.Load() != state {
		panic("sync: WaitGroup misuse: Add called concurrently with Wait")
	}
	// Reset waiters count to 0.
	wg.state.Store(0)
	for ; w != 0; w-- {
		runtime_Semrelease(&wg.sema, false, 0)
	}
}

// Done decrements the WaitGroup counter by one.
func (wg *WaitGroup) Done() {
	wg.Add(-1)
}

// Wait blocks until the WaitGroup counter is zero.
func (wg *WaitGroup) Wait() {
	if race.Enabled {
		race.Disable()
	}
	for {
		state := wg.state.Load()
		v := int32(state >> 32)
		w := uint32(state)
		if v == 0 {
			// Counter is 0, no need to wait.
			if race.Enabled {
				race.Enable()
				race.Acquire(unsafe.Pointer(wg))
			}
			return
		}
		// Increment waiters count.
		if wg.state.CompareAndSwap(state, state+1) {
			if race.Enabled && w == 0 {
				// Wait must be synchronized with the first Add.
				// Need to model this is as a write to race with the read in Add.
				// As a consequence, can do the write only for the first waiter,
				// otherwise concurrent Waits will race with each other.
				race.Write(unsafe.Pointer(&wg.sema))
			}
			runtime_Semacquire(&wg.sema)
			if wg.state.Load() != 0 {
				panic("sync: WaitGroup is reused before previous Wait has returned")
			}
			if race.Enabled {
				race.Enable()
				race.Acquire(unsafe.Pointer(wg))
			}
			return
		}
	}
}
<|endoftext|>


<commit_before>package datastore

import (
	"fmt"
	"time"

	"github.com/gocql/gocql"

	"github.com/jeffpierce/cassabon/config"
	"github.com/jeffpierce/cassabon/logging"
)

type batchWriter struct {
	dbClient  *gocql.Session
	keyspace  string
	batchSize int

	batch     *gocql.Batch
	stmtCount int
	stmt      string
}

// Init
func (bw *batchWriter) Init(dbClient *gocql.Session, keyspace string, batchSize int) {
	bw.dbClient = dbClient
	bw.keyspace = keyspace
	bw.batchSize = batchSize
}

// Size
func (bw *batchWriter) Size() int {
	return bw.stmtCount
}

// Prepare
func (bw *batchWriter) Prepare(table string) {
	bw.batch = gocql.NewBatch(gocql.LoggedBatch)
	bw.stmtCount = 0
	bw.stmt = fmt.Sprintf(
		`INSERT INTO %s.%s (path, time, stat) VALUES (?, ?, ?)`, bw.keyspace, table)
}

// Append
func (bw *batchWriter) Append(path string, ts time.Time, value float64) error {
	bw.batch.Query(bw.stmt, path, ts, value)
	bw.stmtCount++
	if bw.stmtCount >= bw.batchSize {
		return bw.Write()
	} else {
		return nil
	}
}

// Write
func (bw *batchWriter) Write() error {
	if bw.stmtCount > 0 {
		bwt := time.Now()
		bw.stmtCount = 0
		err := bw.dbClient.ExecuteBatch(bw.batch)
		if err != nil {
			// Retry with exponential backoff.
			config.G.Log.System.LogWarn("Retrying MetricManager write...")
			go bw.retryWrite(bw.batch)
			logging.Statsd.Client.Inc("metricmgr.db.retry", 1, 1.0)
		}
		logging.Statsd.Client.TimingDuration("metricmgr.db.write", time.Since(bwt), 1.0)
	}
	return nil
}

func (bw *batchWriter) retryWrite(batch *gocql.Batch) error {
	var i time.Duration
	i = 0
	var err error
	for i < 5 {
		err = bw.dbClient.ExecuteBatch(batch)
		if err == nil {
			return err
		}
		i++
		time.Sleep(i * time.Second)
	}
	// Failed 5x times, give up and log the error.
	logging.Statsd.Client.Inc("metricmgr.db.err.write", 1, 1.0)
	config.G.Log.System.LogError("Could not write batch to database: %v", err.Error())
	return err
}
<commit_msg>Remove unnecessary return value from goroutine<commit_after>package datastore

import (
	"fmt"
	"time"

	"github.com/gocql/gocql"

	"github.com/jeffpierce/cassabon/config"
	"github.com/jeffpierce/cassabon/logging"
)

type batchWriter struct {
	dbClient  *gocql.Session
	keyspace  string
	batchSize int

	batch     *gocql.Batch
	stmtCount int
	stmt      string
}

// Init
func (bw *batchWriter) Init(dbClient *gocql.Session, keyspace string, batchSize int) {
	bw.dbClient = dbClient
	bw.keyspace = keyspace
	bw.batchSize = batchSize
}

// Size
func (bw *batchWriter) Size() int {
	return bw.stmtCount
}

// Prepare
func (bw *batchWriter) Prepare(table string) {
	bw.batch = gocql.NewBatch(gocql.LoggedBatch)
	bw.stmtCount = 0
	bw.stmt = fmt.Sprintf(
		`INSERT INTO %s.%s (path, time, stat) VALUES (?, ?, ?)`, bw.keyspace, table)
}

// Append
func (bw *batchWriter) Append(path string, ts time.Time, value float64) error {
	bw.batch.Query(bw.stmt, path, ts, value)
	bw.stmtCount++
	if bw.stmtCount >= bw.batchSize {
		return bw.Write()
	} else {
		return nil
	}
}

// Write
func (bw *batchWriter) Write() error {
	if bw.stmtCount > 0 {
		bwt := time.Now()
		bw.stmtCount = 0
		err := bw.dbClient.ExecuteBatch(bw.batch)
		if err != nil {
			// Retry with exponential backoff.
			config.G.Log.System.LogWarn("Retrying MetricManager write...")
			go bw.retryWrite(bw.batch)
			logging.Statsd.Client.Inc("metricmgr.db.retry", 1, 1.0)
		}
		logging.Statsd.Client.TimingDuration("metricmgr.db.write", time.Since(bwt), 1.0)
	}
	return nil
}

// retryWrite
func (bw *batchWriter) retryWrite(batch *gocql.Batch) {
	var i time.Duration
	i = 0
	var err error
	for i < 5 {
		err = bw.dbClient.ExecuteBatch(batch)
		if err == nil {
			return
		}
		i++
		time.Sleep(i * time.Second)
	}
	// Failed 5x times, give up and log the error.
	logging.Statsd.Client.Inc("metricmgr.db.err.write", 1, 1.0)
	config.G.Log.System.LogError("Could not write batch to database: %v", err.Error())
}
<|endoftext|>


<commit_before>package swgohgg

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/cookiejar"
	"net/url"

	"github.com/PuerkitoBio/goquery"
)

// Client implements methods to interact with the https://swgoh.gg/ website.
type Client struct {
	hc         *http.Client
	jar        *cookiejar.Jar
	profile    string
	authorized bool
}

// NewClient initializes a new instance of the client, tied to the specified user profile.
func NewClient(profile string) *Client {
	jar, err := cookiejar.New(nil)
	if err != nil {
		// Should never happen.
		panic(err)
	}

	c := &Client{
		hc:      http.DefaultClient,
		profile: profile,
	}
	c.hc.Jar = jar
	return c
}

// Get retrieves the provided URL and returns a parsed goquery.Document.
func (c *Client) Get(url string) (*goquery.Document, error) {
	// Not in cache, fetch from remote site
	resp, err := c.hc.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode == 404 {
		return nil, fmt.Errorf("swgohgg: unable to find collection for profile '%s'", c.profile)
	}
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("swgohgg: unexpected status code %d", resp.StatusCode)
	}
	data, err := ioutil.ReadAll(resp.Body)
	return goquery.NewDocumentFromReader(bytes.NewBuffer(data))
}

// UseHTTPClient allows one to overwrite the default HTTP Client.
// The Client.Jar is replaced before next use.
func (c *Client) UseHTTPClient(hc *http.Client) *Client {
	c.hc = hc
	c.hc.Jar = c.jar
	return c
}

// Profile sets the client profile to a new value.
func (c *Client) Profile(profile string) *Client {
	c.profile = profile
	return c
}

// Login authorizes the bot client using the provided username and password.
func (c *Client) Login(username, password string) (err error) {
	resp, err := c.hc.Get("https://swgoh.gg/accounts/login/")
	if err != nil {
		return err
	}
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		return fmt.Errorf("swgoh.gg: unexpected status code %d: %v", resp.StatusCode, resp.Status)
	}
	loginPage, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return err
	}

	loginForm := make(url.Values)
	loginPage.Find("input").Each(func(i int, s *goquery.Selection) {
		loginForm[s.AttrOr("name", "")] = []string{s.AttrOr("value", "")}
	})
	loginForm["username"] = []string{username}
	loginForm["password"] = []string{password}
	resp, err = c.hc.PostForm("https://swgoh.gg/accounts/login/", loginForm)
	if err != nil {
		return err
	}
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		return fmt.Errorf("swgoh.gg: unexpected status code %d: %v", resp.StatusCode, resp.Status)
	}
	// Logged in!
	c.authorized = true
	return nil
}
<commit_msg>Fixed nil pointer when calling .ReuseClient.<commit_after>package swgohgg

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/cookiejar"
	"net/url"

	"github.com/PuerkitoBio/goquery"
)

// Client implements methods to interact with the https://swgoh.gg/ website.
type Client struct {
	hc         *http.Client
	profile    string
	authorized bool
}

// NewClient initializes a new instance of the client, tied to the specified user profile.
func NewClient(profile string) *Client {
	jar, err := cookiejar.New(nil)
	if err != nil {
		// Should never happen.
		panic(err)
	}

	c := &Client{
		hc:      http.DefaultClient,
		profile: profile,
	}
	c.hc.Jar = jar
	return c
}

// Get retrieves the provided URL and returns a parsed goquery.Document.
func (c *Client) Get(url string) (*goquery.Document, error) {
	// Not in cache, fetch from remote site
	resp, err := c.hc.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	if resp.StatusCode == 404 {
		return nil, fmt.Errorf("swgohgg: unable to find collection for profile '%s'", c.profile)
	}
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("swgohgg: unexpected status code %d", resp.StatusCode)
	}
	data, err := ioutil.ReadAll(resp.Body)
	return goquery.NewDocumentFromReader(bytes.NewBuffer(data))
}

// UseHTTPClient allows one to overwrite the default HTTP Client.
// The Client.Jar is replaced before next use.
func (c *Client) UseHTTPClient(hc *http.Client) *Client {
	hc.Jar = c.hc.Jar
	c.hc = hc
	return c
}

// Profile sets the client profile to a new value.
func (c *Client) Profile(profile string) *Client {
	c.profile = profile
	return c
}

// Login authorizes the bot client using the provided username and password.
func (c *Client) Login(username, password string) (err error) {
	resp, err := c.hc.Get("https://swgoh.gg/accounts/login/")
	if err != nil {
		return err
	}
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		return fmt.Errorf("swgoh.gg: unexpected status code %d: %v", resp.StatusCode, resp.Status)
	}
	loginPage, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		return err
	}

	loginForm := make(url.Values)
	loginPage.Find("input").Each(func(i int, s *goquery.Selection) {
		loginForm[s.AttrOr("name", "")] = []string{s.AttrOr("value", "")}
	})
	loginForm["username"] = []string{username}
	loginForm["password"] = []string{password}
	resp, err = c.hc.PostForm("https://swgoh.gg/accounts/login/", loginForm)
	if err != nil {
		return err
	}
	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		return fmt.Errorf("swgoh.gg: unexpected status code %d: %v", resp.StatusCode, resp.Status)
	}
	// Logged in!
	c.authorized = true
	return nil
}
<|endoftext|>


<commit_before>// sync_test.go
//
// Copyright (c) 2016, Ayke van Laethem
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package sync

import (
	"io/ioutil"
	"testing"

	"github.com/aykevl/dtsync/tree/memory"
)

type testCase struct {
	file     string
	action   Action
	contents []byte
}

func TestSync(t *testing.T) {
	fs1 := memory.NewRoot()
	fs2 := memory.NewRoot()

	result, err := Sync(fs1, fs2)
	if err != nil {
		t.Fatal("could not start sync:", err)
	}
	result.MarkFullySynced()
	err = result.SaveStatus()
	if err != nil {
		t.Error("could not save replica state:", err)
	}
	if fs1.Size() != 1 {
		t.Error("replica state wasn't saved for fs1")
	}
	if fs2.Size() != 1 {
		t.Error("replica state wasn't saved for fs2")
	}

	testCases := []testCase{
		{"file1.txt", ACTION_COPY, []byte("The quick brown fox...")},
		{"file1.txt", ACTION_UPDATE, []byte("The quick brown fox jumps over the lazy dog.")},
		{"file1.txt", ACTION_REMOVE, nil},
	}

	runTests(t, fs1, fs2, false, testCases)
	runTests(t, fs1, fs2, true, testCases)
	runTests(t, fs2, fs1, false, testCases)
	runTests(t, fs2, fs1, true, testCases)
}

func runTests(t *testing.T, fs1, fs2 *memory.Entry, swap bool, cases []testCase) {
	// The number of files currently in the filesystems.
	// It starts at 1, as there are status files.
	fileCount := int64(1)

	for _, tc := range cases {
		statusBefore := readStatuses(t, fs1, fs2)

		var err error
		switch tc.action {
		case ACTION_COPY: // add
			fileCount++
			_, err = fs1.AddRegular(tc.file, tc.contents)
		case ACTION_UPDATE:
			child := getFile(fs1, tc.file)
			if child == nil {
				t.Fatalf("could not find file %s to update", tc.file)
			}
			child.SetContents(tc.contents)
		case ACTION_REMOVE:
			fileCount--
			child := getFile(fs1, tc.file)
			if child == nil {
				t.Fatalf("could not find file %s to remove", tc.file)
			}
			err = fs1.Remove(child)
		default:
			t.Fatalf("unknown action: %d", tc.action)
		}
		if err != nil {
			t.Fatalf("could not %s file %s: %s", tc.action, tc.file, err)
		}

		var result *Result
		if swap {
			result, err = Sync(fs2, fs1)
		} else {
			result, err = Sync(fs1, fs2)
		}
		if err != nil {
			t.Fatalf("could not sync after: %s %s: %s", tc.action, tc.file, err)
		}

		if err := result.SyncAll(); err != nil {
			t.Errorf("could not sync all: %s", err)
		}
		result.MarkFullySynced()
		if err := result.SaveStatus(); err != nil {
			t.Errorf("could not save status: %s", err)
		}

		if !fsEqual(fs1, fs2) {
			t.Errorf("directory trees are not equal after: %s %s", tc.action, tc.file)
		}

		if fs1.Size() != fileCount || fs2.Size() != fileCount {
			t.Errorf("unexpected number of files after sync (expected %d): fs1=%d fs2=%d", fileCount, fs1.Size(), fs2.Size())
		}

		if t.Failed() {
			t.Logf("Action: %s %s", tc.action, tc.file)
			t.Logf("Status before, side 1\n%s", string(statusBefore[0]))
			t.Logf("Status before, side 2\n%s", string(statusBefore[1]))
			statusAfter := readStatuses(t, fs1, fs2)
			t.Logf("Status after, side 1\n%s", string(statusAfter[0]))
			t.Logf("Status after, side 2\n%s", string(statusAfter[1]))
		}
		if t.Failed() {
			t.FailNow()
		}
	}
}

func getFile(parent *memory.Entry, name string) *memory.Entry {
	list, err := parent.List()
	assert(err)
	for _, child := range list {
		if child.Name() == name {
			return child.(*memory.Entry)
		}
	}
	return nil
}

func fsEqual(fs1, fs2 *memory.Entry) bool {
	list1, err := fs1.List()
	assert(err)
	list2, err := fs2.List()
	assert(err)
	if len(list1) != len(list2) {
		return false
	}
	for i := 0; i < len(list1); i++ {
		file1 := list1[i].(*memory.Entry)
		file2 := list2[i].(*memory.Entry)
		if file1.Name() == STATUS_FILE && file2.Name() == STATUS_FILE {
			continue
		}
		if !file1.Equal(file2) {
			return false
		}
	}
	return true
}

// readStatuses returns the contents of the status files in the provided
// directories.
func readStatuses(t *testing.T, fs1, fs2 *memory.Entry) (statusData [2][]byte) {
	for i, fs := range []*memory.Entry{fs1, fs2} {
		statusFile, err := fs.GetFile(STATUS_FILE)
		if err != nil {
			t.Fatal("could not get status file:", err)
		}
		status, err := ioutil.ReadAll(statusFile)
		statusData[i] = status
		assert(err)
	}
	return
}

// Assert panicks when the error is non-nil.
// This is a convenience function for situations that should be impossible to
// happen.
func assert(err error) {
	if err != nil {
		panic("assert: " + err.Error())
	}
}

func TestLeastName(t *testing.T) {
	testCases := []struct{
		input  []string
		output string
	}{
		{[]string{""}, ""},
		{[]string{"a"}, "a"},
		{[]string{"a", "b"}, "a"},
		{[]string{"b", "a"}, "a"},
		{[]string{"a", ""}, "a"},
		{[]string{"", "a"}, "a"},
		{[]string{"a", "", "b"}, "a"},
		{[]string{"", "a", "b"}, "a"},
		{[]string{"", "b", "a"}, "a"},
		{[]string{"a", "", "b"}, "a"},
		{[]string{"aba", "abc"}, "aba"},
		{[]string{"a", "aba"}, "a"},
	}
	for _, tc := range testCases {
		name := leastName(tc.input)
		if name != tc.output {
			t.Errorf("expected %#v but got %#v for input %#v", tc.output, name, tc.input)
		}
	}
}
<commit_msg>Only try to use the result when scanning succeeded<commit_after>// sync_test.go
//
// Copyright (c) 2016, Ayke van Laethem
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
//    notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
//    notice, this list of conditions and the following disclaimer in the
//    documentation and/or other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
// TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
// PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package sync

import (
	"io/ioutil"
	"testing"

	"github.com/aykevl/dtsync/tree/memory"
)

type testCase struct {
	file     string
	action   Action
	contents []byte
}

func TestSync(t *testing.T) {
	fs1 := memory.NewRoot()
	fs2 := memory.NewRoot()

	result, err := Sync(fs1, fs2)
	if err != nil {
		t.Fatal("could not start sync:", err)
	}
	result.MarkFullySynced()
	err = result.SaveStatus()
	if err != nil {
		t.Error("could not save replica state:", err)
	}
	if fs1.Size() != 1 {
		t.Error("replica state wasn't saved for fs1")
	}
	if fs2.Size() != 1 {
		t.Error("replica state wasn't saved for fs2")
	}

	testCases := []testCase{
		{"file1.txt", ACTION_COPY, []byte("The quick brown fox...")},
		{"file1.txt", ACTION_UPDATE, []byte("The quick brown fox jumps over the lazy dog.")},
		{"file1.txt", ACTION_REMOVE, nil},
	}

	runTests(t, fs1, fs2, false, testCases)
	runTests(t, fs1, fs2, true, testCases)
	runTests(t, fs2, fs1, false, testCases)
	runTests(t, fs2, fs1, true, testCases)
}

func runTests(t *testing.T, fs1, fs2 *memory.Entry, swap bool, cases []testCase) {
	// The number of files currently in the filesystems.
	// It starts at 1, as there are status files.
	fileCount := int64(1)

	for _, tc := range cases {
		statusBefore := readStatuses(t, fs1, fs2)

		var err error
		switch tc.action {
		case ACTION_COPY: // add
			fileCount++
			_, err = fs1.AddRegular(tc.file, tc.contents)
		case ACTION_UPDATE:
			child := getFile(fs1, tc.file)
			if child == nil {
				t.Fatalf("could not find file %s to update", tc.file)
			}
			child.SetContents(tc.contents)
		case ACTION_REMOVE:
			fileCount--
			child := getFile(fs1, tc.file)
			if child == nil {
				t.Fatalf("could not find file %s to remove", tc.file)
			}
			err = fs1.Remove(child)
		default:
			t.Fatalf("unknown action: %d", tc.action)
		}
		if err != nil {
			t.Fatalf("could not %s file %s: %s", tc.action, tc.file, err)
		}

		var result *Result
		if swap {
			result, err = Sync(fs2, fs1)
		} else {
			result, err = Sync(fs1, fs2)
		}
		if err != nil {
			t.Errorf("could not sync after: %s %s: %s", tc.action, tc.file, err)
		} else {
			if err := result.SyncAll(); err != nil {
				t.Errorf("could not sync all: %s", err)
			}
			if !fsEqual(fs1, fs2) {
				t.Errorf("directory trees are not equal after: %s %s", tc.action, tc.file)
			}
			result.MarkFullySynced()
			if err := result.SaveStatus(); err != nil {
				t.Errorf("could not save status: %s", err)
			}

			if fs1.Size() != fileCount || fs2.Size() != fileCount {
				t.Errorf("unexpected number of files after first sync (expected %d): fs1=%d fs2=%d", fileCount, fs1.Size(), fs2.Size())
			}
		}

		if t.Failed() {
			t.Logf("Action: %s %s", tc.action, tc.file)
			t.Logf("Status before, side 1\n%s", string(statusBefore[0]))
			t.Logf("Status before, side 2\n%s", string(statusBefore[1]))
			statusAfter := readStatuses(t, fs1, fs2)
			t.Logf("Status after, side 1\n%s", string(statusAfter[0]))
			t.Logf("Status after, side 2\n%s", string(statusAfter[1]))
		}
		if t.Failed() {
			t.FailNow()
		}
	}
}

func getFile(parent *memory.Entry, name string) *memory.Entry {
	list, err := parent.List()
	assert(err)
	for _, child := range list {
		if child.Name() == name {
			return child.(*memory.Entry)
		}
	}
	return nil
}

func fsEqual(fs1, fs2 *memory.Entry) bool {
	list1, err := fs1.List()
	assert(err)
	list2, err := fs2.List()
	assert(err)
	if len(list1) != len(list2) {
		return false
	}
	for i := 0; i < len(list1); i++ {
		file1 := list1[i].(*memory.Entry)
		file2 := list2[i].(*memory.Entry)
		if file1.Name() == STATUS_FILE && file2.Name() == STATUS_FILE {
			continue
		}
		if !file1.Equal(file2) {
			return false
		}
	}
	return true
}

// readStatuses returns the contents of the status files in the provided
// directories.
func readStatuses(t *testing.T, fs1, fs2 *memory.Entry) (statusData [2][]byte) {
	for i, fs := range []*memory.Entry{fs1, fs2} {
		statusFile, err := fs.GetFile(STATUS_FILE)
		if err != nil {
			t.Fatal("could not get status file:", err)
		}
		status, err := ioutil.ReadAll(statusFile)
		statusData[i] = status
		assert(err)
	}
	return
}

// Assert panicks when the error is non-nil.
// This is a convenience function for situations that should be impossible to
// happen.
func assert(err error) {
	if err != nil {
		panic("assert: " + err.Error())
	}
}

func TestLeastName(t *testing.T) {
	testCases := []struct{
		input  []string
		output string
	}{
		{[]string{""}, ""},
		{[]string{"a"}, "a"},
		{[]string{"a", "b"}, "a"},
		{[]string{"b", "a"}, "a"},
		{[]string{"a", ""}, "a"},
		{[]string{"", "a"}, "a"},
		{[]string{"a", "", "b"}, "a"},
		{[]string{"", "a", "b"}, "a"},
		{[]string{"", "b", "a"}, "a"},
		{[]string{"a", "", "b"}, "a"},
		{[]string{"aba", "abc"}, "aba"},
		{[]string{"a", "aba"}, "a"},
	}
	for _, tc := range testCases {
		name := leastName(tc.input)
		if name != tc.output {
			t.Errorf("expected %#v but got %#v for input %#v", tc.output, name, tc.input)
		}
	}
}
<|endoftext|>