<commit_before>package agent

import (
	"errors"
	"fmt"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
	"github.com/aws/aws-sdk-go/aws/endpoints"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/buildkite/agent/logger"
)

type credentialsProvider struct {
	retrieved bool
}

func (e *credentialsProvider) Retrieve() (creds credentials.Value, err error) {
	e.retrieved = false

	creds.AccessKeyID = os.Getenv("BUILDKITE_S3_ACCESS_KEY_ID")
	if creds.AccessKeyID == "" {
		creds.AccessKeyID = os.Getenv("BUILDKITE_S3_ACCESS_KEY")
	}

	creds.SecretAccessKey = os.Getenv("BUILDKITE_S3_SECRET_ACCESS_KEY")
	if creds.SecretAccessKey == "" {
		creds.SecretAccessKey = os.Getenv("BUILDKITE_S3_SECRET_KEY")
	}

	if creds.AccessKeyID == "" {
		err = errors.New("BUILDKITE_S3_ACCESS_KEY_ID or BUILDKITE_S3_ACCESS_KEY not found in environment")
	}
	if creds.SecretAccessKey == "" {
		err = errors.New("BUILDKITE_S3_SECRET_ACCESS_KEY or BUILDKITE_S3_SECRET_KEY not found in environment")
	}

	e.retrieved = true
	return
}

func (e *credentialsProvider) IsExpired() bool {
	return !e.retrieved
}

func awsS3Credentials() *credentials.Credentials {
	return credentials.NewChainCredentials(
		[]credentials.Provider{
			&credentialsProvider{},
			&credentials.EnvProvider{},
			&ec2rolecreds.EC2RoleProvider{},
		})
}

func awsS3RegionFromEnv() (region string, err error) {
	regionName := "us-east-1"
	if os.Getenv("BUILDKITE_S3_DEFAULT_REGION") != "" {
		regionName = os.Getenv("BUILDKITE_S3_DEFAULT_REGION")
	} else {
		var err error
		regionName, err = awsRegion()
		if err != nil {
			return "", err
		}
	}

	// Check to make sure the region exists.
	resolver := endpoints.DefaultResolver()
	partitions := resolver.(endpoints.EnumPartitions).Partitions()

	for _, p := range partitions {
		for id := range p.Regions() {
			if id == regionName {
				return regionName, nil
			}
		}
	}

	return "", fmt.Errorf("Unknown AWS S3 Region %q", regionName)
}

func newS3Client(bucket string) (*s3.S3, error) {
	// Generate the AWS config used by the S3 client
	region, err := awsS3RegionFromEnv()
	if err != nil {
		return nil, err
	}

	sess, err := session.NewSession(&aws.Config{
		Credentials: awsS3Credentials(),
		Region:      aws.String(region),
	})
	if err != nil {
		return nil, err
	}

	logger.Debug("Authorizing S3 credentials and finding bucket `%s` in region `%s`...", bucket, region)

	s3client := s3.New(sess)

	// Test the authentication by trying to list the first 0 objects in the bucket.
	_, err = s3client.ListObjects(&s3.ListObjectsInput{
		Bucket:  aws.String(bucket),
		MaxKeys: aws.Int64(0),
	})
	if err != nil {
		if err == credentials.ErrNoValidProvidersFoundInChain {
			return nil, fmt.Errorf("Could not find a valid authentication strategy to connect to S3. Try setting BUILDKITE_S3_ACCESS_KEY and BUILDKITE_S3_SECRET_KEY")
		}
		return nil, fmt.Errorf("Failed to authenticate to bucket `%s` in region `%s` (%s)", bucket, region, err.Error())
	}

	return s3client, nil
}
<commit_msg>EC2RoleProvider causes segfaults without a Client<commit_after>package agent

import (
	"errors"
	"fmt"
	"os"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
	"github.com/aws/aws-sdk-go/aws/ec2metadata"
	"github.com/aws/aws-sdk-go/aws/endpoints"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/buildkite/agent/logger"
)

type credentialsProvider struct {
	retrieved bool
}

func (e *credentialsProvider) Retrieve() (creds credentials.Value, err error) {
	e.retrieved = false

	creds.AccessKeyID = os.Getenv("BUILDKITE_S3_ACCESS_KEY_ID")
	if creds.AccessKeyID == "" {
		creds.AccessKeyID = os.Getenv("BUILDKITE_S3_ACCESS_KEY")
	}

	creds.SecretAccessKey = os.Getenv("BUILDKITE_S3_SECRET_ACCESS_KEY")
	if creds.SecretAccessKey == "" {
		creds.SecretAccessKey = os.Getenv("BUILDKITE_S3_SECRET_KEY")
	}

	if creds.AccessKeyID == "" {
		err = errors.New("BUILDKITE_S3_ACCESS_KEY_ID or BUILDKITE_S3_ACCESS_KEY not found in environment")
	}
	if creds.SecretAccessKey == "" {
		err = errors.New("BUILDKITE_S3_SECRET_ACCESS_KEY or BUILDKITE_S3_SECRET_KEY not found in environment")
	}

	e.retrieved = true
	return
}

func (e *credentialsProvider) IsExpired() bool {
	return !e.retrieved
}

func awsS3RegionFromEnv() (region string, err error) {
	regionName := "us-east-1"
	if os.Getenv("BUILDKITE_S3_DEFAULT_REGION") != "" {
		regionName = os.Getenv("BUILDKITE_S3_DEFAULT_REGION")
	} else {
		var err error
		regionName, err = awsRegion()
		if err != nil {
			return "", err
		}
	}

	// Check to make sure the region exists.
	resolver := endpoints.DefaultResolver()
	partitions := resolver.(endpoints.EnumPartitions).Partitions()

	for _, p := range partitions {
		for id := range p.Regions() {
			if id == regionName {
				return regionName, nil
			}
		}
	}

	return "", fmt.Errorf("Unknown AWS S3 Region %q", regionName)
}

func awsS3Session(region string) (*session.Session, error) {
	// Chicken and egg... but this is kinda how they do it in the sdk
	sess, err := session.NewSession()
	if err != nil {
		return nil, err
	}

	sess.Config.Region = aws.String(region)

	sess.Config.Credentials = credentials.NewChainCredentials(
		[]credentials.Provider{
			&credentialsProvider{},
			&credentials.EnvProvider{},
			&ec2rolecreds.EC2RoleProvider{
				Client: ec2metadata.New(sess),
			},
		})

	return sess, nil
}

func newS3Client(bucket string) (*s3.S3, error) {
	region, err := awsS3RegionFromEnv()
	if err != nil {
		return nil, err
	}

	sess, err := awsS3Session(region)
	if err != nil {
		return nil, err
	}

	logger.Debug("Authorizing S3 credentials and finding bucket `%s` in region `%s`...", bucket, region)

	s3client := s3.New(sess)

	// Test the authentication by trying to list the first 0 objects in the bucket.
	_, err = s3client.ListObjects(&s3.ListObjectsInput{
		Bucket:  aws.String(bucket),
		MaxKeys: aws.Int64(0),
	})
	if err != nil {
		if err == credentials.ErrNoValidProvidersFoundInChain {
			return nil, fmt.Errorf("Could not find a valid authentication strategy to connect to S3. Try setting BUILDKITE_S3_ACCESS_KEY and BUILDKITE_S3_SECRET_KEY")
		}
		return nil, fmt.Errorf("Failed to authenticate to bucket `%s` in region `%s` (%s)", bucket, region, err.Error())
	}

	return s3client, nil
}
<|endoftext|>


<commit_before>package glog

import (
	"fmt"
	"net/http"

	"gopkg.in/airbrake/gobrake.v1"
)

var Gobrake *gobrake.Notifier

func notifyAirbrake(s severity, format string, args ...interface{}) {
	if Gobrake == nil {
		return
	}
	if s < errorLog {
		return
	}

	var msg string
	if format != "" {
		msg = fmt.Sprintf(format, args...)
	} else {
		msg = fmt.Sprint(args...)
	}

	var req *http.Request
	for _, arg := range args {
		if v, ok := arg.(requester); ok {
			req = v.Request()
			break
		}
	}

	foundErr := false
	for _, arg := range args {
		err, ok := arg.(error)
		if !ok {
			continue
		}
		foundErr = true

		notice := Gobrake.Notice(err, req, 5)
		notice.Env["glog_message"] = msg
		go Gobrake.SendNotice(notice)
	}

	if !foundErr {
		notice := Gobrake.Notice(msg, req, 5)
		go Gobrake.SendNotice(notice)
	}
}

type requester interface {
	Request() *http.Request
}
<commit_msg>Add GobrakeSeverity option.<commit_after>package glog

import (
	"fmt"
	"net/http"

	"gopkg.in/airbrake/gobrake.v1"
)

var Gobrake *gobrake.Notifier

// Minimum log severity that will be sent to Airbrake.
var GobrakeSeverity = errorLog

func notifyAirbrake(s severity, format string, args ...interface{}) {
	if Gobrake == nil {
		return
	}
	if s < GobrakeSeverity {
		return
	}

	var msg string
	if format != "" {
		msg = fmt.Sprintf(format, args...)
	} else {
		msg = fmt.Sprint(args...)
	}

	var req *http.Request
	for _, arg := range args {
		if v, ok := arg.(requester); ok {
			req = v.Request()
			break
		}
	}

	foundErr := false
	for _, arg := range args {
		err, ok := arg.(error)
		if !ok {
			continue
		}
		foundErr = true

		notice := Gobrake.Notice(err, req, 5)
		notice.Env["glog_message"] = msg
		go Gobrake.SendNotice(notice)
	}

	if !foundErr {
		notice := Gobrake.Notice(msg, req, 5)
		go Gobrake.SendNotice(notice)
	}
}

type requester interface {
	Request() *http.Request
}
<|endoftext|>


<commit_before>package fsutil

import (
	"path/filepath"
	"testing"

	"github.com/rjeczalik/tools/fs/memfs"
)

var trees = []memfs.FS{
	0: memfs.Must(memfs.TabTree([]byte(".\ndata\n\tgithub.com\n\t\tuser\n\t\t" +
		"\texample\n\t\t\t\t.git/\n\t\t\t\tdir\n\t\t\t\t\tdir.txt\n\t\t\t\tas" +
		"sets\n\t\t\t\t\tjs\n\t\t\t\t\t\tapp.js\n\t\t\t\t\t\tlink.js\n\t\t\t" +
		"\t\tcss\n\t\t\t\t\t\tdefault.css\nsrc\n\tgithub.com\n\t\tuser\n\t\t" +
		"\texample\n\t\t\t\t.git/\n\t\t\t\tdir\n\t\t\t\t\tdir.go\n\t\t\t\tex" +
		"ample.go"))),
	1: memfs.Must(memfs.TabTree([]byte(".\ndata\n\tgithub.com\n\t\tuser\n\t" +
		"\t\texample\n\t\t\t\tdir\n\t\t\t\t\tdir.dat\n\t\t\t\tfirst\n\t\t\t\t" +
		"\tcss\n\t\t\t\t\t\tfirst.css\n\t\t\t\t\tjs\n\t\t\t\t\t\tfirst.js\n\t" +
		"\t\t\tsecond\n\t\t\t\t\tcss\n\t\t\t\t\t\tsecond.css\n\t\t\t\t\tjs\n" +
		"\t\t\t\t\t\tsecond.js\nsrc\n\tgithub.com\n\t\tuser\n\t\t\texample\n" +
		"\t\t\t\tdir\n\t\t\t\t\tdir.go\n\t\t\t\texample.go"))),
	2: memfs.Must(memfs.TabTree([]byte(".\nschema\n\tlicstat\n\t\tschema\n\t" +
		"\t\tdatabasequery\n\t\t\t\treqaddaliasls.json\n\t\t\t\treqdeletef.j" +
		"son\n\t\t\t\treqdeletels.json\n\t\t\t\treqmergels.json\n\t\t\t\treq" +
		"querystatus.json\n\t\t\tdefinitions.json\n\t\t\tgeneralinfo\n\t\t\t" +
		"\treqinstallpath.json\n\t\t\tlicense\n\t\t\t\treqlicensedetail.json" +
		"\n\t\t\tmonitorconf\n\t\t\t\treqaddls.json\n\t\t\t\treqcheckls.json" +
		"\n\t\t\t\treqeditls.json\n\t\t\t\treqremovels.json\n\t\t\t\treqstat" +
		"usls.json\nsrc\n\tlicstat\n\t\tschema\n\t\t\tschema.go\n\t\t\ttmp/"))),
}

func equal(lhs, cas []string) bool {
	if len(lhs) != len(cas) {
		return false
	}
	for i := range cas {
		cas[i] = filepath.FromSlash(cas[i])
	}
LOOP:
	for i := range lhs {
		for j := range cas {
			if lhs[i] == cas[j] {
				continue LOOP
			}
		}
		return false
	}
	return true
}

func TestReadpaths(t *testing.T) {
	t.Skip("TODO(rjeczalik)")
}

func TestReaddirpaths(t *testing.T) {
	cases := map[string][]string{
		filepath.FromSlash("/data/github.com/user/example"): {
			"assets",
			"dir",
		},
		filepath.FromSlash("/src/github.com/user/example"): {
			"dir",
		},
	}
	c := Control{FS: trees[0]}
	for dir, cas := range cases {
		for _, b := range [...]bool{false, true} {
			if c.Hidden = b; b {
				cas = append(cas, ".git")
			}
			names := c.Readdirpaths(dir)
			if names == nil {
				t.Errorf("want names!=nil (dir=%q,hidden=%v)", dir, b)
				continue
			}
			if !equal(names, cas) {
				t.Errorf("want names=%v; got %v (dir=%q,hidden=%v)", cas, names, dir, b)
			}
		}
	}
}

func TestIntersect(t *testing.T) {
	cases := [...]struct {
		c    Control
		dirs []string
		src  string
		dst  string
	}{
		0: {
			Control{FS: trees[0]},
			[]string{
				"github.com/user/example",
				"github.com/user/example/dir",
			},
			"/src", "/data",
		},
		1: {
			Control{FS: trees[0], Hidden: true},
			[]string{
				"github.com/user/example",
				"github.com/user/example/dir",
				"github.com/user/example/.git",
			},
			"/src", "/data",
		},
		2: {
			Control{FS: trees[2]},
			[]string{
				"licstat/schema",
			},
			"/src", "/schema",
		},
	}
	for i, cas := range cases {
		dirs := cas.c.Intersect(
			filepath.FromSlash(cas.src),
			filepath.FromSlash(cas.dst),
		)
		if len(dirs) == 0 {
			t.Errorf("want len(dirs)!=0 (i=%d)", i)
			continue
		}
		if !equal(dirs, cas.dirs) {
			t.Errorf("want dirs=%v; got %v (i=%d)", cas.dirs, dirs, i)
		}
	}
}

func TestFind(t *testing.T) {
	t.Skip("TODO(rjeczalik)")
}
<commit_msg>fs/fsutil: Add a test-case for TestIntersect<commit_after>package fsutil

import (
	"path/filepath"
	"testing"

	"github.com/rjeczalik/tools/fs/memfs"
)

var trees = []memfs.FS{
	0: memfs.Must(memfs.TabTree([]byte(".\ndata\n\tgithub.com\n\t\tuser\n\t\t" +
		"\texample\n\t\t\t\t.git/\n\t\t\t\tdir\n\t\t\t\t\tdir.txt\n\t\t\t\tas" +
		"sets\n\t\t\t\t\tjs\n\t\t\t\t\t\tapp.js\n\t\t\t\t\t\tlink.js\n\t\t\t" +
		"\t\tcss\n\t\t\t\t\t\tdefault.css\nsrc\n\tgithub.com\n\t\tuser\n\t\t" +
		"\texample\n\t\t\t\t.git/\n\t\t\t\tdir\n\t\t\t\t\tdir.go\n\t\t\t\tex" +
		"ample.go"))),
	1: memfs.Must(memfs.TabTree([]byte(".\ndata\n\tgithub.com\n\t\tuser\n\t" +
		"\t\texample\n\t\t\t\tdir\n\t\t\t\t\tdir.dat\n\t\t\t\tfirst\n\t\t\t\t" +
		"\tcss\n\t\t\t\t\t\tfirst.css\n\t\t\t\t\tjs\n\t\t\t\t\t\tfirst.js\n\t" +
		"\t\t\tsecond\n\t\t\t\t\tcss\n\t\t\t\t\t\tsecond.css\n\t\t\t\t\tjs\n" +
		"\t\t\t\t\t\tsecond.js\nsrc\n\tgithub.com\n\t\tuser\n\t\t\texample\n" +
		"\t\t\t\tdir\n\t\t\t\t\tdir.go\n\t\t\t\texample.go"))),
	2: memfs.Must(memfs.TabTree([]byte(".\nschema\n\tlicstat\n\t\tschema\n\t" +
		"\t\tdatabasequery\n\t\t\t\treqaddaliasls.json\n\t\t\t\treqdeletef.j" +
		"son\n\t\t\t\treqdeletels.json\n\t\t\t\treqmergels.json\n\t\t\t\treq" +
		"querystatus.json\n\t\t\tdefinitions.json\n\t\t\tgeneralinfo\n\t\t\t" +
		"\treqinstallpath.json\n\t\t\tlicense\n\t\t\t\treqlicensedetail.json" +
		"\n\t\t\tmonitorconf\n\t\t\t\treqaddls.json\n\t\t\t\treqcheckls.json" +
		"\n\t\t\t\treqeditls.json\n\t\t\t\treqremovels.json\n\t\t\t\treqstat" +
		"usls.json\nsrc\n\tlicstat\n\t\tschema\n\t\t\tschema.go\n\t\t\ttmp/"))),
}

func equal(lhs, cas []string) bool {
	if len(lhs) != len(cas) {
		return false
	}
	for i := range cas {
		cas[i] = filepath.FromSlash(cas[i])
	}
LOOP:
	for i := range lhs {
		for j := range cas {
			if lhs[i] == cas[j] {
				continue LOOP
			}
		}
		return false
	}
	return true
}

func TestReadpaths(t *testing.T) {
	t.Skip("TODO(rjeczalik)")
}

func TestReaddirpaths(t *testing.T) {
	cases := map[string][]string{
		filepath.FromSlash("/data/github.com/user/example"): {
			"assets",
			"dir",
		},
		filepath.FromSlash("/src/github.com/user/example"): {
			"dir",
		},
	}
	c := Control{FS: trees[0]}
	for dir, cas := range cases {
		for _, b := range [...]bool{false, true} {
			if c.Hidden = b; b {
				cas = append(cas, ".git")
			}
			names := c.Readdirpaths(dir)
			if names == nil {
				t.Errorf("want names!=nil (dir=%q,hidden=%v)", dir, b)
				continue
			}
			if !equal(names, cas) {
				t.Errorf("want names=%v; got %v (dir=%q,hidden=%v)", cas, names, dir, b)
			}
		}
	}
}

func TestIntersect(t *testing.T) {
	cases := [...]struct {
		c    Control
		dirs []string
		src  string
		dst  string
	}{
		0: {
			Control{FS: trees[0]},
			[]string{
				"github.com/user/example",
				"github.com/user/example/dir",
			},
			"/src", "/data",
		},
		1: {
			Control{FS: trees[0], Hidden: true},
			[]string{
				"github.com/user/example",
				"github.com/user/example/dir",
				"github.com/user/example/.git",
			},
			"/src", "/data",
		},
		2: {
			Control{FS: trees[2]},
			[]string{
				"licstat/schema",
			},
			"/src", "/schema",
		},
		3: {
			Control{FS: trees[2], Hidden: true},
			[]string{
				"licstat/schema",
			},
			"/src", "/schema",
		},
		4: {
			Control{FS: trees[1]},
			[]string{
				"github.com/user/example",
				"github.com/user/example/dir",
			},
			"/src", "/data",
		},
		5: {
			Control{FS: trees[1], Hidden: true},
			[]string{
				"github.com/user/example",
				"github.com/user/example/dir",
			},
			"/src", "/data",
		},
	}
	for i, cas := range cases {
		dirs := cas.c.Intersect(
			filepath.FromSlash(cas.src),
			filepath.FromSlash(cas.dst),
		)
		if len(dirs) == 0 {
			t.Errorf("want len(dirs)!=0 (i=%d)", i)
			continue
		}
		if !equal(dirs, cas.dirs) {
			t.Errorf("want dirs=%v; got %v (i=%d)", cas.dirs, dirs, i)
		}
	}
}

func TestFind(t *testing.T) {
	t.Skip("TODO(rjeczalik)")
}
<|endoftext|>


<commit_before>package glog

import (
	"fmt"
	"net/http"

	"gopkg.in/airbrake/gobrake.v2"
)

// Gobrake is an instance of Airbrake Go Notifier that is used to send
// logs to Airbrake.
var Gobrake *gobrake.Notifier

// Minimum log severity that will be sent to Airbrake.
//
// Valid names are "INFO", "WARNING", "ERROR", and "FATAL".  If the name is not
// recognized, "ERROR" severity is used.
//
// TODO: replace with SetGobrakeSeverity
var GobrakeSeverity = "ERROR"

type requester interface {
	Request() *http.Request
}

func notifyAirbrake(depth int, s severity, format string, args ...interface{}) {
	if Gobrake == nil {
		return
	}

	severity, ok := severityByName(GobrakeSeverity)
	if !ok {
		severity = errorLog
	}
	if s < severity {
		return
	}

	var msg string
	if format != "" {
		msg = fmt.Sprintf(format, args...)
	} else {
		msg = fmt.Sprint(args...)
	}

	var req *http.Request
	for _, arg := range args {
		if v, ok := arg.(requester); ok {
			req = v.Request()
			break
		}
	}

	for _, arg := range args {
		err, ok := arg.(error)
		if !ok {
			continue
		}

		notice := Gobrake.Notice(err, req, depth)
		notice.Errors[0].Message = msg
		Gobrake.SendNoticeAsync(notice)
		return
	}

	notice := Gobrake.Notice(msg, req, depth)
	Gobrake.SendNoticeAsync(notice)
}
<commit_msg>Save severity in notice.Context[severity].<commit_after>package glog

import (
	"fmt"
	"net/http"

	"gopkg.in/airbrake/gobrake.v2"
)

// Gobrake is an instance of Airbrake Go Notifier that is used to send
// logs to Airbrake.
var Gobrake *gobrake.Notifier

// Minimum log severity that will be sent to Airbrake.
//
// Valid names are "INFO", "WARNING", "ERROR", and "FATAL".  If the name is not
// recognized, "ERROR" severity is used.
//
// TODO: replace with SetGobrakeSeverity
var GobrakeSeverity = "ERROR"

type requester interface {
	Request() *http.Request
}

func notifyAirbrake(depth int, s severity, format string, args ...interface{}) {
	if Gobrake == nil {
		return
	}

	severity, ok := severityByName(GobrakeSeverity)
	if !ok {
		severity = errorLog
	}
	if s < severity {
		return
	}

	var msg string
	if format != "" {
		msg = fmt.Sprintf(format, args...)
	} else {
		msg = fmt.Sprint(args...)
	}

	var req *http.Request
	for _, arg := range args {
		if v, ok := arg.(requester); ok {
			req = v.Request()
			break
		}
	}

	for _, arg := range args {
		err, ok := arg.(error)
		if !ok {
			continue
		}

		notice := Gobrake.Notice(err, req, depth)
		notice.Errors[0].Message = msg
		notice.Context["severity"] = severityName[s]
		Gobrake.SendNoticeAsync(notice)
		return
	}

	notice := Gobrake.Notice(msg, req, depth)
	notice.Context["severity"] = severityName[s]
	Gobrake.SendNoticeAsync(notice)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"runtime"
	"strings"
	"time"

	"github.com/abh/geodns/Godeps/_workspace/src/github.com/rcrowley/go-metrics"
	"github.com/abh/geodns/Godeps/_workspace/src/github.com/stathat/go"
)

func (zs *Zones) statHatPoster() {

	if len(Config.StatHat.ApiKey) == 0 {
		return
	}

	stathatGroups := append(serverGroups, "total", serverID)
	suffix := strings.Join(stathatGroups, ",")

	lastCounts := map[string]int64{}
	lastEdnsCounts := map[string]int64{}

	for name, zone := range *zs {
		if zone.Logging.StatHat == true {
			lastCounts[name] = zone.Metrics.Queries.Count()
			lastEdnsCounts[name] = zone.Metrics.EdnsQueries.Count()
		}
	}

	for {
		time.Sleep(60 * time.Second)

		for name, zone := range *zs {

			count := zone.Metrics.Queries.Count()
			newCount := count - lastCounts[name]
			lastCounts[name] = count

			if zone.Logging != nil && zone.Logging.StatHat == true {

				apiKey := zone.Logging.StatHatAPI
				if len(apiKey) == 0 {
					apiKey = Config.StatHat.ApiKey
				}
				if len(apiKey) == 0 {
					continue
				}
				stathat.PostEZCount("zone "+name+" queries~"+suffix, Config.StatHat.ApiKey, int(newCount))

				ednsCount := zone.Metrics.EdnsQueries.Count()
				newEdnsCount := ednsCount - lastEdnsCounts[name]
				lastEdnsCounts[name] = ednsCount
				stathat.PostEZCount("zone "+name+" edns queries~"+suffix, Config.StatHat.ApiKey, int(newEdnsCount))

			}
		}
	}
}

func statHatPoster() {

	qCounter := metrics.Get("queries").(metrics.Meter)
	lastQueryCount := qCounter.Count()
	stathatGroups := append(serverGroups, "total", serverID)
	suffix := strings.Join(stathatGroups, ",")
	// stathat.Verbose = true

	for {
		time.Sleep(60 * time.Second)

		if !Config.Flags.HasStatHat {
			log.Println("No stathat configuration")
			continue
		}

		log.Println("Posting to stathat")

		current := qCounter.Count()
		newQueries := current - lastQueryCount
		lastQueryCount = current

		stathat.PostEZCount("queries~"+suffix, Config.StatHat.ApiKey, int(newQueries))
		stathat.PostEZValue("goroutines "+serverID, Config.StatHat.ApiKey, float64(runtime.NumGoroutine()))

	}
}
<commit_msg>StatHat configuration can change at runtime<commit_after>package main

import (
	"log"
	"runtime"
	"strings"
	"time"

	"github.com/abh/geodns/Godeps/_workspace/src/github.com/rcrowley/go-metrics"
	"github.com/abh/geodns/Godeps/_workspace/src/github.com/stathat/go"
)

func (zs *Zones) statHatPoster() {

	if len(Config.StatHat.ApiKey) == 0 {
		return
	}

	stathatGroups := append(serverGroups, "total", serverID)
	suffix := strings.Join(stathatGroups, ",")

	lastCounts := map[string]int64{}
	lastEdnsCounts := map[string]int64{}

	for name, zone := range *zs {
		lastCounts[name] = zone.Metrics.Queries.Count()
		lastEdnsCounts[name] = zone.Metrics.EdnsQueries.Count()
	}

	for {
		time.Sleep(60 * time.Second)

		for name, zone := range *zs {

			count := zone.Metrics.Queries.Count()
			newCount := count - lastCounts[name]
			lastCounts[name] = count

			if zone.Logging != nil && zone.Logging.StatHat == true {

				apiKey := zone.Logging.StatHatAPI
				if len(apiKey) == 0 {
					apiKey = Config.StatHat.ApiKey
				}
				if len(apiKey) == 0 {
					continue
				}
				stathat.PostEZCount("zone "+name+" queries~"+suffix, Config.StatHat.ApiKey, int(newCount))

				ednsCount := zone.Metrics.EdnsQueries.Count()
				newEdnsCount := ednsCount - lastEdnsCounts[name]
				lastEdnsCounts[name] = ednsCount
				stathat.PostEZCount("zone "+name+" edns queries~"+suffix, Config.StatHat.ApiKey, int(newEdnsCount))

			}
		}
	}
}

func statHatPoster() {

	qCounter := metrics.Get("queries").(metrics.Meter)
	lastQueryCount := qCounter.Count()
	stathatGroups := append(serverGroups, "total", serverID)
	suffix := strings.Join(stathatGroups, ",")
	// stathat.Verbose = true

	for {
		time.Sleep(60 * time.Second)

		if !Config.Flags.HasStatHat {
			log.Println("No stathat configuration")
			continue
		}

		log.Println("Posting to stathat")

		current := qCounter.Count()
		newQueries := current - lastQueryCount
		lastQueryCount = current

		stathat.PostEZCount("queries~"+suffix, Config.StatHat.ApiKey, int(newQueries))
		stathat.PostEZValue("goroutines "+serverID, Config.StatHat.ApiKey, float64(runtime.NumGoroutine()))

	}
}
<|endoftext|>


<commit_before>package main

import (
        "log"
        "github.com/jmhodges/levigo"
        "os"
        "os/signal"
        "syscall"
)

var (
        dataStore string // the filepath kludge should store data in
        dbOpts *levigo.Options
        ldb *levigo.DB
)

func init() {
        dataStore = "data"
        dbOpts = levigo.NewOptions()
        dbOpts.SetCache(levigo.NewLRUCache(3<<30))
        dbOpts.SetCreateIfMissing(true)
}

func main() {
        var err error
        sigc := make(chan os.Signal, 1)

        log.Println("starting kludge")
        ldb, err = levigo.Open(dataStore, dbOpts)
        if err != nil {
                log.Fatal("Failed to start kludge backend: ", err.Error())
        }
        defer ldb.Close()

        signal.Notify(sigc, os.Kill, os.Interrupt, syscall.SIGTERM)
        <-sigc

        log.Println("kludge is shutting down")
}
<commit_msg>handle startup/shutdown of listener and worker pool.<commit_after>package main

import (
	"flag"
	"github.com/gokyle/goconfig"
	"github.com/jmhodges/levigo"
	"log"
	"os"
	"os/signal"
	"strconv"
	"syscall"
	"time"
)

var (
	dataStore  string // the filepath kludge should store data in
	dbOpts     *levigo.Options
	ldb        *levigo.DB
	listenAddr = ":5987"
	poolSize   = 4
	reqBuf     = 16
)

func init() {
	configFile := flag.String("f", "etc/kludge/backendrc",
		"path to configuration file")
	flag.Parse()

	var cfg map[string]string
	if cfgmap, err := goconfig.ParseFile(*configFile); err != nil {
		log.Fatal(err.Error())
	} else {
		cfg = cfgmap["default"]
	}

	dataStore = cfg["datastore"]
	if dataStore == "" {
		log.Fatal("no datastore specified")
	}

	if cfgAddr, ok := cfg["listen"]; ok {
		listenAddr = cfgAddr
	}

	if cfgReqBuf, ok := cfg["request_buffer"]; ok {
		var err error

		reqBuf, err = strconv.Atoi(cfgReqBuf)
		if err != nil {
			log.Printf("invalid value %s for request buffer: %s",
				cfgReqBuf, err.Error())
		}
	}

	if cfgPSize, ok := cfg["pool_size"]; ok {
		var err error

		poolSize, err = strconv.Atoi(cfgPSize)
		if err != nil {
			log.Printf("invalid value %s for pool size: %s",
				cfgPSize, err.Error())
		}
	}
	dbOpts = levigo.NewOptions()
	dbOpts.SetCache(levigo.NewLRUCache(3 << 30))
	dbOpts.SetCreateIfMissing(true)
}

func main() {
	var err error
	sigc := make(chan os.Signal, 1)

	log.Println("starting kludge")
	ldb, err = levigo.Open(dataStore, dbOpts)
	if err != nil {
		log.Fatal("Failed to start kludge backend: ", err.Error())
	}
	defer ldb.Close()

	go startPool()
	go listener()
	signal.Notify(sigc, os.Kill, os.Interrupt, syscall.SIGTERM)
	<-sigc

	// the worker pool is managed in pool.go.
	if reqQ != nil {
		close(reqQ)
		log.Println("giving workers time to complete")
		<-time.After(250 * time.Millisecond)
	}
	log.Println("kludge is shutting down")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/glendc/cgreader"
	"strings"
)

type Vector struct {
	x, y int
	icon string
}

func (v Vector) GetMapCoordinates() string {
	return fmt.Sprintf("%d;%d", v.x, v.y)
}

func (v Vector) GetMapIcon() string {
	return v.icon
}

type Ragnarok struct {
	thor, target, dimensions Vector
	energy                   int
	trail                    []Vector
}

func GetDirection(a, b string, x, y int) <-chan string {
	ch := make(chan string)
	go func() {
		difference := x - y
		switch {
		case difference < 0:
			ch <- a
		case difference > 0:
			ch <- b
		default:
			ch <- ""
		}
		close(ch)
	}()
	return ch
}

func (ragnarok *Ragnarok) ParseInitialData(ch <-chan string) {
	fmt.Sscanf(
		<-ch,
		"%d %d %d %d %d %d %d \n",
		&ragnarok.dimensions.x,
		&ragnarok.dimensions.y,
		&ragnarok.thor.x,
		&ragnarok.thor.y,
		&ragnarok.target.x,
		&ragnarok.target.y,
		&ragnarok.energy)

	ragnarok.thor.icon, ragnarok.target.icon = "H", "T"
}

func (ragnarok *Ragnarok) GetInput() (ch chan string) {
	ch = make(chan string)
	go func() {
		ch <- fmt.Sprintf("%d", ragnarok.energy)
	}()
	return
}

func (ragnarok *Ragnarok) Update(ch <-chan string) string {
	channel_b := GetDirection("N", "S", ragnarok.target.y, ragnarok.thor.y)
	channel_a := GetDirection("E", "W", ragnarok.thor.x, ragnarok.target.x)

	result_b := <-channel_b
	result_a := <-channel_a

	return fmt.Sprint(result_b + result_a)
}

func (ragnarok *Ragnarok) SetOutput(output string) string {
	ragnarok.trail = append(ragnarok.trail, Vector{ragnarok.thor.x, ragnarok.thor.y, "+"})

	if strings.Contains(output, "N") {
		ragnarok.thor.y -= 1
	} else if strings.Contains(output, "S") {
		ragnarok.thor.y += 1
	}

	if strings.Contains(output, "E") {
		ragnarok.thor.x += 1
	} else if strings.Contains(output, "W") {
		ragnarok.thor.x -= 1
	}

	ragnarok.energy -= 1

	trail := append(ragnarok.trail, ragnarok.thor, ragnarok.target)

	map_info := make([]cgreader.MapObject, len(trail))
	for i, v := range trail {
		map_info[i] = cgreader.MapObject(v)
	}

	cgreader.DrawMap(
		ragnarok.dimensions.x,
		ragnarok.dimensions.y,
		".",
		map_info...)

	return fmt.Sprintf(
		"Target = (%d,%d)\nThor = (%d,%d)\nEnergy = %d",
		ragnarok.target.x,
		ragnarok.target.y,
		ragnarok.thor.x,
		ragnarok.thor.y,
		ragnarok.energy)
}

func (ragnarok *Ragnarok) LoseConditionCheck() bool {
	if ragnarok.energy <= 0 {
		return true
	}

	x, y := ragnarok.thor.x, ragnarok.thor.y
	dx, dy := ragnarok.dimensions.x, ragnarok.dimensions.y

	if x < 0 || x >= dx || y < 0 || y >= dy {
		return true
	}

	return false
}

func (ragnarok *Ragnarok) WinConditionCheck() bool {
	return ragnarok.target.x == ragnarok.thor.x &&
		ragnarok.target.y == ragnarok.thor.y
}

func main() {
	cgreader.RunTargetProgram("../../input/ragnarok_3.txt", true, &Ragnarok{})
}
<commit_msg>Ragnarok is now correctly solved<commit_after>package main

import (
	"fmt"
	"github.com/glendc/cgreader"
	"strings"
)

type Vector struct {
	x, y int
	icon string
}

func (v Vector) GetMapCoordinates() string {
	return fmt.Sprintf("%d;%d", v.x, v.y)
}

func (v Vector) GetMapIcon() string {
	return v.icon
}

type Ragnarok struct {
	thor, target, dimensions Vector
	energy                   int
	trail                    []Vector
}

func GetDirection(x, y int) <-chan int {
	ch := make(chan int)
	go func() {
		difference := x - y
		switch {
		case difference < 0:
			ch <- -1
		case difference > 0:
			ch <- 1
		default:
			ch <- 0
		}
		close(ch)
	}()
	return ch
}

func GetDirectionLetter(a, b string, v int) string {
	switch v {
	default:
		return ""
	case -1:
		return a
	case 1:
		return b
	}
}

var TX, TY, PX, PY, E int

func (ragnarok *Ragnarok) ParseInitialData(ch <-chan string) {
	fmt.Sscanf(
		<-ch,
		"%d %d %d %d %d %d %d \n",
		&ragnarok.dimensions.x,
		&ragnarok.dimensions.y,
		&ragnarok.thor.x,
		&ragnarok.thor.y,
		&ragnarok.target.x,
		&ragnarok.target.y,
		&ragnarok.energy)

	TX, TY = ragnarok.thor.x, ragnarok.thor.y
	PX, PY = ragnarok.target.x, ragnarok.target.y
	E = ragnarok.energy

	ragnarok.thor.icon, ragnarok.target.icon = "H", "T"
}

func (ragnarok *Ragnarok) GetInput() (ch chan string) {
	ch = make(chan string)
	go func() {
		ch <- fmt.Sprintf("%d", ragnarok.energy)
	}()
	return
}

func (ragnarok *Ragnarok) Update(ch <-chan string) string {
	fmt.Sscanf(<-ch, "%d", &E)

	chx := GetDirection(PX, TX)
	chy := GetDirection(PY, TY)

	dx, dy := <-chx, <-chy
	x := GetDirectionLetter("W", "E", dx)
	y := GetDirectionLetter("N", "S", dy)

	TX, TY = TX+dx, TY+dy

	return y+x
}

func (ragnarok *Ragnarok) SetOutput(output string) string {
	ragnarok.trail = append(ragnarok.trail, Vector{ragnarok.thor.x, ragnarok.thor.y, "+"})

	if strings.Contains(output, "N") {
		ragnarok.thor.y -= 1
	} else if strings.Contains(output, "S") {
		ragnarok.thor.y += 1
	}

	if strings.Contains(output, "E") {
		ragnarok.thor.x += 1
	} else if strings.Contains(output, "W") {
		ragnarok.thor.x -= 1
	}

	ragnarok.energy -= 1

	trail := append(ragnarok.trail, ragnarok.thor, ragnarok.target)

	map_info := make([]cgreader.MapObject, len(trail))
	for i, v := range trail {
		map_info[i] = cgreader.MapObject(v)
	}

	cgreader.DrawMap(
		ragnarok.dimensions.x,
		ragnarok.dimensions.y,
		".",
		map_info...)

	return fmt.Sprintf(
		"Target = (%d,%d)\nThor = (%d,%d)\nEnergy = %d",
		ragnarok.target.x,
		ragnarok.target.y,
		ragnarok.thor.x,
		ragnarok.thor.y,
		ragnarok.energy)
}

func (ragnarok *Ragnarok) LoseConditionCheck() bool {
	if ragnarok.energy <= 0 {
		return true
	}

	x, y := ragnarok.thor.x, ragnarok.thor.y
	dx, dy := ragnarok.dimensions.x, ragnarok.dimensions.y

	if x < 0 || x >= dx || y < 0 || y >= dy {
		return true
	}

	return false
}

func (ragnarok *Ragnarok) WinConditionCheck() bool {
	return ragnarok.target.x == ragnarok.thor.x &&
		ragnarok.target.y == ragnarok.thor.y
}

func main() {
	cgreader.RunTargetProgram("../../input/ragnarok_3.txt", true, &Ragnarok{})
}
<|endoftext|>


<commit_before>package gostruct

import (
	"fmt"
	"reflect"
	"strconv"
	"time"

	"github.com/PuerkitoBio/goquery"
)

// some parts of this code are stolen^Winspired from
//   https://github.com/vrischmann/envconfig

func Fetch(target interface{}, url string) error {
	doc, err := goquery.NewDocument(url)
	if err != nil {
		return err
	}
	return Populate(target, doc)
}

func Populate(target interface{}, doc *goquery.Document) error {
	value := reflect.ValueOf(target)

	if value.Kind() != reflect.Ptr {
		return fmt.Errorf("value '%s' is not a pointer", target)
	}

	elem := value.Elem()

	switch elem.Kind() {
	case reflect.Ptr:
		elem.Set(reflect.New(elem.Type().Elem()))
		return populateStruct(elem.Elem(), doc.Selection)
	case reflect.Struct:
		return populateStruct(elem, doc.Selection)
	default:
		return fmt.Errorf("value '%s' must be a pointer on struct", target)
	}
}

func populateStruct(target reflect.Value, doc *goquery.Selection) (err error) {
	fieldsCount := target.NumField()
	targetType := target.Type()

	for i := 0; i < fieldsCount; i++ {
		field := target.Field(i)
		sel := targetType.Field(i).Tag.Get("gostruct")
		if sel == "" {
			continue
		}

		subdoc := doc.Find(sel)

	doPopulate:
		switch field.Kind() {
		case reflect.Ptr:
			field.Set(reflect.New(field.Type().Elem()))
			field = field.Elem()
			goto doPopulate
		default:
			err = setField(field, subdoc)
		}

		if err != nil {
			break
		}
	}

	return
}

var (
	durationType  = reflect.TypeOf(new(time.Duration)).Elem()
	byteSliceType = reflect.TypeOf([]byte(nil))
)

func isDurationField(t reflect.Type) bool {
	return t.AssignableTo(durationType)
}

func setField(field reflect.Value, doc *goquery.Selection) error {
	if !field.CanSet() {
		// unexported field: don't do anything
		return nil
	}

	ftype := field.Type()
	kind := ftype.Kind()

	// types which take the whole selection
	switch kind {
	case reflect.Struct:
		return populateStruct(field, doc)
	case reflect.Slice:
		if ftype == byteSliceType {
			return setByteSliceValue(field, doc)
		}
		return setSliceValue(field, doc)
	case reflect.String:
		return setStringValue(field, doc)
	case reflect.Bool:
		return setBoolValue(field, doc)
	}

	text := doc.First().Text()

	// types which take only the first element's text

	if isDurationField(ftype) {
		return setDurationValue(field, text)
	}

	switch kind {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return setIntValue(field, text)
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		return setUintValue(field, text)
	case reflect.Float32, reflect.Float64:
		return setFloatValue(field, text)
	default:
		return fmt.Errorf("Unsupported field type: '%v'", ftype)
	}
}

func setStringValue(field reflect.Value, sel *goquery.Selection) error {
	field.SetString(sel.Text())
	return nil
}

func setBoolValue(field reflect.Value, sel *goquery.Selection) error {
	// this one is tricky because there are multiple possible interpretations:
	// - set to true only if there are elements matching the selector
	// - set to true if the selection's text is not empty (this is what we're
	//   doing here)
	// - set to the resulting value of `strconf.ParseBool` called on the
	//   selection's text
	field.SetBool(sel.Text() != "")
	return nil
}

func setIntValue(field reflect.Value, s string) error {
	if s == "" {
		field.SetInt(0)
		return nil
	}

	val, err := strconv.ParseInt(s, 10, 64)
	if err == nil {
		field.SetInt(val)
	}

	return err
}

func setUintValue(field reflect.Value, s string) error {
	if s == "" {
		field.SetUint(0)
		return nil
	}

	val, err := strconv.ParseUint(s, 10, 64)
	if err == nil {
		field.SetUint(val)
	}

	return err
}

func setFloatValue(field reflect.Value, s string) error {
	if s == "" {
		field.SetFloat(0)
		return nil
	}

	val, err := strconv.ParseFloat(s, 64)
	if err == nil {
		field.SetFloat(val)
	}

	return err
}

func setDurationValue(field reflect.Value, s string) error {
	val, err := time.ParseDuration(s)
	if err == nil {
		field.SetInt(int64(val))
	}

	return err
}

// this one is like setStringValue except that we convert the string in a byte
// slice
func setByteSliceValue(field reflect.Value, sel *goquery.Selection) error {
	field.SetBytes([]byte(sel.Text()))
	return nil
}

func setSliceValue(field reflect.Value, sel *goquery.Selection) error {
	count := sel.Length()

	eltype := field.Type().Elem()
	capacity := field.Cap()

	if count > capacity {
		capacity = count
	}

	slice := reflect.MakeSlice(field.Type(), 0, capacity)

	var err error

	sel.EachWithBreak(func(i int, subSel *goquery.Selection) bool {
		el := reflect.New(eltype).Elem()

		if err = setField(el, subSel); err != nil {
			return false
		}

		slice = reflect.Append(slice, el)

		return true
	})

	if err != nil {
		return err
	}

	field.Set(slice)

	return nil
}
<commit_msg>skip fields with '-' tag<commit_after>package gostruct

import (
	"fmt"
	"reflect"
	"strconv"
	"time"

	"github.com/PuerkitoBio/goquery"
)

// some parts of this code are stolen^Winspired from
//   https://github.com/vrischmann/envconfig

func Fetch(target interface{}, url string) error {
	doc, err := goquery.NewDocument(url)
	if err != nil {
		return err
	}
	return Populate(target, doc)
}

func Populate(target interface{}, doc *goquery.Document) error {
	value := reflect.ValueOf(target)

	if value.Kind() != reflect.Ptr {
		return fmt.Errorf("value '%s' is not a pointer", target)
	}

	elem := value.Elem()

	switch elem.Kind() {
	case reflect.Ptr:
		elem.Set(reflect.New(elem.Type().Elem()))
		return populateStruct(elem.Elem(), doc.Selection)
	case reflect.Struct:
		return populateStruct(elem, doc.Selection)
	default:
		return fmt.Errorf("value '%s' must be a pointer on struct", target)
	}
}

func populateStruct(target reflect.Value, doc *goquery.Selection) (err error) {
	fieldsCount := target.NumField()
	targetType := target.Type()

	for i := 0; i < fieldsCount; i++ {
		field := target.Field(i)
		sel := targetType.Field(i).Tag.Get("gostruct")
		if sel == "" || sel == "-" {
			continue
		}

		subdoc := doc.Find(sel)

	doPopulate:
		switch field.Kind() {
		case reflect.Ptr:
			field.Set(reflect.New(field.Type().Elem()))
			field = field.Elem()
			goto doPopulate
		default:
			err = setField(field, subdoc)
		}

		if err != nil {
			break
		}
	}

	return
}

var (
	durationType  = reflect.TypeOf(new(time.Duration)).Elem()
	byteSliceType = reflect.TypeOf([]byte(nil))
)

func isDurationField(t reflect.Type) bool {
	return t.AssignableTo(durationType)
}

func setField(field reflect.Value, doc *goquery.Selection) error {
	if !field.CanSet() {
		// unexported field: don't do anything
		return nil
	}

	ftype := field.Type()
	kind := ftype.Kind()

	// types which take the whole selection
	switch kind {
	case reflect.Struct:
		return populateStruct(field, doc)
	case reflect.Slice:
		if ftype == byteSliceType {
			return setByteSliceValue(field, doc)
		}
		return setSliceValue(field, doc)
	case reflect.String:
		return setStringValue(field, doc)
	case reflect.Bool:
		return setBoolValue(field, doc)
	}

	text := doc.First().Text()

	// types which take only the first element's text

	if isDurationField(ftype) {
		return setDurationValue(field, text)
	}

	switch kind {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return setIntValue(field, text)
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		return setUintValue(field, text)
	case reflect.Float32, reflect.Float64:
		return setFloatValue(field, text)
	default:
		return fmt.Errorf("Unsupported field type: '%v'", ftype)
	}
}

func setStringValue(field reflect.Value, sel *goquery.Selection) error {
	field.SetString(sel.Text())
	return nil
}

func setBoolValue(field reflect.Value, sel *goquery.Selection) error {
	// this one is tricky because there are multiple possible interpretations:
	// - set to true only if there are elements matching the selector
	// - set to true if the selection's text is not empty (this is what we're
	//   doing here)
	// - set to the resulting value of `strconf.ParseBool` called on the
	//   selection's text
	field.SetBool(sel.Text() != "")
	return nil
}

func setIntValue(field reflect.Value, s string) error {
	if s == "" {
		field.SetInt(0)
		return nil
	}

	val, err := strconv.ParseInt(s, 10, 64)
	if err == nil {
		field.SetInt(val)
	}

	return err
}

func setUintValue(field reflect.Value, s string) error {
	if s == "" {
		field.SetUint(0)
		return nil
	}

	val, err := strconv.ParseUint(s, 10, 64)
	if err == nil {
		field.SetUint(val)
	}

	return err
}

func setFloatValue(field reflect.Value, s string) error {
	if s == "" {
		field.SetFloat(0)
		return nil
	}

	val, err := strconv.ParseFloat(s, 64)
	if err == nil {
		field.SetFloat(val)
	}

	return err
}

func setDurationValue(field reflect.Value, s string) error {
	val, err := time.ParseDuration(s)
	if err == nil {
		field.SetInt(int64(val))
	}

	return err
}

// this one is like setStringValue except that we convert the string in a byte
// slice
func setByteSliceValue(field reflect.Value, sel *goquery.Selection) error {
	field.SetBytes([]byte(sel.Text()))
	return nil
}

func setSliceValue(field reflect.Value, sel *goquery.Selection) error {
	count := sel.Length()

	eltype := field.Type().Elem()
	capacity := field.Cap()

	if count > capacity {
		capacity = count
	}

	slice := reflect.MakeSlice(field.Type(), 0, capacity)

	var err error

	sel.EachWithBreak(func(i int, subSel *goquery.Selection) bool {
		el := reflect.New(eltype).Elem()

		if err = setField(el, subSel); err != nil {
			return false
		}

		slice = reflect.Append(slice, el)

		return true
	})

	if err != nil {
		return err
	}

	field.Set(slice)

	return nil
}
<|endoftext|>


<commit_before>package indicators_test

import (
	. "github.com/onsi/ginkgo"
	"github.com/thetruetrade/gotrade"
	. "github.com/thetruetrade/gotrade/indicators"
)

var _ = Describe("when calculating a linear regression intercept (linearregintercept) with DOHLCV source data", func() {
	var (
		period    int = 3
		indicator *LinRegInt
		inputs    IndicatorWithFloatBoundsSharedSpecInputs
	)

	BeforeEach(func() {
		indicator, _ = NewLinRegInt(period, gotrade.UseClosePrice)

		inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
			func() float64 {
				return GetFloatDataMax(indicator.Data)
			},
			func() float64 {
				return GetFloatDataMin(indicator.Data)
			})
	})

	Context("and the indicator has not yet received any ticks", func() {
		ShouldBeAnInitialisedIndicator(&inputs)

		ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
	})

	Context("and the indicator has received less ticks than the lookback period", func() {

		BeforeEach(func() {
			for i := 0; i < indicator.GetLookbackPeriod(); i++ {
				indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
			}
		})

		ShouldBeAnIndicatorThatHasReceivedFewerTicksThanItsLookbackPeriod(&inputs)

		ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
	})

	Context("and the indicator has received ticks equal to the lookback period", func() {

		BeforeEach(func() {
			for i := 0; i <= indicator.GetLookbackPeriod(); i++ {
				indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
			}
		})

		ShouldBeAnIndicatorThatHasReceivedTicksEqualToItsLookbackPeriod(&inputs)

		ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
	})

	Context("and the indicator has received more ticks than the lookback period", func() {

		BeforeEach(func() {
			for i := range sourceDOHLCVData {
				indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
			}
		})

		ShouldBeAnIndicatorThatHasReceivedMoreTicksThanItsLookbackPeriod(&inputs)

		ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
	})

	Context("and the indicator has recieved all of its ticks", func() {
		BeforeEach(func() {
			for i := 0; i < len(sourceDOHLCVData); i++ {
				indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
			}
		})

		ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

		ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
	})
})
<commit_msg>#77 achieve 100% test coverage for indicators - linregint<commit_after>package indicators_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/thetruetrade/gotrade"
	"github.com/thetruetrade/gotrade/indicators"
)

var _ = Describe("when calculating a linear regression intercept (linearregintercept) with DOHLCV source data", func() {
	var (
		period    int = 3
		indicator *indicators.LinRegInt
		inputs    IndicatorWithFloatBoundsSharedSpecInputs
		stream    *fakeDOHLCVStreamSubscriber
	)

	Context("given the indicator is created via the standard constructor", func() {
		BeforeEach(func() {
			indicator, _ = indicators.NewLinRegInt(period, gotrade.UseClosePrice)

			inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
				func() float64 {
					return GetFloatDataMax(indicator.Data)
				},
				func() float64 {
					return GetFloatDataMin(indicator.Data)
				})
		})

		Context("and the indicator has not yet received any ticks", func() {
			ShouldBeAnInitialisedIndicator(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has received less ticks than the lookback period", func() {

			BeforeEach(func() {
				for i := 0; i < indicator.GetLookbackPeriod(); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedFewerTicksThanItsLookbackPeriod(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has received ticks equal to the lookback period", func() {

			BeforeEach(func() {
				for i := 0; i <= indicator.GetLookbackPeriod(); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedTicksEqualToItsLookbackPeriod(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
		})

		Context("and the indicator has received more ticks than the lookback period", func() {

			BeforeEach(func() {
				for i := range sourceDOHLCVData {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedMoreTicksThanItsLookbackPeriod(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
		})

		Context("and the indicator has recieved all of its ticks", func() {
			BeforeEach(func() {
				for i := 0; i < len(sourceDOHLCVData); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
		})

	})

	Context("given the indicator is created via the constructor with defaulted parameters", func() {
		BeforeEach(func() {
			indicator, _ = indicators.NewDefaultLinRegInt()
			inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
				func() float64 {
					return GetFloatDataMax(indicator.Data)
				},
				func() float64 {
					return GetFloatDataMin(indicator.Data)
				})
		})

		Context("and the indicator has not yet received any ticks", func() {
			ShouldBeAnInitialisedIndicator(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has recieved all of its ticks", func() {
			BeforeEach(func() {
				for i := 0; i < len(sourceDOHLCVData); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
		})
	})

	Context("given the indicator is created via the constructor with fixed source length", func() {
		BeforeEach(func() {
			indicator, _ = indicators.NewLinRegIntWithSrcLen(uint(len(sourceDOHLCVData)), 4, gotrade.UseClosePrice)
			inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
				func() float64 {
					return GetFloatDataMax(indicator.Data)
				},
				func() float64 {
					return GetFloatDataMin(indicator.Data)
				})
		})

		It("should have pre-allocated storge for the output data", func() {
			Expect(cap(indicator.Data)).To(Equal(len(sourceDOHLCVData) - indicator.GetLookbackPeriod()))
		})

		Context("and the indicator has not yet received any ticks", func() {
			ShouldBeAnInitialisedIndicator(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has recieved all of its ticks", func() {
			BeforeEach(func() {
				for i := 0; i < len(sourceDOHLCVData); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)

			It("no new storage capcity should have been allocated", func() {
				Expect(len(indicator.Data)).To(Equal(cap(indicator.Data)))
			})
		})
	})

	Context("given the indicator is created via the constructor with defaulted parameters and fixed source length", func() {
		BeforeEach(func() {
			indicator, _ = indicators.NewDefaultLinRegIntWithSrcLen(uint(len(sourceDOHLCVData)))
			inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
				func() float64 {
					return GetFloatDataMax(indicator.Data)
				},
				func() float64 {
					return GetFloatDataMin(indicator.Data)
				})
		})

		It("should have pre-allocated storge for the output data", func() {
			Expect(cap(indicator.Data)).To(Equal(len(sourceDOHLCVData) - indicator.GetLookbackPeriod()))
		})

		Context("and the indicator has not yet received any ticks", func() {
			ShouldBeAnInitialisedIndicator(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has recieved all of its ticks", func() {
			BeforeEach(func() {
				for i := 0; i < len(sourceDOHLCVData); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)

			It("no new storage capcity should have been allocated", func() {
				Expect(len(indicator.Data)).To(Equal(cap(indicator.Data)))
			})
		})
	})

	Context("given the indicator is created via the constructor for use with a price stream", func() {
		BeforeEach(func() {
			stream = newFakeDOHLCVStreamSubscriber()
			indicator, _ = indicators.NewLinRegIntForStream(stream, 4, gotrade.UseClosePrice)
			inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
				func() float64 {
					return GetFloatDataMax(indicator.Data)
				},
				func() float64 {
					return GetFloatDataMin(indicator.Data)
				})
		})

		It("should have requested to be attached to the stream", func() {
			Expect(stream.lastCallToAddTickSubscriptionArg).To(Equal(indicator))
		})

		Context("and the indicator has not yet received any ticks", func() {
			ShouldBeAnInitialisedIndicator(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has recieved all of its ticks", func() {
			BeforeEach(func() {
				for i := 0; i < len(sourceDOHLCVData); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
		})
	})

	Context("given the indicator is created via the constructor for use with a price stream with defaulted parameters", func() {
		BeforeEach(func() {
			stream = newFakeDOHLCVStreamSubscriber()
			indicator, _ = indicators.NewDefaultLinRegIntForStream(stream)
			inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
				func() float64 {
					return GetFloatDataMax(indicator.Data)
				},
				func() float64 {
					return GetFloatDataMin(indicator.Data)
				})
		})

		It("should have requested to be attached to the stream", func() {
			Expect(stream.lastCallToAddTickSubscriptionArg).To(Equal(indicator))
		})

		Context("and the indicator has not yet received any ticks", func() {
			ShouldBeAnInitialisedIndicator(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has recieved all of its ticks", func() {
			BeforeEach(func() {
				for i := 0; i < len(sourceDOHLCVData); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)
		})
	})

	Context("given the indicator is created via the constructor for use with a price stream with fixed source length", func() {
		BeforeEach(func() {
			stream = newFakeDOHLCVStreamSubscriber()
			indicator, _ = indicators.NewLinRegIntForStreamWithSrcLen(uint(len(sourceDOHLCVData)), stream, 4, gotrade.UseClosePrice)
			inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
				func() float64 {
					return GetFloatDataMax(indicator.Data)
				},
				func() float64 {
					return GetFloatDataMin(indicator.Data)
				})
		})

		It("should have pre-allocated storge for the output data", func() {
			Expect(cap(indicator.Data)).To(Equal(len(sourceDOHLCVData) - indicator.GetLookbackPeriod()))
		})

		It("should have requested to be attached to the stream", func() {
			Expect(stream.lastCallToAddTickSubscriptionArg).To(Equal(indicator))
		})

		Context("and the indicator has not yet received any ticks", func() {
			ShouldBeAnInitialisedIndicator(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has recieved all of its ticks", func() {
			BeforeEach(func() {
				for i := 0; i < len(sourceDOHLCVData); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)

			It("no new storage capcity should have been allocated", func() {
				Expect(len(indicator.Data)).To(Equal(cap(indicator.Data)))
			})
		})
	})

	Context("given the indicator is created via the constructor for use with a price stream with fixed source length with defaulted parmeters", func() {
		BeforeEach(func() {
			stream = newFakeDOHLCVStreamSubscriber()
			indicator, _ = indicators.NewDefaultLinRegIntForStreamWithSrcLen(uint(len(sourceDOHLCVData)), stream)
			inputs = NewIndicatorWithFloatBoundsSharedSpecInputs(indicator, len(sourceDOHLCVData), indicator,
				func() float64 {
					return GetFloatDataMax(indicator.Data)
				},
				func() float64 {
					return GetFloatDataMin(indicator.Data)
				})
		})

		It("should have pre-allocated storge for the output data", func() {
			Expect(cap(indicator.Data)).To(Equal(len(sourceDOHLCVData) - indicator.GetLookbackPeriod()))
		})

		It("should have requested to be attached to the stream", func() {
			Expect(stream.lastCallToAddTickSubscriptionArg).To(Equal(indicator))
		})

		Context("and the indicator has not yet received any ticks", func() {
			ShouldBeAnInitialisedIndicator(&inputs)

			ShouldNotHaveAnyFloatBoundsSetYet(&inputs)
		})

		Context("and the indicator has recieved all of its ticks", func() {
			BeforeEach(func() {
				for i := 0; i < len(sourceDOHLCVData); i++ {
					indicator.ReceiveDOHLCVTick(sourceDOHLCVData[i], i+1)
				}
			})

			ShouldBeAnIndicatorThatHasReceivedAllOfItsTicks(&inputs)

			ShouldHaveFloatBoundsSetToMinMaxOfResults(&inputs)

			It("no new storage capcity should have been allocated", func() {
				Expect(len(indicator.Data)).To(Equal(cap(indicator.Data)))
			})
		})
	})
})
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"flag"
	"log"
	"net/http"
	"net/smtp"
	"net/url"
	"runtime"
	"strconv"
	"text/template"
	"time"
)

type flags struct {
	url        url.URL
	caption    string
	from_Gmail string
	to_mail    string
	password   string
}

func main() {

	runtime.GOMAXPROCS(runtime.NumCPU()*2 + 1)

	log.Printf("Hawk is flying...")

	f, err := getFlags()
	if err != nil {
		log.Fatalf("flags parsing fail: %v", err)
	}

	for {

		go func(url url.URL) {

			c := make(chan error, 1)
			go looking(url, c)

			t := time.Now()

			err := error(nil)

			select {

			case <-time.After(1 * time.Second):
				err = errors.New("timeout")

			case err = <-c:
				log.Printf("looking: " + time.Since(t).String())
				close(c)
			}

			if err != nil {

				go sendGMail(f, err)
				log.Printf("result: %v", err)

				err, ok := <-c
				if ok {
					go sendGMail(f, err)
					log.Printf("result: %v", err)
				}
			}

		}(f.url)

		time.Sleep(time.Second * time.Duration(10))
	}
}

func getFlags() (flags, error) {

	// parse
	u := flag.String("u", "http://localhost:8080", "hawk url")
	c := flag.String("c", "cobra", "caption")
	f := flag.String("f", "sender@gmail.com", "gmail sender")
	t := flag.String("t", "receiver@example.com", "email receiver")
	p := flag.String("p", "123456", "gmail password")

	flag.Parse()

	// url
	ur, err := url.Parse(*u)
	if err != nil {
		return flags{}, err
	}

	// caption
	ca := *c

	// from_Gmail
	fr := *f

	// to_mail
	to := *t

	//password
	pw := *p

	return flags{*ur, ca, fr, to, pw}, nil
}

func looking(url url.URL, c chan error) {

	req, err := http.NewRequest("GET", url.String(), nil)
	if err != nil {
		c <- err
		return
	}

	req.Header.Set("Connection", "close")
	req.Header.Set("User-Agent", "HawkEye")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		c <- err
		return
	}

	if err := resp.Body.Close(); err != nil {
		c <- err
		return
	}

	if resp.StatusCode != 200 {
		c <- errors.New("http resp: " + strconv.Itoa(resp.StatusCode) + " " + resp.Status)
		return
	}

	c <- nil
}

func sendGMail(f flags, e error) {

	auth := smtp.PlainAuth(
		"",
		f.from_Gmail,
		f.password,
		"smtp.gmail.com",
	)

	type SmtpTemplateData struct {
		From    string
		To      string
		Subject string
		Body    string
	}

	const emailTemplate = `From: {{.From}}
To: {{.To}}
Subject: {{.Subject}}

{{.Body}}
`

	var err error
	var doc bytes.Buffer

	context := &SmtpTemplateData{
		f.from_Gmail,
		f.to_mail,
		f.caption + " " + time.Now().Format("01/02 15:04:05"),
		e.Error(),
	}

	t := template.New("emailTemplate")
	t, err = t.Parse(emailTemplate)
	if err != nil {
		log.Printf("error trying to parse mail template")
		return
	}
	err = t.Execute(&doc, context)
	if err != nil {
		log.Printf("error trying to execute mail template")
		return
	}

	err = smtp.SendMail(
		"smtp.gmail.com:587",
		auth,
		f.from_Gmail,
		[]string{f.to_mail},
		doc.Bytes(),
	)
	if err != nil {
		log.Printf("smtp.SendMail err: " + err.Error())
		return
	}
}
<commit_msg>Fix err == nil bug.<commit_after>package main

import (
	"bytes"
	"errors"
	"flag"
	"log"
	"net/http"
	"net/smtp"
	"net/url"
	"runtime"
	"strconv"
	"text/template"
	"time"
)

type flags struct {
	url        url.URL
	caption    string
	from_Gmail string
	to_mail    string
	password   string
}

func main() {

	runtime.GOMAXPROCS(runtime.NumCPU()*2 + 1)

	log.Printf("Hawk is flying...")

	f, err := getFlags()
	if err != nil {
		log.Fatalf("flags parsing fail: %v", err)
	}

	for {

		go func(url url.URL) {

			c := make(chan error, 1)
			go looking(url, c)

			t := time.Now()

			err := error(nil)

			select {

			case <-time.After(1 * time.Second):
				err = errors.New("timeout")

			case err = <-c:
				log.Printf("looking: " + time.Since(t).String())
				close(c)
			}

			if err != nil {

				go sendGMail(f, err)
				log.Printf("result: %v", err)

				err, ok := <-c
				if ok {

					if err == nil {
						err = errors.New("timeout without error")
					}

					go sendGMail(f, err)
					log.Printf("result: %v", err)
				}
			}

		}(f.url)

		time.Sleep(time.Second * time.Duration(10))
	}
}

func getFlags() (flags, error) {

	// parse
	u := flag.String("u", "http://localhost:8080", "hawk url")
	c := flag.String("c", "cobra", "caption")
	f := flag.String("f", "sender@gmail.com", "gmail sender")
	t := flag.String("t", "receiver@example.com", "email receiver")
	p := flag.String("p", "123456", "gmail password")

	flag.Parse()

	// url
	ur, err := url.Parse(*u)
	if err != nil {
		return flags{}, err
	}

	// caption
	ca := *c

	// from_Gmail
	fr := *f

	// to_mail
	to := *t

	//password
	pw := *p

	return flags{*ur, ca, fr, to, pw}, nil
}

func looking(url url.URL, c chan error) {

	req, err := http.NewRequest("GET", url.String(), nil)
	if err != nil {
		c <- err
		return
	}

	req.Header.Set("Connection", "close")
	req.Header.Set("User-Agent", "HawkEye")

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		c <- err
		return
	}

	if err := resp.Body.Close(); err != nil {
		c <- err
		return
	}

	if resp.StatusCode != 200 {
		c <- errors.New("http resp: " + strconv.Itoa(resp.StatusCode) + " " + resp.Status)
		return
	}

	c <- nil
}

func sendGMail(f flags, e error) {

	auth := smtp.PlainAuth(
		"",
		f.from_Gmail,
		f.password,
		"smtp.gmail.com",
	)

	type SmtpTemplateData struct {
		From    string
		To      string
		Subject string
		Body    string
	}

	const emailTemplate = `From: {{.From}}
To: {{.To}}
Subject: {{.Subject}}

{{.Body}}
`

	var err error
	var doc bytes.Buffer

	context := &SmtpTemplateData{
		f.from_Gmail,
		f.to_mail,
		f.caption + " " + time.Now().Format("01/02 15:04:05"),
		e.Error(),
	}

	t := template.New("emailTemplate")
	t, err = t.Parse(emailTemplate)
	if err != nil {
		log.Printf("error trying to parse mail template")
		return
	}
	err = t.Execute(&doc, context)
	if err != nil {
		log.Printf("error trying to execute mail template")
		return
	}

	err = smtp.SendMail(
		"smtp.gmail.com:587",
		auth,
		f.from_Gmail,
		[]string{f.to_mail},
		doc.Bytes(),
	)
	if err != nil {
		log.Printf("smtp.SendMail err: " + err.Error())
		return
	}
}
<|endoftext|>


<commit_before>package hdfs

import (
	"errors"
	//"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"sync"
)

func (h *WebHdfs) GetToLocal(path string, destination string, permission string) error {
	d, err := h.Get(path)
	if err != nil {
		return err
	}
	if permission == "" {
		permission = "755"
	}
	iperm, _ := strconv.Atoi(permission)
	err = ioutil.WriteFile(destination, d, os.FileMode(iperm))
	if err != nil {
		return err
	}
	return nil
}

func (h *WebHdfs) Get(path string) ([]byte, error) {
	r, err := h.call("GET", path, OP_OPEN, nil)
	if err != nil {
		return nil, err
	}
	if r.StatusCode != 307 {
		return nil, errors.New("Invalid Response Header on OP_OPEN: " + r.Status)
	}

	location := r.Header["Location"][0]
	r, err = h.call("GET", location, OP_OPEN, nil)
	if err != nil {
		return nil, err
	}
	if r.StatusCode != 200 {
		return nil, errors.New(r.Status)
	}
	d, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()
	return d, nil
}

func mergeMapString(source map[string]string, adds map[string]string) map[string]string {
	if source == nil {
		source = make(map[string]string)
	}
	if adds != nil {
		for k, v := range adds {
			source[k] = v
		}
	}
	return source
}

func (h *WebHdfs) Put(localfile string, destination string, permission string, parms map[string]string) error {
	if permission == "" {
		permission = "755"
	}
	parms = mergeMapString(parms, map[string]string{"permission": permission})
	r, err := h.call("PUT", destination, OP_CREATE, parms)
	if err != nil {
		return err
	}
	if r.StatusCode != 307 {
		return errors.New("Invalid Response Header on OP_CREATE: " + r.Status)
	}

	location := r.Header["Location"][0]

	r, err = h.callPayload("PUT", location, OP_CREATE, localfile, nil)
	if err != nil {
		return err
	}
	if r.StatusCode != 201 {
		return errors.New(r.Status)
	}
	return nil
}

func (h *WebHdfs) Puts(paths []string, destinationFolder string, permission string, parms map[string]string) map[string]error {
	var es map[string]error
	if permission == "" {
		permission = "755"
	}

	fileCount := len(paths)

	//parms = mergeMapString(parms, map[string]string{"permission": strconv.Itoa(permission)})
	ipool := 0
	iprocessing := 0
	iread := 0
	files := []string{}
	for _, path := range paths {
		ipool = ipool + 1
		iread = iread + 1
		files = append(files, path)
		if ipool == h.Config.PoolSize || iread == fileCount {
			wg := sync.WaitGroup{}
			wg.Add(ipool)

			for _, f := range files {
				go func(path string, swg *sync.WaitGroup) {
					defer swg.Done()
					iprocessing = iprocessing + 1
					_, filename := filepath.Split(path)
					newfilename := filepath.Join(destinationFolder, filename)
					e := h.Put(path, newfilename, permission, parms)
					//var e error
					if e != nil {
						if es == nil {
							es = make(map[string]error)
							es[path] = e
						}
						//fmt.Println(path, "=> ", newfilename, " ... FAIL => ", e.Error(), " | Processing ", iprocessing, " of ", fileCount)
					} else {
						//fmt.Println(path, "=> ", newfilename, " ... SUCCESS | Processing ", iprocessing, " of ", fileCount)
					}
				}(f, &wg)
			}

			wg.Wait()
			ipool = 0
			files = []string{}
		}
	}

	return es
}

func (h *WebHdfs) Append(localfile string, destination string) error {
	r, err := h.call("POST", destination, OP_APPEND, nil)
	if err != nil {
		return err
	}
	if r.StatusCode != 307 {
		return errors.New("Invalid Response Header on OP_APPEND: " + r.Status)
	}

	location := r.Header["Location"][0]

	r, err = h.callPayload("POST", location, OP_APPEND, localfile, nil)
	if err != nil {
		return err
	}
	if r.StatusCode != 201 {
		return errors.New(r.Status)
	}
	return nil
}

func (h *WebHdfs) SetOwner(path string, owner string, group string) error {
	ownerInfo := map[string]string{}
	if owner != "" {
		ownerInfo["owner"] = owner
	}
	if group != "" {
		ownerInfo["group"] = group
	}
	r, e := h.call("PUT", path, OP_SETOWNER, ownerInfo)
	if e != nil {
		return e
	}
	if r.StatusCode != 200 {
		return errors.New("Invalid Response Header on OP_SETOWNER: " + r.Status)
	}
	return nil
}

func (h *WebHdfs) SetPermission(path string, permission string) error {
	if permission == "" {
		permission = "755"
	}

	parms := map[string]string{}
	parms["permission"] = permission

	r, e := h.call("PUT", path, OP_SETPERMISSION, parms)
	if e != nil {
		return e
	}
	if r.StatusCode != 200 {
		return errors.New("Invalid Response Header on OP_SETPERMISSION: " + r.Status)
	}
	return nil
}

func (h *WebHdfs) CreateNewFile(path, filename, permission string) error {
	if permission == "" {
		permission = "755"
	}

	parms := map[string]string{}
	parms["permission"] = permission

	var fullpath string

	if string(path[len(path)-1]) == "/" {
		fullpath = path + filename
	} else {
		fullpath = path + "/" + filename
	}

	log.Println(fullpath)

	r, e := h.callPayload("PUT", path, OP_CREATE, filename, parms)
	if e != nil {
		return e
	}
	if r.StatusCode != 200 {
		return errors.New("Invalid Response Header on OP_CREATE: " + r.Status)
	}
	return nil
}
<commit_msg>add new file<commit_after>package hdfs

import (
	"errors"
	//"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"sync"
)

func (h *WebHdfs) GetToLocal(path string, destination string, permission string) error {
	d, err := h.Get(path)
	if err != nil {
		return err
	}
	if permission == "" {
		permission = "755"
	}
	iperm, _ := strconv.Atoi(permission)
	err = ioutil.WriteFile(destination, d, os.FileMode(iperm))
	if err != nil {
		return err
	}
	return nil
}

func (h *WebHdfs) Get(path string) ([]byte, error) {
	r, err := h.call("GET", path, OP_OPEN, nil)
	if err != nil {
		return nil, err
	}
	if r.StatusCode != 307 {
		return nil, errors.New("Invalid Response Header on OP_OPEN: " + r.Status)
	}

	location := r.Header["Location"][0]
	r, err = h.call("GET", location, OP_OPEN, nil)
	if err != nil {
		return nil, err
	}
	if r.StatusCode != 200 {
		return nil, errors.New(r.Status)
	}
	d, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()
	return d, nil
}

func mergeMapString(source map[string]string, adds map[string]string) map[string]string {
	if source == nil {
		source = make(map[string]string)
	}
	if adds != nil {
		for k, v := range adds {
			source[k] = v
		}
	}
	return source
}

func (h *WebHdfs) Put(localfile string, destination string, permission string, parms map[string]string) error {
	if permission == "" {
		permission = "755"
	}
	parms = mergeMapString(parms, map[string]string{"permission": permission})
	r, err := h.call("PUT", destination, OP_CREATE, parms)
	if err != nil {
		return err
	}
	if r.StatusCode != 307 {
		return errors.New("Invalid Response Header on OP_CREATE: " + r.Status)
	}

	location := r.Header["Location"][0]

	r, err = h.callPayload("PUT", location, OP_CREATE, localfile, nil)
	if err != nil {
		return err
	}
	if r.StatusCode != 201 {
		return errors.New(r.Status)
	}
	return nil
}

func (h *WebHdfs) Puts(paths []string, destinationFolder string, permission string, parms map[string]string) map[string]error {
	var es map[string]error
	if permission == "" {
		permission = "755"
	}

	fileCount := len(paths)

	//parms = mergeMapString(parms, map[string]string{"permission": strconv.Itoa(permission)})
	ipool := 0
	iprocessing := 0
	iread := 0
	files := []string{}
	for _, path := range paths {
		ipool = ipool + 1
		iread = iread + 1
		files = append(files, path)
		if ipool == h.Config.PoolSize || iread == fileCount {
			wg := sync.WaitGroup{}
			wg.Add(ipool)

			for _, f := range files {
				go func(path string, swg *sync.WaitGroup) {
					defer swg.Done()
					iprocessing = iprocessing + 1
					_, filename := filepath.Split(path)
					newfilename := filepath.Join(destinationFolder, filename)
					e := h.Put(path, newfilename, permission, parms)
					//var e error
					if e != nil {
						if es == nil {
							es = make(map[string]error)
							es[path] = e
						}
						//fmt.Println(path, "=> ", newfilename, " ... FAIL => ", e.Error(), " | Processing ", iprocessing, " of ", fileCount)
					} else {
						//fmt.Println(path, "=> ", newfilename, " ... SUCCESS | Processing ", iprocessing, " of ", fileCount)
					}
				}(f, &wg)
			}

			wg.Wait()
			ipool = 0
			files = []string{}
		}
	}

	return es
}

func (h *WebHdfs) Append(localfile string, destination string) error {
	r, err := h.call("POST", destination, OP_APPEND, nil)
	if err != nil {
		return err
	}
	if r.StatusCode != 307 {
		return errors.New("Invalid Response Header on OP_APPEND: " + r.Status)
	}

	location := r.Header["Location"][0]

	r, err = h.callPayload("POST", location, OP_APPEND, localfile, nil)
	if err != nil {
		return err
	}
	if r.StatusCode != 201 {
		return errors.New(r.Status)
	}
	return nil
}

func (h *WebHdfs) SetOwner(path string, owner string, group string) error {
	ownerInfo := map[string]string{}
	if owner != "" {
		ownerInfo["owner"] = owner
	}
	if group != "" {
		ownerInfo["group"] = group
	}
	r, e := h.call("PUT", path, OP_SETOWNER, ownerInfo)
	if e != nil {
		return e
	}
	if r.StatusCode != 200 {
		return errors.New("Invalid Response Header on OP_SETOWNER: " + r.Status)
	}
	return nil
}

func (h *WebHdfs) SetPermission(path string, permission string) error {
	if permission == "" {
		permission = "755"
	}

	parms := map[string]string{}
	parms["permission"] = permission

	r, e := h.call("PUT", path, OP_SETPERMISSION, parms)
	if e != nil {
		return e
	}
	if r.StatusCode != 200 {
		return errors.New("Invalid Response Header on OP_SETPERMISSION: " + r.Status)
	}
	return nil
}

func (h *WebHdfs) CreateNewFile(path, filename, permission string) error {
	if permission == "" {
		permission = "755"
	}

	parms := map[string]string{}
	parms["permission"] = permission

	var fullpath string

	if string(path[len(path)-1]) == "/" {
		fullpath = path + filename
	} else {
		fullpath = path + "/" + filename
	}

	log.Println(fullpath)

	r, e := h.call("POST", fullpath, OP_CREATE, parms)
	if e != nil {
		return e
	}
	if r.StatusCode != 200 {
		return errors.New("Invalid Response Header on OP_CREATE: " + r.Status)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/dustin/gomemcached"
	"github.com/dustin/gomemcached/client"
)

var heartFreq = flag.Duration("heartbeat", 10*time.Second,
	"Heartbeat frequency")
var reconcileFreq = flag.Duration("reconcile", 24*time.Hour,
	"Reconciliation frequency")

type AboutNode struct {
	Addr     string    `json:"addr"`
	Type     string    `json:"type"`
	Time     time.Time `json:"time"`
	BindAddr string    `json:"bindaddr"`
	Hash     string    `json:"hash"`
}

type PeriodicJob struct {
	period time.Duration
	f      func() error
}

var periodicJobs = map[string]PeriodicJob{
	"checkStaleNodes": PeriodicJob{
		time.Minute * 5,
		checkStaleNodes,
	},
}

func getNodeAddress(sid string) (string, error) {
	sidkey := "/" + sid
	aboutSid := AboutNode{}
	err := couchbase.Get(sidkey, &aboutSid)
	if err != nil {
		return "", err
	}
	if strings.HasPrefix(aboutSid.BindAddr, ":") {
		return aboutSid.Addr + aboutSid.BindAddr, nil
	}
	return aboutSid.BindAddr, nil
}

type JobMarker struct {
	Node    string    `json:"node"`
	Started time.Time `json:"started"`
	Type    string    `json:"type"`
}

// Run a named task if we know one hasn't in the last t seconds.
func runNamedGlobalTask(name string, t time.Duration, f func() error) bool {
	key := "/@" + name

	jm := JobMarker{
		Node:    serverId,
		Started: time.Now(),
		Type:    "job",
	}

	err := couchbase.Do(key, func(mc *memcached.Client, vb uint16) error {
		data, err := json.Marshal(&jm)
		if err != nil {
			log.Fatalf("Can't jsonify a JobMarker: %v", err)
		}
		resp, err := mc.Add(vb, key, 0, int(t.Seconds()), data)
		if err != nil {
			return err
		}
		if resp.Status != gomemcached.SUCCESS {
			return fmt.Errorf("Wanted success, got %v", resp.Status)
		}
		return nil
	})

	if err == nil {
		err = f()
		if err != nil {
			log.Printf("Error running periodic task %#v: %v", name, err)
		}
	}

	return true
}

func heartbeat() {
	for {
		u, err := url.Parse(*couchbaseServer)
		c, err := net.Dial("tcp", u.Host)
		localAddr := ""
		if err == nil {
			localAddr = strings.Split(c.LocalAddr().String(), ":")[0]
			c.Close()
		}

		aboutMe := AboutNode{
			Addr:     localAddr,
			Type:     "storage",
			Time:     time.Now().UTC(),
			BindAddr: *bindAddr,
			Hash:     *hashType,
		}

		err = couchbase.Set("/"+serverId, aboutMe)
		if err != nil {
			log.Printf("Failed to record a heartbeat: %v", err)
		}
		time.Sleep(*heartFreq)
	}
}

func reconcile() error {
	explen := getHash().Size() * 2
	return filepath.Walk(*root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() && !strings.HasPrefix(info.Name(), "tmp") &&
			len(info.Name()) == explen {
			// I can do way more efficient stuff than this.
			recordBlobOwnership(info.Name(), info.Size())
			return err
		}
		return nil
	})
}

func reconcileLoop() {
	if *reconcileFreq == 0 {
		return
	}
	for {
		err := reconcile()
		if err != nil {
			log.Printf("Error in reconciliation loop: %v", err)
		}
		time.Sleep(*reconcileFreq)
	}
}

func checkStaleNodes() error {
	// TODO:  Make this not lie.
	log.Printf("Checking stale nodes")
	return nil
}

func runPeriodicJob(name string, job PeriodicJob) {
	for {
		if runNamedGlobalTask(name, job.period, job.f) {
			log.Printf("Attempted job %v", name)
		} else {
			log.Printf("Didn't run job %v", name)
		}
		time.Sleep(job.period + time.Second)
	}
}

func runPeriodicJobs() {
	for n, j := range periodicJobs {
		go runPeriodicJob(n, j)
	}
}
<commit_msg>Dead node cleaner.<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/dustin/gomemcached"
	"github.com/dustin/gomemcached/client"
)

var heartFreq = flag.Duration("heartbeat", 10*time.Second,
	"Heartbeat frequency")
var reconcileFreq = flag.Duration("reconcile", 24*time.Hour,
	"Reconciliation frequency")

type AboutNode struct {
	Addr     string    `json:"addr"`
	Type     string    `json:"type"`
	Time     time.Time `json:"time"`
	BindAddr string    `json:"bindaddr"`
	Hash     string    `json:"hash"`
}

type PeriodicJob struct {
	period time.Duration
	f      func() error
}

var periodicJobs = map[string]PeriodicJob{
	"checkStaleNodes": PeriodicJob{
		time.Minute * 5,
		checkStaleNodes,
	},
}

func getNodeAddress(sid string) (string, error) {
	sidkey := "/" + sid
	aboutSid := AboutNode{}
	err := couchbase.Get(sidkey, &aboutSid)
	if err != nil {
		return "", err
	}
	if strings.HasPrefix(aboutSid.BindAddr, ":") {
		return aboutSid.Addr + aboutSid.BindAddr, nil
	}
	return aboutSid.BindAddr, nil
}

type JobMarker struct {
	Node    string    `json:"node"`
	Started time.Time `json:"started"`
	Type    string    `json:"type"`
}

// Run a named task if we know one hasn't in the last t seconds.
func runNamedGlobalTask(name string, t time.Duration, f func() error) bool {
	key := "/@" + name

	jm := JobMarker{
		Node:    serverId,
		Started: time.Now(),
		Type:    "job",
	}

	err := couchbase.Do(key, func(mc *memcached.Client, vb uint16) error {
		data, err := json.Marshal(&jm)
		if err != nil {
			log.Fatalf("Can't jsonify a JobMarker: %v", err)
		}
		resp, err := mc.Add(vb, key, 0, int(t.Seconds()), data)
		if err != nil {
			return err
		}
		if resp.Status != gomemcached.SUCCESS {
			return fmt.Errorf("Wanted success, got %v", resp.Status)
		}
		return nil
	})

	if err == nil {
		err = f()
		if err != nil {
			log.Printf("Error running periodic task %#v: %v", name, err)
		}
	}

	return true
}

func heartbeat() {
	for {
		u, err := url.Parse(*couchbaseServer)
		c, err := net.Dial("tcp", u.Host)
		localAddr := ""
		if err == nil {
			localAddr = strings.Split(c.LocalAddr().String(), ":")[0]
			c.Close()
		}

		aboutMe := AboutNode{
			Addr:     localAddr,
			Type:     "storage",
			Time:     time.Now().UTC(),
			BindAddr: *bindAddr,
			Hash:     *hashType,
		}

		err = couchbase.Set("/"+serverId, aboutMe)
		if err != nil {
			log.Printf("Failed to record a heartbeat: %v", err)
		}
		time.Sleep(*heartFreq)
	}
}

func reconcile() error {
	explen := getHash().Size() * 2
	return filepath.Walk(*root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() && !strings.HasPrefix(info.Name(), "tmp") &&
			len(info.Name()) == explen {
			// I can do way more efficient stuff than this.
			recordBlobOwnership(info.Name(), info.Size())
			return err
		}
		return nil
	})
}

func reconcileLoop() {
	if *reconcileFreq == 0 {
		return
	}
	for {
		err := reconcile()
		if err != nil {
			log.Printf("Error in reconciliation loop: %v", err)
		}
		time.Sleep(*reconcileFreq)
	}
}

func removeBlobOwnershipRecord(h, node string) {
	log.Printf("Cleaning up %v from %v", h, node)

	k := "/" + h
	err := couchbase.Do(k, func(mc *memcached.Client, vb uint16) error {
		_, err := mc.CAS(vb, k, func(in []byte) []byte {
			ownership := BlobOwnership{}
			err := json.Unmarshal(in, &ownership)
			if err == nil {
				delete(ownership.Nodes, node)
			} else {
				return nil
			}

			rv, err := json.Marshal(&ownership)
			if err != nil {
				log.Fatalf("Error marshaling blob ownership: %v", err)
			}
			return rv
		}, 0)
		return err
	})
	if err != nil {
		log.Printf("Error cleaning %v from %v", node, h)
	}
}

func cleanupNode(node string) {
	log.Printf("Cleaning up node %v", node)
	vres, err := couchbase.View("cbfs", "node_blobs",
		map[string]interface{}{
			"key":    `"` + node + `"`,
			"limit":  1000,
			"reduce": false,
			"stale":  false,
		})
	if err != nil {
		log.Printf("Error executing node_blobs view: %v", err)
		return
	}
	foundRows := 0
	for _, r := range vres.Rows {
		removeBlobOwnershipRecord(r.ID[1:], node)
		foundRows++
	}
	if foundRows == 0 {
		log.Printf("Removing node record: %v", node)
		err = couchbase.Delete("/" + node)
		if err != nil {
			log.Printf("Error deleting %v node record: %v", node, err)
		}
	}
}

func checkStaleNodes() error {
	// TODO:  Make this not lie.
	log.Printf("Checking stale nodes")
	vres, err := couchbase.View("cbfs", "nodes", map[string]interface{}{
		"stale": false})
	if err != nil {
		return err
	}
	for _, r := range vres.Rows {
		ks, ok := r.Key.(string)
		if !ok {
			log.Printf("Wrong key type returned from view: %#v", r)
			continue
		}
		t, err := time.Parse(time.RFC3339Nano, ks)
		if err != nil {
			log.Printf("Error parsing time from %v", r)
			continue
		}
		d := time.Since(t)

		if d > *heartFreq*10 {
			node := r.ID[1:]
			if node == serverId {
				log.Printf("Would've cleaned up myself after %v",
					d)
				continue
			}
			log.Printf("  Node %v missed heartbeat schedule: %v", node, d)
			go cleanupNode(node)
		}
	}
	return nil
}

func runPeriodicJob(name string, job PeriodicJob) {
	for {
		if runNamedGlobalTask(name, job.period, job.f) {
			log.Printf("Attempted job %v", name)
		} else {
			log.Printf("Didn't run job %v", name)
		}
		time.Sleep(job.period + time.Second)
	}
}

func runPeriodicJobs() {
	for n, j := range periodicJobs {
		go runPeriodicJob(n, j)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lucicfg

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"regexp"
	"strings"
	"testing"

	"github.com/sergi/go-diff/diffmatchpatch"

	"go.starlark.net/resolve"
	"go.starlark.net/starlark"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/starlark/builtins"
	"go.chromium.org/luci/starlark/interpreter"
	"go.chromium.org/luci/starlark/starlarktest"
)

func init() {
	// Enable not-yet-standard features.
	resolve.AllowLambda = true
	resolve.AllowNestedDef = true
	resolve.AllowFloat = true
	resolve.AllowSet = true
}

// TestAllStarlark loads and executes all test scripts (testdata/*.star).
func TestAllStarlark(t *testing.T) {
	t.Parallel()

	starlarktest.RunTests(t, starlarktest.Options{
		TestsDir: "testdata",
		Skip:     "support",

		Executor: func(t *testing.T, path string, predeclared starlark.StringDict) error {
			blob, err := ioutil.ReadFile(path)
			if err != nil {
				return err
			}
			body := string(blob)

			// Read "mocked" `-var name=value` assignments.
			presetVars := map[string]string{}
			presetVarsBlock := readCommentBlock(body, "Prepare CLI vars as:")
			for _, line := range strings.Split(presetVarsBlock, "\n") {
				if line = strings.TrimSpace(line); line != "" {
					chunks := strings.SplitN(line, "=", 2)
					if len(chunks) != 2 {
						t.Errorf("Bad CLI var declaration %q", line)
						return nil
					}
					presetVars[chunks[0]] = chunks[1]
				}
			}

			expectErrExct := readCommentBlock(body, "Expect errors:")
			expectErrLike := readCommentBlock(body, "Expect errors like:")
			expectCfg := readCommentBlock(body, "Expect configs:")
			if expectErrExct != "" && expectErrLike != "" {
				t.Errorf("Cannot use 'Expect errors' and 'Expect errors like' at the same time")
				return nil
			}

			// We treat tests that compare the generator output to some expected
			// output as "integration tests", and everything else is a unit tests.
			// See below for why this is important.
			integrationTest := expectErrExct != "" || expectErrLike != "" || expectCfg != ""

			state, err := Generate(context.Background(), Inputs{
				// Use file system loader so test scripts can load supporting scripts
				// (from '**/support/*' which is skipped by the test runner). This also
				// makes error messages have the original scripts full name. Note that
				// 'go test' executes tests with cwd set to corresponding package
				// directories, regardless of what cwd was when 'go test' was called.
				Code:  interpreter.FileSystemLoader("."),
				Entry: filepath.ToSlash(path),
				Vars:  presetVars,

				// Expose 'assert' module, hook up error reporting to 't'.
				testPredeclared: predeclared,
				testThreadModifier: func(th *starlark.Thread) {
					starlarktest.HookThread(th, t)
				},

				// Don't spit out "# This file is generated by lucicfg" headers.
				testOmitHeader: true,

				// Failure collector interferes with assert.fails() in a bad way.
				// assert.fails() captures errors, but it doesn't clear the failure
				// collector state, so we may end up in a situation when the script
				// fails with one error (some native starlark error, e.g. invalid
				// function call, not 'fail'), but the failure collector remembers
				// another (stale!) error, emitted by 'fail' before and caught by
				// assert.fails(). This results in invalid error message at the end
				// of the script execution.
				//
				// Unfortunately, it is not easy to modify assert.fails() without
				// forking it. So instead we do a cheesy thing and disable the failure
				// collector if the file under test appears to be unit-testy (rather
				// than integration-testy). We define integration tests to be tests
				// that examine the output of the generator using "Expect ..." blocks
				// (see above), and unit tests are tests that use asserts.
				//
				// Disabling the failure collector results in fail(..., trace=t)
				// ignoring the custom stack trace 't'. But unit tests don't generally
				// check the stack trace (only the error message), so it's not a big
				// deal for them.
				testDisableFailureCollector: !integrationTest,

				// Do not put frequently changing version string into test outputs.
				testVersion: "1.1.1",
			})

			// If test was expected to fail on Starlark side, make sure it did, in
			// an expected way.
			if expectErrExct != "" || expectErrLike != "" {
				allErrs := strings.Builder{}
				var skip bool
				errors.Walk(err, func(err error) bool {
					if skip {
						skip = false
						return true
					}

					if bt, ok := err.(BacktracableError); ok {
						allErrs.WriteString(bt.Backtrace())
						// We need to skip Unwrap from starlark.EvalError
						_, skip = err.(*starlark.EvalError)
					} else {
						switch err.(type) {
						case errors.MultiError, errors.Wrapped:
							return true
						}

						allErrs.WriteString(err.Error())
					}
					allErrs.WriteString("\n\n")
					return true
				})

				// Strip line and column numbers from backtraces.
				normalized := builtins.NormalizeStacktrace(allErrs.String())

				if expectErrExct != "" {
					errorOnDiff(t, normalized, expectErrExct)
				} else {
					errorOnPatternMismatch(t, normalized, expectErrLike)
				}
				return nil
			}

			// Otherwise just report all errors to Mr. T.
			errors.WalkLeaves(err, func(err error) bool {
				if bt, ok := err.(BacktracableError); ok {
					t.Errorf("%s\n", bt.Backtrace())
				} else {
					t.Errorf("%s\n", err)
				}
				return true
			})
			if err != nil {
				return nil // the error has been reported already
			}

			// If was expecting to see some configs, assert we did see them.
			if expectCfg != "" {
				got := bytes.Buffer{}
				for _, f := range state.Output.Files() {
					fmt.Fprintf(&got, "=== %s\n", f)
					if blob, err := state.Output.Data[f].Bytes(); err != nil {
						t.Errorf("Serializing %s: %s", f, err)
					} else {
						fmt.Fprintf(&got, string(blob))
					}
					fmt.Fprintf(&got, "===\n\n")
				}
				errorOnDiff(t, got.String(), expectCfg)
			}

			return nil
		},
	})
}

// readCommentBlock reads a comment block that start with "# <hdr>\n".
//
// Returns empty string if there's no such block.
func readCommentBlock(script, hdr string) string {
	scanner := bufio.NewScanner(strings.NewReader(script))
	for scanner.Scan() && scanner.Text() != "# "+hdr {
		continue
	}
	sb := strings.Builder{}
	for scanner.Scan() {
		if line := scanner.Text(); strings.HasPrefix(line, "#") {
			sb.WriteString(strings.TrimPrefix(line[1:], " "))
			sb.WriteRune('\n')
		} else {
			break // the comment block has ended
		}
	}
	return sb.String()
}

// errorOnDiff emits an error to T if got != exp.
func errorOnDiff(t *testing.T, got, exp string) {
	t.Helper()

	got = strings.TrimSpace(got)
	exp = strings.TrimSpace(exp)
	switch {
	case got == "":
		t.Errorf("Got nothing, but was expecting:\n\n%s\n", exp)
	case got != exp:
		dmp := diffmatchpatch.New()
		diffs := dmp.DiffMain(exp, got, false)
		t.Errorf(
			"Got:\n\n%s\n\nWas expecting:\n\n%s\n\nDiff:\n\n%s\n",
			got, exp, dmp.DiffPrettyText(diffs))
	}
}

// errorOnMismatch emits an error to T if got doesn't match a pattern pat.
//
// The pattern is syntax is:
//   * A line "[space]...[space]" matches zero or more arbitrary lines.
//   * Trigram "???" matches [0-9a-zA-Z]+.
//   * The rest should match as is.
func errorOnPatternMismatch(t *testing.T, got, pat string) {
	t.Helper()

	got = strings.TrimSpace(got)
	pat = strings.TrimSpace(pat)

	re := strings.Builder{}
	re.WriteRune('^')
	for _, line := range strings.Split(pat, "\n") {
		if strings.TrimSpace(line) == "..." {
			re.WriteString(`(.*\n)*`)
		} else {
			for line != "" {
				idx := strings.Index(line, "???")
				if idx == -1 {
					re.WriteString(regexp.QuoteMeta(line))
					break
				}
				re.WriteString(regexp.QuoteMeta(line[:idx]))
				re.WriteString(`[0-9a-zA-Z]+`)
				line = line[idx+3:]
			}
			re.WriteString(`\n`)
		}
	}
	re.WriteRune('$')

	if exp := regexp.MustCompile(re.String()); !exp.MatchString(got + "\n") {
		t.Errorf("Got:\n\n%s\n\nWas expecting pattern:\n\n%s\n\n", got, pat)
		t.Errorf("Regexp: %s", re.String())
	}
}
<commit_msg>[lucicfg] Teach tests to regenerate expected config output.<commit_after>// Copyright 2018 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lucicfg

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"testing"

	"github.com/sergi/go-diff/diffmatchpatch"

	"go.starlark.net/resolve"
	"go.starlark.net/starlark"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/starlark/builtins"
	"go.chromium.org/luci/starlark/interpreter"
	"go.chromium.org/luci/starlark/starlarktest"
)

// If this env var is 1, the test will regenerate the "Expect configs:" part of
// test *.star files.
const RegenEnvVar = "LUCICFG_TEST_REGEN"

const (
	expectConfigsHeader    = "Expect configs:"
	expectErrorsHeader     = "Expect errors:"
	expectErrorsLikeHeader = "Expect errors like:"
)

func init() {
	// Enable not-yet-standard features.
	resolve.AllowLambda = true
	resolve.AllowNestedDef = true
	resolve.AllowFloat = true
	resolve.AllowSet = true
}

// TestAllStarlark loads and executes all test scripts (testdata/*.star).
func TestAllStarlark(t *testing.T) {
	t.Parallel()

	gotExpectationErrors := false

	starlarktest.RunTests(t, starlarktest.Options{
		TestsDir: "testdata",
		Skip:     "support",

		Executor: func(t *testing.T, path string, predeclared starlark.StringDict) error {
			blob, err := ioutil.ReadFile(path)
			if err != nil {
				return err
			}
			body := string(blob)

			// Read "mocked" `-var name=value` assignments.
			presetVars := map[string]string{}
			presetVarsBlock := readCommentBlock(body, "Prepare CLI vars as:")
			for _, line := range strings.Split(presetVarsBlock, "\n") {
				if line = strings.TrimSpace(line); line != "" {
					chunks := strings.SplitN(line, "=", 2)
					if len(chunks) != 2 {
						t.Errorf("Bad CLI var declaration %q", line)
						return nil
					}
					presetVars[chunks[0]] = chunks[1]
				}
			}

			expectErrExct := readCommentBlock(body, expectErrorsHeader)
			expectErrLike := readCommentBlock(body, expectErrorsLikeHeader)
			expectCfg := readCommentBlock(body, expectConfigsHeader)
			if expectErrExct != "" && expectErrLike != "" {
				t.Errorf("Cannot use %q and %q at the same time", expectErrorsHeader, expectErrorsLikeHeader)
				return nil
			}

			// We treat tests that compare the generator output to some expected
			// output as "integration tests", and everything else is a unit tests.
			// See below for why this is important.
			integrationTest := expectErrExct != "" || expectErrLike != "" || expectCfg != ""

			state, err := Generate(context.Background(), Inputs{
				// Use file system loader so test scripts can load supporting scripts
				// (from '**/support/*' which is skipped by the test runner). This also
				// makes error messages have the original scripts full name. Note that
				// 'go test' executes tests with cwd set to corresponding package
				// directories, regardless of what cwd was when 'go test' was called.
				Code:  interpreter.FileSystemLoader("."),
				Entry: filepath.ToSlash(path),
				Vars:  presetVars,

				// Expose 'assert' module, hook up error reporting to 't'.
				testPredeclared: predeclared,
				testThreadModifier: func(th *starlark.Thread) {
					starlarktest.HookThread(th, t)
				},

				// Don't spit out "# This file is generated by lucicfg" headers.
				testOmitHeader: true,

				// Failure collector interferes with assert.fails() in a bad way.
				// assert.fails() captures errors, but it doesn't clear the failure
				// collector state, so we may end up in a situation when the script
				// fails with one error (some native starlark error, e.g. invalid
				// function call, not 'fail'), but the failure collector remembers
				// another (stale!) error, emitted by 'fail' before and caught by
				// assert.fails(). This results in invalid error message at the end
				// of the script execution.
				//
				// Unfortunately, it is not easy to modify assert.fails() without
				// forking it. So instead we do a cheesy thing and disable the failure
				// collector if the file under test appears to be unit-testy (rather
				// than integration-testy). We define integration tests to be tests
				// that examine the output of the generator using "Expect ..." blocks
				// (see above), and unit tests are tests that use asserts.
				//
				// Disabling the failure collector results in fail(..., trace=t)
				// ignoring the custom stack trace 't'. But unit tests don't generally
				// check the stack trace (only the error message), so it's not a big
				// deal for them.
				testDisableFailureCollector: !integrationTest,

				// Do not put frequently changing version string into test outputs.
				testVersion: "1.1.1",
			})

			// If test was expected to fail on Starlark side, make sure it did, in
			// an expected way.
			if expectErrExct != "" || expectErrLike != "" {
				allErrs := strings.Builder{}
				var skip bool
				errors.Walk(err, func(err error) bool {
					if skip {
						skip = false
						return true
					}

					if bt, ok := err.(BacktracableError); ok {
						allErrs.WriteString(bt.Backtrace())
						// We need to skip Unwrap from starlark.EvalError
						_, skip = err.(*starlark.EvalError)
					} else {
						switch err.(type) {
						case errors.MultiError, errors.Wrapped:
							return true
						}

						allErrs.WriteString(err.Error())
					}
					allErrs.WriteString("\n\n")
					return true
				})

				// Strip line and column numbers from backtraces.
				normalized := builtins.NormalizeStacktrace(allErrs.String())

				if expectErrExct != "" {
					errorOnDiff(t, normalized, expectErrExct)
				} else {
					errorOnPatternMismatch(t, normalized, expectErrLike)
				}
				return nil
			}

			// Otherwise just report all errors to Mr. T.
			errors.WalkLeaves(err, func(err error) bool {
				if bt, ok := err.(BacktracableError); ok {
					t.Errorf("%s\n", bt.Backtrace())
				} else {
					t.Errorf("%s\n", err)
				}
				return true
			})
			if err != nil {
				return nil // the error has been reported already
			}

			// If was expecting to see some configs, assert we did see them.
			if expectCfg != "" {
				got := bytes.Buffer{}
				for idx, f := range state.Output.Files() {
					if idx != 0 {
						fmt.Fprintf(&got, "\n\n")
					}
					fmt.Fprintf(&got, "=== %s\n", f)
					if blob, err := state.Output.Data[f].Bytes(); err != nil {
						t.Errorf("Serializing %s: %s", f, err)
					} else {
						fmt.Fprintf(&got, string(blob))
					}
					fmt.Fprintf(&got, "===")
				}
				if os.Getenv(RegenEnvVar) == "1" {
					if err := updateExpected(path, got.String()); err != nil {
						t.Errorf("Failed to updated %q: %s", path, err)
					}
				} else if errorOnDiff(t, got.String(), expectCfg) {
					gotExpectationErrors = true
				}
			}

			return nil
		},
	})

	if gotExpectationErrors {
		t.Errorf("\n\n"+
			"========================================================\n"+
			"If you want to update expectations stored in *.star run:\n"+
			"$ %s=1 go test .\n"+
			"========================================================", RegenEnvVar)
	}
}

// readCommentBlock reads a comment block that start with "# <hdr>\n".
//
// Returns empty string if there's no such block.
func readCommentBlock(script, hdr string) string {
	scanner := bufio.NewScanner(strings.NewReader(script))
	for scanner.Scan() && scanner.Text() != "# "+hdr {
		continue
	}
	sb := strings.Builder{}
	for scanner.Scan() {
		if line := scanner.Text(); strings.HasPrefix(line, "#") {
			sb.WriteString(strings.TrimPrefix(line[1:], " "))
			sb.WriteRune('\n')
		} else {
			break // the comment block has ended
		}
	}
	return sb.String()
}

// updateExpected updates the expected generated config stored in the comment
// block at the end of the *.star file.
func updateExpected(path, exp string) error {
	blob, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}

	idx := bytes.Index(blob, []byte(fmt.Sprintf("# %s\n", expectConfigsHeader)))
	if idx == -1 {
		return errors.Reason("doesn't have `Expect configs` comment block").Err()
	}
	blob = blob[:idx]

	blob = append(blob, []byte(fmt.Sprintf("# %s\n", expectConfigsHeader))...)
	blob = append(blob, []byte("#\n")...)
	for _, line := range strings.Split(exp, "\n") {
		if len(line) == 0 {
			blob = append(blob, '#')
		} else {
			blob = append(blob, []byte("# ")...)
			blob = append(blob, []byte(line)...)
		}
		blob = append(blob, '\n')
	}

	return ioutil.WriteFile(path, blob, 0666)
}

// errorOnDiff emits an error to T and returns true if got != exp.
func errorOnDiff(t *testing.T, got, exp string) bool {
	t.Helper()

	got = strings.TrimSpace(got)
	exp = strings.TrimSpace(exp)

	switch {
	case got == "":
		t.Errorf("Got nothing, but was expecting:\n\n%s\n", exp)
		return true
	case got != exp:
		dmp := diffmatchpatch.New()
		diffs := dmp.DiffMain(exp, got, false)
		t.Errorf(
			"Got:\n\n%s\n\nWas expecting:\n\n%s\n\nDiff:\n\n%s\n",
			got, exp, dmp.DiffPrettyText(diffs))
		return true
	}

	return false
}

// errorOnMismatch emits an error to T if got doesn't match a pattern pat.
//
// The pattern is syntax is:
//   * A line "[space]...[space]" matches zero or more arbitrary lines.
//   * Trigram "???" matches [0-9a-zA-Z]+.
//   * The rest should match as is.
func errorOnPatternMismatch(t *testing.T, got, pat string) {
	t.Helper()

	got = strings.TrimSpace(got)
	pat = strings.TrimSpace(pat)

	re := strings.Builder{}
	re.WriteRune('^')
	for _, line := range strings.Split(pat, "\n") {
		if strings.TrimSpace(line) == "..." {
			re.WriteString(`(.*\n)*`)
		} else {
			for line != "" {
				idx := strings.Index(line, "???")
				if idx == -1 {
					re.WriteString(regexp.QuoteMeta(line))
					break
				}
				re.WriteString(regexp.QuoteMeta(line[:idx]))
				re.WriteString(`[0-9a-zA-Z]+`)
				line = line[idx+3:]
			}
			re.WriteString(`\n`)
		}
	}
	re.WriteRune('$')

	if exp := regexp.MustCompile(re.String()); !exp.MatchString(got + "\n") {
		t.Errorf("Got:\n\n%s\n\nWas expecting pattern:\n\n%s\n\n", got, pat)
		t.Errorf("Regexp: %s", re.String())
	}
}
<|endoftext|>