<commit_before>package sweep

// bucket contains a set of inputs that are not mutually exclusive.
type bucket pendingInputs

// tryAdd tries to add a new input to this bucket.
func (b bucket) tryAdd(input *pendingInput) bool {
	exclusiveGroup := input.params.ExclusiveGroup
	if exclusiveGroup != nil {
		for _, input := range b {
			existingGroup := input.params.ExclusiveGroup
			if existingGroup != nil &&
				*existingGroup == *exclusiveGroup {

				return false
			}
		}
	}

	b[*input.OutPoint()] = input

	return true
}

// bucketList is a list of buckets that contain non-mutually exclusive inputs.
type bucketList struct {
	buckets []bucket
}

// add adds a new input. If the input is not accepted by any of the existing
// buckets, a new bucket will be created.
func (b *bucketList) add(input *pendingInput) {
	for _, existingBucket := range b.buckets {
		if existingBucket.tryAdd(input) {
			return
		}
	}

	// Create a new bucket and add the input. It is not necessary to check
	// the return value of tryAdd because it will always succeed on an empty
	// bucket.
	newBucket := make(bucket)
	newBucket.tryAdd(input)
	b.buckets = append(b.buckets, newBucket)
}
<commit_msg>sweep: do not combine exclusive and non-exclusive inputs<commit_after>package sweep

// bucket contains a set of inputs that are not mutually exclusive.
type bucket pendingInputs

// tryAdd tries to add a new input to this bucket.
func (b bucket) tryAdd(input *pendingInput) bool {
	exclusiveGroup := input.params.ExclusiveGroup
	if exclusiveGroup != nil {
		for _, input := range b {
			existingGroup := input.params.ExclusiveGroup

			// Don't add an exclusive group input if other inputs
			// are non-exclusive. The exclusive group input may be
			// invalid (for example in the case of commitment
			// anchors) and could thereby block sweeping of the
			// other inputs.
			if existingGroup == nil {
				return false
			}

			// Don't combine inputs from the same exclusive group.
			// Because only one input is valid, this may result in
			// txes that are always invalid.
			if *existingGroup == *exclusiveGroup {
				return false
			}
		}
	}

	b[*input.OutPoint()] = input

	return true
}

// bucketList is a list of buckets that contain non-mutually exclusive inputs.
type bucketList struct {
	buckets []bucket
}

// add adds a new input. If the input is not accepted by any of the existing
// buckets, a new bucket will be created.
func (b *bucketList) add(input *pendingInput) {
	for _, existingBucket := range b.buckets {
		if existingBucket.tryAdd(input) {
			return
		}
	}

	// Create a new bucket and add the input. It is not necessary to check
	// the return value of tryAdd because it will always succeed on an empty
	// bucket.
	newBucket := make(bucket)
	newBucket.tryAdd(input)
	b.buckets = append(b.buckets, newBucket)
}
<|endoftext|>


<commit_before>package main

// Code responsible for handling an incoming event from github

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
)

var jsonLogFile, _ = os.Create("logs/json.log")
var jsonLog = log.New(jsonLogFile, "", log.LstdFlags)

// This function is called whenever an event happens on github.
func handleEvent(eventType string, document []byte) (err error) {

	// log.Println("Incoming request:", string(document))
	jsonLog.Println("Incoming request:", string(document))

	switch eventType {
	case "push":

		var event PushEvent
		err = json.Unmarshal(document, &event)
		if err != nil {
			return
		}

		log.Printf("Received PushEvent %#+v", event)

		if event.Deleted {
			// When a branch is deleted we get a "push" event we don't care
			// about (after = "0000")
			return
		}

		err = eventPush(event)
		if err != nil {
			return
		}

	default:
		log.Println("Unhandled event:", eventType)
	}

	return
}

// HTTP handler for /hook
// It is expecting a POST with a JSON payload according to
// http://developer.github.com/v3/activity/events/
func handleHook(w http.ResponseWriter, r *http.Request) {

	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		fmt.Fprintf(w, "Expected JSON POST payload.\n")
		return
	}

	request, err := ioutil.ReadAll(r.Body)
	check(err)

	var buf bytes.Buffer
	// r.Header.Write(&buf)
	// log.Println("Incoming request headers: ", string(buf.Bytes()))
	// buf.Reset()

	err = json.Indent(&buf, request, "", "  ")
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Expected valid JSON POST payload.\n")
		log.Println("Not a valid JSON payload. NOOP.")
		return
	}

	if len(r.Header["X-Github-Event"]) != 1 {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Expected X-Github-Event header.\n")
		log.Println("No X-Github-Event header. NOOP")
		return
	}
	eventType := r.Header["X-Github-Event"][0]
	data := buf.Bytes()

	// Check to see if we have data from somewhere which is not github
	j, err := ParseJustNongithub(request)
	if !j.NonGithub.Wait {
		go func() {
			err := handleEvent(eventType, data)
			if err != nil {
				log.Printf("Error processing %v %v %q", eventType, string(data), err)
			}
		}()

		w.WriteHeader(http.StatusOK)
		fmt.Fprintf(w, "OK. Not waiting for build.\n")
		return
	}

	// Handle the event
	err = handleEvent(eventType, data)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Error handling event: %q\n", err)
		log.Printf("Error handling event: %q", err)
		return
	}

	w.WriteHeader(http.StatusOK)
	fmt.Fprintf(w, "OK\n")
}

// Invoked when a respository we are watching changes
func runTang(repo, repo_path, logPath string, event PushEvent) (err error) {

	sha := event.After
	ref := event.Ref

	// TODO(pwaller): do tee in go.
	// c := `./tang.hook |& tee $TANG_LOGPATH; exit ${PIPESTATUS[0]}`
	// cmd := Command(repo_path, "bash", "-c", c)
	cmd := Command(repo_path, "./tang.hook")

	tang_logfile, err := os.Create(logPath)
	if err != nil {
		return
	}
	defer tang_logfile.Close()

	out := io.MultiWriter(os.Stdout, tang_logfile)
	cmd.Stdout = out
	cmd.Stderr = out

	cmd.Env = append(os.Environ(),
		"TANG_SHA="+sha, "TANG_REF="+ref, "TANG_LOGPATH="+logPath)
	err = cmd.Run()

	return
}

// Invoked when there is a push event to github.
func eventPush(event PushEvent) (err error) {
	if event.Repository.Name == "" {
		return ErrEmptyRepoName
	}

	if event.Repository.Organization == "" {
		return ErrEmptyRepoOrganization
	}

	if _, ok := allowedPushersSet[event.Pusher.Name]; !ok {
		log.Printf("Ignoring %q, not allowed", event.Pusher.Name)
		return ErrUserNotAllowed
	}

	gh_repo := path.Join(event.Repository.Organization, event.Repository.Name)

	log.Println("Push to", event.Repository.Url, event.Ref, "after", event.After)

	// The name of the subdirectory where the git
	// mirror is (or will appear, if it hasn't been
	// cloned yet).
	git_dir := path.Join(GIT_BASE_DIR, gh_repo)

	// Update our local mirror
	err = gitLocalMirror(event.Repository.Url, git_dir)
	if err != nil {
		err = fmt.Errorf("Failed to update git mirror: %q", err)
		infoURL := "http://services.scraperwiki.com/tang/"
		s := GithubStatus{"failure", infoURL, err.Error()}
		updateStatus(gh_repo, event.After, s)
		return
	}

	// Check if we there is a tang hook
	tang_hook_present, err := gitHaveFile(git_dir, event.After, "tang.hook")
	if err != nil || !tang_hook_present || event.NonGithub.NoBuild {
		// Bail out, error, no tang.hook or instructed not to build it.
		return
	}

	// Dereference event.After, always. Not needed for github but useful for
	// `tang-event`, where we don't know the sha beforehand.
	sha, err := gitRevParse(git_dir, event.After)
	if err != nil {
		err = fmt.Errorf("gitRevParse: %q", err)
		return
	}

	// Only use 6 characters of sha for the name of the
	// directory checked out for this repository by tang.
	short_sha := sha[:6]
	checkout_dir := path.Join("checkout", short_sha)

	// Checkout the target sha
	err = gitCheckout(git_dir, checkout_dir, event.After)
	if err != nil {
		return
	}

	log.Println("Created", checkout_dir)

	pwd, err := os.Getwd()
	if err != nil {
		err = fmt.Errorf("runTang/getwd %q", err)
		return
	}

	logDir := path.Join("logs", short_sha)
	err = os.MkdirAll(logDir, 0777)
	if err != nil {
		err = fmt.Errorf("runTang/MkdirAll(%q): ", logDir, err)
		return
	}

	logPath := path.Join(logDir, "log.txt")
	fullLogPath := path.Join(pwd, logPath)

	// TODO(pwaller): One day this will have more information, e.g, QA link.
	infoURL := "http://services.scraperwiki.com/tang/" + logPath

	// Set the state of the commit to "in progress" (seen as yellow in
	// a github pull request)
	status := GithubStatus{"pending", infoURL, "Running"}
	updateStatus(gh_repo, event.After, status)

	// Run the tang script for the repository, if there is one.
	repo_workdir := path.Join(git_dir, checkout_dir)
	err = runTang(gh_repo, repo_workdir, fullLogPath, event)

	if err == nil {
		// All OK, send along a green
		s := GithubStatus{"success", infoURL, "Tests passed"}
		updateStatus(gh_repo, event.After, s)
		return
	}

	// Not OK, send along red.
	s := GithubStatus{"failure", infoURL, err.Error()}
	updateStatus(gh_repo, event.After, s)
	return
}
<commit_msg>Start logging earlier<commit_after>package main

// Code responsible for handling an incoming event from github

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
)

var jsonLogFile, _ = os.Create("logs/json.log")
var jsonLog = log.New(jsonLogFile, "", log.LstdFlags)

// This function is called whenever an event happens on github.
func handleEvent(eventType string, document []byte) (err error) {

	// log.Println("Incoming request:", string(document))
	jsonLog.Println("Incoming request:", string(document))

	switch eventType {
	case "push":

		var event PushEvent
		err = json.Unmarshal(document, &event)
		if err != nil {
			return
		}

		log.Printf("Received PushEvent %#+v", event)

		if event.Deleted {
			// When a branch is deleted we get a "push" event we don't care
			// about (after = "0000")
			return
		}

		err = eventPush(event)
		if err != nil {
			return
		}

	default:
		log.Println("Unhandled event:", eventType)
	}

	return
}

// HTTP handler for /hook
// It is expecting a POST with a JSON payload according to
// http://developer.github.com/v3/activity/events/
func handleHook(w http.ResponseWriter, r *http.Request) {

	if r.Method != "POST" {
		w.WriteHeader(http.StatusMethodNotAllowed)
		fmt.Fprintf(w, "Expected JSON POST payload.\n")
		return
	}

	request, err := ioutil.ReadAll(r.Body)
	check(err)

	var buf bytes.Buffer
	// r.Header.Write(&buf)
	// log.Println("Incoming request headers: ", string(buf.Bytes()))
	// buf.Reset()

	err = json.Indent(&buf, request, "", "  ")
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Expected valid JSON POST payload.\n")
		log.Println("Not a valid JSON payload. NOOP.")
		return
	}

	if len(r.Header["X-Github-Event"]) != 1 {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Expected X-Github-Event header.\n")
		log.Println("No X-Github-Event header. NOOP")
		return
	}
	eventType := r.Header["X-Github-Event"][0]
	data := buf.Bytes()

	// Check to see if we have data from somewhere which is not github
	j, err := ParseJustNongithub(request)
	if !j.NonGithub.Wait {
		go func() {
			err := handleEvent(eventType, data)
			if err != nil {
				log.Printf("Error processing %v %v %q", eventType, string(data), err)
			}
		}()

		w.WriteHeader(http.StatusOK)
		fmt.Fprintf(w, "OK. Not waiting for build.\n")
		return
	}

	// Handle the event
	err = handleEvent(eventType, data)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Error handling event: %q\n", err)
		log.Printf("Error handling event: %q", err)
		return
	}

	w.WriteHeader(http.StatusOK)
	fmt.Fprintf(w, "OK\n")
}

// Invoked when a respository we are watching changes
func runTang(repo, repo_path string, log io.Writer, event PushEvent) (err error) {

	sha := event.After
	ref := event.Ref

	cmd := Command(repo_path, "./tang.hook")

	out := io.MultiWriter(os.Stdout, log)
	cmd.Stdout = out
	cmd.Stderr = out

	cmd.Env = append(os.Environ(),
		"TANG_SHA="+sha, "TANG_REF="+ref)
	err = cmd.Run()

	return
}

func getLogPath(shortSha string) (logPath, diskLogPath string, err error) {

	pwd, err := os.Getwd()
	if err != nil {
		err = fmt.Errorf("getLogPath/getwd %q", err)
		return
	}

	logDir := path.Join("logs", shortSha)
	err = os.MkdirAll(logDir, 0777)
	if err != nil {
		err = fmt.Errorf("getLogPath/MkdirAll(%q): ", logDir, err)
		return
	}

	logPath = path.Join(logDir, "log.txt")
	diskLogPath = path.Join(pwd, logPath)
	return
}

// Invoked when there is a push event to github.
func eventPush(event PushEvent) (err error) {
	if event.Repository.Name == "" {
		return ErrEmptyRepoName
	}

	if event.Repository.Organization == "" {
		return ErrEmptyRepoOrganization
	}

	if _, ok := allowedPushersSet[event.Pusher.Name]; !ok {
		log.Printf("Ignoring %q, not allowed", event.Pusher.Name)
		return ErrUserNotAllowed
	}

	gh_repo := path.Join(event.Repository.Organization, event.Repository.Name)

	log.Println("Push to", event.Repository.Url, event.Ref, "after", event.After)

	logPath, diskLogPath, err := getLogPath(shortSha)
	if err != nil {
		return
	}

	tangLog, err := os.Create(diskLogPath)
	if err != nil {
		return
	}
	defer tangLog.Close()

	// The name of the subdirectory where the git
	// mirror is (or will appear, if it hasn't been
	// cloned yet).
	git_dir := path.Join(GIT_BASE_DIR, gh_repo)

	// Update our local mirror
	err = gitLocalMirror(event.Repository.Url, git_dir)
	if err != nil {
		err = fmt.Errorf("Failed to update git mirror: %q", err)
		infoURL := "http://services.scraperwiki.com/tang/"
		s := GithubStatus{"failure", infoURL, err.Error()}
		updateStatus(gh_repo, event.After, s)
		return
	}

	// Check if we there is a tang hook
	tang_hook_present, err := gitHaveFile(git_dir, event.After, "tang.hook")
	if err != nil || !tang_hook_present || event.NonGithub.NoBuild {
		// Bail out, error, no tang.hook or instructed not to build it.
		return
	}

	// Dereference event.After, always. Not needed for github but useful for
	// `tang-event`, where we don't know the sha beforehand.
	BREAKAGE // TODO(pwaller): Hmm. Need to thing about this if we want to start
	// logging immediately. Was a nice idea but I'm not so sure if a good one
	// anymore.
	sha, err := gitRevParse(git_dir, event.After)
	if err != nil {
		err = fmt.Errorf("gitRevParse: %q", err)
		return
	}

	// Only use 6 characters of sha for the name of the
	// directory checked out for this repository by tang.
	shortSha := sha[:6]
	checkout_dir := path.Join("checkout", shortSha)

	// Checkout the target sha
	err = gitCheckout(git_dir, checkout_dir, event.After)
	if err != nil {
		return
	}

	log.Println("Created", checkout_dir)

	// TODO(pwaller): One day this will have more information, e.g, QA link.
	infoURL := "http://services.scraperwiki.com/tang/" + logPath

	// Set the state of the commit to "in progress" (seen as yellow in
	// a github pull request)
	status := GithubStatus{"pending", infoURL, "Running"}
	updateStatus(gh_repo, event.After, status)

	// Run the tang script for the repository, if there is one.
	repo_workdir := path.Join(git_dir, checkout_dir)
	err = runTang(gh_repo, repo_workdir, tangLog, event)

	if err == nil {
		// All OK, send along a green
		s := GithubStatus{"success", infoURL, "Tests passed"}
		updateStatus(gh_repo, event.After, s)
		return
	}

	// Not OK, send along red.
	s := GithubStatus{"failure", infoURL, err.Error()}
	updateStatus(gh_repo, event.After, s)
	return
}
<|endoftext|>


<commit_before>// Package Fetch allows the querying of nested data through javascript-style accessors

package fetch

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
	"unicode"
	"unicode/utf8"
)

const (
	itemError = iota
	itemBeginArray
	itemEndArray
	itemString
	itemNumber
	itemDot
	itemField
	itemSpace
	fieldDot
	fieldMap
	fieldArray
)

var ident = map[int]string{
	itemError:      "itemError",
	itemBeginArray: "itemBeginArray",
	itemEndArray:   "itemEndArray",
	itemString:     "itemString",
	itemNumber:     "itemNumber",
	itemDot:        "itemDot",
	itemField:      "itemField",
	itemSpace:      "itemSpace",
	fieldDot:       "fieldDot",
	fieldMap:       "fieldMap",
	fieldArray:     "fieldArray",
}

const eof = -1

type itemType int
type stateFn func(*query) stateFn

type item struct {
	typ itemType
	pos int
	val string
}

type fieldType int

type field struct {
	typ   fieldType
	index int
	key   string
}

type query struct {
	state   stateFn
	pos     int
	width   int
	input   string
	start   int
	lastPos int
	items   chan item
	fields  []field
}

func (l *query) run() {
	for l.state = startLex; l.state != nil; {
		l.state = l.state(l)
	}
}

func (l *query) next() rune {
	if int(l.pos) >= len(l.input) {
		l.width = 0
		return eof
	}
	r, w := utf8.DecodeRuneInString(l.input[l.pos:])
	l.width = w
	l.pos += l.width
	return r
}

func (l *query) peek() rune {
	r := l.next()
	l.backup()
	return r
}

func (l *query) backup() {
	l.pos -= l.width
}

func (l *query) emit(t itemType) {
	l.items <- item{t, l.start, l.input[l.start:l.pos]}
	l.start = l.pos
}

func (l *query) ignore() {
	l.start = l.pos
}

func (l *query) errorf(format string, args ...interface{}) stateFn {
	l.items <- item{itemError, l.start, fmt.Sprintf(format, args...)}
	return nil
}

func (l *query) accept(valid string) bool {
	if strings.IndexRune(valid, l.next()) >= 0 {
		return true
	}
	l.backup()
	return false
}

func (l *query) acceptRun(valid string) {
	for strings.IndexRune(valid, l.next()) >= 0 {
	}
	l.backup()
}

func startLex(l *query) stateFn {
	c := l.next()
	switch {
	case c == '[':
		l.emit(itemBeginArray)
		return startLex
	case c == ']':
		l.emit(itemEndArray)
		return startLex
	case c == '"':
		return lexQuote
	case c == '\'':
		return lexSQuote
	case c == '.':
		if !isAlphaNumeric(l.peek()) {
			l.emit(itemDot)
		} else {
			return lexField
		}
		return startLex
	case '0' <= c && c <= '9':
		l.backup()
		return lexNumber
	case c == eof:
		l.emit(eof)
		return nil
	case isAlphaNumeric(c):
		l.emit(itemError)
	case !isAlphaNumeric(c):
		l.emit(itemError)
		return startLex
	}

	return startLex
}

func lexField(l *query) stateFn {
Loop:
	for {
		switch r := l.next(); {
		case isAlphaNumeric(r):
		default:
			l.backup()
			word := l.input[l.start:l.pos]
			if !l.atTerminator() {
				return l.errorf("bad character %#U", r)
			}
			switch {
			case word[0] == '.':
				l.emit(itemField)
			default:
				l.emit(itemError)
			}
			break Loop
		}
	}
	return startLex
}

func lexQuote(l *query) stateFn {
Loop:
	for {
		switch l.next() {
		case '\\':
			if r := l.next(); r != eof && r != '\n' {
				break
			}
			fallthrough
		case eof, '\n':
			return l.errorf("unterminated quoted string")
		case '"':
			break Loop
		}
	}
	l.emit(itemString)
	return startLex
}

func lexSQuote(l *query) stateFn {
Loop:
	for {
		switch l.next() {
		case '\\':
			if r := l.next(); r != eof && r != '\n' {
				break
			}
			fallthrough
		case eof, '\n':
			return l.errorf("unterminated quoted string")
		case '\'':
			break Loop
		}
	}
	l.emit(itemString)
	return startLex
}

func lexNumber(l *query) stateFn {
	if !l.scanNumber() {
		return l.errorf("bad number syntax: %q", l.input[l.start:l.pos])
	}
	l.emit(itemNumber)
	return startLex
}

func lexSpace(l *query) stateFn {
	for isSpace(l.peek()) {
		l.next()
	}
	l.emit(itemSpace)
	return startLex
}

func (l *query) atTerminator() bool {
	r := l.peek()
	if isSpace(r) || isEndOfLine(r) {
		return true
	}
	switch r {
	case eof, '.', ',', '|', ':', ')', '(', '[', ']', '{', '}', '+', '-', '/', '*':
		return true
	}
	return false
}

func (l *query) runField() error {
	accessor := false
	pos := 0
	var i *field
	for pos < len(l.input) {
		c := l.nextItem()

		switch c.typ {
		case itemField:
			l.fields = append(l.fields, field{typ: fieldMap, key: c.val[1:]})
		case itemBeginArray:
			if accessor {
				return errors.New(fmt.Sprintf("Unexpected token %s at position %d", c.val, c.pos))
			}
			accessor = true
		case itemString:
			if i != nil || !accessor {
				return errors.New(fmt.Sprintf("Unexpected token %s at position %d", c.val, c.pos))
			}

			k := c.val[1:]
			k = k[:len(k)-1]
			i = &field{
				typ: fieldMap,
				key: k,
			}
		case itemNumber:
			if i != nil || !accessor {
				return errors.New(fmt.Sprintf("Unexpected token %s at position %d", c.val, c.pos))
			}
			index, err := strconv.Atoi(c.val)
			if err != nil {
				return err
			}

			i = &field{
				typ:   fieldArray,
				index: index,
			}
		case itemEndArray:
			if i == nil || !accessor {
				return errors.New(fmt.Sprintf("Unexpected token %s at position %d", c.val, c.pos))
			}

			l.fields = append(l.fields, *i)
			i = nil
			accessor = false
		case eof:
			return nil
		case itemDot:
			if pos == 0 {
				break
			}
			fallthrough
		default:
			return errors.New(fmt.Sprintf("Unexpected token %s at position %d", c.val, c.pos))
		}
		pos += len(c.val)
	}

	return nil
}

func (l *query) nextItem() item {
	item := <-l.items
	l.lastPos = item.pos
	return item
}

func (l *query) scanNumber() bool {
	digits := "0123456789"
	l.acceptRun(digits)
	return true
}

func isSpace(r rune) bool {
	return r == ' ' || r == '\t'
}

func isEndOfLine(r rune) bool {
	return r == '\r' || r == '\n'
}

func isAlphaNumeric(r rune) bool {
	return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
}

func mapValue(o interface{}, key string) (interface{}, error) {
	n, ok := o.(map[string]interface{})
	if !ok {
		return nil, errors.New("Not of type object")
	}
	p, ok := n[key]
	if !ok {
		return nil, errors.New(fmt.Sprintf("Key (%s) does not exist", key))
	}
	return p, nil
}

func indexValue(o interface{}, index int) (interface{}, error) {
	n, ok := o.([]interface{})
	if !ok {
		return nil, errors.New("Not of type array")
	}
	if index > len(n) {
		return nil, errors.New(fmt.Sprintf("Index (%d) out of range", index))
	}
	return n[index], nil
}

// Converts a query string into a *Fetch.Query.
// Fetch.Parse is similar to jq, in that in order to reference the base object,
// you must begin a query with '.'
// For example, a query string of '.' will return an entire object, a query string
// of '.foo' will return the value of key foo on the root of the object. Every
// subsequent field can be accessed through javascript-style dot/bracket notation.
// for example, .foo[0] would return the first element of array foo, and
// .["foo"][0] would do the same as well.
func Parse(input string) (*query, error) {
	l := &query{
		input:  input,
		items:  make(chan item),
		fields: []field{},
	}

	go l.run()
	err := l.runField()
	if err != nil {
		return nil, err
	}

	return l, nil
}

// Executes a *Fetch.Query on some data. Returns the result of the query.
func Run(l *query, o interface{}) (interface{}, error) {
	var err error
	for _, v := range l.fields {
		switch v.typ {
		case fieldMap:
			o, err = mapValue(o, v.key)
			if err != nil {
				return nil, err
			}
		case fieldArray:
			o, err = indexValue(o, v.index)
			if err != nil {
				return nil, err
			}
		}
	}
	return o, nil
}

// A convenience function that runs both Parse() and Run() automatically.
// It is highly recommended that you parse your query ahead of time
// with Fetch.Parse() and follow up with Fetch.Run() instead.
func Fetch(input string, obj interface{}) (interface{}, error) {
	l, err := Parse(input)
	if err != nil {
		return nil, err
	}
	return Run(l, obj)
}
<commit_msg>wording<commit_after>// Package Fetch allows the querying of nested data through javascript-style accessors

package fetch

import (
	"errors"
	"fmt"
	"strconv"
	"strings"
	"unicode"
	"unicode/utf8"
)

const (
	itemError = iota
	itemBeginArray
	itemEndArray
	itemString
	itemNumber
	itemDot
	itemField
	itemSpace
	fieldDot
	fieldMap
	fieldArray
)

var ident = map[int]string{
	itemError:      "itemError",
	itemBeginArray: "itemBeginArray",
	itemEndArray:   "itemEndArray",
	itemString:     "itemString",
	itemNumber:     "itemNumber",
	itemDot:        "itemDot",
	itemField:      "itemField",
	itemSpace:      "itemSpace",
	fieldDot:       "fieldDot",
	fieldMap:       "fieldMap",
	fieldArray:     "fieldArray",
}

const eof = -1

type itemType int
type stateFn func(*query) stateFn

type item struct {
	typ itemType
	pos int
	val string
}

type fieldType int

type field struct {
	typ   fieldType
	index int
	key   string
}

type query struct {
	state   stateFn
	pos     int
	width   int
	input   string
	start   int
	lastPos int
	items   chan item
	fields  []field
}

func (l *query) run() {
	for l.state = startLex; l.state != nil; {
		l.state = l.state(l)
	}
}

func (l *query) next() rune {
	if int(l.pos) >= len(l.input) {
		l.width = 0
		return eof
	}
	r, w := utf8.DecodeRuneInString(l.input[l.pos:])
	l.width = w
	l.pos += l.width
	return r
}

func (l *query) peek() rune {
	r := l.next()
	l.backup()
	return r
}

func (l *query) backup() {
	l.pos -= l.width
}

func (l *query) emit(t itemType) {
	l.items <- item{t, l.start, l.input[l.start:l.pos]}
	l.start = l.pos
}

func (l *query) ignore() {
	l.start = l.pos
}

func (l *query) errorf(format string, args ...interface{}) stateFn {
	l.items <- item{itemError, l.start, fmt.Sprintf(format, args...)}
	return nil
}

func (l *query) accept(valid string) bool {
	if strings.IndexRune(valid, l.next()) >= 0 {
		return true
	}
	l.backup()
	return false
}

func (l *query) acceptRun(valid string) {
	for strings.IndexRune(valid, l.next()) >= 0 {
	}
	l.backup()
}

func startLex(l *query) stateFn {
	c := l.next()
	switch {
	case c == '[':
		l.emit(itemBeginArray)
		return startLex
	case c == ']':
		l.emit(itemEndArray)
		return startLex
	case c == '"':
		return lexQuote
	case c == '\'':
		return lexSQuote
	case c == '.':
		if !isAlphaNumeric(l.peek()) {
			l.emit(itemDot)
		} else {
			return lexField
		}
		return startLex
	case '0' <= c && c <= '9':
		l.backup()
		return lexNumber
	case c == eof:
		l.emit(eof)
		return nil
	case isAlphaNumeric(c):
		l.emit(itemError)
	case !isAlphaNumeric(c):
		l.emit(itemError)
		return startLex
	}

	return startLex
}

func lexField(l *query) stateFn {
Loop:
	for {
		switch r := l.next(); {
		case isAlphaNumeric(r):
		default:
			l.backup()
			word := l.input[l.start:l.pos]
			if !l.atTerminator() {
				return l.errorf("bad character %#U", r)
			}
			switch {
			case word[0] == '.':
				l.emit(itemField)
			default:
				l.emit(itemError)
			}
			break Loop
		}
	}
	return startLex
}

func lexQuote(l *query) stateFn {
Loop:
	for {
		switch l.next() {
		case '\\':
			if r := l.next(); r != eof && r != '\n' {
				break
			}
			fallthrough
		case eof, '\n':
			return l.errorf("unterminated quoted string")
		case '"':
			break Loop
		}
	}
	l.emit(itemString)
	return startLex
}

func lexSQuote(l *query) stateFn {
Loop:
	for {
		switch l.next() {
		case '\\':
			if r := l.next(); r != eof && r != '\n' {
				break
			}
			fallthrough
		case eof, '\n':
			return l.errorf("unterminated quoted string")
		case '\'':
			break Loop
		}
	}
	l.emit(itemString)
	return startLex
}

func lexNumber(l *query) stateFn {
	if !l.scanNumber() {
		return l.errorf("bad number syntax: %q", l.input[l.start:l.pos])
	}
	l.emit(itemNumber)
	return startLex
}

func lexSpace(l *query) stateFn {
	for isSpace(l.peek()) {
		l.next()
	}
	l.emit(itemSpace)
	return startLex
}

func (l *query) atTerminator() bool {
	r := l.peek()
	if isSpace(r) || isEndOfLine(r) {
		return true
	}
	switch r {
	case eof, '.', ',', '|', ':', ')', '(', '[', ']', '{', '}', '+', '-', '/', '*':
		return true
	}
	return false
}

func (l *query) runField() error {
	accessor := false
	pos := 0
	var i *field
	for pos < len(l.input) {
		c := l.nextItem()

		switch c.typ {
		case itemField:
			l.fields = append(l.fields, field{typ: fieldMap, key: c.val[1:]})
		case itemBeginArray:
			if accessor {
				return errors.New(fmt.Sprintf("Unexpected token %s at position %d", c.val, c.pos))
			}
			accessor = true
		case itemString:
			if i != nil || !accessor {
				return errors.New(fmt.Sprintf("Unexpected token %s at position %d", c.val, c.pos))
			}

			k := c.val[1:]
			k = k[:len(k)-1]
			i = &field{
				typ: fieldMap,
				key: k,
			}
		case itemNumber:
			if i != nil || !accessor {
				return errors.New(fmt.Sprintf("Unexpected token %s at position %d", c.val, c.pos))
			}
			index, err := strconv.Atoi(c.val)
			if err != nil {
				return err
			}

			i = &field{
				typ:   fieldArray,
				index: index,
			}
		case itemEndArray:
			if i == nil || !accessor {
				return errors.New(fmt.Sprintf("Unexpected token %s at position %d", c.val, c.pos))
			}

			l.fields = append(l.fields, *i)
			i = nil
			accessor = false
		case eof:
			return nil
		case itemDot:
			if pos == 0 {
				break
			}
			fallthrough
		default:
			return errors.New(fmt.Sprintf("Unexpected token %s at position %d", c.val, c.pos))
		}
		pos += len(c.val)
	}

	return nil
}

func (l *query) nextItem() item {
	item := <-l.items
	l.lastPos = item.pos
	return item
}

func (l *query) scanNumber() bool {
	digits := "0123456789"
	l.acceptRun(digits)
	return true
}

func isSpace(r rune) bool {
	return r == ' ' || r == '\t'
}

func isEndOfLine(r rune) bool {
	return r == '\r' || r == '\n'
}

func isAlphaNumeric(r rune) bool {
	return r == '_' || unicode.IsLetter(r) || unicode.IsDigit(r)
}

func mapValue(o interface{}, key string) (interface{}, error) {
	n, ok := o.(map[string]interface{})
	if !ok {
		return nil, errors.New("Not of type object")
	}
	p, ok := n[key]
	if !ok {
		return nil, errors.New(fmt.Sprintf("Key (%s) does not exist", key))
	}
	return p, nil
}

func indexValue(o interface{}, index int) (interface{}, error) {
	n, ok := o.([]interface{})
	if !ok {
		return nil, errors.New("Not of type array")
	}
	if index > len(n) {
		return nil, errors.New(fmt.Sprintf("Index (%d) out of range", index))
	}
	return n[index], nil
}

// Converts a query string into a *Fetch.Query.
// Fetch.Parse is similar to jq, in that in order to reference the base value,
// you must begin a query with '.'
// For example, a query string of '.' will return an entire value, a query string
// of '.foo' will return the value of key foo on the root of the value. Every
// subsequent field can be accessed through javascript-style dot/bracket notation.
// for example, .foo[0] would return the first element of array foo, and
// .["foo"][0] would do the same as well.
func Parse(input string) (*query, error) {
	l := &query{
		input:  input,
		items:  make(chan item),
		fields: []field{},
	}

	go l.run()
	err := l.runField()
	if err != nil {
		return nil, err
	}

	return l, nil
}

// Executes a *Fetch.Query on some data. Returns the result of the query.
func Run(l *query, o interface{}) (interface{}, error) {
	var err error
	for _, v := range l.fields {
		switch v.typ {
		case fieldMap:
			o, err = mapValue(o, v.key)
			if err != nil {
				return nil, err
			}
		case fieldArray:
			o, err = indexValue(o, v.index)
			if err != nil {
				return nil, err
			}
		}
	}
	return o, nil
}

// A convenience function that runs both Parse() and Run() automatically.
// It is highly recommended that you parse your query ahead of time
// with Fetch.Parse() and follow up with Fetch.Run() instead.
func Fetch(input string, obj interface{}) (interface{}, error) {
	l, err := Parse(input)
	if err != nil {
		return nil, err
	}
	return Run(l, obj)
}
<|endoftext|>


<commit_before>package sensu

import (
	"os"
	"reflect"
	"strings"
	"testing"

	"github.com/upfluence/sensu-client-go/Godeps/_workspace/src/github.com/upfluence/sensu-go/sensu/check"
	stdClient "github.com/upfluence/sensu-client-go/Godeps/_workspace/src/github.com/upfluence/sensu-go/sensu/client"
)

func validateStringParameter(
	actualRabbitMqUri string,
	expectedRabbitMqUri string,
	parameterName string,
	t *testing.T) {

	if actualRabbitMqUri != expectedRabbitMqUri {
		t.Errorf(
			"Expected %s to be \"%s\" but got \"%s\" instead!",
			parameterName,
			expectedRabbitMqUri,
			actualRabbitMqUri,
		)
	}
}

func TestRabbitMQURIDefaultValue(t *testing.T) {
	validateStringParameter(
		(&Config{}).RabbitMQURI(),
		"amqp://guest:guest@localhost:5672/%2f",
		"RabbitMQ URI",
		t,
	)
}

func TestRabbitMQURIFromEnvVar(t *testing.T) {
	expectedRabbitMqUri := "amqp://user:password@example.com:5672"

	os.Setenv("RABBITMQ_URI", expectedRabbitMqUri)

	validateStringParameter(
		(&Config{}).RabbitMQURI(),
		expectedRabbitMqUri,
		"RabbitMQ URI",
		t,
	)
}

func TestRabbitMQURIFromConfig(t *testing.T) {
	expectedRabbitMqUri := "amqp://user:password@example.com:5672"

	config := Config{config: &configPayload{RabbitMQURI: &expectedRabbitMqUri}}

	validateStringParameter(
		config.RabbitMQURI(),
		expectedRabbitMqUri,
		"RabbitMQ URI",
		t,
	)
}

var expectedClient = &stdClient.Client{
	Name:          "test_client",
	Address:       "10.0.0.42",
	Subscriptions: strings.Split("email,messenger", ","),
}

func validateClient(actualClient *stdClient.Client, t *testing.T) {
	validateStringParameter(
		actualClient.Name,
		expectedClient.Name,
		"client name",
		t,
	)

	validateStringParameter(
		actualClient.Address,
		expectedClient.Address,
		"client address",
		t,
	)

	if !reflect.DeepEqual(
		actualClient.Subscriptions,
		expectedClient.Subscriptions,
	) {

		t.Errorf(
			"Expected client subscriptions to be \"%v\" but got \"%v\" instead!",
			expectedClient.Subscriptions,
			actualClient.Subscriptions,
		)
	}
}

func TestExpectedClientFromConfig(t *testing.T) {
	config := Config{config: &configPayload{Client: expectedClient}}

	validateClient(config.Client(), t)
}

func TestExpectedClientFromEnvVars(t *testing.T) {
	os.Setenv("SENSU_CLIENT_NAME", expectedClient.Name)
	os.Setenv("SENSU_CLIENT_ADDRESS", expectedClient.Address)
	os.Setenv(
		"SENSU_CLIENT_SUBSCRIPTIONS",
		strings.Join(expectedClient.Subscriptions, ","),
	)

	validateClient((&Config{}).Client(), t)
}

func TestChecksFromConfig(t *testing.T) {
	expectedCheckCount := 2
	config := Config{
		config: &configPayload{
			Checks: []*check.Check{&check.Check{}, &check.Check{}},
		},
	}

	actualCheckCount := len(config.Checks())

	if expectedCheckCount != actualCheckCount {
		t.Errorf(
			"Expected check count to be \"%d\" but got \"%d\" instead!",
			expectedCheckCount,
			actualCheckCount,
		)
	}
}
<commit_msg>Add unit test for 202414e and do some refactoring<commit_after>package sensu

import (
	"os"
	"reflect"
	"strings"
	"testing"

	"github.com/upfluence/sensu-client-go/Godeps/_workspace/src/github.com/upfluence/sensu-go/sensu/check"
	stdClient "github.com/upfluence/sensu-client-go/Godeps/_workspace/src/github.com/upfluence/sensu-go/sensu/client"
)

func validateStringParameter(
	actualRabbitMqUri string,
	expectedRabbitMqUri string,
	parameterName string,
	t *testing.T) {

	if actualRabbitMqUri != expectedRabbitMqUri {
		t.Errorf(
			"Expected %s to be \"%s\" but got \"%s\" instead!",
			parameterName,
			expectedRabbitMqUri,
			actualRabbitMqUri,
		)
	}
}

func TestRabbitMQURIDefaultValue(t *testing.T) {
	validateStringParameter(
		(&Config{}).RabbitMQURI(),
		"amqp://guest:guest@localhost:5672/%2f",
		"RabbitMQ URI",
		t,
	)
}

func TestRabbitMQURIFromEnvVar(t *testing.T) {
	expectedRabbitMqUri := "amqp://user:password@example.com:5672"

	os.Setenv("RABBITMQ_URI", expectedRabbitMqUri)
	defer os.Unsetenv("RABBITMQ_URI")

	validateStringParameter(
		(&Config{}).RabbitMQURI(),
		expectedRabbitMqUri,
		"RabbitMQ URI",
		t,
	)
}

func TestRabbitMQURIFromConfig(t *testing.T) {
	expectedRabbitMqUri := "amqp://user:password@example.com:5672"

	config := Config{config: &configPayload{RabbitMQURI: &expectedRabbitMqUri}}

	validateStringParameter(
		config.RabbitMQURI(),
		expectedRabbitMqUri,
		"RabbitMQ URI",
		t,
	)
}

func validateClient(actualClient *stdClient.Client, expectedClient *stdClient.Client, t *testing.T) {
	validateStringParameter(
		actualClient.Name,
		expectedClient.Name,
		"client name",
		t,
	)

	validateStringParameter(
		actualClient.Address,
		expectedClient.Address,
		"client address",
		t,
	)

	if !reflect.DeepEqual(
		actualClient.Subscriptions,
		expectedClient.Subscriptions,
	) {
		t.Errorf(
			"Expected client subscriptions to be \"%#v\" but got \"%#v\" instead!",
			expectedClient.Subscriptions,
			actualClient.Subscriptions,
		)
	}
}

var dummyClient = &stdClient.Client{
	Name:          "test_client",
	Address:       "10.0.0.42",
	Subscriptions: strings.Split("email,messenger", ","),
}

func TestExpectedClientFromConfig(t *testing.T) {
	config := Config{config: &configPayload{Client: dummyClient}}

	validateClient(config.Client(), dummyClient, t)
}

func TestExpectedClientFromEnvVars(t *testing.T) {
	os.Setenv("SENSU_CLIENT_NAME", dummyClient.Name)
	defer os.Unsetenv("SENSU_CLIENT_NAME")

	os.Setenv("SENSU_CLIENT_ADDRESS", dummyClient.Address)
	defer os.Unsetenv("SENSU_CLIENT_ADDRESS")

	os.Setenv(
		"SENSU_CLIENT_SUBSCRIPTIONS",
		strings.Join(dummyClient.Subscriptions, ","),
	)
	defer os.Unsetenv("SENSU_CLIENT_SUBSCRIPTIONS")

	validateClient((&Config{}).Client(), dummyClient, t)
}

func TestExpectedClientFromEnvVarsNoSubscriptions(t *testing.T) {
	dummyClientNoSubscriptions := dummyClient
	dummyClientNoSubscriptions.Subscriptions = []string{}

	os.Setenv("SENSU_CLIENT_NAME", dummyClientNoSubscriptions.Name)
	defer os.Unsetenv("SENSU_CLIENT_NAME")

	os.Setenv("SENSU_CLIENT_ADDRESS", dummyClientNoSubscriptions.Address)
	defer os.Unsetenv("SENSU_CLIENT_ADDRESS")

	validateClient((&Config{}).Client(), dummyClientNoSubscriptions, t)
}

func TestChecksFromConfig(t *testing.T) {
	expectedCheckCount := 2
	config := Config{
		config: &configPayload{
			Checks: []*check.Check{&check.Check{}, &check.Check{}},
		},
	}

	actualCheckCount := len(config.Checks())

	if expectedCheckCount != actualCheckCount {
		t.Errorf(
			"Expected check count to be \"%d\" but got \"%d\" instead!",
			expectedCheckCount,
			actualCheckCount,
		)
	}
}
<|endoftext|>


<commit_before>/*
	Used by the Android Compile Server to interact with the cloud datastore.
*/

package util

import (
	"context"
	"errors"
	"fmt"
	"sort"
	"time"

	"cloud.google.com/go/datastore"
	"go.skia.org/infra/go/ds"
	"golang.org/x/oauth2"
	"google.golang.org/api/iterator"
	"google.golang.org/api/option"

	"go.skia.org/infra/go/sklog"
)

var (
	ErrAnotherInstanceRunningTask        = errors.New("Another instance has picked up this task")
	ErrThisInstanceRunningTask           = errors.New("This instance is already running this task")
	ErrThisInstanceOwnsTaskButNotRunning = errors.New("This instance has picked up this task but it is not running yet")
)

type AndroidCompileInstance struct {
	MirrorLastSynced     string `json:"mirror_last_synced"`
	MirrorUpdateDuration string `json:"mirror_update_duration"`
	ForceMirrorUpdate    bool   `json:"force_mirror_update"`
	Name                 string `json:"name"`
}

type CompileTask struct {
	Issue    int    `json:"issue"`
	PatchSet int    `json:"patchset"`
	Hash     string `json:"hash"`

	LunchTarget string `json:"lunch_target"`
	MMMATargets string `json:"mmma_targets"`

	Checkout string `json:"checkout"`

	Created   time.Time `json:"created"`
	Completed time.Time `json:"completed"`

	WithPatchSucceeded bool `json:"withpatch_success"`
	NoPatchSucceeded   bool `json:"nopatch_success"`

	WithPatchLog string `json:"withpatch_log"`
	NoPatchLog   string `json:"nopatch_log"`

	CompileServerInstance string `json:"compile_server_instance"`
	IsMasterBranch        bool   `json:"is_master_branch"`
	Done                  bool   `json:"done"`
	// Write Error only to Google storage and not the datastore because sometimes
	// the error can be large and cause failures when writing to datastore.
	Error        string `json:"error" datastore:"-"`
	InfraFailure bool   `json:"infra_failure"`
}

type sortTasks []*CompileTask

func (a sortTasks) Len() int      { return len(a) }
func (a sortTasks) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a sortTasks) Less(i, j int) bool {
	return a[i].Created.Before(a[j].Created)
}

// ClaimAndAddCompileTask adds the compile task to the datastore and marks it
// as being owned by the specified instance.
// The function throws the following custom errors:
// * ErrThisInstanceOwnsTaskButNotRunning - Thrown when the specified instance
//     owns the task but it is not running yet.
// * ErrThisInstanceRunningTask - Thrown when the specified instance owns the task
// *   and it is currently running.
// * ErrAnotherInstanceRunningTask - Thrown when another instance (not the specified
//     instance) owns the task.
func ClaimAndAddCompileTask(taskFromGS *CompileTask, currentInstance string) error {
	var err error
	_, err = ds.DS.RunInTransaction(context.Background(), func(tx *datastore.Transaction) error {
		var taskFromDS CompileTask
		// Use the task from GS to construct the Key and look in Datastore.
		k := GetTaskDSKey(taskFromGS.LunchTarget, taskFromGS.Issue, taskFromGS.PatchSet)
		if err := tx.Get(k, &taskFromDS); err != nil && err != datastore.ErrNoSuchEntity {
			return err
		}
		if taskFromDS.Done {
			sklog.Infof("%s exists in Datastore and is completed but there was a new request for it. Running it..", k)
		} else if taskFromDS.CompileServerInstance != "" {
			if taskFromDS.CompileServerInstance == currentInstance {
				if taskFromDS.Checkout == "" {
					sklog.Infof("%s has already been picked up by this instance but task is not running.", k)
					return ErrThisInstanceOwnsTaskButNotRunning
				} else {
					sklog.Infof("%s has already been picked up by this instance", k)
					return ErrThisInstanceRunningTask
				}
			} else {
				sklog.Infof("%s has been picked up by %s", k, taskFromDS.CompileServerInstance)
				return ErrAnotherInstanceRunningTask
			}
		}
		// Populate some taskFromGS properties before adding to datastore.
		taskFromGS.CompileServerInstance = currentInstance
		taskFromGS.Created = time.Now()
		if _, err := tx.Put(k, taskFromGS); err != nil {
			return err
		}
		return nil
	})
	return err
}

// AddUnownedCompileTask adds the task to the datastore without an owner instance.
// Task is added to the datastore if it does not already exist in the datastore or
// if it exists but is marked as completed.
func AddUnownedCompileTask(taskFromGS *CompileTask) error {
	var err error
	_, err = ds.DS.RunInTransaction(context.Background(), func(tx *datastore.Transaction) error {
		var taskFromDS CompileTask
		k := GetTaskDSKey(taskFromGS.LunchTarget, taskFromGS.Issue, taskFromGS.PatchSet)
		if err := tx.Get(k, &taskFromDS); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// If task does not exist then add it as a pending task.
				taskFromGS.Created = time.Now()
				if _, err := tx.Put(k, taskFromGS); err != nil {
					return err
				}
			} else {
				return err
			}
		}
		if taskFromDS.Done {
			// Task is in Datastore and has completed, but a new request
			// has come in so override the old task.
			taskFromGS.Created = time.Now()
			if _, err := tx.Put(k, taskFromGS); err != nil {
				return err
			}
		}
		return nil
	})
	return err
}

// GetPendingCompileTasks returns slices of unowned tasks and currently running
// (but not yet completed) tasks.
func GetPendingCompileTasks(ownedByInstance string) ([]*CompileTask, []*CompileTask, error) {
	// Pending tasks that have not been picked up by an instance yet.
	unownedPendingTasks := []*CompileTask{}
	// Pending tasks that have been picked up by an instance.
	ownedPendingTasks := []*CompileTask{}

	q := ds.NewQuery(ds.COMPILE_TASK).EventualConsistency().Filter("Done =", false)
	it := ds.DS.Run(context.TODO(), q)
	for {
		t := &CompileTask{}
		_, err := it.Next(t)
		if err == iterator.Done {
			break
		} else if err != nil {
			return nil, nil, fmt.Errorf("Failed to retrieve list of tasks: %s", err)
		}
		if t.CompileServerInstance == "" {
			unownedPendingTasks = append(unownedPendingTasks, t)
		} else {
			if ownedByInstance == "" {
				ownedPendingTasks = append(ownedPendingTasks, t)
			} else if t.CompileServerInstance == ownedByInstance {
				ownedPendingTasks = append(ownedPendingTasks, t)
			}
		}
	}
	sort.Sort(sortTasks(unownedPendingTasks))
	sort.Sort(sortTasks(ownedPendingTasks))

	return unownedPendingTasks, ownedPendingTasks, nil
}

func DatastoreInit(project, ns string, ts oauth2.TokenSource) error {
	return ds.InitWithOpt(project, ns, option.WithTokenSource(ts))
}

func UpdateInstanceInDS(ctx context.Context, hostname, mirrorLastSynced string, mirrorUpdateDuration time.Duration, forceMirrorUpdate bool) error {
	k := GetInstanceDSKey(hostname)
	i := AndroidCompileInstance{
		MirrorLastSynced:     mirrorLastSynced,
		MirrorUpdateDuration: mirrorUpdateDuration.String(),
		ForceMirrorUpdate:    forceMirrorUpdate,
		Name:                 hostname,
	}
	_, err := ds.DS.Put(ctx, k, &i)
	return err
}

func GetAllCompileInstances(ctx context.Context) ([]*AndroidCompileInstance, error) {
	var instances []*AndroidCompileInstance
	q := ds.NewQuery(ds.ANDROID_COMPILE_INSTANCES)
	_, err := ds.DS.GetAll(ctx, q, &instances)
	return instances, err
}

func SetForceMirrorUpdateOnAllInstances(ctx context.Context) error {
	var instances []*AndroidCompileInstance
	q := ds.NewQuery(ds.ANDROID_COMPILE_INSTANCES)
	if _, err := ds.DS.GetAll(ctx, q, &instances); err != nil {
		return err
	}
	for _, i := range instances {
		i.ForceMirrorUpdate = true
		if _, err := ds.DS.Put(ctx, GetInstanceDSKey(i.Name), i); err != nil {
			return err
		}
	}
	return nil
}

func GetForceMirrorUpdateBool(ctx context.Context, hostname string) (bool, error) {
	k := GetInstanceDSKey(hostname)
	var i AndroidCompileInstance
	if err := ds.DS.Get(ctx, k, &i); err != nil {
		return false, err
	}
	return i.ForceMirrorUpdate, nil
}

func GetInstanceDSKey(hostname string) *datastore.Key {
	k := ds.NewKey(ds.ANDROID_COMPILE_INSTANCES)
	k.Name = hostname
	return k
}

func GetTaskDSKey(lunchTarget string, issue, patchset int) *datastore.Key {
	k := ds.NewKey(ds.COMPILE_TASK)
	k.Name = fmt.Sprintf("%s-%d-%d", lunchTarget, issue, patchset)
	return k
}

func UpdateTaskInDS(ctx context.Context, t *CompileTask) (*datastore.Key, error) {
	k := GetTaskDSKey(t.LunchTarget, t.Issue, t.PatchSet)
	return ds.DS.Put(ctx, k, t)
}
<commit_msg>[Android compile server] Use omitempty for "Error" datastore field.<commit_after>/*
	Used by the Android Compile Server to interact with the cloud datastore.
*/

package util

import (
	"context"
	"errors"
	"fmt"
	"sort"
	"time"

	"cloud.google.com/go/datastore"
	"go.skia.org/infra/go/ds"
	"golang.org/x/oauth2"
	"google.golang.org/api/iterator"
	"google.golang.org/api/option"

	"go.skia.org/infra/go/sklog"
)

var (
	ErrAnotherInstanceRunningTask        = errors.New("Another instance has picked up this task")
	ErrThisInstanceRunningTask           = errors.New("This instance is already running this task")
	ErrThisInstanceOwnsTaskButNotRunning = errors.New("This instance has picked up this task but it is not running yet")
)

type AndroidCompileInstance struct {
	MirrorLastSynced     string `json:"mirror_last_synced"`
	MirrorUpdateDuration string `json:"mirror_update_duration"`
	ForceMirrorUpdate    bool   `json:"force_mirror_update"`
	Name                 string `json:"name"`
}

type CompileTask struct {
	Issue    int    `json:"issue"`
	PatchSet int    `json:"patchset"`
	Hash     string `json:"hash"`

	LunchTarget string `json:"lunch_target"`
	MMMATargets string `json:"mmma_targets"`

	Checkout string `json:"checkout"`

	Created   time.Time `json:"created"`
	Completed time.Time `json:"completed"`

	WithPatchSucceeded bool `json:"withpatch_success"`
	NoPatchSucceeded   bool `json:"nopatch_success"`

	WithPatchLog string `json:"withpatch_log"`
	NoPatchLog   string `json:"nopatch_log"`

	CompileServerInstance string `json:"compile_server_instance"`
	IsMasterBranch        bool   `json:"is_master_branch"`
	Done                  bool   `json:"done"`
	// Write Error only to Google storage and not the datastore because sometimes
	// the error can be large and cause failures when writing to datastore.
	Error        string `json:"error" datastore:"-,omitempty"`
	InfraFailure bool   `json:"infra_failure"`
}

type sortTasks []*CompileTask

func (a sortTasks) Len() int      { return len(a) }
func (a sortTasks) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a sortTasks) Less(i, j int) bool {
	return a[i].Created.Before(a[j].Created)
}

// ClaimAndAddCompileTask adds the compile task to the datastore and marks it
// as being owned by the specified instance.
// The function throws the following custom errors:
// * ErrThisInstanceOwnsTaskButNotRunning - Thrown when the specified instance
//     owns the task but it is not running yet.
// * ErrThisInstanceRunningTask - Thrown when the specified instance owns the task
// *   and it is currently running.
// * ErrAnotherInstanceRunningTask - Thrown when another instance (not the specified
//     instance) owns the task.
func ClaimAndAddCompileTask(taskFromGS *CompileTask, currentInstance string) error {
	var err error
	_, err = ds.DS.RunInTransaction(context.Background(), func(tx *datastore.Transaction) error {
		var taskFromDS CompileTask
		// Use the task from GS to construct the Key and look in Datastore.
		k := GetTaskDSKey(taskFromGS.LunchTarget, taskFromGS.Issue, taskFromGS.PatchSet)
		if err := tx.Get(k, &taskFromDS); err != nil && err != datastore.ErrNoSuchEntity {
			return err
		}
		if taskFromDS.Done {
			sklog.Infof("%s exists in Datastore and is completed but there was a new request for it. Running it..", k)
		} else if taskFromDS.CompileServerInstance != "" {
			if taskFromDS.CompileServerInstance == currentInstance {
				if taskFromDS.Checkout == "" {
					sklog.Infof("%s has already been picked up by this instance but task is not running.", k)
					return ErrThisInstanceOwnsTaskButNotRunning
				} else {
					sklog.Infof("%s has already been picked up by this instance", k)
					return ErrThisInstanceRunningTask
				}
			} else {
				sklog.Infof("%s has been picked up by %s", k, taskFromDS.CompileServerInstance)
				return ErrAnotherInstanceRunningTask
			}
		}
		// Populate some taskFromGS properties before adding to datastore.
		taskFromGS.CompileServerInstance = currentInstance
		taskFromGS.Created = time.Now()
		if _, err := tx.Put(k, taskFromGS); err != nil {
			return err
		}
		return nil
	})
	return err
}

// AddUnownedCompileTask adds the task to the datastore without an owner instance.
// Task is added to the datastore if it does not already exist in the datastore or
// if it exists but is marked as completed.
func AddUnownedCompileTask(taskFromGS *CompileTask) error {
	var err error
	_, err = ds.DS.RunInTransaction(context.Background(), func(tx *datastore.Transaction) error {
		var taskFromDS CompileTask
		k := GetTaskDSKey(taskFromGS.LunchTarget, taskFromGS.Issue, taskFromGS.PatchSet)
		if err := tx.Get(k, &taskFromDS); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// If task does not exist then add it as a pending task.
				taskFromGS.Created = time.Now()
				if _, err := tx.Put(k, taskFromGS); err != nil {
					return err
				}
			} else {
				return err
			}
		}
		if taskFromDS.Done {
			// Task is in Datastore and has completed, but a new request
			// has come in so override the old task.
			taskFromGS.Created = time.Now()
			if _, err := tx.Put(k, taskFromGS); err != nil {
				return err
			}
		}
		return nil
	})
	return err
}

// GetPendingCompileTasks returns slices of unowned tasks and currently running
// (but not yet completed) tasks.
func GetPendingCompileTasks(ownedByInstance string) ([]*CompileTask, []*CompileTask, error) {
	// Pending tasks that have not been picked up by an instance yet.
	unownedPendingTasks := []*CompileTask{}
	// Pending tasks that have been picked up by an instance.
	ownedPendingTasks := []*CompileTask{}

	q := ds.NewQuery(ds.COMPILE_TASK).EventualConsistency().Filter("Done =", false)
	it := ds.DS.Run(context.TODO(), q)
	for {
		t := &CompileTask{}
		_, err := it.Next(t)
		if err == iterator.Done {
			break
		} else if err != nil {
			return nil, nil, fmt.Errorf("Failed to retrieve list of tasks: %s", err)
		}
		if t.CompileServerInstance == "" {
			unownedPendingTasks = append(unownedPendingTasks, t)
		} else {
			if ownedByInstance == "" {
				ownedPendingTasks = append(ownedPendingTasks, t)
			} else if t.CompileServerInstance == ownedByInstance {
				ownedPendingTasks = append(ownedPendingTasks, t)
			}
		}
	}
	sort.Sort(sortTasks(unownedPendingTasks))
	sort.Sort(sortTasks(ownedPendingTasks))

	return unownedPendingTasks, ownedPendingTasks, nil
}

func DatastoreInit(project, ns string, ts oauth2.TokenSource) error {
	return ds.InitWithOpt(project, ns, option.WithTokenSource(ts))
}

func UpdateInstanceInDS(ctx context.Context, hostname, mirrorLastSynced string, mirrorUpdateDuration time.Duration, forceMirrorUpdate bool) error {
	k := GetInstanceDSKey(hostname)
	i := AndroidCompileInstance{
		MirrorLastSynced:     mirrorLastSynced,
		MirrorUpdateDuration: mirrorUpdateDuration.String(),
		ForceMirrorUpdate:    forceMirrorUpdate,
		Name:                 hostname,
	}
	_, err := ds.DS.Put(ctx, k, &i)
	return err
}

func GetAllCompileInstances(ctx context.Context) ([]*AndroidCompileInstance, error) {
	var instances []*AndroidCompileInstance
	q := ds.NewQuery(ds.ANDROID_COMPILE_INSTANCES)
	_, err := ds.DS.GetAll(ctx, q, &instances)
	return instances, err
}

func SetForceMirrorUpdateOnAllInstances(ctx context.Context) error {
	var instances []*AndroidCompileInstance
	q := ds.NewQuery(ds.ANDROID_COMPILE_INSTANCES)
	if _, err := ds.DS.GetAll(ctx, q, &instances); err != nil {
		return err
	}
	for _, i := range instances {
		i.ForceMirrorUpdate = true
		if _, err := ds.DS.Put(ctx, GetInstanceDSKey(i.Name), i); err != nil {
			return err
		}
	}
	return nil
}

func GetForceMirrorUpdateBool(ctx context.Context, hostname string) (bool, error) {
	k := GetInstanceDSKey(hostname)
	var i AndroidCompileInstance
	if err := ds.DS.Get(ctx, k, &i); err != nil {
		return false, err
	}
	return i.ForceMirrorUpdate, nil
}

func GetInstanceDSKey(hostname string) *datastore.Key {
	k := ds.NewKey(ds.ANDROID_COMPILE_INSTANCES)
	k.Name = hostname
	return k
}

func GetTaskDSKey(lunchTarget string, issue, patchset int) *datastore.Key {
	k := ds.NewKey(ds.COMPILE_TASK)
	k.Name = fmt.Sprintf("%s-%d-%d", lunchTarget, issue, patchset)
	return k
}

func UpdateTaskInDS(ctx context.Context, t *CompileTask) (*datastore.Key, error) {
	k := GetTaskDSKey(t.LunchTarget, t.Issue, t.PatchSet)
	return ds.DS.Put(ctx, k, t)
}
<|endoftext|>


<commit_before>//
// Copyright 2014-2020 Cristian Maglie. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// Testing code idea and fix thanks to @angri
// https://github.com/bugst/go-serial/pull/42
//

package serial

import (
	"context"
	"os/exec"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
)

func TestSerialReadAndCloseConcurrency(t *testing.T) {

	// Run this test with race detector to actually test that
	// the correct multitasking behaviour is happening.

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	cmd := exec.CommandContext(ctx, "socat", "STDIO", "pty,link=/tmp/faketty")
	require.NoError(t, cmd.Start())
	go cmd.Wait()
	// let our fake serial port node to appear
	time.Sleep(time.Millisecond * 100)

	port, err := Open("/tmp/faketty", &Mode{})
	require.NoError(t, err)
	buf := make([]byte, 100)
	go port.Read(buf)
	// let port.Read to start
	time.Sleep(time.Millisecond * 1)
	port.Close()
}

func TestDoubleCloseIsNoop(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	cmd := exec.CommandContext(ctx, "socat", "STDIO", "pty,link=/tmp/faketty")
	require.NoError(t, cmd.Start())
	go cmd.Wait()
	// let our fake serial port node to appear
	time.Sleep(time.Millisecond * 100)

	port, err := Open("/tmp/faketty", &Mode{})
	require.NoError(t, err)
	require.NoError(t, port.Close())
	require.NoError(t, port.Close())
}
<commit_msg>Wait for socat to be ready before running full test.<commit_after>//
// Copyright 2014-2020 Cristian Maglie. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//
// Testing code idea and fix thanks to @angri
// https://github.com/bugst/go-serial/pull/42
//

package serial

import (
	"context"
	"os/exec"
	"testing"
	"time"

	"github.com/stretchr/testify/require"
)

func startSocatAndWaitForPort(t *testing.T, ctx context.Context) *exec.Cmd {
	cmd := exec.CommandContext(ctx, "socat", "-D", "STDIO", "pty,link=/tmp/faketty")
	r, err := cmd.StderrPipe()
	require.NoError(t, err)
	require.NoError(t, cmd.Start())
	// Let our fake serial port node appear.
	// socat will write to stderr before starting transfer phase;
	// we don't really care what, just that it did, because then it's ready.
	buf := make([]byte, 1024)
	_, err = r.Read(buf)
	require.NoError(t, err)
	return cmd
}

func TestSerialReadAndCloseConcurrency(t *testing.T) {

	// Run this test with race detector to actually test that
	// the correct multitasking behaviour is happening.

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	cmd := startSocatAndWaitForPort(t, ctx)
	go cmd.Wait()

	port, err := Open("/tmp/faketty", &Mode{})
	require.NoError(t, err)
	buf := make([]byte, 100)
	go port.Read(buf)
	// let port.Read to start
	time.Sleep(time.Millisecond * 1)
	port.Close()
}

func TestDoubleCloseIsNoop(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	cmd := startSocatAndWaitForPort(t, ctx)
	go cmd.Wait()

	port, err := Open("/tmp/faketty", &Mode{})
	require.NoError(t, err)
	require.NoError(t, port.Close())
	require.NoError(t, port.Close())
}
<|endoftext|>


<commit_before>/* Copyright (C) 2013 CompleteDB LLC.
 *
 * This program is free software: you this.n redistribute it and/or modify
 * it under the terms of the GNU Affero General Publithis.License as
 * published by the Free Software Foundation, either version 3 of the
 * Lithis.nse, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Publithis.License for more details.
 *
 * You should have rethis.ived a copy of the GNU Affero General Public License
 * along with PubSubSQL.  If not, see <http://www.gnu.org/lithis.nses/>.
 */

package pubsubsql

import (
	"os"
	"time"
)

type Controller struct {
	network  *network
	requests chan *requestItem
	stoper   *Stoper
}

func (this *Controller) Run() {
	if !config.processCommandLine(os.Args[1:]) {
		return
	}
	// stoper
	this.stoper = NewStoper()
	// process commands
	switch config.COMMAND {
	case "start":
		this.runAsServer()
	case "cli":
		this.runAsClient()
	case "help":
		println("help")
	case "status":
		println("status")
	}
}

func (this *Controller) runAsClient() {
	client := newCli()
	client.run()
}

func (this *Controller) runAsServer() {
	// requests
	this.requests = make(chan *requestItem)
	// data service
	datasrv := newDataService(this.stoper)
	go datasrv.run()
	// router
	router := newRequestRouter(datasrv)
	router.controllerRequests = this.requests
	// network context
	context := new(networkContext)
	context.stoper = this.stoper
	context.router = router
	// network
	this.network = newNetwork(context)
	this.network.start(config.netAddress())
	info("started")
	// watch for quit input
	go this.readInput()
	// wait for command or stop event
	ok := true
	for ok {
		select {
		case <-this.stoper.GetChan():
			ok = false
		case item := <-this.requests:
			this.onCommandRequest(item)
		}
	}
	// shutdown
	this.network.stop()
	this.stoper.Stop(0)
	this.stoper.Wait(time.Millisecond * config.WAIT_MILLISECOND_SERVER_SHUTDOWN)
	info("stoped")
}

func (this *Controller) onCommandRequest(item *requestItem) {
	switch item.req.(type) {
	case *cmdStatusRequest:
		loginfo("client connection:", item.sender.connectionId, "requested server status ")
		res := &cmdStatusResponse{connections: this.network.connectionCount()}
		item.sender.send(res)
	case *cmdStopRequest:
		loginfo("client connection:", item.sender.connectionId, "requested to stop the server ")
		this.stoper.Stop(0)
	}
}

func (this *Controller) readInput() {
	// we do not join the stoper because there is no way to return from blocking readLine
	// closing Stdin does not do anything
	cin := newLineReader("q")
	for cin.readLine() {
	}
	this.stoper.Stop(0)
	debug("controller done readInput")
}
<commit_msg>controller add implement help command issue #5<commit_after>/* Copyright (C) 2013 CompleteDB LLC.
 *
 * This program is free software: you this.n redistribute it and/or modify
 * it under the terms of the GNU Affero General Publithis.License as
 * published by the Free Software Foundation, either version 3 of the
 * Lithis.nse, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Publithis.License for more details.
 *
 * You should have rethis.ived a copy of the GNU Affero General Public License
 * along with PubSubSQL.  If not, see <http://www.gnu.org/lithis.nses/>.
 */

package pubsubsql

import (
	"os"
	"time"
	"fmt"
)

type Controller struct {
	network  *network
	requests chan *requestItem
	stoper   *Stoper
}

func (this *Controller) Run() {
	if !config.processCommandLine(os.Args[1:]) {
		return
	}
	// stoper
	this.stoper = NewStoper()
	// process commands
	switch config.COMMAND {
	case "start":
		this.runAsServer()
	case "cli":
		this.runAsClient()
	case "help":
		this.helpCommand()
	}
}

func (this *Controller) helpCommand() {
	fmt.Println("")
	fmt.Println("commands:")
	fmt.Println(validCommandsUsageString())
	fmt.Println("")
	fmt.Println("options:")
	config.flags.PrintDefaults()
}

func (this *Controller) runAsClient() {
	client := newCli()
	client.run()
}

func (this *Controller) runAsServer() {
	// requests
	this.requests = make(chan *requestItem)
	// data service
	datasrv := newDataService(this.stoper)
	go datasrv.run()
	// router
	router := newRequestRouter(datasrv)
	router.controllerRequests = this.requests
	// network context
	context := new(networkContext)
	context.stoper = this.stoper
	context.router = router
	// network
	this.network = newNetwork(context)
	this.network.start(config.netAddress())
	info("started")
	// watch for quit input
	go this.readInput()
	// wait for command or stop event
	ok := true
	for ok {
		select {
		case <-this.stoper.GetChan():
			ok = false
		case item := <-this.requests:
			this.onCommandRequest(item)
		}
	}
	// shutdown
	this.network.stop()
	this.stoper.Stop(0)
	this.stoper.Wait(time.Millisecond * config.WAIT_MILLISECOND_SERVER_SHUTDOWN)
	info("stoped")
}

func (this *Controller) onCommandRequest(item *requestItem) {
	switch item.req.(type) {
	case *cmdStatusRequest:
		loginfo("client connection:", item.sender.connectionId, "requested server status ")
		res := &cmdStatusResponse{connections: this.network.connectionCount()}
		item.sender.send(res)
	case *cmdStopRequest:
		loginfo("client connection:", item.sender.connectionId, "requested to stop the server ")
		this.stoper.Stop(0)
	}
}

func (this *Controller) readInput() {
	// we do not join the stoper because there is no way to return from blocking readLine
	// closing Stdin does not do anything
	cin := newLineReader("q")
	for cin.readLine() {
	}
	this.stoper.Stop(0)
	debug("controller done readInput")
}
<|endoftext|>


<commit_before>package handler

import (
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/drone/drone/server/database"
	"github.com/drone/drone/server/pubsub"
	"github.com/drone/drone/server/session"
	"github.com/drone/drone/shared/model"
	"github.com/gorilla/pat"

	"github.com/gorilla/websocket"
)

const (
	// Time allowed to write the message to the client.
	writeWait = 10 * time.Second

	// Time allowed to read the next pong message from the client.
	pongWait = 60 * time.Second

	// Send pings to client with this period. Must be less than pongWait.
	pingPeriod = (pongWait * 9) / 10
)

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
}

type WsHandler struct {
	pubsub  *pubsub.PubSub
	commits database.CommitManager
	perms   database.PermManager
	repos   database.RepoManager
	sess    session.Session
}

func NewWsHandler(repos database.RepoManager, commits database.CommitManager, perms database.PermManager, sess session.Session, pubsub *pubsub.PubSub) *WsHandler {
	return &WsHandler{pubsub, commits, perms, repos, sess}
}

// WsUser will upgrade the connection to a Websocket and will stream
// all events to the browser pertinent to the authenticated user. If the user
// is not authenticated, only public events are streamed.
func (h *WsHandler) WsUser(w http.ResponseWriter, r *http.Request) error {
	// get the user form the session
	user := h.sess.UserCookie(r)

	// upgrade the websocket
	ws, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		return badRequest{err}
	}

	// register a channel for global events
	channel := h.pubsub.Register("_global")
	sub := channel.Subscribe()

	ticker := time.NewTicker(pingPeriod)
	defer func() {
		ticker.Stop()
		sub.Close()
		ws.Close()
	}()

	go func() {
		for {
			select {
			case msg := <-sub.Read():
				work, ok := msg.(*model.Request)
				if !ok {
					break
				}

				// user must have read access to the repository
				// in order to pass this message along
				if role := h.perms.Find(user, work.Repo); !role.Read {
					break
				}

				ws.SetWriteDeadline(time.Now().Add(writeWait))
				err := ws.WriteJSON(work)
				if err != nil {
					ws.Close()
					return
				}
			case <-sub.CloseNotify():
				ws.Close()
				return
			case <-ticker.C:
				ws.SetWriteDeadline(time.Now().Add(writeWait))
				err := ws.WriteMessage(websocket.PingMessage, []byte{})
				if err != nil {
					ws.Close()
					return
				}
			}
		}
	}()

	readWebsocket(ws)
	return nil

}

// WsConsole will upgrade the connection to a Websocket and will stream
// the build output to the browser.
func (h *WsHandler) WsConsole(w http.ResponseWriter, r *http.Request) error {
	var commitID, _ = strconv.Atoi(r.FormValue(":id"))

	commit, err := h.commits.Find(int64(commitID))
	if err != nil {
		return notFound{err}
	}
	repo, err := h.repos.Find(commit.RepoID)
	if err != nil {
		return notFound{err}
	}
	user := h.sess.UserCookie(r)
	if ok, _ := h.perms.Read(user, repo); !ok {
		return notFound{err}
	}

	// find a channel that we can subscribe to
	// and listen for stream updates.
	channel := h.pubsub.Lookup(commit.ID)
	if channel == nil {
		return notFound{}
	}
	sub := channel.Subscribe()
	defer sub.Close()

	// upgrade the websocket
	ws, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		return badRequest{err}
	}

	ticker := time.NewTicker(pingPeriod)
	defer func() {
		ticker.Stop()
		ws.Close()
	}()

	go func() {
		for {
			select {
			case msg := <-sub.Read():
				data, ok := msg.([]byte)
				if !ok {
					break
				}
				ws.SetWriteDeadline(time.Now().Add(writeWait))
				err := ws.WriteMessage(websocket.TextMessage, data)
				if err != nil {
					log.Printf("websocket for commit %d closed. Err: %s\n", commitID, err)
					ws.Close()
					return
				}
			case <-sub.CloseNotify():
				log.Printf("websocket for commit %d closed by client\n", commitID)
				ws.Close()
				return
			case <-ticker.C:
				ws.SetWriteDeadline(time.Now().Add(writeWait))
				err := ws.WriteMessage(websocket.PingMessage, []byte{})
				if err != nil {
					log.Printf("websocket for commit %d closed. Err: %s\n", commitID, err)
					ws.Close()
					return
				}
			}
		}
	}()

	readWebsocket(ws)
	return nil
}

// readWebsocket will block while reading the websocket data
func readWebsocket(ws *websocket.Conn) {
	defer ws.Close()
	ws.SetReadLimit(512)
	ws.SetReadDeadline(time.Now().Add(pongWait))
	ws.SetPongHandler(func(string) error {
		ws.SetReadDeadline(time.Now().Add(pongWait))
		return nil
	})
	for {
		_, _, err := ws.ReadMessage()
		if err != nil {
			break
		}
	}
}

// Ping is a method that is being used for internal testing and
// will be removed prior to release
func (h *WsHandler) Ping(w http.ResponseWriter, r *http.Request) error {
	channel := h.pubsub.Register("_global")
	msg := model.Request{
		Repo:   &model.Repo{ID: 1, Private: false, Host: "github.com", Owner: "drone", Name: "drone"},
		Commit: &model.Commit{ID: 1, Status: "Started", Branch: "master", Sha: "113f4917ff9174945388d86395f902cd154074cb", Message: "Remove branches by SCM hook", Author: "bradrydzewski", Gravatar: "8c58a0be77ee441bb8f8595b7f1b4e87"},
	}
	channel.Publish(&msg)
	w.WriteHeader(http.StatusOK)
	return nil
}

func (h *WsHandler) Register(r *pat.Router) {
	r.Post("/ws/ping", errorHandler(h.Ping))
	r.Get("/ws/user", errorHandler(h.WsUser))
	r.Get("/ws/stdout/{id}", errorHandler(h.WsConsole))
}
<commit_msg>removed ping method that was for testing only<commit_after>package handler

import (
	"log"
	"net/http"
	"strconv"
	"time"

	"github.com/drone/drone/server/database"
	"github.com/drone/drone/server/pubsub"
	"github.com/drone/drone/server/session"
	"github.com/drone/drone/shared/model"
	"github.com/gorilla/pat"

	"github.com/gorilla/websocket"
)

const (
	// Time allowed to write the message to the client.
	writeWait = 10 * time.Second

	// Time allowed to read the next pong message from the client.
	pongWait = 60 * time.Second

	// Send pings to client with this period. Must be less than pongWait.
	pingPeriod = (pongWait * 9) / 10
)

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
}

type WsHandler struct {
	pubsub  *pubsub.PubSub
	commits database.CommitManager
	perms   database.PermManager
	repos   database.RepoManager
	sess    session.Session
}

func NewWsHandler(repos database.RepoManager, commits database.CommitManager, perms database.PermManager, sess session.Session, pubsub *pubsub.PubSub) *WsHandler {
	return &WsHandler{pubsub, commits, perms, repos, sess}
}

// WsUser will upgrade the connection to a Websocket and will stream
// all events to the browser pertinent to the authenticated user. If the user
// is not authenticated, only public events are streamed.
func (h *WsHandler) WsUser(w http.ResponseWriter, r *http.Request) error {
	// get the user form the session
	user := h.sess.UserCookie(r)

	// upgrade the websocket
	ws, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		return badRequest{err}
	}

	// register a channel for global events
	channel := h.pubsub.Register("_global")
	sub := channel.Subscribe()

	ticker := time.NewTicker(pingPeriod)
	defer func() {
		ticker.Stop()
		sub.Close()
		ws.Close()
	}()

	go func() {
		for {
			select {
			case msg := <-sub.Read():
				work, ok := msg.(*model.Request)
				if !ok {
					break
				}

				// user must have read access to the repository
				// in order to pass this message along
				if role := h.perms.Find(user, work.Repo); !role.Read {
					break
				}

				ws.SetWriteDeadline(time.Now().Add(writeWait))
				err := ws.WriteJSON(work)
				if err != nil {
					ws.Close()
					return
				}
			case <-sub.CloseNotify():
				ws.Close()
				return
			case <-ticker.C:
				ws.SetWriteDeadline(time.Now().Add(writeWait))
				err := ws.WriteMessage(websocket.PingMessage, []byte{})
				if err != nil {
					ws.Close()
					return
				}
			}
		}
	}()

	readWebsocket(ws)
	return nil

}

// WsConsole will upgrade the connection to a Websocket and will stream
// the build output to the browser.
func (h *WsHandler) WsConsole(w http.ResponseWriter, r *http.Request) error {
	var commitID, _ = strconv.Atoi(r.FormValue(":id"))

	commit, err := h.commits.Find(int64(commitID))
	if err != nil {
		return notFound{err}
	}
	repo, err := h.repos.Find(commit.RepoID)
	if err != nil {
		return notFound{err}
	}
	user := h.sess.UserCookie(r)
	if ok, _ := h.perms.Read(user, repo); !ok {
		return notFound{err}
	}

	// find a channel that we can subscribe to
	// and listen for stream updates.
	channel := h.pubsub.Lookup(commit.ID)
	if channel == nil {
		return notFound{}
	}
	sub := channel.Subscribe()
	defer sub.Close()

	// upgrade the websocket
	ws, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		return badRequest{err}
	}

	ticker := time.NewTicker(pingPeriod)
	defer func() {
		ticker.Stop()
		ws.Close()
	}()

	go func() {
		for {
			select {
			case msg := <-sub.Read():
				data, ok := msg.([]byte)
				if !ok {
					break
				}
				ws.SetWriteDeadline(time.Now().Add(writeWait))
				err := ws.WriteMessage(websocket.TextMessage, data)
				if err != nil {
					log.Printf("websocket for commit %d closed. Err: %s\n", commitID, err)
					ws.Close()
					return
				}
			case <-sub.CloseNotify():
				log.Printf("websocket for commit %d closed by client\n", commitID)
				ws.Close()
				return
			case <-ticker.C:
				ws.SetWriteDeadline(time.Now().Add(writeWait))
				err := ws.WriteMessage(websocket.PingMessage, []byte{})
				if err != nil {
					log.Printf("websocket for commit %d closed. Err: %s\n", commitID, err)
					ws.Close()
					return
				}
			}
		}
	}()

	readWebsocket(ws)
	return nil
}

// readWebsocket will block while reading the websocket data
func readWebsocket(ws *websocket.Conn) {
	defer ws.Close()
	ws.SetReadLimit(512)
	ws.SetReadDeadline(time.Now().Add(pongWait))
	ws.SetPongHandler(func(string) error {
		ws.SetReadDeadline(time.Now().Add(pongWait))
		return nil
	})
	for {
		_, _, err := ws.ReadMessage()
		if err != nil {
			break
		}
	}
}

func (h *WsHandler) Register(r *pat.Router) {
	r.Get("/ws/user", errorHandler(h.WsUser))
	r.Get("/ws/stdout/{id}", errorHandler(h.WsConsole))
}
<|endoftext|>


<commit_before>package operation

const Version string = "0.1.3"
<commit_msg>version 0.1.4<commit_after>package operation

const Version string = "0.1.4"
<|endoftext|>


<commit_before>package utils

import (
	"strings"
)

// RoleList is a list of roles
type RoleList []string

// Len returns the length of the list
func (r RoleList) Len() int {
	return len(r)
}

// Less returns true if the item at i should be sorted
// before the item at j. It's an unstable partial ordering
// based on the number of segments, separated by "/", in
// the role name
func (r RoleList) Less(i, j int) bool {
	segsI := strings.Split(r[i], "/")
	segsJ := strings.Split(r[j], "/")
	return len(segsI) < len(segsJ)
}

// Swap the items at 2 locations in the list
func (r RoleList) Swap(i, j int) {
	r[i], r[j] = r[j], r[i]
}
<commit_msg>make roles sort alphabetically when equivalent number of segments Signed-off-by: David Lawrence <david.lawrence@docker.com> (github: endophage)<commit_after>package utils

import (
	"strings"
)

// RoleList is a list of roles
type RoleList []string

// Len returns the length of the list
func (r RoleList) Len() int {
	return len(r)
}

// Less returns true if the item at i should be sorted
// before the item at j. It's an unstable partial ordering
// based on the number of segments, separated by "/", in
// the role name
func (r RoleList) Less(i, j int) bool {
	segsI := strings.Split(r[i], "/")
	segsJ := strings.Split(r[j], "/")
	if len(segsI) == len(segsJ) {
		return strings.Compare(r[i], r[j]) == -1
	}
	return len(segsI) < len(segsJ)
}

// Swap the items at 2 locations in the list
func (r RoleList) Swap(i, j int) {
	r[i], r[j] = r[j], r[i]
}
<|endoftext|>


<commit_before>package types

import (
	"testing"
	"time"
)

// example from site: http://www.silisoftware.com/tools/date.php
func TestTime(t *testing.T) {
	ft := FileTime{0xEC2DE500, 0x01CF9BCF}
	expect := time.Unix(1404949554, 0)
	if !ft.Time().Equal(expect) {
		t.Errorf("Date type: expect %v to equal %v", ft, expect)
	}
}
<commit_msg>update<commit_after>package types

import (
	"testing"
	"time"
)

// example from site: http://www.silisoftware.com/tools/date.php
func TestTime(t *testing.T) {
	ft := FileTime{0xEC2DE500, 0x01CF9BCF}
	expect := time.Unix(1404949554, 0)
	if !ft.Time().Equal(expect) {
		t.Errorf("Date type: expect %v to equal %v", ft.Time(), expect)
	}
}
<|endoftext|>


<commit_before>package dbs

import (
	"database/sql"
	"os"
	"sync"
	"testing"
	"time"

	dockertest "gopkg.in/ory-am/dockertest.v2"
)

var (
	testMySQLImagePull sync.Once
)

func prepareMySQLTestContainer(t *testing.T) (cid dockertest.ContainerID, retURL string) {
	if os.Getenv("MYSQL_URL") != "" {
		return "", os.Getenv("MYSQL_URL")
	}

	// Without this the checks for whether the container has started seem to
	// never actually pass. There's really no reason to expose the test
	// containers, so don't.
	dockertest.BindDockerToLocalhost = "yep"

	testImagePull.Do(func() {
		dockertest.Pull("mysql")
	})

	cid, connErr := dockertest.ConnectToMySQL(60, 500*time.Millisecond, func(connURL string) bool {
		// This will cause a validation to run
		connProducer := &sqlConnectionProducer{}
		connProducer.ConnectionURL = connURL
		connProducer.config = &DatabaseConfig{
			DatabaseType: mySQLTypeName,
		}

		conn, err := connProducer.connection()
		if err != nil {
			return false
		}
		if err := conn.(*sql.DB).Ping(); err != nil {
			return false
		}

		connProducer.Close()

		retURL = connURL
		return true
	})

	if connErr != nil {
		t.Fatalf("could not connect to database: %v", connErr)
	}

	return
}

func TestMySQL_Initialize(t *testing.T) {
	cid, connURL := prepareMySQLTestContainer(t)
	if cid != "" {
		defer cleanupTestContainer(t, cid)
	}

	conf := &DatabaseConfig{
		DatabaseType: mySQLTypeName,
		ConnectionDetails: map[string]interface{}{
			"connection_url": connURL,
		},
	}

	dbRaw, err := BuiltinFactory(conf, nil)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Deconsturct the middleware chain to get the underlying mysql object
	dbMetrics := dbRaw.(*databaseMetricsMiddleware)
	db := dbMetrics.next.(*MySQL)

	err = dbRaw.Initialize(conf.ConnectionDetails)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	connProducer := db.ConnectionProducer.(*sqlConnectionProducer)
	if !connProducer.initalized {
		t.Fatal("Database should be initalized")
	}

	err = dbRaw.Close()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if connProducer.db != nil {
		t.Fatal("db object should be nil")
	}
}

func TestMySQL_CreateUser(t *testing.T) {
	cid, connURL := prepareMySQLTestContainer(t)
	if cid != "" {
		defer cleanupTestContainer(t, cid)
	}

	conf := &DatabaseConfig{
		DatabaseType: mySQLTypeName,
		ConnectionDetails: map[string]interface{}{
			"connection_url": connURL,
		},
	}

	db, err := BuiltinFactory(conf, nil)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	err = db.Initialize(conf.ConnectionDetails)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	username, err := db.GenerateUsername("test")
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	password, err := db.GeneratePassword()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	expiration, err := db.GenerateExpiration(time.Minute)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Test with no configured Creation Statememt
	err = db.CreateUser(Statements{}, username, password, expiration)
	if err == nil {
		t.Fatal("Expected error when no creation statement is provided")
	}

	statements := Statements{
		CreationStatements: testMySQLRoleWildCard,
	}

	err = db.CreateUser(statements, username, password, expiration)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	username, err = db.GenerateUsername("test")
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	password, err = db.GeneratePassword()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	expiration, err = db.GenerateExpiration(time.Minute)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	statements.CreationStatements = testMySQLRoleHost
	err = db.CreateUser(statements, username, password, expiration)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	username, err = db.GenerateUsername("test")
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	password, err = db.GeneratePassword()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	expiration, err = db.GenerateExpiration(time.Minute)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	/*	statements.CreationStatements = testBlockStatementRole
		err = db.CreateUser(statements, username, password, expiration)
		if err != nil {
			t.Fatalf("err: %s", err)
		}*/
}

func TestMySQL_RenewUser(t *testing.T) {
	cid, connURL := prepareMySQLTestContainer(t)
	if cid != "" {
		defer cleanupTestContainer(t, cid)
	}

	conf := &DatabaseConfig{
		DatabaseType: mySQLTypeName,
		ConnectionDetails: map[string]interface{}{
			"connection_url": connURL,
		},
	}

	db, err := BuiltinFactory(conf, nil)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	err = db.Initialize(conf.ConnectionDetails)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	username, err := db.GenerateUsername("test")
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	password, err := db.GeneratePassword()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	expiration, err := db.GenerateExpiration(time.Minute)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	statements := Statements{
		CreationStatements: testMySQLRoleWildCard,
	}

	err = db.CreateUser(statements, username, password, expiration)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	expiration, err = db.GenerateExpiration(time.Minute)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	err = db.RenewUser(statements, username, expiration)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
}

func TestMySQL_RevokeUser(t *testing.T) {
	cid, connURL := prepareMySQLTestContainer(t)
	if cid != "" {
		defer cleanupTestContainer(t, cid)
	}

	conf := &DatabaseConfig{
		DatabaseType: mySQLTypeName,
		ConnectionDetails: map[string]interface{}{
			"connection_url": connURL,
		},
	}

	db, err := BuiltinFactory(conf, nil)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	err = db.Initialize(conf.ConnectionDetails)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	username, err := db.GenerateUsername("test")
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	password, err := db.GeneratePassword()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	expiration, err := db.GenerateExpiration(time.Minute)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	statements := Statements{
		CreationStatements: testMySQLRoleWildCard,
	}

	err = db.CreateUser(statements, username, password, expiration)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	expiration, err = db.GenerateExpiration(time.Minute)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Test default revoke statememts
	err = db.RevokeUser(statements, username)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	username, err = db.GenerateUsername("test")
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	password, err = db.GeneratePassword()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	expiration, err = db.GenerateExpiration(time.Minute)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	statements.CreationStatements = testMySQLRoleHost
	err = db.CreateUser(statements, username, password, expiration)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Test custom revoke statements
	statements.RevocationStatements = testMySQLRevocationSQL
	err = db.RevokeUser(statements, username)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

}

const testMySQLRoleWildCard = `
CREATE USER '{{name}}'@'%' IDENTIFIED BY '{{password}}';
GRANT SELECT ON *.* TO '{{name}}'@'%';
`
const testMySQLRoleHost = `
CREATE USER '{{name}}'@'10.1.1.2' IDENTIFIED BY '{{password}}';
GRANT SELECT ON *.* TO '{{name}}'@'10.1.1.2';
`
const testMySQLRevocationSQL = `
REVOKE ALL PRIVILEGES, GRANT OPTION FROM '{{name}}'@'10.1.1.2';
DROP USER '{{name}}'@'10.1.1.2';
`
<commit_msg>Remove unsused code block<commit_after>package dbs

import (
	"database/sql"
	"os"
	"sync"
	"testing"
	"time"

	dockertest "gopkg.in/ory-am/dockertest.v2"
)

var (
	testMySQLImagePull sync.Once
)

func prepareMySQLTestContainer(t *testing.T) (cid dockertest.ContainerID, retURL string) {
	if os.Getenv("MYSQL_URL") != "" {
		return "", os.Getenv("MYSQL_URL")
	}

	// Without this the checks for whether the container has started seem to
	// never actually pass. There's really no reason to expose the test
	// containers, so don't.
	dockertest.BindDockerToLocalhost = "yep"

	testImagePull.Do(func() {
		dockertest.Pull("mysql")
	})

	cid, connErr := dockertest.ConnectToMySQL(60, 500*time.Millisecond, func(connURL string) bool {
		// This will cause a validation to run
		connProducer := &sqlConnectionProducer{}
		connProducer.ConnectionURL = connURL
		connProducer.config = &DatabaseConfig{
			DatabaseType: mySQLTypeName,
		}

		conn, err := connProducer.connection()
		if err != nil {
			return false
		}
		if err := conn.(*sql.DB).Ping(); err != nil {
			return false
		}

		connProducer.Close()

		retURL = connURL
		return true
	})

	if connErr != nil {
		t.Fatalf("could not connect to database: %v", connErr)
	}

	return
}

func TestMySQL_Initialize(t *testing.T) {
	cid, connURL := prepareMySQLTestContainer(t)
	if cid != "" {
		defer cleanupTestContainer(t, cid)
	}

	conf := &DatabaseConfig{
		DatabaseType: mySQLTypeName,
		ConnectionDetails: map[string]interface{}{
			"connection_url": connURL,
		},
	}

	dbRaw, err := BuiltinFactory(conf, nil)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Deconsturct the middleware chain to get the underlying mysql object
	dbMetrics := dbRaw.(*databaseMetricsMiddleware)
	db := dbMetrics.next.(*MySQL)

	err = dbRaw.Initialize(conf.ConnectionDetails)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	connProducer := db.ConnectionProducer.(*sqlConnectionProducer)
	if !connProducer.initalized {
		t.Fatal("Database should be initalized")
	}

	err = dbRaw.Close()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	if connProducer.db != nil {
		t.Fatal("db object should be nil")
	}
}

func TestMySQL_CreateUser(t *testing.T) {
	cid, connURL := prepareMySQLTestContainer(t)
	if cid != "" {
		defer cleanupTestContainer(t, cid)
	}

	conf := &DatabaseConfig{
		DatabaseType: mySQLTypeName,
		ConnectionDetails: map[string]interface{}{
			"connection_url": connURL,
		},
	}

	db, err := BuiltinFactory(conf, nil)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	err = db.Initialize(conf.ConnectionDetails)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	username, err := db.GenerateUsername("test")
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	password, err := db.GeneratePassword()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	expiration, err := db.GenerateExpiration(time.Minute)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Test with no configured Creation Statememt
	err = db.CreateUser(Statements{}, username, password, expiration)
	if err == nil {
		t.Fatal("Expected error when no creation statement is provided")
	}

	statements := Statements{
		CreationStatements: testMySQLRoleWildCard,
	}

	err = db.CreateUser(statements, username, password, expiration)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	username, err = db.GenerateUsername("test")
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	password, err = db.GeneratePassword()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	expiration, err = db.GenerateExpiration(time.Minute)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	statements.CreationStatements = testMySQLRoleHost
	err = db.CreateUser(statements, username, password, expiration)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	username, err = db.GenerateUsername("test")
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	password, err = db.GeneratePassword()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	expiration, err = db.GenerateExpiration(time.Minute)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
}

func TestMySQL_RenewUser(t *testing.T) {
	cid, connURL := prepareMySQLTestContainer(t)
	if cid != "" {
		defer cleanupTestContainer(t, cid)
	}

	conf := &DatabaseConfig{
		DatabaseType: mySQLTypeName,
		ConnectionDetails: map[string]interface{}{
			"connection_url": connURL,
		},
	}

	db, err := BuiltinFactory(conf, nil)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	err = db.Initialize(conf.ConnectionDetails)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	username, err := db.GenerateUsername("test")
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	password, err := db.GeneratePassword()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	expiration, err := db.GenerateExpiration(time.Minute)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	statements := Statements{
		CreationStatements: testMySQLRoleWildCard,
	}

	err = db.CreateUser(statements, username, password, expiration)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	expiration, err = db.GenerateExpiration(time.Minute)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	err = db.RenewUser(statements, username, expiration)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
}

func TestMySQL_RevokeUser(t *testing.T) {
	cid, connURL := prepareMySQLTestContainer(t)
	if cid != "" {
		defer cleanupTestContainer(t, cid)
	}

	conf := &DatabaseConfig{
		DatabaseType: mySQLTypeName,
		ConnectionDetails: map[string]interface{}{
			"connection_url": connURL,
		},
	}

	db, err := BuiltinFactory(conf, nil)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	err = db.Initialize(conf.ConnectionDetails)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	username, err := db.GenerateUsername("test")
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	password, err := db.GeneratePassword()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	expiration, err := db.GenerateExpiration(time.Minute)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	statements := Statements{
		CreationStatements: testMySQLRoleWildCard,
	}

	err = db.CreateUser(statements, username, password, expiration)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	expiration, err = db.GenerateExpiration(time.Minute)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Test default revoke statememts
	err = db.RevokeUser(statements, username)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	username, err = db.GenerateUsername("test")
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	password, err = db.GeneratePassword()
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	expiration, err = db.GenerateExpiration(time.Minute)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	statements.CreationStatements = testMySQLRoleHost
	err = db.CreateUser(statements, username, password, expiration)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Test custom revoke statements
	statements.RevocationStatements = testMySQLRevocationSQL
	err = db.RevokeUser(statements, username)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

}

const testMySQLRoleWildCard = `
CREATE USER '{{name}}'@'%' IDENTIFIED BY '{{password}}';
GRANT SELECT ON *.* TO '{{name}}'@'%';
`
const testMySQLRoleHost = `
CREATE USER '{{name}}'@'10.1.1.2' IDENTIFIED BY '{{password}}';
GRANT SELECT ON *.* TO '{{name}}'@'10.1.1.2';
`
const testMySQLRevocationSQL = `
REVOKE ALL PRIVILEGES, GRANT OPTION FROM '{{name}}'@'10.1.1.2';
DROP USER '{{name}}'@'10.1.1.2';
`
<|endoftext|>


<commit_before>package test

import (
	"errors"
	"strings"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestDataSource_dataSourceCount(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers: testAccProviders,
		CheckDestroy: func(s *terraform.State) error {
			return nil
		},
		Steps: []resource.TestStep{
			{
				Config: strings.TrimSpace(`
data "test_data_source" "test" {
  count = 3
  input = "count-${count.index}"
}

resource "test_resource" "foo" {
  required     = "yep"
  required_map = {
    key = "value"
  }

  list = ["${data.test_data_source.test.*.output}"]
}
				`),
				Check: func(s *terraform.State) error {
					res, hasRes := s.RootModule().Resources["test_resource.foo"]
					if !hasRes {
						return errors.New("No test_resource.foo in state")
					}
					if res.Primary.Attributes["list.#"] != "3" {
						return errors.New("Wrong list.#, expected 3")
					}
					if res.Primary.Attributes["list.0"] != "count-0" {
						return errors.New("Wrong list.0, expected count-0")
					}
					if res.Primary.Attributes["list.1"] != "count-1" {
						return errors.New("Wrong list.0, expected count-1")
					}
					if res.Primary.Attributes["list.2"] != "count-2" {
						return errors.New("Wrong list.0, expected count-2")
					}
					return nil
				},
			},
		},
	})
}
<commit_msg>providers/test: additional testing via integration tests<commit_after>package test

import (
	"errors"
	"fmt"
	"strings"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestDataSource_dataSourceCount(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers: testAccProviders,
		CheckDestroy: func(s *terraform.State) error {
			return nil
		},
		Steps: []resource.TestStep{
			{
				Config: strings.TrimSpace(`
data "test_data_source" "test" {
  count = 3
  input = "count-${count.index}"
}

resource "test_resource" "foo" {
  required     = "yep"
  required_map = {
    key = "value"
  }

  list = ["${data.test_data_source.test.*.output}"]
}
				`),
				Check: func(s *terraform.State) error {
					res, hasRes := s.RootModule().Resources["test_resource.foo"]
					if !hasRes {
						return errors.New("No test_resource.foo in state")
					}
					if res.Primary.Attributes["list.#"] != "3" {
						return errors.New("Wrong list.#, expected 3")
					}
					if res.Primary.Attributes["list.0"] != "count-0" {
						return errors.New("Wrong list.0, expected count-0")
					}
					if res.Primary.Attributes["list.1"] != "count-1" {
						return errors.New("Wrong list.0, expected count-1")
					}
					if res.Primary.Attributes["list.2"] != "count-2" {
						return errors.New("Wrong list.0, expected count-2")
					}
					return nil
				},
			},
		},
	})
}

// Test that the output of a data source can be used as the value for
// a "count" in a real resource. This would fail with "count cannot be computed"
// at some point.
func TestDataSource_valueAsResourceCount(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		Providers: testAccProviders,
		CheckDestroy: func(s *terraform.State) error {
			return nil
		},
		Steps: []resource.TestStep{
			{
				Config: strings.TrimSpace(`
data "test_data_source" "test" {
  input = "4"
}

resource "test_resource" "foo" {
  count = "${data.test_data_source.test.output}"

  required     = "yep"
  required_map = {
    key = "value"
  }
}
				`),
				Check: func(s *terraform.State) error {
					count := 0
					for k, _ := range s.RootModule().Resources {
						if strings.HasPrefix(k, "test_resource.foo.") {
							count++
						}
					}

					if count != 4 {
						return fmt.Errorf("bad count: %d", count)
					}
					return nil
				},
			},
		},
	})
}
<|endoftext|>