<commit_before>package topicfeed

import (
	"encoding/json"
	"fmt"
	"socialapi/models"

	"github.com/koding/bongo"
	"github.com/koding/logging"
	"github.com/kylemcc/twitter-text-go/extract"
	"github.com/streadway/amqp"

	verbalexpressions "github.com/VerbalExpressions/GoVerbalExpressions"
)

// s := "naber #foo hede #bar dede gel # baz #123 #-`3sdf"
// will find [foo, bar, 123]
// will not find [' baz', '-`3sdf']
// here is the regex -> (?m)(?:#)(\w+)
var topicRegex = verbalexpressions.New().
	Find("#").
	BeginCapture().
	Word().
	EndCapture().
	Regex()

// extend this regex with https://github.com/twitter/twitter-text-rb/blob/eacf388136891eb316f1c110da8898efb8b54a38/lib/twitter-text/regex.rb
// to support all languages

type Controller struct {
	log logging.Logger
}

func New(log logging.Logger) *Controller {
	return &Controller{
		log: log,
	}
}

func (t *Controller) DefaultErrHandler(delivery amqp.Delivery, err error) bool {
	if delivery.Redelivered {
		t.log.Error("Redelivered message gave error again, putting to maintenance queue", err)
		delivery.Ack(false)
		return true
	}

	t.log.Error("an error occurred putting message back to queue", err)
	delivery.Nack(false, true)
	return false
}

func (f *Controller) MessageSaved(data *models.ChannelMessage) error {
	if res, _ := isEligible(data); !res {
		return nil
	}

	topics := extractTopics(data.Body)
	if len(topics) == 0 {
		return nil
	}

	c, err := models.ChannelById(data.InitialChannelId)
	if err != nil {
		f.log.Error("Error on models.ChannelById", data.InitialChannelId, err)
		return err
	}

	return ensureChannelMessages(c, data, topics)
}

func ensureChannelMessages(parentChannel *models.Channel, data *models.ChannelMessage, topics []string) error {
	for _, topic := range topics {
		tc, err := fetchTopicChannel(parentChannel.GroupName, topic)
		if err != nil && err != bongo.RecordNotFound {
			return err
		}

		if err == bongo.RecordNotFound {
			tc, err = createTopicChannel(data.AccountId, parentChannel.GroupName, topic, parentChannel.PrivacyConstant)
			if err != nil {
				return err
			}
		}

		_, err = tc.EnsureMessage(data.Id, true)
		// safely skip
		if err == models.ErrMessageAlreadyInTheChannel {
			continue
		}

		if err != nil {
			return err
		}
	}

	return nil
}

func extractTopics(body string) []string {
	flattened := make([]string, 0)

	// extract twitter style hashtags
	res := extract.ExtractHashtags(body)
	if res == nil {
		return flattened
	}

	topics := map[string]struct{}{}
	// remove duplicate tag usages
	for _, e := range res {
		if hashTag, ok := e.Hashtag(); ok {
			topics[hashTag] = struct{}{}
		}
	}

	// filter unwanted topics
	topics = filterTopics(topics)

	for topic := range topics {
		flattened = append(flattened, topic)
	}

	return flattened
}

func filterTopics(topics map[string]struct{}) map[string]struct{} {
	blacklistedTopics := []string{
		// public topic is used for group channel, if user adds `public` tag
		// into the message, do not try to add it to the group channel again
		"public",
	}

	filteredTopics := make(map[string]struct{})

	for topic, _ := range topics {
		blacklisted := false
		// check if the topic is in blacklisted topics
		for _, blacklistedTopic := range blacklistedTopics {
			if topic == blacklistedTopic {
				blacklisted = true
				break
			}
		}

		// merge -not blacklisted- topics
		if !blacklisted {
			filteredTopics[topic] = struct{}{}
		}
	}

	return filteredTopics

}
func (f *Controller) MessageUpdated(data *models.ChannelMessage) error {
	if res, _ := isEligible(data); !res {
		return nil
	}

	f.log.Debug("udpate message %s", data.Id)
	// fetch message's current topics from the db
	channels, err := fetchMessageChannels(data.Id)
	if err != nil {
		return err
	}

	// get current topics from
	topics := extractTopics(data.Body)
	if topics == nil {
		return nil
	}

	// if message and the topics dont have any item, we can safely return
	if len(channels) == 0 && len(topics) == 0 {
		return nil
	}

	excludedChannelId := data.InitialChannelId

	res := getTopicDiff(channels, topics, excludedChannelId)

	// add messages
	if len(res["added"]) > 0 {
		initialChannel, err := models.ChannelById(data.InitialChannelId)
		if err != nil {
			return err
		}

		if err := ensureChannelMessages(initialChannel, data, res["added"]); err != nil {
			return err
		}
	}

	// delete messages
	if len(res["deleted"]) > 0 {
		if err := deleteChannelMessages(channels, data, res["deleted"]); err != nil {
			return err
		}
	}

	return nil
}

func deleteChannelMessages(channels []models.Channel, data *models.ChannelMessage, toBeDeletedTopics []string) error {
	for _, channel := range channels {
		for _, topic := range toBeDeletedTopics {
			if channel.Name != topic {
				continue
			}

			cml := models.NewChannelMessageList()
			selector := map[string]interface{}{
				"message_id": data.Id,
				"channel_id": channel.Id,
			}

			if err := cml.DeleteMessagesBySelector(selector); err != nil {
				return err
			}
		}
	}
	return nil
}

func fetchMessageChannels(messageId int64) ([]models.Channel, error) {
	cml := models.NewChannelMessageList()
	return cml.FetchMessageChannels(messageId)
}

func getTopicDiff(channels []models.Channel, topics []string, excludedChannelId int64) map[string][]string {
	res := make(map[string][]string)

	// aggregate all channel names into map
	channelNames := map[string]struct{}{}
	for _, channel := range channels {
		if excludedChannelId != channel.GetId() {
			channelNames[channel.Name] = struct{}{}
		}
	}

	// range over new topics, bacause we are gonna remove
	// unused channels
	for _, topic := range topics {
		found := false
		for channelName := range channelNames {
			if channelName == topic {
				found = true
			}
		}
		if !found {
			res["added"] = append(res["added"], topic)
		} else {
			// if we have topic in channels
			// do remove it because at the end we are gonna mark
			// channels as deleted which are still in channelNames
			delete(channelNames, topic)
		}
	}
	// flatten the deleted channel names
	for channelName := range channelNames {
		res["deleted"] = append(res["deleted"], channelName)
	}

	return res
}

func (f *Controller) MessageDeleted(data *models.ChannelMessage) error {
	if res, _ := isEligible(data); !res {
		return nil
	}

	cml := models.NewChannelMessageList()
	selector := map[string]interface{}{
		"message_id": data.Id,
	}

	if err := cml.DeleteMessagesBySelector(selector); err != nil {
		return err
	}
	return nil
}

func mapMessage(data []byte) (*models.ChannelMessage, error) {
	cm := models.NewChannelMessage()
	if err := json.Unmarshal(data, cm); err != nil {
		return nil, err
	}

	return cm, nil
}

func isEligible(cm *models.ChannelMessage) (bool, error) {
	if cm.InitialChannelId == 0 {
		return false, nil
	}

	if cm.TypeConstant != models.ChannelMessage_TYPE_POST {
		return false, nil
	}

	return true, nil
}

// todo add caching here
func fetchTopicChannel(groupName, channelName string) (*models.Channel, error) {
	c := models.NewChannel()

	selector := map[string]interface{}{
		"group_name":    groupName,
		"name":          channelName,
		"type_constant": models.Channel_TYPE_TOPIC,
	}

	err := c.One(bongo.NewQS(selector))
	if err != nil {
		return nil, err
	}

	return c, nil
}

func createTopicChannel(creatorId int64, groupName, channelName, privacy string) (*models.Channel, error) {
	c := models.NewChannel()
	c.Name = channelName
	c.CreatorId = creatorId
	c.GroupName = groupName
	c.Purpose = fmt.Sprintf("Channel for %s topic", channelName)
	c.TypeConstant = models.Channel_TYPE_TOPIC
	c.PrivacyConstant = privacy
	if err := c.Create(); err != nil {
		return nil, err
	}

	return c, nil
}
<commit_msg>topicfeed: fix error message format<commit_after>package topicfeed

import (
	"encoding/json"
	"fmt"
	"socialapi/models"

	"github.com/koding/bongo"
	"github.com/koding/logging"
	"github.com/kylemcc/twitter-text-go/extract"
	"github.com/streadway/amqp"

	verbalexpressions "github.com/VerbalExpressions/GoVerbalExpressions"
)

// s := "naber #foo hede #bar dede gel # baz #123 #-`3sdf"
// will find [foo, bar, 123]
// will not find [' baz', '-`3sdf']
// here is the regex -> (?m)(?:#)(\w+)
var topicRegex = verbalexpressions.New().
	Find("#").
	BeginCapture().
	Word().
	EndCapture().
	Regex()

// extend this regex with https://github.com/twitter/twitter-text-rb/blob/eacf388136891eb316f1c110da8898efb8b54a38/lib/twitter-text/regex.rb
// to support all languages

type Controller struct {
	log logging.Logger
}

func New(log logging.Logger) *Controller {
	return &Controller{
		log: log,
	}
}

func (t *Controller) DefaultErrHandler(delivery amqp.Delivery, err error) bool {
	if delivery.Redelivered {
		t.log.Error("Redelivered message gave error again, putting to maintenance queue: %s", err)
		delivery.Ack(false)
		return true
	}

	t.log.Error("an error occurred putting message back to queue: %s", err)
	delivery.Nack(false, true)
	return false
}

func (f *Controller) MessageSaved(data *models.ChannelMessage) error {
	if res, _ := isEligible(data); !res {
		return nil
	}

	topics := extractTopics(data.Body)
	if len(topics) == 0 {
		return nil
	}

	c, err := models.ChannelById(data.InitialChannelId)
	if err != nil {
		f.log.Error("Error on models.ChannelById", data.InitialChannelId, err)
		return err
	}

	return ensureChannelMessages(c, data, topics)
}

func ensureChannelMessages(parentChannel *models.Channel, data *models.ChannelMessage, topics []string) error {
	for _, topic := range topics {
		tc, err := fetchTopicChannel(parentChannel.GroupName, topic)
		if err != nil && err != bongo.RecordNotFound {
			return err
		}

		if err == bongo.RecordNotFound {
			tc, err = createTopicChannel(data.AccountId, parentChannel.GroupName, topic, parentChannel.PrivacyConstant)
			if err != nil {
				return err
			}
		}

		_, err = tc.EnsureMessage(data.Id, true)
		// safely skip
		if err == models.ErrMessageAlreadyInTheChannel {
			continue
		}

		if err != nil {
			return err
		}
	}

	return nil
}

func extractTopics(body string) []string {
	flattened := make([]string, 0)

	// extract twitter style hashtags
	res := extract.ExtractHashtags(body)
	if res == nil {
		return flattened
	}

	topics := map[string]struct{}{}
	// remove duplicate tag usages
	for _, e := range res {
		if hashTag, ok := e.Hashtag(); ok {
			topics[hashTag] = struct{}{}
		}
	}

	// filter unwanted topics
	topics = filterTopics(topics)

	for topic := range topics {
		flattened = append(flattened, topic)
	}

	return flattened
}

func filterTopics(topics map[string]struct{}) map[string]struct{} {
	blacklistedTopics := []string{
		// public topic is used for group channel, if user adds `public` tag
		// into the message, do not try to add it to the group channel again
		"public",
	}

	filteredTopics := make(map[string]struct{})

	for topic, _ := range topics {
		blacklisted := false
		// check if the topic is in blacklisted topics
		for _, blacklistedTopic := range blacklistedTopics {
			if topic == blacklistedTopic {
				blacklisted = true
				break
			}
		}

		// merge -not blacklisted- topics
		if !blacklisted {
			filteredTopics[topic] = struct{}{}
		}
	}

	return filteredTopics

}
func (f *Controller) MessageUpdated(data *models.ChannelMessage) error {
	if res, _ := isEligible(data); !res {
		return nil
	}

	f.log.Debug("udpate message %s", data.Id)
	// fetch message's current topics from the db
	channels, err := fetchMessageChannels(data.Id)
	if err != nil {
		return err
	}

	// get current topics from
	topics := extractTopics(data.Body)
	if topics == nil {
		return nil
	}

	// if message and the topics dont have any item, we can safely return
	if len(channels) == 0 && len(topics) == 0 {
		return nil
	}

	excludedChannelId := data.InitialChannelId

	res := getTopicDiff(channels, topics, excludedChannelId)

	// add messages
	if len(res["added"]) > 0 {
		initialChannel, err := models.ChannelById(data.InitialChannelId)
		if err != nil {
			return err
		}

		if err := ensureChannelMessages(initialChannel, data, res["added"]); err != nil {
			return err
		}
	}

	// delete messages
	if len(res["deleted"]) > 0 {
		if err := deleteChannelMessages(channels, data, res["deleted"]); err != nil {
			return err
		}
	}

	return nil
}

func deleteChannelMessages(channels []models.Channel, data *models.ChannelMessage, toBeDeletedTopics []string) error {
	for _, channel := range channels {
		for _, topic := range toBeDeletedTopics {
			if channel.Name != topic {
				continue
			}

			cml := models.NewChannelMessageList()
			selector := map[string]interface{}{
				"message_id": data.Id,
				"channel_id": channel.Id,
			}

			if err := cml.DeleteMessagesBySelector(selector); err != nil {
				return err
			}
		}
	}
	return nil
}

func fetchMessageChannels(messageId int64) ([]models.Channel, error) {
	cml := models.NewChannelMessageList()
	return cml.FetchMessageChannels(messageId)
}

func getTopicDiff(channels []models.Channel, topics []string, excludedChannelId int64) map[string][]string {
	res := make(map[string][]string)

	// aggregate all channel names into map
	channelNames := map[string]struct{}{}
	for _, channel := range channels {
		if excludedChannelId != channel.GetId() {
			channelNames[channel.Name] = struct{}{}
		}
	}

	// range over new topics, bacause we are gonna remove
	// unused channels
	for _, topic := range topics {
		found := false
		for channelName := range channelNames {
			if channelName == topic {
				found = true
			}
		}
		if !found {
			res["added"] = append(res["added"], topic)
		} else {
			// if we have topic in channels
			// do remove it because at the end we are gonna mark
			// channels as deleted which are still in channelNames
			delete(channelNames, topic)
		}
	}
	// flatten the deleted channel names
	for channelName := range channelNames {
		res["deleted"] = append(res["deleted"], channelName)
	}

	return res
}

func (f *Controller) MessageDeleted(data *models.ChannelMessage) error {
	if res, _ := isEligible(data); !res {
		return nil
	}

	cml := models.NewChannelMessageList()
	selector := map[string]interface{}{
		"message_id": data.Id,
	}

	if err := cml.DeleteMessagesBySelector(selector); err != nil {
		return err
	}
	return nil
}

func mapMessage(data []byte) (*models.ChannelMessage, error) {
	cm := models.NewChannelMessage()
	if err := json.Unmarshal(data, cm); err != nil {
		return nil, err
	}

	return cm, nil
}

func isEligible(cm *models.ChannelMessage) (bool, error) {
	if cm.InitialChannelId == 0 {
		return false, nil
	}

	if cm.TypeConstant != models.ChannelMessage_TYPE_POST {
		return false, nil
	}

	return true, nil
}

// todo add caching here
func fetchTopicChannel(groupName, channelName string) (*models.Channel, error) {
	c := models.NewChannel()

	selector := map[string]interface{}{
		"group_name":    groupName,
		"name":          channelName,
		"type_constant": models.Channel_TYPE_TOPIC,
	}

	err := c.One(bongo.NewQS(selector))
	if err != nil {
		return nil, err
	}

	return c, nil
}

func createTopicChannel(creatorId int64, groupName, channelName, privacy string) (*models.Channel, error) {
	c := models.NewChannel()
	c.Name = channelName
	c.CreatorId = creatorId
	c.GroupName = groupName
	c.Purpose = fmt.Sprintf("Channel for %s topic", channelName)
	c.TypeConstant = models.Channel_TYPE_TOPIC
	c.PrivacyConstant = privacy
	if err := c.Create(); err != nil {
		return nil, err
	}

	return c, nil
}
<|endoftext|>


<commit_before>package gitmedia

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

const Version = "0.2.1"

var (
	LargeSizeThreshold = 5 * 1024 * 1024
	TempDir            = filepath.Join(os.TempDir(), "git-media")
	UserAgent          string
	LocalWorkingDir    string
	LocalGitDir        string
	LocalMediaDir      string
	LocalLogDir        string
	checkedTempDir     string
)

func TempFile(prefix string) (*os.File, error) {
	if checkedTempDir != TempDir {
		if err := os.MkdirAll(TempDir, 0774); err != nil {
			return nil, err
		}
		checkedTempDir = TempDir
	}

	return ioutil.TempFile(TempDir, prefix)
}

func ResetTempDir() error {
	checkedTempDir = ""
	return os.RemoveAll(TempDir)
}

func LocalMediaPath(sha string) (string, error) {
	path := filepath.Join(LocalMediaDir, sha[0:2], sha[2:4])
	if err := os.MkdirAll(path, 0744); err != nil {
		return "", fmt.Errorf("Error trying to create local media directory in '%s': %s", path, err)
	}

	return filepath.Join(path, sha), nil
}

func Environ() []string {
	osEnviron := os.Environ()
	env := make([]string, 4, len(osEnviron)+4)
	env[0] = fmt.Sprintf("LocalWorkingDir=%s", LocalWorkingDir)
	env[1] = fmt.Sprintf("LocalGitDir=%s", LocalGitDir)
	env[2] = fmt.Sprintf("LocalMediaDir=%s", LocalMediaDir)
	env[3] = fmt.Sprintf("TempDir=%s", TempDir)

	for _, e := range osEnviron {
		if !strings.Contains(e, "GIT_") {
			continue
		}
		env = append(env, e)
	}

	return env
}

func InRepo() bool {
	return LocalWorkingDir != ""
}

func init() {
	var err error
	LocalWorkingDir, LocalGitDir, err = resolveGitDir()
	if err == nil {
		LocalMediaDir = filepath.Join(LocalGitDir, "media")
		LocalLogDir = filepath.Join(LocalMediaDir, "logs")
		TempDir = filepath.Join(LocalMediaDir, "tmp")
		queueDir = setupQueueDir()

		if err := os.MkdirAll(TempDir, 0744); err != nil {
			panic(fmt.Errorf("Error trying to create temp directory in '%s': %s", TempDir, err))
		}
	}

	UserAgent = fmt.Sprintf("git media v%s (go v%s)", Version,
		strings.Replace(runtime.Version(), "go", "", 1))
}

func resolveGitDir() (string, string, error) {
	wd, err := os.Getwd()
	if err != nil {
		return "", "", err
	}

	return recursiveResolveGitDir(wd)
}

func recursiveResolveGitDir(dir string) (string, string, error) {
	var cleanDir = filepath.Clean(dir)
	if cleanDir[len(cleanDir)-1] == os.PathSeparator {
		return "", "", fmt.Errorf("Git repository not found")
	}

	if filepath.Base(dir) == gitExt {
		return filepath.Dir(dir), dir, nil
	}

	gitDir := filepath.Join(dir, gitExt)
	if info, err := os.Stat(gitDir); err == nil {
		if info.IsDir() {
			return dir, gitDir, nil
		} else {
			return processDotGitFile(gitDir)
		}
	}

	return recursiveResolveGitDir(filepath.Dir(dir))
}

func processDotGitFile(file string) (string, string, error) {
	f, err := os.Open(file)
	defer f.Close()

	if err != nil {
		return "", "", err
	}

	data := make([]byte, 512)
	n, err := f.Read(data)
	if err != nil {
		return "", "", err
	}

	contents := string(data[0:n])
	wd, _ := os.Getwd()
	if strings.HasPrefix(contents, gitPtrPrefix) {
		dir := strings.TrimSpace(strings.Split(contents, gitPtrPrefix)[1])
		absDir, _ := filepath.Abs(dir)
		return wd, absDir, nil
	}

	return wd, "", nil
}

const (
	gitExt       = ".git"
	gitPtrPrefix = "gitdir: "
)
<commit_msg>v0.2.2<commit_after>package gitmedia

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
)

const Version = "0.2.2"

var (
	LargeSizeThreshold = 5 * 1024 * 1024
	TempDir            = filepath.Join(os.TempDir(), "git-media")
	UserAgent          string
	LocalWorkingDir    string
	LocalGitDir        string
	LocalMediaDir      string
	LocalLogDir        string
	checkedTempDir     string
)

func TempFile(prefix string) (*os.File, error) {
	if checkedTempDir != TempDir {
		if err := os.MkdirAll(TempDir, 0774); err != nil {
			return nil, err
		}
		checkedTempDir = TempDir
	}

	return ioutil.TempFile(TempDir, prefix)
}

func ResetTempDir() error {
	checkedTempDir = ""
	return os.RemoveAll(TempDir)
}

func LocalMediaPath(sha string) (string, error) {
	path := filepath.Join(LocalMediaDir, sha[0:2], sha[2:4])
	if err := os.MkdirAll(path, 0744); err != nil {
		return "", fmt.Errorf("Error trying to create local media directory in '%s': %s", path, err)
	}

	return filepath.Join(path, sha), nil
}

func Environ() []string {
	osEnviron := os.Environ()
	env := make([]string, 4, len(osEnviron)+4)
	env[0] = fmt.Sprintf("LocalWorkingDir=%s", LocalWorkingDir)
	env[1] = fmt.Sprintf("LocalGitDir=%s", LocalGitDir)
	env[2] = fmt.Sprintf("LocalMediaDir=%s", LocalMediaDir)
	env[3] = fmt.Sprintf("TempDir=%s", TempDir)

	for _, e := range osEnviron {
		if !strings.Contains(e, "GIT_") {
			continue
		}
		env = append(env, e)
	}

	return env
}

func InRepo() bool {
	return LocalWorkingDir != ""
}

func init() {
	var err error
	LocalWorkingDir, LocalGitDir, err = resolveGitDir()
	if err == nil {
		LocalMediaDir = filepath.Join(LocalGitDir, "media")
		LocalLogDir = filepath.Join(LocalMediaDir, "logs")
		TempDir = filepath.Join(LocalMediaDir, "tmp")
		queueDir = setupQueueDir()

		if err := os.MkdirAll(TempDir, 0744); err != nil {
			panic(fmt.Errorf("Error trying to create temp directory in '%s': %s", TempDir, err))
		}
	}

	UserAgent = fmt.Sprintf("git media v%s (go v%s)", Version,
		strings.Replace(runtime.Version(), "go", "", 1))
}

func resolveGitDir() (string, string, error) {
	wd, err := os.Getwd()
	if err != nil {
		return "", "", err
	}

	return recursiveResolveGitDir(wd)
}

func recursiveResolveGitDir(dir string) (string, string, error) {
	var cleanDir = filepath.Clean(dir)
	if cleanDir[len(cleanDir)-1] == os.PathSeparator {
		return "", "", fmt.Errorf("Git repository not found")
	}

	if filepath.Base(dir) == gitExt {
		return filepath.Dir(dir), dir, nil
	}

	gitDir := filepath.Join(dir, gitExt)
	if info, err := os.Stat(gitDir); err == nil {
		if info.IsDir() {
			return dir, gitDir, nil
		} else {
			return processDotGitFile(gitDir)
		}
	}

	return recursiveResolveGitDir(filepath.Dir(dir))
}

func processDotGitFile(file string) (string, string, error) {
	f, err := os.Open(file)
	defer f.Close()

	if err != nil {
		return "", "", err
	}

	data := make([]byte, 512)
	n, err := f.Read(data)
	if err != nil {
		return "", "", err
	}

	contents := string(data[0:n])
	wd, _ := os.Getwd()
	if strings.HasPrefix(contents, gitPtrPrefix) {
		dir := strings.TrimSpace(strings.Split(contents, gitPtrPrefix)[1])
		absDir, _ := filepath.Abs(dir)
		return wd, absDir, nil
	}

	return wd, "", nil
}

const (
	gitExt       = ".git"
	gitPtrPrefix = "gitdir: "
)
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"os/exec"
	"strings"
	"time"

	"github.com/brotherlogic/goserver/utils"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	pbb "github.com/brotherlogic/buildserver/proto"
	pbfc "github.com/brotherlogic/filecopier/proto"
	pb "github.com/brotherlogic/gobuildslave/proto"
	pbvt "github.com/brotherlogic/versiontracker/proto"
)

const (
	pendWait = time.Minute
)

var (
	ackQueueLen = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "gobuildslave_ackqueuelen",
		Help: "The size of the ack queue",
	})
)

func (s *Server) procAcks() {
	for job := range s.ackChan {
		ackQueueLen.Set(float64(len(s.ackChan)))
		ctx, cancel := utils.ManualContext("gobuildslaveack", time.Minute)
		conn, err := s.FDialSpecificServer(ctx, "versiontracker", s.Registry.GetIdentifier())
		if err != nil {
			s.DLog(ctx, fmt.Sprintf("Dial error: (%v), %v\n", job.GetJob(), err))
			s.ackChan <- job
		} else {

			client := pbvt.NewVersionTrackerServiceClient(conn)
			_, err = client.NewJob(ctx, &pbvt.NewJobRequest{Version: &pbb.Version{Job: job.GetJob()}})
			if err != nil {
				s.ackChan <- job
			}
			conn.Close()
		}
		cancel()

		// Don't rush the system
		time.Sleep(time.Second * 10)
	}
}

func (s *Server) runTransition(ctx context.Context, job *pb.JobAssignment) {
	s.DLog(ctx, fmt.Sprintf("TRANS: %v\n", job))
	startState := job.State
	job.LastUpdateTime = time.Now().Unix()
	switch job.State {
	case pb.State_WARMUP:
		res, err := exec.Command("md5sum", fmt.Sprintf("/home/simon/gobuild/bin/%v", job.GetJob().GetName())).Output()
		if err != nil {
			s.Log(fmt.Sprintf("Error reading md5sum: %v", err))
		}
		elems := strings.Fields(string(res))
		job.RunningVersion = elems[0]
		s.ackChan <- job
		job.State = pb.State_ACKNOWLEDGED
	case pb.State_ACKNOWLEDGED:
		key := s.scheduleBuild(ctx, job)
		job.SubState = fmt.Sprintf("SCHED: %v @ %v", key, time.Now())
		if !job.Job.Bootstrap {
			if key != "" {
				job.State = pb.State_BUILT
				job.RunningVersion = key
				s.ackChan <- job
			} else {
				// Bootstrap this job since we don't have an initial version
				if job.Job.PartialBootstrap {
					job.Job.Bootstrap = true
				}
			}
		} else {
			job.CommandKey = key
			job.State = pb.State_BUILDING
		}
	case pb.State_BUILDING:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILD(%v): %v", job.CommandKey, s.scheduler.getState(job.CommandKey))
		s.stateMutex.Unlock()
		s.scheduler.wait(job.CommandKey)
		job.State = pb.State_BUILT
	case pb.State_BUILT:
		job.SubState = "Getting Output"
		output, _ := s.scheduler.getOutput(job.CommandKey)
		job.SubState = "Entering Lock"
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILT(%v): (%v): %v", job.CommandKey, len(output), output)
		s.stateMutex.Unlock()
		if job.Job.Bootstrap && len(output) > 0 {
			if job.BuildFail == 5 {
				job.SubState = "Sending Report"
				s.deliverCrashReport(ctx, job, output)
				job.BuildFail = 0
			}
			job.BuildFail++
			job.State = pb.State_DIED
		} else {
			job.State = pb.State_VERSION_CHECK
			// Don't check version on pb jobs
			if job.GetJob().GetPartialBootstrap() {
				s.doCopy(job)
				job.BuildFail = 0
				job.SubState = "Scheduling A Run post copy"
				key := s.scheduleRun(job)
				job.CommandKey = key
				job.StartTime = time.Now().Unix()
				job.State = pb.State_PENDING
				if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
					s.pendingMap[time.Now().Weekday()] = make(map[string]int)
				}
				s.pendingMap[time.Now().Weekday()][job.Job.Name]++
			}
		}
		job.SubState = "Out of case"
	case pb.State_VERSION_CHECK:
		s.doCopy(job)
		s.loadCurrentVersions()
		version, err := s.getLatestVersion(ctx, job.GetJob().Name, job.GetJob().GetGoPath())
		if err != nil {
			s.Log(fmt.Sprintf("Error getting version: %v", err))
			break
		}

		res, err := exec.Command("md5sum", fmt.Sprintf("/home/simon/gobuild/bin/%v", job.GetJob().GetName())).Output()
		if err != nil {
			s.Log(fmt.Sprintf("Error reading md5sum: %v", err))
		}
		elems := strings.Fields(string(res))
		job.RunningVersion = elems[0]
		s.ackChan <- job

		s.versionsMutex.Lock()
		defer s.versionsMutex.Unlock()
		if elems[0] != version.GetVersion() {
			s.Log(fmt.Sprintf("Bad version on %v for %v -> %v vs %v", s.Registry.Identifier, job.GetJob().GetName(), elems[0], s.versions[job.GetJob().GetName()].Version))
			job.SubState = fmt.Sprintf("Dealing With Version Mismatch: %v", job.BuildFail)

			conn, err := s.FDialSpecificServer(ctx, "versiontracker", s.Registry.Identifier)
			if err != nil {
				s.Log(fmt.Sprintf("Unable to dial vt: %v", err))
			} else {
				defer conn.Close()
				vtc := pbvt.NewVersionTrackerServiceClient(conn)
				_, err = vtc.NewVersion(ctx, &pbvt.NewVersionRequest{
					Version: version,
				})
				s.Log(fmt.Sprintf("Requested new version: %v", err))
			}

			// Don't let the job sit here
			job.BuildFail++
			if job.BuildFail > 10 {
				job.State = pb.State_ACKNOWLEDGED

			}
			break
		}

		job.BuildFail = 0
		job.SubState = "Scheduling of the Run"
		key := s.scheduleRun(job)
		job.CommandKey = key
		job.StartTime = time.Now().Unix()
		job.State = pb.State_PENDING
		if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
			s.pendingMap[time.Now().Weekday()] = make(map[string]int)
		}
		s.pendingMap[time.Now().Weekday()][job.Job.Name]++

	case pb.State_PENDING:
		res, err := exec.Command("md5sum", fmt.Sprintf("/home/simon/gobuild/bin/%v", job.GetJob().GetName())).Output()
		if err != nil {
			s.Log(fmt.Sprintf("Error reading md5sum: %v", err))
		}
		elems := strings.Fields(string(res))
		job.RunningVersion = elems[0]
		s.ackChan <- job

		if job.Job.PartialBootstrap && job.Job.Bootstrap {
			job.Job.Bootstrap = false
		}
		s.stateMutex.Lock()
		out, _ := s.scheduler.getOutput(job.CommandKey)
		s.stateMap[job.Job.Name] = fmt.Sprintf("OUTPUT = %v", out)
		s.stateMutex.Unlock()
		if time.Now().Add(-time.Minute).Unix() > job.StartTime {
			var err error
			if job.Job.Name == "discovery" {
				err = s.runOnChange()
			}
			code := status.Convert(err).Code()

			//Unavailable allows the job to die here
			if code == codes.OK || code == codes.Unavailable {
				// Validate that the job is alive
				if !s.isJobAlive(ctx, job) {
					s.Log(fmt.Sprintf("Job %v is not alive", job))
				}
				job.State = pb.State_RUNNING
			} else {
				s.Log(fmt.Sprintf("Cannot reregister: %v", err))
			}
		}
	case pb.State_RUNNING:
		output, errout := s.scheduler.getOutput(job.CommandKey)
		output2, _ := s.scheduler.getErrOutput(job.CommandKey)
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("ROUTPUT = %v, %v", output, s.scheduler.getStatus(job.CommandKey))
		job.Status = s.scheduler.getStatus(job.CommandKey)
		s.stateMutex.Unlock()
		if len(job.CommandKey) > 0 {
			s.scheduler.wait(job.CommandKey)
			s.stateMap[job.Job.Name] = fmt.Sprintf("ONLOCk = (%v, %v)", job, output)
			s.stateMutex.Lock()
			s.stateMap[job.Job.Name] = fmt.Sprintf("COMPLETE = (%v, %v)", job, output)
			s.stateMutex.Unlock()
			s.deliverCrashReport(ctx, job, fmt.Sprintf("%v%v", output, output2))
			job.State = pb.State_DIED
		}

		if s.Registry != nil {
			entry, err := s.FFindSpecificServer(ctx, job.Job.Name, s.Registry.Identifier)
			if err != nil {
				if job.DiscoverCount > 30 {
					output2, errout2 := s.scheduler.getErrOutput(job.CommandKey)
					s.RaiseIssue("Cannot Discover Running Server", fmt.Sprintf("%v on %v is not discoverable, despite running (%v) the output says %v (%v), %v, %v", job.Job.Name, s.Registry.Identifier, err, output, errout, output2, errout2))
				}
				job.DiscoverCount++
				s.Log(fmt.Sprintf("Missing discover for %+v, %v -> %v", job, entry, err))
			} else {
				job.Port = entry.GetPort()
				job.DiscoverCount = 0
			}
		}

		// Restart this job if we need to
		if !job.Job.Bootstrap {
			if time.Now().Sub(time.Unix(job.LastVersionPull, 0)) > time.Minute*5 {
				version, err := s.getVersion(ctx, job.Job)
				job.LastVersionPull = time.Now().Unix()

				if err == nil && version.Version != job.RunningVersion {
					s.stateMutex.Lock()
					s.stateMap[job.Job.Name] = fmt.Sprintf("VERSION_MISMATCH = %v,%v", version, job.RunningVersion)
					s.stateMutex.Unlock()
					s.scheduler.killJob(job.CommandKey)
				}
			}
		}
	case pb.State_BRINK_OF_DEATH:
		if s.version.confirm(ctx, job.Job.Name) {
			s.scheduler.killJob(job.CommandKey)
			job.State = pb.State_ACKNOWLEDGED
		}
	case pb.State_DIED:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("DIED %v", job.CommandKey)
		s.stateMutex.Unlock()
		job.State = pb.State_ACKNOWLEDGED
	}

	if job.State != startState {
		job.LastTransitionTime = time.Now().Unix()
	}
}

type translator interface {
	build(job *pb.Job) *exec.Cmd
	run(job *pb.Job) *exec.Cmd
}

type checker interface {
	isAlive(ctx context.Context, job *pb.JobAssignment) bool
}

func (s *Server) getVersion(ctx context.Context, job *pb.Job) (*pbb.Version, error) {
	version, err := s.builder.build(ctx, job)
	if err != nil {
		return &pbb.Version{}, err
	}

	return version, nil
}

func updateJob(err error, job *pb.JobAssignment, resp *pbfc.CopyResponse) {
	if err == nil {
		job.QueuePos = resp.IndexInQueue
	}

}

// scheduleBuild builds out the job, returning the current version
func (s *Server) scheduleBuild(ctx context.Context, job *pb.JobAssignment) string {
	if job.Job.Bootstrap {
		c := s.translator.build(job.Job)
		// Block builds to prevent clashes
		return s.scheduler.Schedule(&rCommand{command: c, base: job.Job.Name, block: true})
	}

	val, err := s.builder.build(ctx, job.Job)
	if err != nil {
		s.DLog(ctx, fmt.Sprintf("BUILD Error: %v\n", err))
		return ""
	}
	return val.Version
}

func (s *Server) doCopy(job *pb.JobAssignment) {
	//Copy over any existing new versions

	key := s.scheduler.Schedule(&rCommand{command: exec.Command("mv", "$GOPATH/bin/"+job.GetJob().GetName()+".new", "$GOPATH/bin/"+job.GetJob().GetName()), base: job.GetJob().GetName()})
	s.scheduler.wait(key)

	key = s.scheduler.Schedule(&rCommand{command: exec.Command("mv", "$GOPATH/bin/"+job.GetJob().GetName()+".nversion", "$GOPATH/bin/"+job.GetJob().GetName()+".version"), base: job.GetJob().GetName() + ".version"})
	s.scheduler.wait(key)
}

func (s *Server) scheduleRun(job *pb.JobAssignment) string {
	// Wait a while before starting the job JIC
	time.Sleep(time.Second * 2)

	c := s.translator.run(job.GetJob())
	return s.scheduler.Schedule(&rCommand{command: c, base: job.GetJob().GetName()})
}
<commit_msg>Include running version<commit_after>package main

import (
	"context"
	"fmt"
	"os/exec"
	"strings"
	"time"

	"github.com/brotherlogic/goserver/utils"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	pbb "github.com/brotherlogic/buildserver/proto"
	pbfc "github.com/brotherlogic/filecopier/proto"
	pb "github.com/brotherlogic/gobuildslave/proto"
	pbvt "github.com/brotherlogic/versiontracker/proto"
)

const (
	pendWait = time.Minute
)

var (
	ackQueueLen = promauto.NewGauge(prometheus.GaugeOpts{
		Name: "gobuildslave_ackqueuelen",
		Help: "The size of the ack queue",
	})
)

func (s *Server) procAcks() {
	for job := range s.ackChan {
		ackQueueLen.Set(float64(len(s.ackChan)))
		ctx, cancel := utils.ManualContext("gobuildslaveack", time.Minute)
		conn, err := s.FDialSpecificServer(ctx, "versiontracker", s.Registry.GetIdentifier())
		if err != nil {
			s.DLog(ctx, fmt.Sprintf("Dial error: (%v), %v\n", job.GetJob(), err))
			s.ackChan <- job
		} else {

			client := pbvt.NewVersionTrackerServiceClient(conn)
			_, err = client.NewJob(ctx, &pbvt.NewJobRequest{Version: &pbb.Version{Version: job.GetRunningVersion(), Job: job.GetJob()}})
			if err != nil {
				s.ackChan <- job
			}
			conn.Close()
		}
		cancel()

		// Don't rush the system
		time.Sleep(time.Second * 10)
	}
}

func (s *Server) runTransition(ctx context.Context, job *pb.JobAssignment) {
	s.DLog(ctx, fmt.Sprintf("TRANS: %v\n", job))
	startState := job.State
	job.LastUpdateTime = time.Now().Unix()
	switch job.State {
	case pb.State_WARMUP:
		res, err := exec.Command("md5sum", fmt.Sprintf("/home/simon/gobuild/bin/%v", job.GetJob().GetName())).Output()
		if err != nil {
			s.Log(fmt.Sprintf("Error reading md5sum: %v", err))
		}
		elems := strings.Fields(string(res))
		job.RunningVersion = elems[0]
		s.ackChan <- job
		job.State = pb.State_ACKNOWLEDGED
	case pb.State_ACKNOWLEDGED:
		key := s.scheduleBuild(ctx, job)
		job.SubState = fmt.Sprintf("SCHED: %v @ %v", key, time.Now())
		if !job.Job.Bootstrap {
			if key != "" {
				job.State = pb.State_BUILT
				job.RunningVersion = key
				s.ackChan <- job
			} else {
				// Bootstrap this job since we don't have an initial version
				if job.Job.PartialBootstrap {
					job.Job.Bootstrap = true
				}
			}
		} else {
			job.CommandKey = key
			job.State = pb.State_BUILDING
		}
	case pb.State_BUILDING:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILD(%v): %v", job.CommandKey, s.scheduler.getState(job.CommandKey))
		s.stateMutex.Unlock()
		s.scheduler.wait(job.CommandKey)
		job.State = pb.State_BUILT
	case pb.State_BUILT:
		job.SubState = "Getting Output"
		output, _ := s.scheduler.getOutput(job.CommandKey)
		job.SubState = "Entering Lock"
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("BUILT(%v): (%v): %v", job.CommandKey, len(output), output)
		s.stateMutex.Unlock()
		if job.Job.Bootstrap && len(output) > 0 {
			if job.BuildFail == 5 {
				job.SubState = "Sending Report"
				s.deliverCrashReport(ctx, job, output)
				job.BuildFail = 0
			}
			job.BuildFail++
			job.State = pb.State_DIED
		} else {
			job.State = pb.State_VERSION_CHECK
			// Don't check version on pb jobs
			if job.GetJob().GetPartialBootstrap() {
				s.doCopy(job)
				job.BuildFail = 0
				job.SubState = "Scheduling A Run post copy"
				key := s.scheduleRun(job)
				job.CommandKey = key
				job.StartTime = time.Now().Unix()
				job.State = pb.State_PENDING
				if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
					s.pendingMap[time.Now().Weekday()] = make(map[string]int)
				}
				s.pendingMap[time.Now().Weekday()][job.Job.Name]++
			}
		}
		job.SubState = "Out of case"
	case pb.State_VERSION_CHECK:
		s.doCopy(job)
		s.loadCurrentVersions()
		version, err := s.getLatestVersion(ctx, job.GetJob().Name, job.GetJob().GetGoPath())
		if err != nil {
			s.Log(fmt.Sprintf("Error getting version: %v", err))
			break
		}

		res, err := exec.Command("md5sum", fmt.Sprintf("/home/simon/gobuild/bin/%v", job.GetJob().GetName())).Output()
		if err != nil {
			s.Log(fmt.Sprintf("Error reading md5sum: %v", err))
		}
		elems := strings.Fields(string(res))
		job.RunningVersion = elems[0]
		s.ackChan <- job

		s.versionsMutex.Lock()
		defer s.versionsMutex.Unlock()
		if elems[0] != version.GetVersion() {
			s.Log(fmt.Sprintf("Bad version on %v for %v -> %v vs %v", s.Registry.Identifier, job.GetJob().GetName(), elems[0], s.versions[job.GetJob().GetName()].Version))
			job.SubState = fmt.Sprintf("Dealing With Version Mismatch: %v", job.BuildFail)

			conn, err := s.FDialSpecificServer(ctx, "versiontracker", s.Registry.Identifier)
			if err != nil {
				s.Log(fmt.Sprintf("Unable to dial vt: %v", err))
			} else {
				defer conn.Close()
				vtc := pbvt.NewVersionTrackerServiceClient(conn)
				_, err = vtc.NewVersion(ctx, &pbvt.NewVersionRequest{
					Version: version,
				})
				s.Log(fmt.Sprintf("Requested new version: %v", err))
			}

			// Don't let the job sit here
			job.BuildFail++
			if job.BuildFail > 10 {
				job.State = pb.State_ACKNOWLEDGED

			}
			break
		}

		job.BuildFail = 0
		job.SubState = "Scheduling of the Run"
		key := s.scheduleRun(job)
		job.CommandKey = key
		job.StartTime = time.Now().Unix()
		job.State = pb.State_PENDING
		if _, ok := s.pendingMap[time.Now().Weekday()]; !ok {
			s.pendingMap[time.Now().Weekday()] = make(map[string]int)
		}
		s.pendingMap[time.Now().Weekday()][job.Job.Name]++

	case pb.State_PENDING:
		res, err := exec.Command("md5sum", fmt.Sprintf("/home/simon/gobuild/bin/%v", job.GetJob().GetName())).Output()
		if err != nil {
			s.Log(fmt.Sprintf("Error reading md5sum: %v", err))
		}
		elems := strings.Fields(string(res))
		job.RunningVersion = elems[0]
		s.ackChan <- job

		if job.Job.PartialBootstrap && job.Job.Bootstrap {
			job.Job.Bootstrap = false
		}
		s.stateMutex.Lock()
		out, _ := s.scheduler.getOutput(job.CommandKey)
		s.stateMap[job.Job.Name] = fmt.Sprintf("OUTPUT = %v", out)
		s.stateMutex.Unlock()
		if time.Now().Add(-time.Minute).Unix() > job.StartTime {
			var err error
			if job.Job.Name == "discovery" {
				err = s.runOnChange()
			}
			code := status.Convert(err).Code()

			//Unavailable allows the job to die here
			if code == codes.OK || code == codes.Unavailable {
				// Validate that the job is alive
				if !s.isJobAlive(ctx, job) {
					s.Log(fmt.Sprintf("Job %v is not alive", job))
				}
				job.State = pb.State_RUNNING
			} else {
				s.Log(fmt.Sprintf("Cannot reregister: %v", err))
			}
		}
	case pb.State_RUNNING:
		output, errout := s.scheduler.getOutput(job.CommandKey)
		output2, _ := s.scheduler.getErrOutput(job.CommandKey)
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("ROUTPUT = %v, %v", output, s.scheduler.getStatus(job.CommandKey))
		job.Status = s.scheduler.getStatus(job.CommandKey)
		s.stateMutex.Unlock()
		if len(job.CommandKey) > 0 {
			s.scheduler.wait(job.CommandKey)
			s.stateMap[job.Job.Name] = fmt.Sprintf("ONLOCk = (%v, %v)", job, output)
			s.stateMutex.Lock()
			s.stateMap[job.Job.Name] = fmt.Sprintf("COMPLETE = (%v, %v)", job, output)
			s.stateMutex.Unlock()
			s.deliverCrashReport(ctx, job, fmt.Sprintf("%v%v", output, output2))
			job.State = pb.State_DIED
		}

		if s.Registry != nil {
			entry, err := s.FFindSpecificServer(ctx, job.Job.Name, s.Registry.Identifier)
			if err != nil {
				if job.DiscoverCount > 30 {
					output2, errout2 := s.scheduler.getErrOutput(job.CommandKey)
					s.RaiseIssue("Cannot Discover Running Server", fmt.Sprintf("%v on %v is not discoverable, despite running (%v) the output says %v (%v), %v, %v", job.Job.Name, s.Registry.Identifier, err, output, errout, output2, errout2))
				}
				job.DiscoverCount++
				s.Log(fmt.Sprintf("Missing discover for %+v, %v -> %v", job, entry, err))
			} else {
				job.Port = entry.GetPort()
				job.DiscoverCount = 0
			}
		}

		// Restart this job if we need to
		if !job.Job.Bootstrap {
			if time.Now().Sub(time.Unix(job.LastVersionPull, 0)) > time.Minute*5 {
				version, err := s.getVersion(ctx, job.Job)
				job.LastVersionPull = time.Now().Unix()

				if err == nil && version.Version != job.RunningVersion {
					s.stateMutex.Lock()
					s.stateMap[job.Job.Name] = fmt.Sprintf("VERSION_MISMATCH = %v,%v", version, job.RunningVersion)
					s.stateMutex.Unlock()
					s.scheduler.killJob(job.CommandKey)
				}
			}
		}
	case pb.State_BRINK_OF_DEATH:
		if s.version.confirm(ctx, job.Job.Name) {
			s.scheduler.killJob(job.CommandKey)
			job.State = pb.State_ACKNOWLEDGED
		}
	case pb.State_DIED:
		s.stateMutex.Lock()
		s.stateMap[job.Job.Name] = fmt.Sprintf("DIED %v", job.CommandKey)
		s.stateMutex.Unlock()
		job.State = pb.State_ACKNOWLEDGED
	}

	if job.State != startState {
		job.LastTransitionTime = time.Now().Unix()
	}
}

type translator interface {
	build(job *pb.Job) *exec.Cmd
	run(job *pb.Job) *exec.Cmd
}

type checker interface {
	isAlive(ctx context.Context, job *pb.JobAssignment) bool
}

func (s *Server) getVersion(ctx context.Context, job *pb.Job) (*pbb.Version, error) {
	version, err := s.builder.build(ctx, job)
	if err != nil {
		return &pbb.Version{}, err
	}

	return version, nil
}

func updateJob(err error, job *pb.JobAssignment, resp *pbfc.CopyResponse) {
	if err == nil {
		job.QueuePos = resp.IndexInQueue
	}

}

// scheduleBuild builds out the job, returning the current version
func (s *Server) scheduleBuild(ctx context.Context, job *pb.JobAssignment) string {
	if job.Job.Bootstrap {
		c := s.translator.build(job.Job)
		// Block builds to prevent clashes
		return s.scheduler.Schedule(&rCommand{command: c, base: job.Job.Name, block: true})
	}

	val, err := s.builder.build(ctx, job.Job)
	if err != nil {
		s.DLog(ctx, fmt.Sprintf("BUILD Error: %v\n", err))
		return ""
	}
	return val.Version
}

func (s *Server) doCopy(job *pb.JobAssignment) {
	//Copy over any existing new versions

	key := s.scheduler.Schedule(&rCommand{command: exec.Command("mv", "$GOPATH/bin/"+job.GetJob().GetName()+".new", "$GOPATH/bin/"+job.GetJob().GetName()), base: job.GetJob().GetName()})
	s.scheduler.wait(key)

	key = s.scheduler.Schedule(&rCommand{command: exec.Command("mv", "$GOPATH/bin/"+job.GetJob().GetName()+".nversion", "$GOPATH/bin/"+job.GetJob().GetName()+".version"), base: job.GetJob().GetName() + ".version"})
	s.scheduler.wait(key)
}

func (s *Server) scheduleRun(job *pb.JobAssignment) string {
	// Wait a while before starting the job JIC
	time.Sleep(time.Second * 2)

	c := s.translator.run(job.GetJob())
	return s.scheduler.Schedule(&rCommand{command: c, base: job.GetJob().GetName()})
}
<|endoftext|>


<commit_before>package gorillamux

import "github.com/ungerik/go-httpx/httperr"

var (
	CatchPanics         bool
	PrettyPrint         bool
	PrettyPrintIndent   = "  "
	DefaultErrorHandler = httperr.HandlerFunc(httperr.DefaultHandlerFunc)
)
<commit_msg>default CatchPanics = true<commit_after>package gorillamux

import "github.com/ungerik/go-httpx/httperr"

var (
	CatchPanics         = true
	PrettyPrint         = true
	PrettyPrintIndent   = "  "
	DefaultErrorHandler = httperr.HandlerFunc(httperr.DefaultHandlerFunc)
)
<|endoftext|>


<commit_before>/*
Copyright (c) 2015-2016 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package importx

import (
	"encoding/json"
	"flag"
	"fmt"
	"path"

	"golang.org/x/net/context"

	"github.com/vmware/govmomi/govc/cli"
	"github.com/vmware/govmomi/ovf"
	"github.com/vmware/govmomi/vim25/types"
)

var (
	// all possible ovf property values
	// the first element being the default value
	allDeploymentOptions         = []string{"small", "medium", "large"}
	allDiskProvisioningOptions   = []string{"thin", "monolithicSparse", "monolithicFlat", "twoGbMaxExtentSparse", "twoGbMaxExtentFlat", "seSparse", "eagerZeroedThick", "thick", "sparse", "flat"}
	allIPAllocationPolicyOptions = []string{"dhcpPolicy", "transientPolicy", "fixedPolicy", "fixedAllocatedPolicy"}
	allIPProtocolOptions         = []string{"IPv4", "IPv6"}
)

type spec struct {
	*ArchiveFlag

	verbose bool
}

func init() {
	cli.Register("import.spec", &spec{})
}

func (cmd *spec) Register(ctx context.Context, f *flag.FlagSet) {
	cmd.ArchiveFlag, ctx = newArchiveFlag(ctx)
	cmd.ArchiveFlag.Register(ctx, f)

	f.BoolVar(&cmd.verbose, "verbose", false, "Verbose spec output")
}

func (cmd *spec) Process(ctx context.Context) error {
	if err := cmd.ArchiveFlag.Process(ctx); err != nil {
		return err
	}
	return nil
}

func (cmd *spec) Usage() string {
	return "PATH_TO_OVF_OR_OVA"
}

func (cmd *spec) Run(ctx context.Context, f *flag.FlagSet) error {
	fpath := ""
	args := f.Args()
	if len(args) == 1 {
		fpath = f.Arg(0)
	}

	if len(fpath) > 0 {
		switch path.Ext(fpath) {
		case ".ovf":
			cmd.Archive = &FileArchive{fpath}
		case "", ".ova":
			cmd.Archive = &TapeArchive{fpath}
			fpath = "*.ovf"
		default:
			return fmt.Errorf("invalid file extension %s", path.Ext(fpath))
		}
	}

	return cmd.Spec(fpath)
}

func (cmd *spec) Map(e *ovf.Envelope) (res []Property) {
	if e == nil {
		return nil
	}

	for _, p := range e.VirtualSystem.Product {
		for i, v := range p.Property {
			d := ""
			if v.Default != nil {
				d = *v.Default
			}

			// From OVF spec, section 9.5.1:
			// key-value-env = [class-value "."] key-value-prod ["." instance-value]
			k := v.Key
			if p.Class != nil {
				k = fmt.Sprintf("%s.%s", *p.Class, k)
			}
			if p.Instance != nil {
				k = fmt.Sprintf("%s.%s", k, *p.Instance)
			}

			np := Property{KeyValue: types.KeyValue{Key: k, Value: d}}
			if cmd.verbose {
				np.Spec = &p.Property[i]
			}

			res = append(res, np)
		}
	}

	return
}

func (cmd *spec) Spec(fpath string) error {
	e, err := cmd.ReadEnvelope(fpath)
	if err != nil {
		return err
	}

	var deploymentOptions = allDeploymentOptions
	if e != nil && e.DeploymentOption != nil && e.DeploymentOption.Configuration != nil {
		deploymentOptions = nil

		// add default first
		for _, c := range e.DeploymentOption.Configuration {
			if c.Default != nil && *c.Default {
				deploymentOptions = append(deploymentOptions, c.ID)
			}
		}

		for _, c := range e.DeploymentOption.Configuration {
			if c.Default == nil || !*c.Default {
				deploymentOptions = append(deploymentOptions, c.ID)
			}
		}
	}

	o := Options{
		Deployment:         deploymentOptions[0],
		DiskProvisioning:   allDiskProvisioningOptions[0],
		IPAllocationPolicy: allIPAllocationPolicyOptions[0],
		IPProtocol:         allIPProtocolOptions[0],
		PowerOn:            false,
		WaitForIP:          false,
		InjectOvfEnv:       false,
		PropertyMapping:    cmd.Map(e)}

	if e.VirtualSystem.Annotation != nil {
		for _, a := range e.VirtualSystem.Annotation {
			o.Annotation += a.Annotation
		}
	}

	if e.Network != nil {
		for _, net := range e.Network.Networks {
			o.NetworkMapping = append(o.NetworkMapping, Network{net.Name, ""})
		}
	}

	if cmd.verbose {
		o.AllDeploymentOptions = deploymentOptions
		o.AllDiskProvisioningOptions = allDiskProvisioningOptions
		o.AllIPAllocationPolicyOptions = allIPAllocationPolicyOptions
		o.AllIPProtocolOptions = allIPProtocolOptions
	}

	j, err := json.Marshal(&o)
	if err != nil {
		return err
	}

	fmt.Println(string(j))
	return nil
}
<commit_msg>Add --indent option to import.spec command.<commit_after>/*
Copyright (c) 2015-2016 VMware, Inc. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package importx

import (
	"encoding/json"
	"flag"
	"fmt"
	"path"

	"golang.org/x/net/context"

	"github.com/vmware/govmomi/govc/cli"
	"github.com/vmware/govmomi/ovf"
	"github.com/vmware/govmomi/vim25/types"
)

var (
	// all possible ovf property values
	// the first element being the default value
	allDeploymentOptions         = []string{"small", "medium", "large"}
	allDiskProvisioningOptions   = []string{"thin", "monolithicSparse", "monolithicFlat", "twoGbMaxExtentSparse", "twoGbMaxExtentFlat", "seSparse", "eagerZeroedThick", "thick", "sparse", "flat"}
	allIPAllocationPolicyOptions = []string{"dhcpPolicy", "transientPolicy", "fixedPolicy", "fixedAllocatedPolicy"}
	allIPProtocolOptions         = []string{"IPv4", "IPv6"}
)

type spec struct {
	*ArchiveFlag

	verbose bool
	indent  bool
}

func init() {
	cli.Register("import.spec", &spec{})
}

func (cmd *spec) Register(ctx context.Context, f *flag.FlagSet) {
	cmd.ArchiveFlag, ctx = newArchiveFlag(ctx)
	cmd.ArchiveFlag.Register(ctx, f)

	f.BoolVar(&cmd.verbose, "verbose", false, "Verbose spec output")
	f.BoolVar(&cmd.indent, "indent", false, "Indent output in a pretty way")
}

func (cmd *spec) Process(ctx context.Context) error {
	if err := cmd.ArchiveFlag.Process(ctx); err != nil {
		return err
	}
	return nil
}

func (cmd *spec) Usage() string {
	return "PATH_TO_OVF_OR_OVA"
}

func (cmd *spec) Run(ctx context.Context, f *flag.FlagSet) error {
	fpath := ""
	args := f.Args()
	if len(args) == 1 {
		fpath = f.Arg(0)
	}

	if len(fpath) > 0 {
		switch path.Ext(fpath) {
		case ".ovf":
			cmd.Archive = &FileArchive{fpath}
		case "", ".ova":
			cmd.Archive = &TapeArchive{fpath}
			fpath = "*.ovf"
		default:
			return fmt.Errorf("invalid file extension %s", path.Ext(fpath))
		}
	}

	return cmd.Spec(fpath)
}

func (cmd *spec) Map(e *ovf.Envelope) (res []Property) {
	if e == nil {
		return nil
	}

	for _, p := range e.VirtualSystem.Product {
		for i, v := range p.Property {
			d := ""
			if v.Default != nil {
				d = *v.Default
			}

			// From OVF spec, section 9.5.1:
			// key-value-env = [class-value "."] key-value-prod ["." instance-value]
			k := v.Key
			if p.Class != nil {
				k = fmt.Sprintf("%s.%s", *p.Class, k)
			}
			if p.Instance != nil {
				k = fmt.Sprintf("%s.%s", k, *p.Instance)
			}

			np := Property{KeyValue: types.KeyValue{Key: k, Value: d}}
			if cmd.verbose {
				np.Spec = &p.Property[i]
			}

			res = append(res, np)
		}
	}

	return
}

func (cmd *spec) Spec(fpath string) error {
	e, err := cmd.ReadEnvelope(fpath)
	if err != nil {
		return err
	}

	var deploymentOptions = allDeploymentOptions
	if e != nil && e.DeploymentOption != nil && e.DeploymentOption.Configuration != nil {
		deploymentOptions = nil

		// add default first
		for _, c := range e.DeploymentOption.Configuration {
			if c.Default != nil && *c.Default {
				deploymentOptions = append(deploymentOptions, c.ID)
			}
		}

		for _, c := range e.DeploymentOption.Configuration {
			if c.Default == nil || !*c.Default {
				deploymentOptions = append(deploymentOptions, c.ID)
			}
		}
	}

	o := Options{
		Deployment:         deploymentOptions[0],
		DiskProvisioning:   allDiskProvisioningOptions[0],
		IPAllocationPolicy: allIPAllocationPolicyOptions[0],
		IPProtocol:         allIPProtocolOptions[0],
		PowerOn:            false,
		WaitForIP:          false,
		InjectOvfEnv:       false,
		PropertyMapping:    cmd.Map(e)}

	if e.VirtualSystem.Annotation != nil {
		for _, a := range e.VirtualSystem.Annotation {
			o.Annotation += a.Annotation
		}
	}

	if e.Network != nil {
		for _, net := range e.Network.Networks {
			o.NetworkMapping = append(o.NetworkMapping, Network{net.Name, ""})
		}
	}

	if cmd.verbose {
		o.AllDeploymentOptions = deploymentOptions
		o.AllDiskProvisioningOptions = allDiskProvisioningOptions
		o.AllIPAllocationPolicyOptions = allIPAllocationPolicyOptions
		o.AllIPProtocolOptions = allIPProtocolOptions
	}

	j := []byte{}
	if cmd.indent {
		j, err = json.MarshalIndent(&o, "", "\t")
	} else {
		j, err = json.Marshal(&o)
	}

	if err != nil {
		return err
	}

	fmt.Println(string(j))
	return nil
}
<|endoftext|>


<commit_before>package rebecca

// This file contains thin exported functions and methods related to
// Transaction only.
//
// For unexported functions see: helpers.go
//
// For Context see: context.go

import (
	"errors"
	"fmt"

	"github.com/waterlink/rebecca/driver"
)

// Transaction is for managing transactions for drivers that allow it
type Transaction struct {
	tx       interface{}
	finished bool
}

// Transact is for abstracting transaction handling. It commits transaction if
// fn returned nil, otherwise it rolls transaction back.
func Transact(fn func(tx *Transaction) error) (err error) {
	tx, err := Begin()
	if err != nil {
		return
	}

	defer func() {
		if p := recover(); p != nil {
			err = fmt.Errorf("%s (recovered)", p)
		}
		if err != nil {
			tx.Rollback()
			return
		}
		err = tx.Commit()
	}()

	return fn(tx)
}

// Begin is for creating proper transaction
func Begin() (*Transaction, error) {
	d, lock := driver.Get()
	defer lock.Unlock()

	tx, err := d.Begin()
	if err != nil {
		return nil, fmt.Errorf("Unable to begin transaction - %s", err)
	}
	return &Transaction{
		tx: tx,
	}, nil
}

// Rollback is for rolling back the transaction
func (tx *Transaction) Rollback() {
	if tx.finished {
		return
	}

	d, lock := driver.Get()
	defer lock.Unlock()

	d.Rollback(tx.tx)
	tx.finished = true
}

// Commit is for committing the transaction
func (tx *Transaction) Commit() error {
	if tx.finished {
		return errors.New("Unable to commit transaction - Current transaction is already finished")
	}

	d, lock := driver.Get()
	defer lock.Unlock()

	if err := d.Commit(tx.tx); err != nil {
		return fmt.Errorf("Unable to commit transaction - %s", err)
	}

	tx.finished = true
	return nil
}

// Get is for fetching one record
func (tx *Transaction) Get(record interface{}, ID interface{}) error {
	return get(tx.tx, record, ID)
}

// Save is for saving one record (either creating or updating)
func (tx *Transaction) Save(record interface{}) error {
	return save(tx.tx, record)
}

// All is for fetching all records
func (tx *Transaction) All(records interface{}) error {
	ctx := tx.Context(&Context{})
	return ctx.All(records)
}

// Where is for fetching specific records
func (tx *Transaction) Where(records interface{}, where string, args ...interface{}) error {
	ctx := tx.Context(&Context{})
	return ctx.Where(records, where, args...)
}

// First is for fetching only one specific record
func (tx *Transaction) First(record interface{}, where string, args ...interface{}) error {
	ctx := tx.Context(&Context{})
	return ctx.First(record, where, args...)
}

// Remove is for removing the record
func (tx *Transaction) Remove(record interface{}) error {
	return remove(tx.tx, record)
}

// Exec is for executing a query within transaction and discarding its result
func (tx *Transaction) Exec(query string, args ...interface{}) error {
	return exec(tx.tx, query, args...)
}

// Context is for instantiating proper context for transaction
func (tx *Transaction) Context(ctx *Context) *Context {
	return &Context{
		Order: ctx.Order,
		Group: ctx.Group,
		Limit: ctx.Limit,
		Skip:  ctx.Skip,
		tx:    tx.tx,
	}
}
<commit_msg>Fix transaction.Transact recover error handling<commit_after>package rebecca

// This file contains thin exported functions and methods related to
// Transaction only.
//
// For unexported functions see: helpers.go
//
// For Context see: context.go

import (
	"errors"
	"fmt"

	"github.com/waterlink/rebecca/driver"
)

// Transaction is for managing transactions for drivers that allow it
type Transaction struct {
	tx       interface{}
	finished bool
}

// Transact is for abstracting transaction handling. It commits transaction if
// fn returned nil, otherwise it rolls transaction back.
func Transact(fn func(tx *Transaction) error) (err error) {
	tx, err := Begin()
	if err != nil {
		return
	}

	defer func() {
		if p := recover(); p != nil {
			switch p := p.(type) {
			case error:
				err = p
			default:
				err = fmt.Errorf("%s (recovered)", p)
			}
		}
		if err != nil {
			tx.Rollback()
			return
		}
		err = tx.Commit()
	}()

	return fn(tx)
}

// Begin is for creating proper transaction
func Begin() (*Transaction, error) {
	d, lock := driver.Get()
	defer lock.Unlock()

	tx, err := d.Begin()
	if err != nil {
		return nil, fmt.Errorf("Unable to begin transaction - %s", err)
	}
	return &Transaction{
		tx: tx,
	}, nil
}

// Rollback is for rolling back the transaction
func (tx *Transaction) Rollback() {
	if tx.finished {
		return
	}

	d, lock := driver.Get()
	defer lock.Unlock()

	d.Rollback(tx.tx)
	tx.finished = true
}

// Commit is for committing the transaction
func (tx *Transaction) Commit() error {
	if tx.finished {
		return errors.New("Unable to commit transaction - Current transaction is already finished")
	}

	d, lock := driver.Get()
	defer lock.Unlock()

	if err := d.Commit(tx.tx); err != nil {
		return fmt.Errorf("Unable to commit transaction - %s", err)
	}

	tx.finished = true
	return nil
}

// Get is for fetching one record
func (tx *Transaction) Get(record interface{}, ID interface{}) error {
	return get(tx.tx, record, ID)
}

// Save is for saving one record (either creating or updating)
func (tx *Transaction) Save(record interface{}) error {
	return save(tx.tx, record)
}

// All is for fetching all records
func (tx *Transaction) All(records interface{}) error {
	ctx := tx.Context(&Context{})
	return ctx.All(records)
}

// Where is for fetching specific records
func (tx *Transaction) Where(records interface{}, where string, args ...interface{}) error {
	ctx := tx.Context(&Context{})
	return ctx.Where(records, where, args...)
}

// First is for fetching only one specific record
func (tx *Transaction) First(record interface{}, where string, args ...interface{}) error {
	ctx := tx.Context(&Context{})
	return ctx.First(record, where, args...)
}

// Remove is for removing the record
func (tx *Transaction) Remove(record interface{}) error {
	return remove(tx.tx, record)
}

// Exec is for executing a query within transaction and discarding its result
func (tx *Transaction) Exec(query string, args ...interface{}) error {
	return exec(tx.tx, query, args...)
}

// Context is for instantiating proper context for transaction
func (tx *Transaction) Context(ctx *Context) *Context {
	return &Context{
		Order: ctx.Order,
		Group: ctx.Group,
		Limit: ctx.Limit,
		Skip:  ctx.Skip,
		tx:    tx.tx,
	}
}
<|endoftext|>


<commit_before>package gogulden

import (
	"encoding/json"
	"time"
)

type Transaction struct {
	TransactionData
	Details []Details `json:"details"`
}

type TransactionItem struct {
	Details
	TransactionData
}

type TransactionData struct {
	Confirmations int       `json:"confirmations"`
	TransactionId string    `json:"txid"`
	Time          time.Time `json:"-"`
}

type Details struct {
	Account             string  `json:"account"`
	Address             string  `json:"address"`
	Category            string  `json:"category"`
	Amount              float32 `json:"amount"`
	Vout                int     `json:"vout"`
	SecuredByCheckpoint bool    `json:"-"`

	SecuredByCheckpointStr string `json:"secured_by_checkpoint"`
}

func (c *Client) Transactions(account string, count int, from string, includeWatchOnly bool) ([]*TransactionItem, error) {
	params := []interface{}{}
	if account != "" {
		params = append(params, account)
	}

	if len(params) == 1 && count > 0 {
		params = append(params, count)
	}

	if len(params) == 2 && from != "" {
		params = append(params, from)
	}

	if len(params) == 3 {
		params = append(params, includeWatchOnly)
	}

	transactions := []*TransactionItem{}
	err := c.runCommand(&transactions, "listtransactions", params...)
	if err != nil {
		return nil, err
	}

	return transactions, nil
}

func (c *Client) GetTransaction(transactionId string) (*Transaction, error) {
	var transaction Transaction
	err := c.runCommand(&transaction, "gettransaction", transactionId)
	if err != nil {
		return nil, err
	}

	return &transaction, nil
}

func (t *Transaction) UnmarshalJSON(data []byte) error {
	type Alias Transaction

	aux := &struct {
		*Alias

		Time int64 `json:"time"`
	}{
		Alias: (*Alias)(t),
	}

	if err := json.Unmarshal(data, &aux); err != nil {
		return err
	}

	for idx := range t.Details {
		d := &t.Details[idx]
		d.SecuredByCheckpoint = d.SecuredByCheckpointStr == "yes"
	}

	t.Time = time.Unix(aux.Time, 0)
	return nil
}

func (t *TransactionItem) UnmarshalJSON(data []byte) error {
	type Alias TransactionItem

	aux := &struct {
		*Alias

		Time int64 `json:"time"`
	}{
		Alias: (*Alias)(t),
	}

	if err := json.Unmarshal(data, &aux); err != nil {
		return err
	}

	t.Details.SecuredByCheckpoint = t.Details.SecuredByCheckpointStr == "yes"
	t.Time = time.Unix(aux.Time, 0)
	return nil
}
<commit_msg>Cleaner way of doing it<commit_after>package gogulden

import (
	"encoding/json"
	"time"
)

type Transaction struct {
	TransactionData
	Details []Details `json:"details"`
}

type TransactionItem struct {
	Details
	TransactionData
}

type TransactionData struct {
	Confirmations int    `json:"confirmations"`
	TransactionId string `json:"txid"`

	UnixTime unixTime `json:"time"`
}

type Details struct {
	Account  string  `json:"account"`
	Address  string  `json:"address"`
	Category string  `json:"category"`
	Amount   float32 `json:"amount"`
	Vout     int     `json:"vout"`

	SecuredByCheckpoint securedByCheckpoint `json:"secured_by_checkpoint"`
}

type securedByCheckpoint bool
type unixTime time.Time

func (c *Client) Transactions(account string, count int, from string, includeWatchOnly bool) ([]*TransactionItem, error) {
	params := []interface{}{}
	if account != "" {
		params = append(params, account)
	}

	if len(params) == 1 && count > 0 {
		params = append(params, count)
	}

	if len(params) == 2 && from != "" {
		params = append(params, from)
	}

	if len(params) == 3 {
		params = append(params, includeWatchOnly)
	}

	transactions := []*TransactionItem{}
	err := c.runCommand(&transactions, "listtransactions", params...)
	if err != nil {
		return nil, err
	}

	return transactions, nil
}

func (c *Client) GetTransaction(transactionId string) (*Transaction, error) {
	var transaction Transaction
	err := c.runCommand(&transaction, "gettransaction", transactionId)
	if err != nil {
		return nil, err
	}

	return &transaction, nil
}

func (t *unixTime) UnmarshalJSON(data []byte) error {
	var timeSeconds int64
	if err := json.Unmarshal(data, &timeSeconds); err != nil {
		return err
	}

	*t = unixTime(time.Unix(timeSeconds, 0))
	return nil
}

func (s *securedByCheckpoint) UnmarshalJSON(data []byte) error {
	var secured string
	if err := json.Unmarshal(data, &secured); err != nil {
		return err
	}

	*s = securedByCheckpoint(secured == "yes")
	return nil
}

func (d *Details) Secured() bool {
	return bool(d.SecuredByCheckpoint)
}

func (td *TransactionData) Time() time.Time {
	return time.Time(td.UnixTime)
}
<|endoftext|>


<commit_before>package data

type ServerProperties struct {
	ID         int
	Properties map[string]string
}

type DefaultMap struct {
	Mode               int
	Name               string
	GameMode           int32
	Seed               int64
	Structures, Cheats bool
}

type SuperFlatMap struct {
	DefaultMap
	GeneratorSettings string
}

type CustomMap struct {
	DefaultMap
	GeneratorSettings struct {
		SeaLevel                uint8   `json:"seaLevel"`
		Caves                   bool    `json:"useCaves"`
		Strongholds             bool    `json:"useStrongholds"`
		Villages                bool    `json:"useVillages"`
		Mineshafts              bool    `json:"useMineShafts"`
		Temples                 bool    `json:"useTemples"`
		OceanMonuments          bool    `json:"useMonuments"` // Needs checking
		Ravines                 bool    `json:"useRavines"`
		Dungeons                bool    `json:"useDungeons"`
		DungeonChance           uint8   `json:"dungeonChance"`
		WaterLake               bool    `json:"useWaterLake"`
		WaterLakeChance         uint8   `json:"waterLakeChance"`
		LaveLake                bool    `json:"useLavaLake"`
		LavaLakeChance          uint8   `json:"lavaLakeChance"`
		LavaOceans              bool    `json:"useLavaOceans"`
		Biome                   int16   `json:"fixedBiome"`
		BiomeSize               uint8   `json:"biomeSize"`
		RiverSize               uint8   `json:"riverSize"`
		DirtSize                uint8   `json:"dirtSize"`
		DirtTries               uint8   `json:"dirtCount"`
		DirtMinHeight           uint8   `json:"dirtMinHeight"`
		DirtMaxHeight           uint8   `json:"dirtMaxHeight"`
		GravelSize              uint8   `json:"gravelSize"`
		GravelTries             uint8   `json:"gravelCount"`
		GravelMinHeight         uint8   `json:"gravelMinHeight"`
		GravelMaxHeight         uint8   `json:"gravelMaxHeight"`
		GraniteSize             uint8   `json:"graniteSize"`
		GraniteTries            uint8   `json:"graniteCount"`
		GraniteMinHeight        uint8   `json:"graniteMinHeight"`
		GraniteMaxHeight        uint8   `json:"graniteMaxHeight"`
		DiortiteSize            uint8   `json:"dioriteSize"`
		DiortiteTries           uint8   `json:"dioriteCount"`
		DiortiteMinHeight       uint8   `json:"dioriteMinHeight"`
		DiortiteMaxHeight       uint8   `json:"dioriteMaxHeight"`
		AndesiteSize            uint8   `json:"andesiteSize"`
		AndesiteTries           uint8   `json:"andesiteCount"`
		AndesiteMinHeight       uint8   `json:"andesiteMinHeight"`
		AndesiteMaxHeight       uint8   `json:"andesiteMaxHeight"`
		CoalSize                uint8   `json:"coalSize"`
		CoalTries               uint8   `json:"coalCount"`
		CoalMinHeight           uint8   `json:"coalMinHeight"`
		CoalMaxHeight           uint8   `json:"coalMaxHeight"`
		IronSize                uint8   `json:"ironSize"`
		IronTries               uint8   `json:"ironCount"`
		IronMinHeight           uint8   `json:"ironMinHeight"`
		IronMaxHeight           uint8   `json:"ironMaxHeight"`
		GoldSize                uint8   `json:"goldSize"`
		GoldTries               uint8   `json:"goldCount"`
		GoldMinHeight           uint8   `json:"goldMinHeight"`
		GoldMaxHeight           uint8   `json:"goldMaxHeight"`
		RedstoneSize            uint8   `json:"redstoneSize"`
		RedstoneTries           uint8   `json:"redstoneCount"`
		RedstoneMinHeight       uint8   `json:"redstoneMinHeight"`
		RedstoneMaxHeight       uint8   `json:"redstoneMaxHeight"`
		DiamondSize             uint8   `json:"diamondSize"`
		DiamondTries            uint8   `json:"diamondCount"`
		DiamondMinHeight        uint8   `json:"diamondMinHeight"`
		DiamondMaxHeight        uint8   `json:"diamondMaxHeight"`
		LapisSize               uint8   `json:"lapisSize"`
		LapisTries              uint8   `json:"lapisCount"`
		LapisCenterHeight       uint8   `json:"lapisCenterHeight"`
		LapisSpread             uint8   `json:"lapisSpread"`
		MainNoiseScaleX         float32 `json:"mainNoiseScaleX"`
		MainNoiseScaleY         float32 `json:"mainNoiseScaleY"`
		MainNoiseScaleZ         float32 `json:"mainNoiseScaleZ"`
		DepthNoiseScaleX        float32 `json:"depthNoiseScaleX"`
		DepthNoiseScaleY        float32 `json:"depthNoiseScaleY"`
		DepthNoiseScaleZ        float32 `json:"depthNoiseScaleZ"`
		DepthNoiseScaleExponent float32 `json:"depthNoiseScaleExponent"`
		BaseSize                float32 `json:"baseSize"`
		CoordinateScale         float32 `json:"coordinateScale"`
		HeightScale             float32 `json:"heightScale"`
		HeightStretch           float32 `json:"stretchY"`
		UpperLimitScale         float32 `json:"upperLimitScale"`
		LowerLimitScale         float32 `json:"lowerLimitScale"`
		BiomeDepthWeight        float32 `json:"biomeDepthWeight"`
		BiomeScaleOffset        float32 `json:"biomeDepthOffset"`
		BiomeScaleWeight        float32 `json:"biomeScaleWeight"`
		BiomeDepthOffset        float32 `json:"biomeScaleOffset"`
	}
}
<commit_msg>Changed values to float64<commit_after>package data

type ServerProperties struct {
	ID         int
	Properties map[string]string
}

type DefaultMap struct {
	Mode               int
	Name               string
	GameMode           int32
	Seed               int64
	Structures, Cheats bool
}

type SuperFlatMap struct {
	DefaultMap
	GeneratorSettings string
}

type CustomMap struct {
	DefaultMap
	GeneratorSettings struct {
		SeaLevel                uint8   `json:"seaLevel"`
		Caves                   bool    `json:"useCaves"`
		Strongholds             bool    `json:"useStrongholds"`
		Villages                bool    `json:"useVillages"`
		Mineshafts              bool    `json:"useMineShafts"`
		Temples                 bool    `json:"useTemples"`
		OceanMonuments          bool    `json:"useMonuments"` // Needs checking
		Ravines                 bool    `json:"useRavines"`
		Dungeons                bool    `json:"useDungeons"`
		DungeonChance           uint8   `json:"dungeonChance"`
		WaterLake               bool    `json:"useWaterLake"`
		WaterLakeChance         uint8   `json:"waterLakeChance"`
		LaveLake                bool    `json:"useLavaLake"`
		LavaLakeChance          uint8   `json:"lavaLakeChance"`
		LavaOceans              bool    `json:"useLavaOceans"`
		Biome                   int16   `json:"fixedBiome"`
		BiomeSize               uint8   `json:"biomeSize"`
		RiverSize               uint8   `json:"riverSize"`
		DirtSize                uint8   `json:"dirtSize"`
		DirtTries               uint8   `json:"dirtCount"`
		DirtMinHeight           uint8   `json:"dirtMinHeight"`
		DirtMaxHeight           uint8   `json:"dirtMaxHeight"`
		GravelSize              uint8   `json:"gravelSize"`
		GravelTries             uint8   `json:"gravelCount"`
		GravelMinHeight         uint8   `json:"gravelMinHeight"`
		GravelMaxHeight         uint8   `json:"gravelMaxHeight"`
		GraniteSize             uint8   `json:"graniteSize"`
		GraniteTries            uint8   `json:"graniteCount"`
		GraniteMinHeight        uint8   `json:"graniteMinHeight"`
		GraniteMaxHeight        uint8   `json:"graniteMaxHeight"`
		DiortiteSize            uint8   `json:"dioriteSize"`
		DiortiteTries           uint8   `json:"dioriteCount"`
		DiortiteMinHeight       uint8   `json:"dioriteMinHeight"`
		DiortiteMaxHeight       uint8   `json:"dioriteMaxHeight"`
		AndesiteSize            uint8   `json:"andesiteSize"`
		AndesiteTries           uint8   `json:"andesiteCount"`
		AndesiteMinHeight       uint8   `json:"andesiteMinHeight"`
		AndesiteMaxHeight       uint8   `json:"andesiteMaxHeight"`
		CoalSize                uint8   `json:"coalSize"`
		CoalTries               uint8   `json:"coalCount"`
		CoalMinHeight           uint8   `json:"coalMinHeight"`
		CoalMaxHeight           uint8   `json:"coalMaxHeight"`
		IronSize                uint8   `json:"ironSize"`
		IronTries               uint8   `json:"ironCount"`
		IronMinHeight           uint8   `json:"ironMinHeight"`
		IronMaxHeight           uint8   `json:"ironMaxHeight"`
		GoldSize                uint8   `json:"goldSize"`
		GoldTries               uint8   `json:"goldCount"`
		GoldMinHeight           uint8   `json:"goldMinHeight"`
		GoldMaxHeight           uint8   `json:"goldMaxHeight"`
		RedstoneSize            uint8   `json:"redstoneSize"`
		RedstoneTries           uint8   `json:"redstoneCount"`
		RedstoneMinHeight       uint8   `json:"redstoneMinHeight"`
		RedstoneMaxHeight       uint8   `json:"redstoneMaxHeight"`
		DiamondSize             uint8   `json:"diamondSize"`
		DiamondTries            uint8   `json:"diamondCount"`
		DiamondMinHeight        uint8   `json:"diamondMinHeight"`
		DiamondMaxHeight        uint8   `json:"diamondMaxHeight"`
		LapisSize               uint8   `json:"lapisSize"`
		LapisTries              uint8   `json:"lapisCount"`
		LapisCenterHeight       uint8   `json:"lapisCenterHeight"`
		LapisSpread             uint8   `json:"lapisSpread"`
		MainNoiseScaleX         float64 `json:"mainNoiseScaleX"`
		MainNoiseScaleY         float64 `json:"mainNoiseScaleY"`
		MainNoiseScaleZ         float64 `json:"mainNoiseScaleZ"`
		DepthNoiseScaleX        float64 `json:"depthNoiseScaleX"`
		DepthNoiseScaleZ        float64 `json:"depthNoiseScaleZ"`
		DepthNoiseScaleExponent float64 `json:"depthNoiseScaleExponent"`
		BaseSize                float64 `json:"baseSize"`
		CoordinateScale         float64 `json:"coordinateScale"`
		HeightScale             float64 `json:"heightScale"`
		HeightStretch           float64 `json:"stretchY"`
		UpperLimitScale         float64 `json:"upperLimitScale"`
		LowerLimitScale         float64 `json:"lowerLimitScale"`
		BiomeDepthWeight        float64 `json:"biomeDepthWeight"`
		BiomeScaleOffset        float64 `json:"biomeDepthOffset"`
		BiomeScaleWeight        float64 `json:"biomeScaleWeight"`
		BiomeDepthOffset        float64 `json:"biomeScaleOffset"`
	}
}
<|endoftext|>


<commit_before>// +build linux

// Copyright 2017 Kinvolk
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elf

import (
	"testing"
)

var testData = []struct {
	succeed       bool
	releaseString string
	kernelVersion int
}{
	{true, "4.1.2-3", 262402},
	{true, "4.8.14-200.fc24.x86_64", 264206},
	{true, "4.1.2-3foo", 262402},
	{true, "4.1.2foo-1", 262402},
	{true, "4.1.2-rkt-v1", 262402},
	{true, "4.1.2rkt-v1", 262402},
	{true, "4.1.2-3 foo", 262402},
	{false, "foo 4.1.2-3", -1},
	{true, "4.1.2", 262402},
	{false, ".4.1.2", -1},
	{false, "4.1.", -1},
	{false, "4.1", -1},
}

func TestKernelVersionFromReleaseString(t *testing.T) {
	for _, test := range testData {
		version, err := kernelVersionFromReleaseString(test.releaseString)
		if err != nil && test.succeed {
			t.Errorf("expected %q to succeed: %s", test.releaseString, err)
		} else if err == nil && !test.succeed {
			t.Errorf("expected %q to fail", test.releaseString)
		}
		if version != test.kernelVersion {
			t.Errorf("expected kernel version %d, got %d", test.kernelVersion, version)
		}
	}
}
<commit_msg>kernel_version_test: fix version type<commit_after>// +build linux

// Copyright 2017 Kinvolk
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package elf

import (
	"testing"
)

var testData = []struct {
	succeed       bool
	releaseString string
	kernelVersion uint32
}{
	{true, "4.1.2-3", 262402},
	{true, "4.8.14-200.fc24.x86_64", 264206},
	{true, "4.1.2-3foo", 262402},
	{true, "4.1.2foo-1", 262402},
	{true, "4.1.2-rkt-v1", 262402},
	{true, "4.1.2rkt-v1", 262402},
	{true, "4.1.2-3 foo", 262402},
	{false, "foo 4.1.2-3", 0},
	{true, "4.1.2", 262402},
	{false, ".4.1.2", 0},
	{false, "4.1.", 0},
	{false, "4.1", 0},
}

func TestKernelVersionFromReleaseString(t *testing.T) {
	for _, test := range testData {
		version, err := kernelVersionFromReleaseString(test.releaseString)
		if err != nil && test.succeed {
			t.Errorf("expected %q to succeed: %s", test.releaseString, err)
		} else if err == nil && !test.succeed {
			t.Errorf("expected %q to fail", test.releaseString)
		}
		if version != test.kernelVersion {
			t.Errorf("expected kernel version %d, got %d", test.kernelVersion, version)
		}
	}
}
<|endoftext|>


<commit_before>package opc

// Raver plaid
//   A rainbowy pattern with moving diagonal black stripes

import (
	"bitbucket.org/davidwallace/go-metal/colorutils"
	"bitbucket.org/davidwallace/go-metal/midi"
	"fmt"
	"time"
)

func getAvailableMidiMessages(midiMessageChan chan *midi.MidiMessage) []*midi.MidiMessage {
	result := make([]*midi.MidiMessage, 0)
	for {
		if len(midiMessageChan) == 0 {
			break
		}
		result = append(result, <-midiMessageChan)
	}
	return result
}

const MIDI_VOLUME_GAIN = 1.5    // multiply incoming midi volumes by this much
const MIDI_BRIGHTNESS_MIN = 0.5 // midi volume 1/127, after MIDI_VOLUME_GAIN, -> this much
const MIDI_BRIGHTNESS_MAX = 1   // midi volume 127, after MIDI_VOLUME_GAIN, -> this much
const SECONDS_TO_FADE = 1

func MakePatternBasicMidi(locations []float64) ByteThread {
	return func(bytesIn chan []byte, bytesOut chan []byte, midiMessageChan chan *midi.MidiMessage) {

		// the current volume of each key, from 0 to 1, after applying MIDI_* adjustments
		keyVolumes := make([]float64, 128)
		// smoothed value: like keyVolumes, but fades away slowly when key is off
		smoothedVolumes := make([]float64, 128)

		last_t := float64(0)
		for bytes := range bytesIn {
			n_pixels := len(bytes) / 3
			t := float64(time.Now().UnixNano())/1.0e9 - 9.4e8
			tDiff := colorutils.Clamp(t-last_t, 0, 5) // limit to max of 5 second to avoid pathological value at startup

			// fetch midi messages and maintain state of keyVolumes
			midiMessages := getAvailableMidiMessages(midiMessageChan)
			for _, m := range midiMessages {
				fmt.Println("        ", m)
				if m.Kind == midi.NOTE_ON {
					if m.Value == 0 {
						keyVolumes[m.Key] = 0
					} else {
						keyVolumes[m.Key] = colorutils.Remap(float64(m.Value)/127*MIDI_VOLUME_GAIN, 0, 1, MIDI_BRIGHTNESS_MIN, MIDI_BRIGHTNESS_MAX)
					}
				}
			}

			// update smoothedVolumes: fade old values and re-apply current states
			for ii, v := range smoothedVolumes {
				smoothedVolumes[ii] = colorutils.Clamp(v-tDiff/SECONDS_TO_FADE, 0, 1)
				if keyVolumes[ii] > smoothedVolumes[ii] {
					smoothedVolumes[ii] = keyVolumes[ii]
				}
			}

			// fill in bytes array
			for ii := 0; ii < n_pixels; ii++ {
				//--------------------------------------------------------------------------------

				if ii < len(keyVolumes) {
					k := keyVolumes[ii]
					s := smoothedVolumes[ii]
					if k > 0 {
						// key is currently down
						bytes[ii*3+0] = colorutils.FloatToByte(k)
						bytes[ii*3+1] = colorutils.FloatToByte(k)
						bytes[ii*3+2] = colorutils.FloatToByte(k)
					} else {
						// key not currently down.  use smoothed value which is fading away over time
						bytes[ii*3+0] = colorutils.FloatToByte(s)
						bytes[ii*3+1] = colorutils.FloatToByte(0)
						bytes[ii*3+2] = colorutils.FloatToByte(0)
					}
				} else {
					// if we have more LEDs than MIDI keys
					bytes[ii*3+0] = 0
					bytes[ii*3+1] = 0
					bytes[ii*3+2] = 0
				}

				//--------------------------------------------------------------------------------
			}

			last_t = t
			bytesOut <- bytes
		}
	}
}
<commit_msg>basic midi upgrade in progress, not working<commit_after>package opc

// Raver plaid
//   A rainbowy pattern with moving diagonal black stripes

import (
	"bitbucket.org/davidwallace/go-metal/colorutils"
	"bitbucket.org/davidwallace/go-metal/midi"
	"fmt"
	"time"
)

const MIDI_VOLUME_GAIN = 1.5    // multiply incoming midi volumes by this much
const MIDI_BRIGHTNESS_MIN = 0.5 // midi volume 1/127, after MIDI_VOLUME_GAIN, -> this much
const MIDI_BRIGHTNESS_MAX = 1   // midi volume 127, after MIDI_VOLUME_GAIN, -> this much
const SECONDS_TO_FADE = 1.5
const FADING_GAIN = 0.5  // fading pixels start at this brightness
const COLOR_BLEEDING_RAD = 3

func getAvailableMidiMessages(midiMessageChan chan *midi.MidiMessage) []*midi.MidiMessage {
	result := make([]*midi.MidiMessage, 0)
	for {
		if len(midiMessageChan) == 0 {
			break
		}
		result = append(result, <-midiMessageChan)
	}
	return result
}

func pitchToRGB(pitch int) (float64, float64, float64) {
    var r, g, b float64
    switch pitch % 12 {
    case 0:
        r = 1
        g = 0
        b = 0
    case 1:
        r = 0.9
        g = 0.4
        b = 0
    case 2:
        r = 0.8
        g = 0.8
        b = 0
    case 3:
        r = 0.4
        g = 0.9
        b = 0
    case 4:
        r = 0
        g = 1
        b = 0
    case 5:
        r = 0
        g = 0.9
        b = 0.4
    case 6:
        r = 0
        g = 0.8
        b = 0.8
    case 7:
        r = 0
        g = 0.4
        b = 0.9
    case 8:
        r = 0
        g = 0
        b = 1
    case 9:
        r = 0.4
        g = 0
        b = 0.9
    case 10:
        r = 0.8
        g = 0
        b = 0.8
    case 11:
        r = 0.9
        g = 0
        b = 0.4
    }
    return r,g,b
}

func MakePatternBasicMidi(locations []float64) ByteThread {
	return func(bytesIn chan []byte, bytesOut chan []byte, midiMessageChan chan *midi.MidiMessage) {

		// the current volume of each key, from 0 to 1, after applying MIDI_* adjustments
		keyVolumes := make([]float64, 128)
		// smoothed value: like keyVolumes, but fades away slowly when key is off
		smoothedVolumes := make([]float64, 128)

		last_t := float64(0)
		for bytes := range bytesIn {
			n_pixels := len(bytes) / 3
			t := float64(time.Now().UnixNano())/1.0e9 - 9.4e8
			tDiff := colorutils.Clamp(t-last_t, 0, 5) // limit to max of 5 second to avoid pathological value at startup

			// fetch midi messages and maintain state of keyVolumes
			midiMessages := getAvailableMidiMessages(midiMessageChan)
			for _, m := range midiMessages {
				fmt.Println("        ", m)
				if m.Kind == midi.NOTE_ON {
					if m.Value == 0 {
						keyVolumes[m.Key] = 0
					} else {
						keyVolumes[m.Key] = colorutils.Remap(float64(m.Value)/127*MIDI_VOLUME_GAIN, 0, 1, MIDI_BRIGHTNESS_MIN, MIDI_BRIGHTNESS_MAX)
					}
				}
			}

			// update smoothedVolumes: fade old values and re-apply current states
			for ii, v := range smoothedVolumes {
				smoothedVolumes[ii] = colorutils.Clamp(v-tDiff/SECONDS_TO_FADE, 0, 1)
				if keyVolumes[ii] > smoothedVolumes[ii] {
					smoothedVolumes[ii] = keyVolumes[ii]
				}
			}

			// fill in bytes array
			for ii := 0; ii < n_pixels; ii++ {
				//--------------------------------------------------------------------------------

				if ii < len(keyVolumes) {
					k := keyVolumes[ii]
					s := smoothedVolumes[ii]

                    r := float64(0)
                    g := float64(0)
                    b := float64(0)
                    pr, pg, pb := pitchToRGB(ii)

					if k > 0 {
						// key is currently down
                        r = pr * k
                        g = pg * k
                        b = pb * k

					} else {
						// key not currently down.  use smoothed value which is fading away over time
                        r = pr * s
                        g = pg * s
                        b = pb * s
					}

                    // color bleeding
                    fmt.Println("")
                    for offset := -COLOR_BLEEDING_RAD; offset <= COLOR_BLEEDING_RAD; offset++ {
                        if (ii+offset < 0) || (ii+offset >= n_pixels) {
                            continue
                        }
                        if keyVolumes[ii+offset] > 0 {
                            brightness := float64(offset)/(float64(COLOR_BLEEDING_RAD)+1)
                            if brightness < 0 { brightness = - brightness}
                            brightness = 1 - brightness
                            fmt.Println(ii, offset, brightness)
                            pr2, pg2, pb2 := pitchToRGB(ii+offset)
                            r += pr2 * keyVolumes[ii+offset] * brightness
                            g += pg2 * keyVolumes[ii+offset] * brightness
                            b += pb2 * keyVolumes[ii+offset] * brightness
                        }
                    }

                    bytes[ii*3+0] = colorutils.FloatToByte(r)
                    bytes[ii*3+1] = colorutils.FloatToByte(g)
                    bytes[ii*3+2] = colorutils.FloatToByte(b)
				} else {
					// if we have more LEDs than MIDI keys
					bytes[ii*3+0] = 0
					bytes[ii*3+1] = 0
					bytes[ii*3+2] = 0
				}

				//--------------------------------------------------------------------------------
			}

			last_t = t
			bytesOut <- bytes
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

package main

import (
	"bytes"
	"context"
	"crypto/md5"
	"crypto/rsa"
	"encoding/binary"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"sync"
	"testing"
	"time"

	"golang.org/x/crypto/ssh"
)

type testLogger struct {
	*testing.T // Already has Fatalf method
}

func (t testLogger) Printf(f string, x ...interface{}) { t.Logf(f, x...) }

func TestTunnel(t *testing.T) {
	rootWG := new(sync.WaitGroup)
	defer rootWG.Wait()
	rootCtx, cancelAll := context.WithCancel(context.Background())
	defer cancelAll()

	// Open all of the TCP sockets needed for the test.
	tcpLn0 := openListener(t) // Start of the chain
	tcpLn1 := openListener(t) // Mid-point of the chain
	tcpLn2 := openListener(t) // End of the chain
	srvLn0 := openListener(t) // Socket for SSH server in reverse mode
	srvLn1 := openListener(t) // Socket for SSH server in forward mode

	tcpLn0.Close() // To be later binded by the reverse tunnel
	tcpLn1.Close() // To be later binded by the forward tunnel
	go closeWhenDone(rootCtx, tcpLn2)
	go closeWhenDone(rootCtx, srvLn0)
	go closeWhenDone(rootCtx, srvLn1)

	// Generate keys for both the servers and clients.
	clientPriv0, clientPub0 := generateKeys(t)
	clientPriv1, clientPub1 := generateKeys(t)
	serverPriv0, serverPub0 := generateKeys(t)
	serverPriv1, serverPub1 := generateKeys(t)

	// Start the SSH servers.
	rootWG.Add(2)
	go func() {
		defer rootWG.Done()
		runServer(t, rootCtx, srvLn0, serverPriv0, clientPub0, clientPub1)
	}()
	go func() {
		defer rootWG.Done()
		runServer(t, rootCtx, srvLn1, serverPriv1, clientPub0, clientPub1)
	}()

	wg := new(sync.WaitGroup)
	defer wg.Wait()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	// Create the tunnel configurations.
	tn0 := tunnel{
		auth:     []ssh.AuthMethod{ssh.PublicKeys(clientPriv0)},
		hostKeys: ssh.FixedHostKey(serverPub0),
		mode:     '<', // Reverse tunnel
		user:     "user0",
		hostAddr: srvLn0.Addr().String(),
		bindAddr: tcpLn0.Addr().String(),
		dialAddr: tcpLn1.Addr().String(),
		log:      testLogger{t},
	}
	tn1 := tunnel{
		auth:     []ssh.AuthMethod{ssh.PublicKeys(clientPriv1)},
		hostKeys: ssh.FixedHostKey(serverPub1),
		mode:     '>', // Forward tunnel
		user:     "user1",
		hostAddr: srvLn1.Addr().String(),
		bindAddr: tcpLn1.Addr().String(),
		dialAddr: tcpLn2.Addr().String(),
		log:      testLogger{t},
	}

	// Start the SSH client tunnels.
	wg.Add(2)
	go tn0.bindTunnel(ctx, wg)
	go tn1.bindTunnel(ctx, wg)

	t.Log("test started")
	done := make(chan bool, 10)

	// Start all the transmitters.
	for i := 0; i < cap(done); i++ {
		i := i
		go func() {
			for {
				rnd := rand.New(rand.NewSource(int64(i)))
				hash := md5.New()
				size := uint32((1 << 10) + rnd.Intn(1<<20))
				buf4 := make([]byte, 4)
				binary.LittleEndian.PutUint32(buf4, size)

				cnStart, err := net.Dial("tcp", tcpLn0.Addr().String())
				if err != nil {
					time.Sleep(10 * time.Millisecond)
					continue
				}
				defer cnStart.Close()
				if _, err := cnStart.Write(buf4); err != nil {
					t.Errorf("write size error: %v", err)
					break
				}
				r := io.LimitReader(rnd, int64(size))
				w := io.MultiWriter(cnStart, hash)
				if _, err := io.Copy(w, r); err != nil {
					t.Errorf("copy error: %v", err)
					break
				}
				if _, err := cnStart.Write(hash.Sum(nil)); err != nil {
					t.Errorf("write hash error: %v", err)
					break
				}
				if err := cnStart.Close(); err != nil {
					t.Errorf("close error: %v", err)
					break
				}
				break
			}
		}()
	}

	// Start all the receivers.
	for i := 0; i < cap(done); i++ {
		go func() {
			for {
				hash := md5.New()
				buf4 := make([]byte, 4)

				cnEnd, err := tcpLn2.Accept()
				if err != nil {
					time.Sleep(10 * time.Millisecond)
					continue
				}
				defer cnEnd.Close()

				if _, err := io.ReadFull(cnEnd, buf4); err != nil {
					t.Errorf("read size error: %v", err)
					break
				}
				size := binary.LittleEndian.Uint32(buf4)
				r := io.LimitReader(cnEnd, int64(size))
				if _, err := io.Copy(hash, r); err != nil {
					t.Errorf("copy error: %v", err)
					break
				}
				wantHash, err := ioutil.ReadAll(cnEnd)
				if err != nil {
					t.Errorf("read hash error: %v", err)
					break
				}
				if err := cnEnd.Close(); err != nil {
					t.Errorf("close error: %v", err)
					break
				}

				if gotHash := hash.Sum(nil); !bytes.Equal(gotHash, wantHash) {
					t.Errorf("hash mismatch:\ngot  %x\nwant %x", gotHash, wantHash)
				}
				break
			}
			done <- true
		}()
	}

	for i := 0; i < cap(done); i++ {
		select {
		case <-done:
		case <-time.After(10 * time.Second):
			t.Errorf("timed out: %d remaining", cap(done)-i)
			return
		}
	}
	t.Log("test complete")
}

// generateKeys generates a random pair of SSH private and public keys.
func generateKeys(t *testing.T) (priv ssh.Signer, pub ssh.PublicKey) {
	rnd := rand.New(rand.NewSource(time.Now().Unix()))
	rsaKey, err := rsa.GenerateKey(rnd, 1024)
	if err != nil {
		t.Fatalf("unable to generate RSA key pair: %v", err)
	}
	priv, err = ssh.NewSignerFromKey(rsaKey)
	if err != nil {
		t.Fatalf("unable to generate signer: %v", err)
	}
	pub, err = ssh.NewPublicKey(&rsaKey.PublicKey)
	if err != nil {
		t.Fatalf("unable to generate public key: %v", err)
	}
	return priv, pub
}

func openListener(t *testing.T) net.Listener {
	ln, err := net.Listen("tcp", fmt.Sprintf(":0"))
	if err != nil {
		t.Fatalf("listen error: %v", err)
	}
	return ln
}
<commit_msg>Remove trivial use of fmt.Sprintf<commit_after>// Copyright 2017, The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

package main

import (
	"bytes"
	"context"
	"crypto/md5"
	"crypto/rsa"
	"encoding/binary"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"sync"
	"testing"
	"time"

	"golang.org/x/crypto/ssh"
)

type testLogger struct {
	*testing.T // Already has Fatalf method
}

func (t testLogger) Printf(f string, x ...interface{}) { t.Logf(f, x...) }

func TestTunnel(t *testing.T) {
	rootWG := new(sync.WaitGroup)
	defer rootWG.Wait()
	rootCtx, cancelAll := context.WithCancel(context.Background())
	defer cancelAll()

	// Open all of the TCP sockets needed for the test.
	tcpLn0 := openListener(t) // Start of the chain
	tcpLn1 := openListener(t) // Mid-point of the chain
	tcpLn2 := openListener(t) // End of the chain
	srvLn0 := openListener(t) // Socket for SSH server in reverse mode
	srvLn1 := openListener(t) // Socket for SSH server in forward mode

	tcpLn0.Close() // To be later binded by the reverse tunnel
	tcpLn1.Close() // To be later binded by the forward tunnel
	go closeWhenDone(rootCtx, tcpLn2)
	go closeWhenDone(rootCtx, srvLn0)
	go closeWhenDone(rootCtx, srvLn1)

	// Generate keys for both the servers and clients.
	clientPriv0, clientPub0 := generateKeys(t)
	clientPriv1, clientPub1 := generateKeys(t)
	serverPriv0, serverPub0 := generateKeys(t)
	serverPriv1, serverPub1 := generateKeys(t)

	// Start the SSH servers.
	rootWG.Add(2)
	go func() {
		defer rootWG.Done()
		runServer(t, rootCtx, srvLn0, serverPriv0, clientPub0, clientPub1)
	}()
	go func() {
		defer rootWG.Done()
		runServer(t, rootCtx, srvLn1, serverPriv1, clientPub0, clientPub1)
	}()

	wg := new(sync.WaitGroup)
	defer wg.Wait()
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	// Create the tunnel configurations.
	tn0 := tunnel{
		auth:     []ssh.AuthMethod{ssh.PublicKeys(clientPriv0)},
		hostKeys: ssh.FixedHostKey(serverPub0),
		mode:     '<', // Reverse tunnel
		user:     "user0",
		hostAddr: srvLn0.Addr().String(),
		bindAddr: tcpLn0.Addr().String(),
		dialAddr: tcpLn1.Addr().String(),
		log:      testLogger{t},
	}
	tn1 := tunnel{
		auth:     []ssh.AuthMethod{ssh.PublicKeys(clientPriv1)},
		hostKeys: ssh.FixedHostKey(serverPub1),
		mode:     '>', // Forward tunnel
		user:     "user1",
		hostAddr: srvLn1.Addr().String(),
		bindAddr: tcpLn1.Addr().String(),
		dialAddr: tcpLn2.Addr().String(),
		log:      testLogger{t},
	}

	// Start the SSH client tunnels.
	wg.Add(2)
	go tn0.bindTunnel(ctx, wg)
	go tn1.bindTunnel(ctx, wg)

	t.Log("test started")
	done := make(chan bool, 10)

	// Start all the transmitters.
	for i := 0; i < cap(done); i++ {
		i := i
		go func() {
			for {
				rnd := rand.New(rand.NewSource(int64(i)))
				hash := md5.New()
				size := uint32((1 << 10) + rnd.Intn(1<<20))
				buf4 := make([]byte, 4)
				binary.LittleEndian.PutUint32(buf4, size)

				cnStart, err := net.Dial("tcp", tcpLn0.Addr().String())
				if err != nil {
					time.Sleep(10 * time.Millisecond)
					continue
				}
				defer cnStart.Close()
				if _, err := cnStart.Write(buf4); err != nil {
					t.Errorf("write size error: %v", err)
					break
				}
				r := io.LimitReader(rnd, int64(size))
				w := io.MultiWriter(cnStart, hash)
				if _, err := io.Copy(w, r); err != nil {
					t.Errorf("copy error: %v", err)
					break
				}
				if _, err := cnStart.Write(hash.Sum(nil)); err != nil {
					t.Errorf("write hash error: %v", err)
					break
				}
				if err := cnStart.Close(); err != nil {
					t.Errorf("close error: %v", err)
					break
				}
				break
			}
		}()
	}

	// Start all the receivers.
	for i := 0; i < cap(done); i++ {
		go func() {
			for {
				hash := md5.New()
				buf4 := make([]byte, 4)

				cnEnd, err := tcpLn2.Accept()
				if err != nil {
					time.Sleep(10 * time.Millisecond)
					continue
				}
				defer cnEnd.Close()

				if _, err := io.ReadFull(cnEnd, buf4); err != nil {
					t.Errorf("read size error: %v", err)
					break
				}
				size := binary.LittleEndian.Uint32(buf4)
				r := io.LimitReader(cnEnd, int64(size))
				if _, err := io.Copy(hash, r); err != nil {
					t.Errorf("copy error: %v", err)
					break
				}
				wantHash, err := ioutil.ReadAll(cnEnd)
				if err != nil {
					t.Errorf("read hash error: %v", err)
					break
				}
				if err := cnEnd.Close(); err != nil {
					t.Errorf("close error: %v", err)
					break
				}

				if gotHash := hash.Sum(nil); !bytes.Equal(gotHash, wantHash) {
					t.Errorf("hash mismatch:\ngot  %x\nwant %x", gotHash, wantHash)
				}
				break
			}
			done <- true
		}()
	}

	for i := 0; i < cap(done); i++ {
		select {
		case <-done:
		case <-time.After(10 * time.Second):
			t.Errorf("timed out: %d remaining", cap(done)-i)
			return
		}
	}
	t.Log("test complete")
}

// generateKeys generates a random pair of SSH private and public keys.
func generateKeys(t *testing.T) (priv ssh.Signer, pub ssh.PublicKey) {
	rnd := rand.New(rand.NewSource(time.Now().Unix()))
	rsaKey, err := rsa.GenerateKey(rnd, 1024)
	if err != nil {
		t.Fatalf("unable to generate RSA key pair: %v", err)
	}
	priv, err = ssh.NewSignerFromKey(rsaKey)
	if err != nil {
		t.Fatalf("unable to generate signer: %v", err)
	}
	pub, err = ssh.NewPublicKey(&rsaKey.PublicKey)
	if err != nil {
		t.Fatalf("unable to generate public key: %v", err)
	}
	return priv, pub
}

func openListener(t *testing.T) net.Listener {
	ln, err := net.Listen("tcp", ":0")
	if err != nil {
		t.Fatalf("listen error: %v", err)
	}
	return ln
}
<|endoftext|>