<commit_before>/*
 * Copyright 2016 Robin Engel
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package moseldserver

import (
	"github.com/bluedevel/mosel/moselserver"
	"github.com/bluedevel/mosel/moseld/server/handler"
	"github.com/bluedevel/mosel/moseld/server/context"
	"github.com/bluedevel/mosel/config"
	"errors"
	"net/url"
	"log"
	"github.com/bluedevel/mosel/commons"
	"fmt"
	"time"
)

// The server started by moseld.
// It collects data from the configured nodes and from locally running scripts and provides it over a rest service.
type moseldServer struct {
	moselserver.MoselServer

	config  moselconfig.MoseldServerConfig
	context *context.MoseldServerContext
}

// Construct a new Instance of a MoselServer for a given configuration.
func NewMoseldServer(config moselconfig.MoseldServerConfig) *moseldServer {
	server := moseldServer{
		config: config,
		context: new(context.MoseldServerContext),
	}

	server.MoselServer = moselserver.MoselServer{
		Config: config.MoselServerConfig,
	}

	server.InitFuncs = append(server.InitFuncs,
		server.initDebs,
		server.initDataPersistence,
		server.initDataCache,
		server.initNodeCache,
	)

	server.Handlers = []moselserver.MoselHandler{
		handler.NewLoginHandler(server.context),
		handler.NewPingHandler(),
		handler.NewDebugHandler(server.context),
		handler.NewNodeInfoHandler(server.context),
		handler.NewInfoHandler(server.context),
	}

	return &server
}

/*
 * Initialize Context
 */

// Initializes dependencies within the server context.
func (server *moseldServer) initDebs() error {
	ctx := server.context
	ctx.MoselServerContext = server.Context

	var err error

	if ctx.DataCache, err =
		context.NewDataCache(); err != nil {
		return err
	}

	persistenceConfig := server.config.PersistenceConfig
	if persistenceConfig.Enabled {
		dataSource, ok := ctx.DataSources[persistenceConfig.DataSource]

		if !ok {
			return fmt.Errorf("Datasource %s not configured", persistenceConfig.DataSource)
		}

		var queries commons.SqlQueries
		var err error
		if queries, err = commons.GetQueries(dataSource.Type); err != nil {
			return fmt.Errorf("No queries configured for sql dialect %s", dataSource.Type)
		}

		ctx.DataPersistence = context.NewSqlDataPersistence(dataSource.Db, queries)
	}

	if ctx.NodeHandler, err =
		context.NewNodeRespHandler(ctx.DataCache, ctx.DataPersistence); err != nil {
		return err
	}

	if ctx.Scripts, err =
		context.NewScriptCache(server.config.Scripts); err != nil {
		return err
	}

	if ctx.ScriptsRunner, err =
		context.NewScriptsRunner(ctx.Scripts, ctx.DataCache, ctx.DataPersistence); err != nil {
		return err
	}

	if ctx.Nodes, err =
		context.NewNodeCache(ctx.NodeHandler, ctx.Scripts); err != nil {
		return err
	}

	return nil
}

// Initialize the node cache withe the configured nodes
func (server *moseldServer) initNodeCache() error {
	for nodeName, nodeConf := range server.config.Node {

		var scripts []string

		//get configured
		if len(nodeConf.Scripts) > 0 {
			//manual config
			scripts = nodeConf.Scripts
		} else {
			//default
			scripts = server.context.Scripts.GetScripts()
		}

		//exclude certain scripts
		for _, exclude := range nodeConf.ScriptsExclude {
			for i, script := range scripts {
				if script == exclude {
					scripts = append(scripts[:i], scripts[i + 1:]...)
				}
			}
		}

		nodeScripts := make([]string, 0)
		localScripts := make([]string, 0)

		//sort scripts by scope
		for _, script := range scripts {
			if scriptConf, err := server.context.Scripts.GetScriptConfig(script); err == nil {
				switch scriptConf.Scope {
				case "node":
					nodeScripts = append(nodeScripts, script)
					break
				case "local":
					localScripts = append(localScripts, script)
					break
				default:
					return errors.New("Unknown script scope " + scriptConf.Scope + " for node " + nodeName)
				}

			} else {
				return err
			}
		}

		log.Printf("Scripts configured for node %s: node=%s local=%s", nodeName, nodeScripts, localScripts)

		//get base url
		var url *url.URL
		var err error
		if url, err = url.Parse(nodeConf.URL); err != nil {
			return err
		}

		//instantiate node
		node, err := context.NewNode(
			nodeName,
			*url,
			nodeConf.User,
			nodeConf.Password,
			nodeScripts,
			server.context.NodeHandler,
			server.context.Scripts)

		if err != nil {
			return err
		}

		server.context.Nodes.Add(node)
		server.context.ScriptsRunner.Run(localScripts, node)
	}

	return nil
}

// Initialize the data persistence
func (server *moseldServer) initDataPersistence() error {
	if server.context.DataPersistence == nil {
		return nil
	}
	log.Println("Init data persistence")
	defer log.Println("Finished initializing data persistence")
	return server.context.DataPersistence.Init()
}

// Initialize the data persistence
func (server *moseldServer) initDataCache() error {
	var storage context.DataCacheStorage
	var err error

	log.Println("Init data cache")

	cacheSize := server.config.DataCache.CacheSize

	if cacheSize == "" {
		storage, err = server.context.DataPersistence.GetAll()
		if err != nil {
			return err
		}
	} else {
		dur, err := time.ParseDuration(cacheSize)
		if err != nil {
			return err
		}

		server.context.DataCache.CacheSize = dur
		storage, err = server.context.DataPersistence.GetAllSince(dur)
		if err != nil {
			return err
		}
	}

	// count points
	pointCount := 0
	for _, points := range storage {
		pointCount += len(points)
	}

	server.context.DataCache.SetStorage(storage)

	// clean up the cache
	go func() {
		for {
			server.context.DataCache.Clean()
			time.Sleep(10 * time.Second)
		}
	}()

	log.Printf("Finished initializing data cache with %d data points", pointCount)
	return nil
}<commit_msg>refactoring exclude from array<commit_after>/*
 * Copyright 2016 Robin Engel
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package moseldserver

import (
	"github.com/bluedevel/mosel/moselserver"
	"github.com/bluedevel/mosel/moseld/server/handler"
	"github.com/bluedevel/mosel/moseld/server/context"
	"github.com/bluedevel/mosel/config"
	"errors"
	"net/url"
	"log"
	"github.com/bluedevel/mosel/commons"
	"fmt"
	"time"
)

// The server started by moseld.
// It collects data from the configured nodes and from locally running scripts and provides it over a rest service.
type moseldServer struct {
	moselserver.MoselServer

	config  moselconfig.MoseldServerConfig
	context *context.MoseldServerContext
}

// Construct a new Instance of a MoselServer for a given configuration.
func NewMoseldServer(config moselconfig.MoseldServerConfig) *moseldServer {
	server := moseldServer{
		config: config,
		context: new(context.MoseldServerContext),
	}

	server.MoselServer = moselserver.MoselServer{
		Config: config.MoselServerConfig,
	}

	server.InitFuncs = append(server.InitFuncs,
		server.initDebs,
		server.initDataPersistence,
		server.initDataCache,
		server.initNodeCache,
	)

	server.Handlers = []moselserver.MoselHandler{
		handler.NewLoginHandler(server.context),
		handler.NewPingHandler(),
		handler.NewDebugHandler(server.context),
		handler.NewNodeInfoHandler(server.context),
		handler.NewInfoHandler(server.context),
	}

	return &server
}

/*
 * Initialize Context
 */

// Initializes dependencies within the server context.
func (server *moseldServer) initDebs() error {
	ctx := server.context
	ctx.MoselServerContext = server.Context

	var err error

	if ctx.DataCache, err =
		context.NewDataCache(); err != nil {
		return err
	}

	persistenceConfig := server.config.PersistenceConfig
	if persistenceConfig.Enabled {
		dataSource, ok := ctx.DataSources[persistenceConfig.DataSource]

		if !ok {
			return fmt.Errorf("Datasource %s not configured", persistenceConfig.DataSource)
		}

		var queries commons.SqlQueries
		var err error
		if queries, err = commons.GetQueries(dataSource.Type); err != nil {
			return fmt.Errorf("No queries configured for sql dialect %s", dataSource.Type)
		}

		ctx.DataPersistence = context.NewSqlDataPersistence(dataSource.Db, queries)
	}

	if ctx.NodeHandler, err =
		context.NewNodeRespHandler(ctx.DataCache, ctx.DataPersistence); err != nil {
		return err
	}

	if ctx.Scripts, err =
		context.NewScriptCache(server.config.Scripts); err != nil {
		return err
	}

	if ctx.ScriptsRunner, err =
		context.NewScriptsRunner(ctx.Scripts, ctx.DataCache, ctx.DataPersistence); err != nil {
		return err
	}

	if ctx.Nodes, err =
		context.NewNodeCache(ctx.NodeHandler, ctx.Scripts); err != nil {
		return err
	}

	return nil
}

// Initialize the node cache withe the configured nodes
func (server *moseldServer) initNodeCache() error {
	for nodeName, nodeConf := range server.config.Node {

		var scripts []string

		//get configured
		if len(nodeConf.Scripts) > 0 {
			//manual config
			scripts = nodeConf.Scripts
		} else {
			//default
			scripts = server.context.Scripts.GetScripts()
		}

		//exclude certain scripts
		arrayExclude(&scripts, nodeConf.ScriptsExclude)

		nodeScripts := make([]string, 0)
		localScripts := make([]string, 0)

		//sort scripts by scope
		for _, script := range scripts {
			if scriptConf, err := server.context.Scripts.GetScriptConfig(script); err == nil {
				switch scriptConf.Scope {
				case "node":
					nodeScripts = append(nodeScripts, script)
					break
				case "local":
					localScripts = append(localScripts, script)
					break
				default:
					return errors.New("Unknown script scope " + scriptConf.Scope + " for node " + nodeName)
				}

			} else {
				return err
			}
		}

		log.Printf("Scripts configured for node %s: node=%s local=%s", nodeName, nodeScripts, localScripts)

		//get base url
		var url *url.URL
		var err error
		if url, err = url.Parse(nodeConf.URL); err != nil {
			return err
		}

		//instantiate node
		node, err := context.NewNode(
			nodeName,
			*url,
			nodeConf.User,
			nodeConf.Password,
			nodeScripts,
			server.context.NodeHandler,
			server.context.Scripts)

		if err != nil {
			return err
		}

		server.context.Nodes.Add(node)
		server.context.ScriptsRunner.Run(localScripts, node)
	}

	return nil
}

func arrayExclude(arr *[]struct{}, exclude []struct{}) () {
	for _, elem := range *arr {
		for i, ex := range exclude {
			if elem == ex {
				*arr = append(arr[:i], arr[i + 1:]...)
			}
		}
	}
}

// Initialize the data persistence
func (server *moseldServer) initDataPersistence() error {
	if server.context.DataPersistence == nil {
		return nil
	}
	log.Println("Init data persistence")
	defer log.Println("Finished initializing data persistence")
	return server.context.DataPersistence.Init()
}

// Initialize the data persistence
func (server *moseldServer) initDataCache() error {
	var storage context.DataCacheStorage
	var err error

	log.Println("Init data cache")

	cacheSize := server.config.DataCache.CacheSize

	if cacheSize == "" {
		storage, err = server.context.DataPersistence.GetAll()
		if err != nil {
			return err
		}
	} else {
		dur, err := time.ParseDuration(cacheSize)
		if err != nil {
			return err
		}

		server.context.DataCache.CacheSize = dur
		storage, err = server.context.DataPersistence.GetAllSince(dur)
		if err != nil {
			return err
		}
	}

	// count points
	pointCount := 0
	for _, points := range storage {
		pointCount += len(points)
	}

	server.context.DataCache.SetStorage(storage)

	// clean up the cache
	go func() {
		for {
			server.context.DataCache.Clean()
			time.Sleep(10 * time.Second)
		}
	}()

	log.Printf("Finished initializing data cache with %d data points", pointCount)
	return nil
}<|endoftext|>


<commit_before>package github

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"

	"github.com/BurntSushi/toml"
	"github.com/github/hub/utils"
	"github.com/howeyc/gopass"
)

var (
	defaultConfigsFile = filepath.Join(os.Getenv("HOME"), ".config", "hub")
)

type Credential struct {
	Host        string `toml:"host"`
	User        string `toml:"user"`
	AccessToken string `toml:"access_token"`
}

type Configs struct {
	Credentials []Credential `toml:"credentials"`
}

func (c *Configs) allCredentials() (creds []Credential) {
	for _, cred := range c.Credentials {
		creds = append(creds, cred)
	}

	return
}

func (c *Configs) PromptFor(host string) *Credential {
	cd := c.find(host)
	if cd == nil {
		user := c.PromptForUser()
		pass := c.PromptForPassword(host, user)

		// Create Client with a stub Credential
		client := Client{Credential: &Credential{Host: host}}
		token, err := client.FindOrCreateToken(user, pass, "")
		if err != nil {
			if ce, ok := err.(*ClientError); ok && ce.Is2FAError() {
				code := c.PromptForOTP()
				token, err = client.FindOrCreateToken(user, pass, code)
			}
		}
		utils.Check(err)

		cd = &Credential{Host: host, User: user, AccessToken: token}
		c.Credentials = append(c.Credentials, *cd)
		err = saveTo(configsFile(), c)
		utils.Check(err)
	}

	return cd
}

func (c *Configs) PromptForUser() (user string) {
	user = os.Getenv("GITHUB_USER")
	if user != "" {
		return
	}

	fmt.Printf("%s username: ", GitHubHost)
	fmt.Scanln(&user)

	return
}

func (c *Configs) PromptForPassword(host, user string) (pass string) {
	pass = os.Getenv("GITHUB_PASSWORD")
	if pass != "" {
		return
	}

	fmt.Printf("%s password for %s (never stored): ", host, user)
	if isTerminal(os.Stdout.Fd()) {
		pass = string(gopass.GetPasswd())
	} else {
		fmt.Scanln(&pass)
	}

	return
}

func (c *Configs) PromptForOTP() string {
	var code string
	fmt.Print("two-factor authentication code: ")
	fmt.Scanln(&code)

	return code
}

func (c *Configs) find(host string) *Credential {
	for _, t := range c.Credentials {
		if t.Host == host {
			return &t
		}
	}

	return nil
}

func saveTo(filename string, v interface{}) error {
	err := os.MkdirAll(filepath.Dir(filename), 0771)
	if err != nil {
		return err
	}

	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()

	enc := toml.NewEncoder(f)
	return enc.Encode(v)
}

func loadFrom(filename string, c *Configs) (err error) {
	_, err = toml.DecodeFile(filename, c)
	return
}

func configsFile() string {
	configsFile := os.Getenv("GH_CONFIG")
	if configsFile == "" {
		configsFile = defaultConfigsFile
	}

	return configsFile
}

func CurrentConfigs() *Configs {
	c := &Configs{}

	configFile := configsFile()
	err := loadFrom(configFile, c)
	utils.Check(err)

	return c
}

func (c *Configs) DefaultCredential() (credential *Credential) {
	if GitHubHostEnv != "" {
		credential = c.PromptFor(GitHubHostEnv)
	} else if len(c.Credentials) > 0 {
		credential = c.selectCredential()
	} else {
		credential = c.PromptFor(DefaultHost())
	}

	return
}

func (c *Configs) selectCredential() *Credential {
	creds := c.allCredentials()
	options := len(creds)

	if options == 1 {
		return &creds[0]
	}

	prompt := "Select host:\n"
	for idx, cred := range creds {
		prompt += fmt.Sprintf(" %d. %s\n", idx+1, cred.Host)
	}
	prompt += fmt.Sprint("> ")

	fmt.Printf(prompt)
	var index string
	fmt.Scanln(&index)

	i, err := strconv.Atoi(index)
	if err != nil || i < 1 || i > options {
		utils.Check(fmt.Errorf("Error: must enter a number [1-%d]", options))
	}

	return &creds[i-1]
}

func (c *Configs) Save() error {
	return saveTo(configsFile(), c)
}

// Public for testing purpose
func CreateTestConfigs(user, token string) *Configs {
	f, _ := ioutil.TempFile("", "test-config")
	defaultConfigsFile = f.Name()

	cred := Credential{
		User:        "jingweno",
		AccessToken: "123",
		Host:        GitHubHost,
	}

	c := &Configs{Credentials: []Credential{cred}}
	saveTo(f.Name(), c)

	return c
}
<commit_msg>Add stub block for loading from YAML<commit_after>package github

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"

	"github.com/BurntSushi/toml"
	"github.com/github/hub/utils"
	"github.com/howeyc/gopass"
)

var (
	defaultConfigsFile = filepath.Join(os.Getenv("HOME"), ".config", "hub")
)

type Credential struct {
	Host        string `toml:"host"`
	User        string `toml:"user"`
	AccessToken string `toml:"access_token"`
}

type Configs struct {
	Credentials []Credential `toml:"credentials"`
}

func (c *Configs) allCredentials() (creds []Credential) {
	for _, cred := range c.Credentials {
		creds = append(creds, cred)
	}

	return
}

func (c *Configs) PromptFor(host string) *Credential {
	cd := c.find(host)
	if cd == nil {
		user := c.PromptForUser()
		pass := c.PromptForPassword(host, user)

		// Create Client with a stub Credential
		client := Client{Credential: &Credential{Host: host}}
		token, err := client.FindOrCreateToken(user, pass, "")
		if err != nil {
			if ce, ok := err.(*ClientError); ok && ce.Is2FAError() {
				code := c.PromptForOTP()
				token, err = client.FindOrCreateToken(user, pass, code)
			}
		}
		utils.Check(err)

		cd = &Credential{Host: host, User: user, AccessToken: token}
		c.Credentials = append(c.Credentials, *cd)
		err = saveTo(configsFile(), c)
		utils.Check(err)
	}

	return cd
}

func (c *Configs) PromptForUser() (user string) {
	user = os.Getenv("GITHUB_USER")
	if user != "" {
		return
	}

	fmt.Printf("%s username: ", GitHubHost)
	fmt.Scanln(&user)

	return
}

func (c *Configs) PromptForPassword(host, user string) (pass string) {
	pass = os.Getenv("GITHUB_PASSWORD")
	if pass != "" {
		return
	}

	fmt.Printf("%s password for %s (never stored): ", host, user)
	if isTerminal(os.Stdout.Fd()) {
		pass = string(gopass.GetPasswd())
	} else {
		fmt.Scanln(&pass)
	}

	return
}

func (c *Configs) PromptForOTP() string {
	var code string
	fmt.Print("two-factor authentication code: ")
	fmt.Scanln(&code)

	return code
}

func (c *Configs) find(host string) *Credential {
	for _, t := range c.Credentials {
		if t.Host == host {
			return &t
		}
	}

	return nil
}

func saveTo(filename string, v interface{}) error {
	err := os.MkdirAll(filepath.Dir(filename), 0771)
	if err != nil {
		return err
	}

	f, err := os.Create(filename)
	if err != nil {
		return err
	}
	defer f.Close()

	enc := toml.NewEncoder(f)
	return enc.Encode(v)
}

func loadFrom(filename string, c *Configs) (err error) {
	_, err = toml.DecodeFile(filename, c)
	return
}

func configsFile() string {
	configsFile := os.Getenv("GH_CONFIG")
	if configsFile == "" {
		configsFile = defaultConfigsFile
	}

	return configsFile
}

func CurrentConfigs() *Configs {
	c := &Configs{}

	configFile := configsFile()
	err := loadFrom(configFile, c)
	if err != nil {
		// load from YAML
	}

	return c
}

func (c *Configs) DefaultCredential() (credential *Credential) {
	if GitHubHostEnv != "" {
		credential = c.PromptFor(GitHubHostEnv)
	} else if len(c.Credentials) > 0 {
		credential = c.selectCredential()
	} else {
		credential = c.PromptFor(DefaultHost())
	}

	return
}

func (c *Configs) selectCredential() *Credential {
	creds := c.allCredentials()
	options := len(creds)

	if options == 1 {
		return &creds[0]
	}

	prompt := "Select host:\n"
	for idx, cred := range creds {
		prompt += fmt.Sprintf(" %d. %s\n", idx+1, cred.Host)
	}
	prompt += fmt.Sprint("> ")

	fmt.Printf(prompt)
	var index string
	fmt.Scanln(&index)

	i, err := strconv.Atoi(index)
	if err != nil || i < 1 || i > options {
		utils.Check(fmt.Errorf("Error: must enter a number [1-%d]", options))
	}

	return &creds[i-1]
}

func (c *Configs) Save() error {
	return saveTo(configsFile(), c)
}

// Public for testing purpose
func CreateTestConfigs(user, token string) *Configs {
	f, _ := ioutil.TempFile("", "test-config")
	defaultConfigsFile = f.Name()

	cred := Credential{
		User:        "jingweno",
		AccessToken: "123",
		Host:        GitHubHost,
	}

	c := &Configs{Credentials: []Credential{cred}}
	saveTo(f.Name(), c)

	return c
}
<|endoftext|>


<commit_before>package common

type StringSet map[string]struct{}

func (ss StringSet) Contains(elem string) bool {
	_, ok := ss[elem]
	return ok
}

func (ss StringSet) Add(elem string) {
	ss[elem] = struct{}{}
}

func (ss StringSet) Remove(elem string) {
	delete(ss, elem)
}

func (ss StringSet) ToList() []string {
	keys := make([]string, 0, len(ss))
	for s := range ss {
		keys = append(keys, s)
	}
	return keys
}

type StringList []string

func (sl *StringList) Remove(elem string) {
	widx := 0
	cpsl := *sl
	for idx, _elem := range cpsl {
		if _elem == elem {
			// ignore this elem by doing nothing
		} else {
			if idx != widx {
				cpsl[widx] = _elem
			}
			widx += 1
		}
	}

	*sl = cpsl[:widx]
}

func (sl *StringList) Append(elem string) {
	*sl = append(*sl, elem)
}

func (sl *StringList) Find(s string) int {
	for idx, elem := range *sl {
		if elem == s {
			return idx
		}
	}
	return -1
}
<commit_msg>fixing lint errors<commit_after>package common

// StringSet is a set of strings
type StringSet map[string]struct{}

// Contains checks if Stringset contains the string
func (ss StringSet) Contains(elem string) bool {
	_, ok := ss[elem]
	return ok
}

// Add adds the string to StringSet
func (ss StringSet) Add(elem string) {
	ss[elem] = struct{}{}
}

// Remove removes the string from StringList
func (ss StringSet) Remove(elem string) {
	delete(ss, elem)
}

// ToList convert StringSet to string slice
func (ss StringSet) ToList() []string {
	keys := make([]string, 0, len(ss))
	for s := range ss {
		keys = append(keys, s)
	}
	return keys
}

// StringList is a list of string (slice)
type StringList []string

// Remove removes the string from StringList
func (sl *StringList) Remove(elem string) {
	widx := 0
	cpsl := *sl
	for idx, _elem := range cpsl {
		if _elem == elem {
			// ignore this elem by doing nothing
		} else {
			if idx != widx {
				cpsl[widx] = _elem
			}
			widx += 1
		}
	}

	*sl = cpsl[:widx]
}

// Append add the string to the end of StringList
func (sl *StringList) Append(elem string) {
	*sl = append(*sl, elem)
}

// Find get the index of string in StringList, returns -1 if not found
func (sl *StringList) Find(s string) int {
	for idx, elem := range *sl {
		if elem == s {
			return idx
		}
	}
	return -1
}
<|endoftext|>


<commit_before>package moves

import (
	"errors"
	"github.com/jkomoros/boardgame"
	"reflect"
)

//The interface that moves that can be handled by DefaultConfig implement.
type defaultConfigMoveType interface {
	//The name for the move type
	MoveTypeName(manager *boardgame.GameManager) string
	//The name for the HelpText
	MoveTypeHelpText(manager *boardgame.GameManager) string
	//Whether the move should be a fix up.
	MoveTypeIsFixUp(manager *boardgame.GameManager) bool
}

//MustDefaultConfig is a wrapper around DefaultConfig that if it errors will
//panic. Only suitable for being used during setup.
func MustDefaultConfig(manager *boardgame.GameManager, exampleStruct boardgame.Move) *boardgame.MoveTypeConfig {
	result, err := DefaultConfig(manager, exampleStruct)

	if err != nil {
		panic("Couldn't DefaultConfig: " + err.Error())
	}

	return result
}

//DefaultConfig is a powerful default MoveTypeConfig generator. In many cases
//you'll implement moves that are very thin embeddings of moves in this
//package. Generating a MoveTypeConfig for each is a pain. This method auto-
//generates the MoveTypeConfig based on an example zero type of your move to
//install. It does some magic to create a more fleshed out move, then consults
//move.MoveTypeName and move.MoveTypeHelpText to generate the name and
//helptext. Moves in this package return reasonable values for those methods,
//based on the configuration you set on the rest of your move. See the package
//doc for an example of use.
func DefaultConfig(manager *boardgame.GameManager, exampleStruct boardgame.Move) (*boardgame.MoveTypeConfig, error) {

	if exampleStruct == nil {
		return nil, errors.New("nil struct provided")
	}

	//We'll create a throw-away move type config first to get a fully-
	//initialized and expanded move (e.g. with all tag-based autoinflation)
	//that we can then pass to the MoveType* methods, so they'll have more to work with.

	throwAwayConfig := newMoveTypeConfig("Temporary Move", "Temporary Move Help Text", false, exampleStruct)

	throwAwayMoveType, err := throwAwayConfig.NewMoveType(manager)

	if err != nil {
		return nil, errors.New("Couldn't create temporary move type: " + err.Error())
	}

	actualExample := throwAwayMoveType.NewMove(manager.ExampleState())

	defaultConfig, ok := actualExample.(defaultConfigMoveType)

	if !ok {
		return nil, errors.New("Example struct didn't have MoveTypeName and MoveTypeHelpText.")
	}

	name := defaultConfig.MoveTypeName(manager)
	helpText := defaultConfig.MoveTypeHelpText(manager)
	isFixUp := defaultConfig.MoveTypeIsFixUp(manager)

	return newMoveTypeConfig(name, helpText, isFixUp, exampleStruct), nil

}

func newMoveTypeConfig(name, helpText string, isFixUp bool, exampleStruct boardgame.Move) *boardgame.MoveTypeConfig {
	val := reflect.ValueOf(exampleStruct)

	//We can accept either pointer or struct types.
	if val.Kind() == reflect.Ptr {
		val = val.Elem()
	}

	typ := val.Type()

	return &boardgame.MoveTypeConfig{
		Name:     name,
		HelpText: helpText,
		MoveConstructor: func() boardgame.Move {
			return reflect.New(typ).Interface().(boardgame.Move)
		},
		IsFixUp: isFixUp,
	}
}
<commit_msg>Define stackPropName helper function to identify the name for a stack in a given function. Part of #528.<commit_after>package moves

import (
	"errors"
	"github.com/jkomoros/boardgame"
	"reflect"
)

//The interface that moves that can be handled by DefaultConfig implement.
type defaultConfigMoveType interface {
	//The name for the move type
	MoveTypeName(manager *boardgame.GameManager) string
	//The name for the HelpText
	MoveTypeHelpText(manager *boardgame.GameManager) string
	//Whether the move should be a fix up.
	MoveTypeIsFixUp(manager *boardgame.GameManager) bool
}

//MustDefaultConfig is a wrapper around DefaultConfig that if it errors will
//panic. Only suitable for being used during setup.
func MustDefaultConfig(manager *boardgame.GameManager, exampleStruct boardgame.Move) *boardgame.MoveTypeConfig {
	result, err := DefaultConfig(manager, exampleStruct)

	if err != nil {
		panic("Couldn't DefaultConfig: " + err.Error())
	}

	return result
}

//DefaultConfig is a powerful default MoveTypeConfig generator. In many cases
//you'll implement moves that are very thin embeddings of moves in this
//package. Generating a MoveTypeConfig for each is a pain. This method auto-
//generates the MoveTypeConfig based on an example zero type of your move to
//install. It does some magic to create a more fleshed out move, then consults
//move.MoveTypeName and move.MoveTypeHelpText to generate the name and
//helptext. Moves in this package return reasonable values for those methods,
//based on the configuration you set on the rest of your move. See the package
//doc for an example of use.
func DefaultConfig(manager *boardgame.GameManager, exampleStruct boardgame.Move) (*boardgame.MoveTypeConfig, error) {

	if exampleStruct == nil {
		return nil, errors.New("nil struct provided")
	}

	//We'll create a throw-away move type config first to get a fully-
	//initialized and expanded move (e.g. with all tag-based autoinflation)
	//that we can then pass to the MoveType* methods, so they'll have more to work with.

	throwAwayConfig := newMoveTypeConfig("Temporary Move", "Temporary Move Help Text", false, exampleStruct)

	throwAwayMoveType, err := throwAwayConfig.NewMoveType(manager)

	if err != nil {
		return nil, errors.New("Couldn't create temporary move type: " + err.Error())
	}

	actualExample := throwAwayMoveType.NewMove(manager.ExampleState())

	defaultConfig, ok := actualExample.(defaultConfigMoveType)

	if !ok {
		return nil, errors.New("Example struct didn't have MoveTypeName and MoveTypeHelpText.")
	}

	name := defaultConfig.MoveTypeName(manager)
	helpText := defaultConfig.MoveTypeHelpText(manager)
	isFixUp := defaultConfig.MoveTypeIsFixUp(manager)

	return newMoveTypeConfig(name, helpText, isFixUp, exampleStruct), nil

}

func newMoveTypeConfig(name, helpText string, isFixUp bool, exampleStruct boardgame.Move) *boardgame.MoveTypeConfig {
	val := reflect.ValueOf(exampleStruct)

	//We can accept either pointer or struct types.
	if val.Kind() == reflect.Ptr {
		val = val.Elem()
	}

	typ := val.Type()

	return &boardgame.MoveTypeConfig{
		Name:     name,
		HelpText: helpText,
		MoveConstructor: func() boardgame.Move {
			return reflect.New(typ).Interface().(boardgame.Move)
		},
		IsFixUp: isFixUp,
	}
}

//stackPropName takes a stack that was returned from a given state, and the
//state it was returned from. It searches through the sub- states in state
//until it finds the name of the property where it resides.
func stackPropName(stack boardgame.MutableStack, state boardgame.MutableState) string {

	if name := stackPropNameInReadSetter(stack, state.MutableGameState().ReadSetter()); name != "" {
		return name
	}

	if name := stackPropNameInReadSetter(stack, state.MutablePlayerStates()[0].ReadSetter()); name != "" {
		return name
	}

	for _, dynamicComponentValues := range state.MutableDynamicComponentValues() {
		if name := stackPropNameInReadSetter(stack, dynamicComponentValues[0].ReadSetter()); name != "" {
			return name
		}
	}

	return ""

}

func stackPropNameInReadSetter(stack boardgame.MutableStack, readSetter boardgame.PropertyReadSetter) string {
	for propName, propType := range readSetter.Props() {
		if propType != boardgame.TypeStack {
			continue
		}
		testStack, err := readSetter.MutableStackProp(propName)
		if err != nil {
			continue
		}
		if testStack == stack {
			return propName
		}
	}
	return ""
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"net/rpc"
	"path/filepath"
	"runtime"
	"sync"
	"sync/atomic"
	"time"

	"github.com/dvyukov/go-fuzz/go-fuzz/internal/writerset"
)

// Master manages persistent fuzzer state like input corpus and crashers.
type Master struct {
	mu           sync.Mutex
	idSeq        int
	slaves       map[int]*MasterSlave
	corpus       *PersistentSet
	suppressions *PersistentSet
	crashers     *PersistentSet

	startTime     time.Time
	lastInput     time.Time
	statExecs     uint64
	statRestarts  uint64
	coverFullness int

	statsWriters *writerset.WriterSet
}

// MasterSlave represents master's view of a slave.
type MasterSlave struct {
	id       int
	procs    int
	pending  []MasterInput
	lastSync time.Time
}

// masterMain is entry function for master.
func masterMain(ln net.Listener) {
	m := &Master{}
	m.statsWriters = writerset.New()
	m.startTime = time.Now()
	m.lastInput = time.Now()
	m.suppressions = newPersistentSet(filepath.Join(*flagWorkdir, "suppressions"))
	m.crashers = newPersistentSet(filepath.Join(*flagWorkdir, "crashers"))
	m.corpus = newPersistentSet(filepath.Join(*flagWorkdir, "corpus"))
	if len(m.corpus.m) == 0 {
		m.corpus.add(Artifact{[]byte{}, 0, false})
	}

	m.slaves = make(map[int]*MasterSlave)
	masterListen(m)

	go masterLoop(m)

	s := rpc.NewServer()
	s.Register(m)
	s.Accept(ln)
}

func masterListen(m *Master) {
	if *flagHTTP != "" {
		http.HandleFunc("/eventsource", m.eventSource)
		http.HandleFunc("/", m.index)

		go func() {
			fmt.Printf("Serving statistics on http://%s/\n", flagHTTP)
			panic(http.ListenAndServe(*flagHTTP, nil))
		}()
	} else {
		runtime.MemProfileRate = 0
	}
}

func masterLoop(m *Master) {
	for range time.NewTicker(3 * time.Second).C {
		if atomic.LoadUint32(&shutdown) != 0 {
			return
		}
		m.mu.Lock()
		// Nuke dead slaves.
		for id, s := range m.slaves {
			if time.Since(s.lastSync) < syncDeadline {
				continue
			}
			log.Printf("slave %v died", s.id)
			delete(m.slaves, id)
		}
		m.mu.Unlock()

		m.broadcastStats()
	}
}

func (m *Master) broadcastStats() {
	stats := m.masterStats()

	// log to stdout
	log.Println(stats.String())

	// write to any http clients
	b, err := json.Marshal(stats)
	if err != nil {
		panic(err)
	}

	fmt.Fprintf(m.statsWriters, "event: ping\ndata: %s\n\n", string(b))
	m.statsWriters.Flush()
}

func (m *Master) eventSource(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/event-stream")
	w.WriteHeader(http.StatusOK)
	<-m.statsWriters.Add(w)
}

func (m *Master) index(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "/" {
		r.URL.Path = "/stats.html"
	}
	http.FileServer(assetFS()).ServeHTTP(w, r)
}

func (m *Master) masterStats() masterStats {
	m.mu.Lock()
	defer m.mu.Unlock()

	stats := masterStats{
		Corpus:           uint64(len(m.corpus.m)),
		Crashers:         uint64(len(m.crashers.m)),
		Uptime:           fmtDuration(time.Since(m.startTime)),
		StartTime:        m.startTime,
		LastNewInputTime: m.lastInput,
		Execs:            m.statExecs,
		Cover:            uint64(m.coverFullness),
	}

	// Print stats line.
	if m.statExecs != 0 {
		stats.RestartsDenom = m.statExecs / m.statRestarts
	}

	for _, s := range m.slaves {
		stats.Slaves += uint64(s.procs)
	}

	return stats
}

type masterStats struct {
	Slaves, Corpus, Crashers, Execs, Cover, RestartsDenom uint64
	LastNewInputTime, StartTime                           time.Time
	Uptime                                                string
}

func (s masterStats) String() string {
	return fmt.Sprintf("slaves: %v, corpus: %v (%v ago), crashers: %v,"+
		" restarts: 1/%v, execs: %v (%.0f/sec), cover: %v, uptime: %v",
		s.Slaves, s.Corpus, fmtDuration(time.Since(s.LastNewInputTime)),
		s.Crashers, s.RestartsDenom, s.Execs, s.ExecsPerSec(), s.Cover,
		s.Uptime,
	)
}

func (s masterStats) ExecsPerSec() float64 {
	return float64(s.Execs) * 1e9 / float64(time.Since(s.StartTime))
}

func fmtDuration(d time.Duration) string {
	if d.Hours() >= 1 {
		return fmt.Sprintf("%vh%vm", int(d.Hours()), int(d.Minutes())%60)
	} else if d.Minutes() >= 1 {
		return fmt.Sprintf("%vm%vs", int(d.Minutes()), int(d.Seconds())%60)
	} else {
		return fmt.Sprintf("%vs", int(d.Seconds()))
	}
}

type ConnectArgs struct {
	Procs int
}

type ConnectRes struct {
	ID     int
	Corpus []MasterInput
}

// MasterInput is description of input that is passed between master and slave.
type MasterInput struct {
	Data      []byte
	Prio      uint64
	Type      int
	Minimized bool
	Smashed   bool
}

// Connect attaches new slave to master.
func (m *Master) Connect(a *ConnectArgs, r *ConnectRes) error {
	m.mu.Lock()
	defer m.mu.Unlock()

	m.idSeq++
	s := &MasterSlave{
		id:       m.idSeq,
		procs:    a.Procs,
		lastSync: time.Now(),
	}
	m.slaves[s.id] = s
	r.ID = s.id
	// Give the slave initial corpus.
	for _, a := range m.corpus.m {
		r.Corpus = append(r.Corpus, MasterInput{a.data, a.meta, execCorpus, !a.user, true})
	}
	return nil
}

type NewInputArgs struct {
	ID   int
	Data []byte
	Prio uint64
}

// NewInput saves new interesting input on master.
func (m *Master) NewInput(a *NewInputArgs, r *int) error {
	m.mu.Lock()
	defer m.mu.Unlock()

	s := m.slaves[a.ID]
	if s == nil {
		return errors.New("unknown slave")
	}

	art := Artifact{a.Data, a.Prio, false}
	if !m.corpus.add(art) {
		return nil
	}
	m.lastInput = time.Now()
	// Queue the input for sending to every slave.
	for _, s1 := range m.slaves {
		s1.pending = append(s1.pending, MasterInput{a.Data, a.Prio, execCorpus, true, s1 != s})
	}

	return nil
}

type NewCrasherArgs struct {
	Data        []byte
	Error       []byte
	Suppression []byte
	Hanging     bool
}

// NewCrasher saves new crasher input on master.
func (m *Master) NewCrasher(a *NewCrasherArgs, r *int) error {
	m.mu.Lock()
	defer m.mu.Unlock()

	if !*flagDup && !m.suppressions.add(Artifact{a.Suppression, 0, false}) {
		return nil // Already have this.
	}
	if !m.crashers.add(Artifact{a.Data, 0, false}) {
		return nil // Already have this.
	}

	// Prepare quoted version of input to simplify creation of standalone reproducers.
	var buf bytes.Buffer
	for i := 0; i < len(a.Data); i += 20 {
		e := i + 20
		if e > len(a.Data) {
			e = len(a.Data)
		}
		fmt.Fprintf(&buf, "\t%q", a.Data[i:e])
		if e != len(a.Data) {
			fmt.Fprintf(&buf, " +")
		}
		fmt.Fprintf(&buf, "\n")
	}
	m.crashers.addDescription(a.Data, buf.Bytes(), "quoted")
	m.crashers.addDescription(a.Data, a.Error, "output")

	return nil
}

type SyncArgs struct {
	ID            int
	Execs         uint64
	Restarts      uint64
	CoverFullness int
}

type SyncRes struct {
	Inputs []MasterInput // new interesting inputs
}

var errUnkownSlave = errors.New("unknown slave")

// Sync is a periodic sync with a slave.
// Slave sends statistics. Master returns new inputs.
func (m *Master) Sync(a *SyncArgs, r *SyncRes) error {
	m.mu.Lock()
	defer m.mu.Unlock()

	s := m.slaves[a.ID]
	if s == nil {
		return errUnkownSlave
	}
	m.statExecs += a.Execs
	m.statRestarts += a.Restarts
	if m.coverFullness < a.CoverFullness {
		m.coverFullness = a.CoverFullness
	}
	s.lastSync = time.Now()
	r.Inputs = s.pending
	s.pending = nil
	return nil
}
<commit_msg>fix missing dereference when printing flagHTTP<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof"
	"net/rpc"
	"path/filepath"
	"runtime"
	"sync"
	"sync/atomic"
	"time"

	"github.com/dvyukov/go-fuzz/go-fuzz/internal/writerset"
)

// Master manages persistent fuzzer state like input corpus and crashers.
type Master struct {
	mu           sync.Mutex
	idSeq        int
	slaves       map[int]*MasterSlave
	corpus       *PersistentSet
	suppressions *PersistentSet
	crashers     *PersistentSet

	startTime     time.Time
	lastInput     time.Time
	statExecs     uint64
	statRestarts  uint64
	coverFullness int

	statsWriters *writerset.WriterSet
}

// MasterSlave represents master's view of a slave.
type MasterSlave struct {
	id       int
	procs    int
	pending  []MasterInput
	lastSync time.Time
}

// masterMain is entry function for master.
func masterMain(ln net.Listener) {
	m := &Master{}
	m.statsWriters = writerset.New()
	m.startTime = time.Now()
	m.lastInput = time.Now()
	m.suppressions = newPersistentSet(filepath.Join(*flagWorkdir, "suppressions"))
	m.crashers = newPersistentSet(filepath.Join(*flagWorkdir, "crashers"))
	m.corpus = newPersistentSet(filepath.Join(*flagWorkdir, "corpus"))
	if len(m.corpus.m) == 0 {
		m.corpus.add(Artifact{[]byte{}, 0, false})
	}

	m.slaves = make(map[int]*MasterSlave)
	masterListen(m)

	go masterLoop(m)

	s := rpc.NewServer()
	s.Register(m)
	s.Accept(ln)
}

func masterListen(m *Master) {
	if *flagHTTP != "" {
		http.HandleFunc("/eventsource", m.eventSource)
		http.HandleFunc("/", m.index)

		go func() {
			fmt.Printf("Serving statistics on http://%s/\n", *flagHTTP)
			panic(http.ListenAndServe(*flagHTTP, nil))
		}()
	} else {
		runtime.MemProfileRate = 0
	}
}

func masterLoop(m *Master) {
	for range time.NewTicker(3 * time.Second).C {
		if atomic.LoadUint32(&shutdown) != 0 {
			return
		}
		m.mu.Lock()
		// Nuke dead slaves.
		for id, s := range m.slaves {
			if time.Since(s.lastSync) < syncDeadline {
				continue
			}
			log.Printf("slave %v died", s.id)
			delete(m.slaves, id)
		}
		m.mu.Unlock()

		m.broadcastStats()
	}
}

func (m *Master) broadcastStats() {
	stats := m.masterStats()

	// log to stdout
	log.Println(stats.String())

	// write to any http clients
	b, err := json.Marshal(stats)
	if err != nil {
		panic(err)
	}

	fmt.Fprintf(m.statsWriters, "event: ping\ndata: %s\n\n", string(b))
	m.statsWriters.Flush()
}

func (m *Master) eventSource(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/event-stream")
	w.WriteHeader(http.StatusOK)
	<-m.statsWriters.Add(w)
}

func (m *Master) index(w http.ResponseWriter, r *http.Request) {
	if r.URL.Path == "/" {
		r.URL.Path = "/stats.html"
	}
	http.FileServer(assetFS()).ServeHTTP(w, r)
}

func (m *Master) masterStats() masterStats {
	m.mu.Lock()
	defer m.mu.Unlock()

	stats := masterStats{
		Corpus:           uint64(len(m.corpus.m)),
		Crashers:         uint64(len(m.crashers.m)),
		Uptime:           fmtDuration(time.Since(m.startTime)),
		StartTime:        m.startTime,
		LastNewInputTime: m.lastInput,
		Execs:            m.statExecs,
		Cover:            uint64(m.coverFullness),
	}

	// Print stats line.
	if m.statExecs != 0 {
		stats.RestartsDenom = m.statExecs / m.statRestarts
	}

	for _, s := range m.slaves {
		stats.Slaves += uint64(s.procs)
	}

	return stats
}

type masterStats struct {
	Slaves, Corpus, Crashers, Execs, Cover, RestartsDenom uint64
	LastNewInputTime, StartTime                           time.Time
	Uptime                                                string
}

func (s masterStats) String() string {
	return fmt.Sprintf("slaves: %v, corpus: %v (%v ago), crashers: %v,"+
		" restarts: 1/%v, execs: %v (%.0f/sec), cover: %v, uptime: %v",
		s.Slaves, s.Corpus, fmtDuration(time.Since(s.LastNewInputTime)),
		s.Crashers, s.RestartsDenom, s.Execs, s.ExecsPerSec(), s.Cover,
		s.Uptime,
	)
}

func (s masterStats) ExecsPerSec() float64 {
	return float64(s.Execs) * 1e9 / float64(time.Since(s.StartTime))
}

func fmtDuration(d time.Duration) string {
	if d.Hours() >= 1 {
		return fmt.Sprintf("%vh%vm", int(d.Hours()), int(d.Minutes())%60)
	} else if d.Minutes() >= 1 {
		return fmt.Sprintf("%vm%vs", int(d.Minutes()), int(d.Seconds())%60)
	} else {
		return fmt.Sprintf("%vs", int(d.Seconds()))
	}
}

type ConnectArgs struct {
	Procs int
}

type ConnectRes struct {
	ID     int
	Corpus []MasterInput
}

// MasterInput is description of input that is passed between master and slave.
type MasterInput struct {
	Data      []byte
	Prio      uint64
	Type      int
	Minimized bool
	Smashed   bool
}

// Connect attaches new slave to master.
func (m *Master) Connect(a *ConnectArgs, r *ConnectRes) error {
	m.mu.Lock()
	defer m.mu.Unlock()

	m.idSeq++
	s := &MasterSlave{
		id:       m.idSeq,
		procs:    a.Procs,
		lastSync: time.Now(),
	}
	m.slaves[s.id] = s
	r.ID = s.id
	// Give the slave initial corpus.
	for _, a := range m.corpus.m {
		r.Corpus = append(r.Corpus, MasterInput{a.data, a.meta, execCorpus, !a.user, true})
	}
	return nil
}

type NewInputArgs struct {
	ID   int
	Data []byte
	Prio uint64
}

// NewInput saves new interesting input on master.
func (m *Master) NewInput(a *NewInputArgs, r *int) error {
	m.mu.Lock()
	defer m.mu.Unlock()

	s := m.slaves[a.ID]
	if s == nil {
		return errors.New("unknown slave")
	}

	art := Artifact{a.Data, a.Prio, false}
	if !m.corpus.add(art) {
		return nil
	}
	m.lastInput = time.Now()
	// Queue the input for sending to every slave.
	for _, s1 := range m.slaves {
		s1.pending = append(s1.pending, MasterInput{a.Data, a.Prio, execCorpus, true, s1 != s})
	}

	return nil
}

type NewCrasherArgs struct {
	Data        []byte
	Error       []byte
	Suppression []byte
	Hanging     bool
}

// NewCrasher saves new crasher input on master.
func (m *Master) NewCrasher(a *NewCrasherArgs, r *int) error {
	m.mu.Lock()
	defer m.mu.Unlock()

	if !*flagDup && !m.suppressions.add(Artifact{a.Suppression, 0, false}) {
		return nil // Already have this.
	}
	if !m.crashers.add(Artifact{a.Data, 0, false}) {
		return nil // Already have this.
	}

	// Prepare quoted version of input to simplify creation of standalone reproducers.
	var buf bytes.Buffer
	for i := 0; i < len(a.Data); i += 20 {
		e := i + 20
		if e > len(a.Data) {
			e = len(a.Data)
		}
		fmt.Fprintf(&buf, "\t%q", a.Data[i:e])
		if e != len(a.Data) {
			fmt.Fprintf(&buf, " +")
		}
		fmt.Fprintf(&buf, "\n")
	}
	m.crashers.addDescription(a.Data, buf.Bytes(), "quoted")
	m.crashers.addDescription(a.Data, a.Error, "output")

	return nil
}

type SyncArgs struct {
	ID            int
	Execs         uint64
	Restarts      uint64
	CoverFullness int
}

type SyncRes struct {
	Inputs []MasterInput // new interesting inputs
}

var errUnkownSlave = errors.New("unknown slave")

// Sync is a periodic sync with a slave.
// Slave sends statistics. Master returns new inputs.
func (m *Master) Sync(a *SyncArgs, r *SyncRes) error {
	m.mu.Lock()
	defer m.mu.Unlock()

	s := m.slaves[a.ID]
	if s == nil {
		return errUnkownSlave
	}
	m.statExecs += a.Execs
	m.statRestarts += a.Restarts
	if m.coverFullness < a.CoverFullness {
		m.coverFullness = a.CoverFullness
	}
	s.lastSync = time.Now()
	r.Inputs = s.pending
	s.pending = nil
	return nil
}
<|endoftext|>


<commit_before>// +build linux darwin

package tpmutil

import (
	"fmt"
	"os"

	"golang.org/x/sys/unix"
)

// poll blocks until the file descriptor is ready for reading or an error occurs.
func poll(f *os.File) error {
	var (
		fds = []unix.PollFd{{
			Fd:     int32(f.Fd()),
			Events: 0x1, // POLLIN
		}}
		timeout = 0 // No timeout
	)

	if _, err := unix.Poll(fds, timeout); err != nil {
		return err
	}

	// Revents is filled in by the kernel.
	// If the expected event happened, Revents should match Events.
	if fds[0].Revents != fds[0].Events {
		return fmt.Errorf("unexpected poll Revents 0x%x", fds[0].Revents)
	}
	return nil
}
<commit_msg>Set indefinite timeout in unix.Poll call (#153)<commit_after>// +build linux darwin

package tpmutil

import (
	"fmt"
	"os"

	"golang.org/x/sys/unix"
)

// poll blocks until the file descriptor is ready for reading or an error occurs.
func poll(f *os.File) error {
	var (
		fds = []unix.PollFd{{
			Fd:     int32(f.Fd()),
			Events: 0x1, // POLLIN
		}}
		timeout = -1 // Indefinite timeout
	)

	if _, err := unix.Poll(fds, timeout); err != nil {
		return err
	}

	// Revents is filled in by the kernel.
	// If the expected event happened, Revents should match Events.
	if fds[0].Revents != fds[0].Events {
		return fmt.Errorf("unexpected poll Revents 0x%x", fds[0].Revents)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"github.com/bitly/nsq/nsq"
	"github.com/bitly/nsq/util"
	"hash/crc32"
	"io"
	"log"
	"net"
	"os"
	"os/signal"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"
)

var (
	showVersion          = flag.Bool("version", false, "print version string")
	httpAddress          = flag.String("http-address", "0.0.0.0:4151", "<addr>:<port> to listen on for HTTP clients")
	tcpAddress           = flag.String("tcp-address", "0.0.0.0:4150", "<addr>:<port> to listen on for TCP clients")
	memQueueSize         = flag.Int64("mem-queue-size", 10000, "number of messages to keep in memory (per topic/channel)")
	maxBytesPerFile      = flag.Int64("max-bytes-per-file", 104857600, "number of bytes per diskqueue file before rolling")
	syncEvery            = flag.Int64("sync-every", 2500, "number of messages between diskqueue syncs")
	msgTimeout           = flag.String("msg-timeout", "60s", "duration to wait before auto-requeing a message")
	maxMessageSize       = flag.Int64("max-message-size", 1024768, "maximum size of a single message in bytes")
	maxBodySize          = flag.Int64("max-body-size", 5*1024768, "maximum size of a single command body")
	maxMsgTimeout        = flag.Duration("max-msg-timeout", 15*time.Minute, "maximum duration before a message will timeout")
	maxHeartbeatInterval = flag.Duration("max-heartbeat-interval", 60*time.Second, "maximum duration of time between heartbeats that a client can configure")
	maxRdyCount          = flag.Int64("max-rdy-count", 2500, "maximum RDY count for a single client")
	dataPath             = flag.String("data-path", "", "path to store disk-backed messages")
	workerId             = flag.Int64("worker-id", 0, "unique identifier (int) for this worker (will default to a hash of hostname)")
	verbose              = flag.Bool("verbose", false, "enable verbose logging")
	statsdAddress        = flag.String("statsd-address", "", "UDP <addr>:<port> of a statsd daemon for writing stats")
	statsdInterval       = flag.Int("statsd-interval", 30, "seconds between pushing to statsd")
	broadcastAddress     = flag.String("broadcast-address", "", "address that will be registered with lookupd, (default to the OS hostname)")
	lookupdTCPAddrs      = util.StringArray{}

	maxOutputBufferSize    = flag.Int64("max-output-buffer-size", 64*1024, "maximum client configurable size (in bytes) for a client output buffer")
	maxOutputBufferTimeout = flag.Duration("max-output-buffer-timeout", 1*time.Second, "maximum duration of time between flushing to clients that a client can configure")
)

func init() {
	flag.Var(&lookupdTCPAddrs, "lookupd-tcp-address", "lookupd TCP address (may be given multiple times)")
}

var nsqd *NSQd
var protocols = map[string]nsq.Protocol{}

func main() {
	flag.Parse()

	hostname, err := os.Hostname()
	if err != nil {
		log.Fatal(err)
	}

	if *showVersion {
		fmt.Println(util.Version("nsqd"))
		return
	}

	if *workerId == 0 {
		h := md5.New()
		io.WriteString(h, hostname)
		*workerId = int64(crc32.ChecksumIEEE(h.Sum(nil)) % 1024)
	}

	tcpAddr, err := net.ResolveTCPAddr("tcp", *tcpAddress)
	if err != nil {
		log.Fatal(err)
	}

	httpAddr, err := net.ResolveTCPAddr("tcp", *httpAddress)
	if err != nil {
		log.Fatal(err)
	}

	if *broadcastAddress == "" {
		*broadcastAddress = hostname
	}

	log.Println(util.Version("nsqd"))
	log.Printf("worker id %d", *workerId)

	exitChan := make(chan int)
	signalChan := make(chan os.Signal, 1)
	go func() {
		<-signalChan
		exitChan <- 1
	}()
	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)

	if *statsdAddress != "" {
		undered := fmt.Sprintf("%s_%d", strings.Replace(*broadcastAddress, ".", "_", -1), httpAddr.Port)
		prefix := fmt.Sprintf("nsq.%s.", undered)
		go statsdLoop(*statsdAddress, prefix, *statsdInterval)
	}

	// for backwards compatibility if --msg-timeout only
	// contains numbers then default to ms
	var msgTimeoutDuration time.Duration
	if regexp.MustCompile(`^[0-9]+$`).MatchString(*msgTimeout) {
		intMsgTimeout, err := strconv.Atoi(*msgTimeout)
		if err != nil {
			log.Fatalf("ERROR: failed to Atoi --msg-timeout %s - %s", *msgTimeout, err.Error())
		}
		msgTimeoutDuration = time.Duration(intMsgTimeout) * time.Millisecond
	} else {
		msgTimeoutDuration, err = time.ParseDuration(*msgTimeout)
		if err != nil {
			log.Fatalf("ERROR: failed to ParseDuration --msg-timeout %s - %s", *msgTimeout, err.Error())
		}
	}

	options := NewNsqdOptions()
	options.maxRdyCount = *maxRdyCount
	options.maxMessageSize = *maxMessageSize
	options.maxBodySize = *maxBodySize
	options.memQueueSize = *memQueueSize
	options.dataPath = *dataPath
	options.maxBytesPerFile = *maxBytesPerFile
	options.syncEvery = *syncEvery
	options.msgTimeout = msgTimeoutDuration
	options.maxMsgTimeout = *maxMsgTimeout
	options.broadcastAddress = *broadcastAddress
	options.maxHeartbeatInterval = *maxHeartbeatInterval
	options.maxOutputBufferSize = *maxOutputBufferSize
	options.maxOutputBufferTimeout = *maxOutputBufferTimeout

	nsqd = NewNSQd(*workerId, options)
	nsqd.tcpAddr = tcpAddr
	nsqd.httpAddr = httpAddr
	nsqd.lookupdTCPAddrs = lookupdTCPAddrs

	nsqd.LoadMetadata()
	err = nsqd.PersistMetadata()
	if err != nil {
		log.Fatalf("ERROR: failed to persist metadata - %s", err.Error())
	}
	nsqd.Main()
	<-exitChan
	nsqd.Exit()
}
<commit_msg>nsqd: re-organize command line flags<commit_after>package main

import (
	"crypto/md5"
	"flag"
	"fmt"
	"github.com/bitly/nsq/nsq"
	"github.com/bitly/nsq/util"
	"hash/crc32"
	"io"
	"log"
	"net"
	"os"
	"os/signal"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"
)

var (
	// basic options
	showVersion      = flag.Bool("version", false, "print version string")
	verbose          = flag.Bool("verbose", false, "enable verbose logging")
	workerId         = flag.Int64("worker-id", 0, "unique identifier (int) for this worker (will default to a hash of hostname)")
	httpAddress      = flag.String("http-address", "0.0.0.0:4151", "<addr>:<port> to listen on for HTTP clients")
	tcpAddress       = flag.String("tcp-address", "0.0.0.0:4150", "<addr>:<port> to listen on for TCP clients")
	broadcastAddress = flag.String("broadcast-address", "", "address that will be registered with lookupd (defaults to the OS hostname)")
	lookupdTCPAddrs  = util.StringArray{}

	// diskqueue options
	dataPath        = flag.String("data-path", "", "path to store disk-backed messages")
	memQueueSize    = flag.Int64("mem-queue-size", 10000, "number of messages to keep in memory (per topic/channel)")
	maxBytesPerFile = flag.Int64("max-bytes-per-file", 104857600, "number of bytes per diskqueue file before rolling")
	syncEvery       = flag.Int64("sync-every", 2500, "number of messages between diskqueue syncs")

	// msg and command options
	msgTimeout     = flag.String("msg-timeout", "60s", "duration to wait before auto-requeing a message")
	maxMsgTimeout  = flag.Duration("max-msg-timeout", 15*time.Minute, "maximum duration before a message will timeout")
	maxMessageSize = flag.Int64("max-message-size", 1024768, "maximum size of a single message in bytes")
	maxBodySize    = flag.Int64("max-body-size", 5*1024768, "maximum size of a single command body")

	// client overridable configuration options
	maxHeartbeatInterval   = flag.Duration("max-heartbeat-interval", 60*time.Second, "maximum client configurable duration of time between client heartbeats")
	maxRdyCount            = flag.Int64("max-rdy-count", 2500, "maximum RDY count for a client")
	maxOutputBufferSize    = flag.Int64("max-output-buffer-size", 64*1024, "maximum client configurable size (in bytes) for a client output buffer")
	maxOutputBufferTimeout = flag.Duration("max-output-buffer-timeout", 1*time.Second, "maximum client configurable duration of time between flushing to a client")

	// statsd integration options
	statsdAddress  = flag.String("statsd-address", "", "UDP <addr>:<port> of a statsd daemon for writing stats")
	statsdInterval = flag.Int("statsd-interval", 30, "seconds between pushing to statsd")
)

func init() {
	flag.Var(&lookupdTCPAddrs, "lookupd-tcp-address", "lookupd TCP address (may be given multiple times)")
}

var nsqd *NSQd
var protocols = map[string]nsq.Protocol{}

func main() {
	flag.Parse()

	hostname, err := os.Hostname()
	if err != nil {
		log.Fatal(err)
	}

	if *showVersion {
		fmt.Println(util.Version("nsqd"))
		return
	}

	if *workerId == 0 {
		h := md5.New()
		io.WriteString(h, hostname)
		*workerId = int64(crc32.ChecksumIEEE(h.Sum(nil)) % 1024)
	}

	tcpAddr, err := net.ResolveTCPAddr("tcp", *tcpAddress)
	if err != nil {
		log.Fatal(err)
	}

	httpAddr, err := net.ResolveTCPAddr("tcp", *httpAddress)
	if err != nil {
		log.Fatal(err)
	}

	if *broadcastAddress == "" {
		*broadcastAddress = hostname
	}

	log.Println(util.Version("nsqd"))
	log.Printf("worker id %d", *workerId)

	exitChan := make(chan int)
	signalChan := make(chan os.Signal, 1)
	go func() {
		<-signalChan
		exitChan <- 1
	}()
	signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)

	if *statsdAddress != "" {
		undered := fmt.Sprintf("%s_%d", strings.Replace(*broadcastAddress, ".", "_", -1), httpAddr.Port)
		prefix := fmt.Sprintf("nsq.%s.", undered)
		go statsdLoop(*statsdAddress, prefix, *statsdInterval)
	}

	// for backwards compatibility if --msg-timeout only
	// contains numbers then default to ms
	var msgTimeoutDuration time.Duration
	if regexp.MustCompile(`^[0-9]+$`).MatchString(*msgTimeout) {
		intMsgTimeout, err := strconv.Atoi(*msgTimeout)
		if err != nil {
			log.Fatalf("ERROR: failed to Atoi --msg-timeout %s - %s", *msgTimeout, err.Error())
		}
		msgTimeoutDuration = time.Duration(intMsgTimeout) * time.Millisecond
	} else {
		msgTimeoutDuration, err = time.ParseDuration(*msgTimeout)
		if err != nil {
			log.Fatalf("ERROR: failed to ParseDuration --msg-timeout %s - %s", *msgTimeout, err.Error())
		}
	}

	options := NewNsqdOptions()
	options.maxRdyCount = *maxRdyCount
	options.maxMessageSize = *maxMessageSize
	options.maxBodySize = *maxBodySize
	options.memQueueSize = *memQueueSize
	options.dataPath = *dataPath
	options.maxBytesPerFile = *maxBytesPerFile
	options.syncEvery = *syncEvery
	options.msgTimeout = msgTimeoutDuration
	options.maxMsgTimeout = *maxMsgTimeout
	options.broadcastAddress = *broadcastAddress
	options.maxHeartbeatInterval = *maxHeartbeatInterval
	options.maxOutputBufferSize = *maxOutputBufferSize
	options.maxOutputBufferTimeout = *maxOutputBufferTimeout

	nsqd = NewNSQd(*workerId, options)
	nsqd.tcpAddr = tcpAddr
	nsqd.httpAddr = httpAddr
	nsqd.lookupdTCPAddrs = lookupdTCPAddrs

	nsqd.LoadMetadata()
	err = nsqd.PersistMetadata()
	if err != nil {
		log.Fatalf("ERROR: failed to persist metadata - %s", err.Error())
	}
	nsqd.Main()
	<-exitChan
	nsqd.Exit()
}
<|endoftext|>