<commit_before>package bot

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"os/exec"
	"reflect"
	"regexp"
	"strings"
	"sync"

	"github.com/ghodss/yaml"
)

// PluginNames can be letters, numbers & underscores only, mainly so
// brain functions can use ':' as a separator.
const pNameRegex = `[\w]+`

var pNameRe = regexp.MustCompile(pNameRegex)
var plugins []*Plugin
var plugIDmap map[string]int

// PluginHelp specifies keywords and help text for the 'bot help system
type PluginHelp struct {
	Keywords []string // match words for 'help XXX'
	Helptext []string // help string to give for the keywords, conventionally starting with (bot) for commands or (hear) when the bot needn't be addressed directly
}

// InputMatcher specifies the command or message to match and what to pass to the plugin
type InputMatcher struct {
	Regex   string         // The regular expression string to match - bot adds ^\w* & \w*$
	Command string         // The name of the command to pass to the plugin with it's arguments
	re      *regexp.Regexp // The compiled regular expression. If the regex doesn't compile, the 'bot will log an error
}

type plugType int

const (
	plugGo plugType = iota
	plugExternal
	plugBuiltin
)

// Plugin specifies the structure of a plugin configuration - plugins should include an example / default config
type Plugin struct {
	name           string          // the name of the plugin, used as a key in to the
	pluginType     plugType        // plugGo, plugExternal, plugBuiltin - determines how commands are routed
	pluginPath     string          // Path to the external executable that expects <channel> <user> <command> <arg> <arg> from regex matches - for Plugtype=plugExternal only
	Disabled       bool            // Set true to disable the plugin
	DisallowDirect bool            // Set this true if this plugin can never be accessed via direct message
	DirectOnly     bool            // Set this true if this plugin ONLY accepts direct messages
	Channels       []string        // Channels where the plugin is active - rifraf like "memes" should probably only be in random, but it's configurable. If empty uses DefaultChannels
	AllChannels    bool            // If the Channels list is empty and AllChannels is true, the plugin should be active in all the channels the bot is in
	RequireAdmin   bool            // Set to only allow administrators to access a plugin
	Users          []string        // If non-empty, list of all the users with access to this plugin
	Help           []PluginHelp    // All the keyword sets / help texts for this plugin
	CommandMatches []InputMatcher  // Input matchers for messages that need to be directed to the 'bot
	ReplyMatchers  []InputMatcher  // Input matchers for replies to questions, only match after a RequestContinuation
	MessageMatches []InputMatcher  // Input matchers for messages the 'bot hears even when it's not being spoken to
	CatchAll       bool            // Whenever the robot is spoken to, but no plugin matches, plugins with CatchAll=true get called with command="catchall" and argument=<full text of message to robot>
	Config         json.RawMessage // Arbitrary Plugin configuration, will be stored and provided in a thread-safe manner via GetPluginConfig()
	config         interface{}     // A pointer to an empty struct that the bot can Unmarshal custom configuration into
	pluginID       string          // 32-char random ID for identifying plugins in callbacks
	lock           sync.Mutex      // For use with the robot's Brain
}

// PluginHandler is the struct a plugin registers for the Gopherbot plugin API.
type PluginHandler struct {
	DefaultConfig string /* A yaml-formatted multiline string defining the default Plugin configuration. It should be liberally commented for use in generating
	local/custom configuration for the plugin. If a Config: section is defined, it should match the structure of the optional Config interface{} */
	Handler func(bot *Robot, command string, args ...string) // The callback function called by the robot whenever a Command is matched
	Config  interface{}                                      // An optional empty struct defining custom configuration for the plugin
}

// pluginHandlers maps from plugin names to PluginV1 (later interface{} with a type selector, maybe)
var pluginHandlers = make(map[string]PluginHandler)

// stopRegistrations is set "true" when the bot is created to prevent registration outside of init functions
var stopRegistrations = false

// initialize sends the "init" command to every plugin
func initializePlugins() {
	b.lock.RLock()
	defer b.lock.RUnlock()
	bot := &Robot{
		User:    b.name,
		Channel: "",
		Format:  Variable,
	}
	for _, plugin := range plugins {
		Log(Info, "Initializing plugin:", plugin.name)
		go callPlugin(bot, plugin, "init")
	}
}

// RegisterPlugin allows plugins to register a PluginHandler in a func init().
// When the bot initializes, it will call each plugin's handler with a command
// "init", empty channel, the bot's username, and no arguments, so the plugin
// can store this information for, e.g., scheduled jobs.
func RegisterPlugin(name string, plug PluginHandler) {
	if stopRegistrations {
		return
	}
	if _, exists := pluginHandlers[name]; exists {
		log.Fatal("Attempted registration of duplicate plugin name:", name)
	}
	pluginHandlers[name] = plug
}

func getExtDefCfg(plugin *Plugin) (*[]byte, error) {
	var fullPath string
	if byte(plugin.pluginPath[0]) == byte("/"[0]) {
		fullPath = plugin.pluginPath
	} else {
		_, err := os.Stat(b.localPath + "/" + plugin.pluginPath)
		if err != nil {
			_, err := os.Stat(b.installPath + "/" + plugin.pluginPath)
			if err != nil {
				err = fmt.Errorf("Couldn't locate external plugin %s: %v", plugin.name, err)
				return nil, err
			}
			fullPath = b.installPath + "/" + plugin.pluginPath
			Log(Debug, "Using stock external plugin:", fullPath)
		} else {
			fullPath = b.localPath + "/" + plugin.pluginPath
			Log(Debug, "Using local external plugin:", fullPath)
		}
	}
	// cmd := exec.Command(fullPath, channel, user, matcher.Command, matches[0][1:]...)
	cfg, err := exec.Command(fullPath, "", "", "", "configure").Output()
	if err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			err = fmt.Errorf("Problem retrieving default configuration for external plugin \"%s\", skipping: \"%v\", output: %s", fullPath, err, exitErr.Stderr)
		} else {
			err = fmt.Errorf("Problem retrieving default configuration for external plugin \"%s\", skipping: \"%v\"", fullPath, err)
		}
		return nil, err
	}
	return &cfg, nil
}

// loadPluginConfig() loads the configuration for all the plugins from
// $GOPHER_CONFIGDIR/plugins/<pluginname>.yaml, assigns a pluginID, and
// stores the resulting array in b.plugins. Bad plugins are skipped and logged.
// Plugin configuration is initially loaded into temporary data structures,
// then stored in the bot package under the global bot lock.
func loadPluginConfig() {
	i := 0

	// Copy some data from the bot under lock
	b.lock.RLock()
	// Get a list of all plugins from the package pluginHandlers var and
	// the list of external plugins
	nump := len(pluginHandlers) + len(b.externalPlugins)
	pnames := make([]string, nump)
	ptypes := make([]plugType, nump)
	eppaths := make(map[string]string) // Paths to external plugins
	pfinder := make(map[string]int)    // keep a map of pluginIDs to identify plugins during a callback
	pset := make(map[string]bool)      // track plugin names

	// builtins come first so indexes match, see loop below
	// Note this doesn't need to be under RLock, but it needs to precede
	// external plugins. This should be fast enough that it doesn't matter.
	for _, plugin := range builtIns {
		pnames[i] = plugin
		pset[plugin] = true
		ptypes[i] = plugBuiltin
		i++
	}

	for index, plug := range b.externalPlugins {
		if len(plug.Name) == 0 || len(plug.Path) == 0 {
			Log(Error, fmt.Sprintf("Skipping external plugin #%d with zero-length Name or Path", index+1))
			nump--
			continue
		}
		if !pNameRe.MatchString(plug.Name) {
			Log(Error, fmt.Sprintf("Plugin name \"%s\" doesn't match plugin name regex \"%s\", skipping", plug.Name, pNameRe.String()))
			nump--
			continue
		}
		if pset[plug.Name] {
			Log(Error, fmt.Sprintf("External plugin #%d, \"%s\" duplicates builtIn, skipping", index, plug.Name))
			nump--
			continue
		}
		pnames[i] = plug.Name
		pset[plug.Name] = true
		ptypes[i] = plugExternal
		eppaths[plug.Name] = plug.Path
		i++
	}
	// shrink slices when plugins were skipped
	pnames = pnames[0:nump]
	ptypes = ptypes[0:nump]
	// copy the list of default channels
	pchan := make([]string, 0, len(b.plugChannels))
	pchan = append(pchan, b.plugChannels...)
	b.lock.RUnlock() // we're done with bot data 'til the end

PlugHandlerLoop:
	for plug := range pluginHandlers {
		if !pNameRe.MatchString(plug) {
			Log(Error, fmt.Sprintf("Plugin name \"%s\" doesn't match plugin name regex \"%s\", skipping", plug, pNameRe.String()))
			nump--
			continue
		}
		if pset[plug] { // have to check builtIns, already loaded
			for _, plugin := range builtIns {
				if plug == plugin {
					continue PlugHandlerLoop // skip it, already loaded
				}
			}
			// Since external plugins can change on reload, just log an error if
			// we get a duplicate plugin name.
			Log(Error, "Plugin name duplicates external, skipping:", plug)
		} else {
			pnames[i] = plug
			ptypes[i] = plugGo
			i++
		}
	}
	// shrink slices when plugins were skipped
	pnames = pnames[0:nump]
	ptypes = ptypes[0:nump]
	plist := make([]*Plugin, 0, nump)

	// Because some plugins may be disabled, pnames and plugins won't necessarily sync
	plugIndex := 0

PlugLoop:
	for i, plug := range pnames {
		var plugin Plugin
		Log(Trace, fmt.Sprintf("Loading plugin #%d - %s, type %d", plugIndex, plug, ptypes[i]))

		plugin.pluginType = ptypes[i]
		if plugin.pluginType == plugExternal {
			// External plugins spit their default config to stdout when called with command="configure"
			plugin.pluginPath = eppaths[plug]
			cfg, err := getExtDefCfg(&plugin)
			if err != nil {
				Log(Error, err)
				continue
			}
			if err := yaml.Unmarshal(*cfg, &plugin); err != nil {
				Log(Error, fmt.Errorf("Problem unmarshalling plugin default config for \"%s\", skipping: %v", plug, err))
				continue
			}
		} else {
			if err := yaml.Unmarshal([]byte(pluginHandlers[plug].DefaultConfig), &plugin); err != nil {
				Log(Error, fmt.Errorf("Problem unmarshalling plugin default config for \"%s\", skipping: %v", plug, err))
				continue
			}
		}
		// getConfigFile overlays the default config with local config
		if err := getConfigFile("plugins/"+plug+".yaml", false, &plugin); err != nil {
			Log(Error, fmt.Errorf("Problem with local configuration for plugin \"%s\", skipping: %v", plug, err))
			continue
		}
		if plugin.Disabled {
			Log(Info, fmt.Sprintf("Plugin \"%s\" is disabled, skipping", plug))
			continue
		}
		Log(Info, "Loaded configuration for plugin", plug)
		// Use bot default plugin channels if none defined, unless AllChannels requested. Admin can override.
		if len(plugin.Channels) == 0 && len(pchan) > 0 && !plugin.AllChannels {
			plugin.Channels = pchan
		}
		Log(Info, fmt.Sprintf("Plugin \"%s\" will be active in channels %q; all channels: %t", plug, plugin.Channels, plugin.AllChannels))
		// Compile the regex's
		for i := range plugin.CommandMatches {
			command := &plugin.CommandMatches[i]
			re, err := regexp.Compile(`^\s*` + strings.Replace(command.Regex, " ", `\s+`, -1) + `\s*$`)
			if err != nil {
				Log(Error, fmt.Errorf("Skipping %s, couldn't compile command regular expression \"%s\": %v", plug, command.Regex, err))
				continue PlugLoop
			}
			command.re = re
		}
		for i := range plugin.ReplyMatchers {
			reply := &plugin.ReplyMatchers[i]
			re, err := regexp.Compile(`^\s*` + strings.Replace(reply.Regex, " ", `\s+`, -1) + `\s*$`)
			if err != nil {
				Log(Error, fmt.Errorf("Skipping %s, couldn't compile reply regular expression \"%s\": %v", plug, reply.Regex, err))
				continue PlugLoop
			}
			reply.re = re
		}
		for i := range plugin.MessageMatches {
			// Note that full message regexes don't get the beginning and end anchors added - the individual plugin
			// will need to do this if necessary.
			message := &plugin.MessageMatches[i]
			re, err := regexp.Compile(strings.Replace(message.Regex, " ", `\s+`, -1))
			if err != nil {
				Log(Error, fmt.Errorf("Skipping %s, couldn't compile message regular expression \"%s\": %v", plug, message.Regex, err))
				continue PlugLoop
			}
			message.re = re
		}
		plugin.name = plug
		// Copy the pointer to the empty config struct / empty struct (when no config)
		pt := reflect.ValueOf(pluginHandlers[plug].Config)
		if pt.Kind() == reflect.Ptr {
			if plugin.Config != nil {
				// reflect magic: create a pointer to a new empty config struct for the plugin
				plugin.config = reflect.New(reflect.Indirect(pt).Type()).Interface()
				if err := json.Unmarshal(plugin.Config, plugin.config); err != nil {
					Log(Error, fmt.Sprintf("Error unmarshalling plugin config json to config: %v", err))
				}
			} else {
				Log(Debug, fmt.Sprintf("Plugin \"%s\" has custom config, but no local custom config provided", plug))
			}
		} else {
			Log(Debug, "config interface isn't a pointer, skipping unmarshal for plugin:", plug)
		}
		// Generate the random id
		p := make([]byte, 16)
		random.Read(p)
		plugin.pluginID = fmt.Sprintf("%x", p)
		pfinder[plugin.pluginID] = plugIndex
		// Store this plugin's config in the temporary list
		Log(Info, fmt.Sprintf("Recorded plugin #%d, \"%s\"", plugIndex, plugin.name))
		plist = append(plist, &plugin)
		plugIndex++
	}

	reInitPlugins := false
	b.lock.Lock()
	plugins = plist
	plugIDmap = pfinder
	// loadPluginConfig is called in newBot, before the connector has started;
	// don't init plugins in that case.
	if b.Connector != nil {
		reInitPlugins = true
	}
	b.lock.Unlock()
	if reInitPlugins {
		initializePlugins()
	}
}
<commit_msg>Replace optional spaces with \s*<commit_after>package bot

import (
	"encoding/json"
	"fmt"
	"log"
	"os"
	"os/exec"
	"reflect"
	"regexp"
	"strings"
	"sync"

	"github.com/ghodss/yaml"
)

// PluginNames can be letters, numbers & underscores only, mainly so
// brain functions can use ':' as a separator.
const pNameRegex = `[\w]+`

var pNameRe = regexp.MustCompile(pNameRegex)
var plugins []*Plugin
var plugIDmap map[string]int

// PluginHelp specifies keywords and help text for the 'bot help system
type PluginHelp struct {
	Keywords []string // match words for 'help XXX'
	Helptext []string // help string to give for the keywords, conventionally starting with (bot) for commands or (hear) when the bot needn't be addressed directly
}

// InputMatcher specifies the command or message to match and what to pass to the plugin
type InputMatcher struct {
	Regex   string         // The regular expression string to match - bot adds ^\w* & \w*$
	Command string         // The name of the command to pass to the plugin with it's arguments
	re      *regexp.Regexp // The compiled regular expression. If the regex doesn't compile, the 'bot will log an error
}

type plugType int

const (
	plugGo plugType = iota
	plugExternal
	plugBuiltin
)

// Plugin specifies the structure of a plugin configuration - plugins should include an example / default config
type Plugin struct {
	name           string          // the name of the plugin, used as a key in to the
	pluginType     plugType        // plugGo, plugExternal, plugBuiltin - determines how commands are routed
	pluginPath     string          // Path to the external executable that expects <channel> <user> <command> <arg> <arg> from regex matches - for Plugtype=plugExternal only
	Disabled       bool            // Set true to disable the plugin
	DisallowDirect bool            // Set this true if this plugin can never be accessed via direct message
	DirectOnly     bool            // Set this true if this plugin ONLY accepts direct messages
	Channels       []string        // Channels where the plugin is active - rifraf like "memes" should probably only be in random, but it's configurable. If empty uses DefaultChannels
	AllChannels    bool            // If the Channels list is empty and AllChannels is true, the plugin should be active in all the channels the bot is in
	RequireAdmin   bool            // Set to only allow administrators to access a plugin
	Users          []string        // If non-empty, list of all the users with access to this plugin
	Help           []PluginHelp    // All the keyword sets / help texts for this plugin
	CommandMatches []InputMatcher  // Input matchers for messages that need to be directed to the 'bot
	ReplyMatchers  []InputMatcher  // Input matchers for replies to questions, only match after a RequestContinuation
	MessageMatches []InputMatcher  // Input matchers for messages the 'bot hears even when it's not being spoken to
	CatchAll       bool            // Whenever the robot is spoken to, but no plugin matches, plugins with CatchAll=true get called with command="catchall" and argument=<full text of message to robot>
	Config         json.RawMessage // Arbitrary Plugin configuration, will be stored and provided in a thread-safe manner via GetPluginConfig()
	config         interface{}     // A pointer to an empty struct that the bot can Unmarshal custom configuration into
	pluginID       string          // 32-char random ID for identifying plugins in callbacks
	lock           sync.Mutex      // For use with the robot's Brain
}

// PluginHandler is the struct a plugin registers for the Gopherbot plugin API.
type PluginHandler struct {
	DefaultConfig string /* A yaml-formatted multiline string defining the default Plugin configuration. It should be liberally commented for use in generating
	local/custom configuration for the plugin. If a Config: section is defined, it should match the structure of the optional Config interface{} */
	Handler func(bot *Robot, command string, args ...string) // The callback function called by the robot whenever a Command is matched
	Config  interface{}                                      // An optional empty struct defining custom configuration for the plugin
}

// pluginHandlers maps from plugin names to PluginV1 (later interface{} with a type selector, maybe)
var pluginHandlers = make(map[string]PluginHandler)

// stopRegistrations is set "true" when the bot is created to prevent registration outside of init functions
var stopRegistrations = false

// initialize sends the "init" command to every plugin
func initializePlugins() {
	b.lock.RLock()
	defer b.lock.RUnlock()
	bot := &Robot{
		User:    b.name,
		Channel: "",
		Format:  Variable,
	}
	for _, plugin := range plugins {
		Log(Info, "Initializing plugin:", plugin.name)
		go callPlugin(bot, plugin, "init")
	}
}

// RegisterPlugin allows plugins to register a PluginHandler in a func init().
// When the bot initializes, it will call each plugin's handler with a command
// "init", empty channel, the bot's username, and no arguments, so the plugin
// can store this information for, e.g., scheduled jobs.
func RegisterPlugin(name string, plug PluginHandler) {
	if stopRegistrations {
		return
	}
	if _, exists := pluginHandlers[name]; exists {
		log.Fatal("Attempted registration of duplicate plugin name:", name)
	}
	pluginHandlers[name] = plug
}

func getExtDefCfg(plugin *Plugin) (*[]byte, error) {
	var fullPath string
	if byte(plugin.pluginPath[0]) == byte("/"[0]) {
		fullPath = plugin.pluginPath
	} else {
		_, err := os.Stat(b.localPath + "/" + plugin.pluginPath)
		if err != nil {
			_, err := os.Stat(b.installPath + "/" + plugin.pluginPath)
			if err != nil {
				err = fmt.Errorf("Couldn't locate external plugin %s: %v", plugin.name, err)
				return nil, err
			}
			fullPath = b.installPath + "/" + plugin.pluginPath
			Log(Debug, "Using stock external plugin:", fullPath)
		} else {
			fullPath = b.localPath + "/" + plugin.pluginPath
			Log(Debug, "Using local external plugin:", fullPath)
		}
	}
	// cmd := exec.Command(fullPath, channel, user, matcher.Command, matches[0][1:]...)
	cfg, err := exec.Command(fullPath, "", "", "", "configure").Output()
	if err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			err = fmt.Errorf("Problem retrieving default configuration for external plugin \"%s\", skipping: \"%v\", output: %s", fullPath, err, exitErr.Stderr)
		} else {
			err = fmt.Errorf("Problem retrieving default configuration for external plugin \"%s\", skipping: \"%v\"", fullPath, err)
		}
		return nil, err
	}
	return &cfg, nil
}

// loadPluginConfig() loads the configuration for all the plugins from
// $GOPHER_CONFIGDIR/plugins/<pluginname>.yaml, assigns a pluginID, and
// stores the resulting array in b.plugins. Bad plugins are skipped and logged.
// Plugin configuration is initially loaded into temporary data structures,
// then stored in the bot package under the global bot lock.
func loadPluginConfig() {
	i := 0

	// Copy some data from the bot under lock
	b.lock.RLock()
	// Get a list of all plugins from the package pluginHandlers var and
	// the list of external plugins
	nump := len(pluginHandlers) + len(b.externalPlugins)
	pnames := make([]string, nump)
	ptypes := make([]plugType, nump)
	eppaths := make(map[string]string) // Paths to external plugins
	pfinder := make(map[string]int)    // keep a map of pluginIDs to identify plugins during a callback
	pset := make(map[string]bool)      // track plugin names

	// builtins come first so indexes match, see loop below
	// Note this doesn't need to be under RLock, but it needs to precede
	// external plugins. This should be fast enough that it doesn't matter.
	for _, plugin := range builtIns {
		pnames[i] = plugin
		pset[plugin] = true
		ptypes[i] = plugBuiltin
		i++
	}

	for index, plug := range b.externalPlugins {
		if len(plug.Name) == 0 || len(plug.Path) == 0 {
			Log(Error, fmt.Sprintf("Skipping external plugin #%d with zero-length Name or Path", index+1))
			nump--
			continue
		}
		if !pNameRe.MatchString(plug.Name) {
			Log(Error, fmt.Sprintf("Plugin name \"%s\" doesn't match plugin name regex \"%s\", skipping", plug.Name, pNameRe.String()))
			nump--
			continue
		}
		if pset[plug.Name] {
			Log(Error, fmt.Sprintf("External plugin #%d, \"%s\" duplicates builtIn, skipping", index, plug.Name))
			nump--
			continue
		}
		pnames[i] = plug.Name
		pset[plug.Name] = true
		ptypes[i] = plugExternal
		eppaths[plug.Name] = plug.Path
		i++
	}
	// shrink slices when plugins were skipped
	pnames = pnames[0:nump]
	ptypes = ptypes[0:nump]
	// copy the list of default channels
	pchan := make([]string, 0, len(b.plugChannels))
	pchan = append(pchan, b.plugChannels...)
	b.lock.RUnlock() // we're done with bot data 'til the end

PlugHandlerLoop:
	for plug := range pluginHandlers {
		if !pNameRe.MatchString(plug) {
			Log(Error, fmt.Sprintf("Plugin name \"%s\" doesn't match plugin name regex \"%s\", skipping", plug, pNameRe.String()))
			nump--
			continue
		}
		if pset[plug] { // have to check builtIns, already loaded
			for _, plugin := range builtIns {
				if plug == plugin {
					continue PlugHandlerLoop // skip it, already loaded
				}
			}
			// Since external plugins can change on reload, just log an error if
			// we get a duplicate plugin name.
			Log(Error, "Plugin name duplicates external, skipping:", plug)
		} else {
			pnames[i] = plug
			ptypes[i] = plugGo
			i++
		}
	}
	// shrink slices when plugins were skipped
	pnames = pnames[0:nump]
	ptypes = ptypes[0:nump]
	plist := make([]*Plugin, 0, nump)

	// Because some plugins may be disabled, pnames and plugins won't necessarily sync
	plugIndex := 0

PlugLoop:
	for i, plug := range pnames {
		var plugin Plugin
		Log(Trace, fmt.Sprintf("Loading plugin #%d - %s, type %d", plugIndex, plug, ptypes[i]))

		plugin.pluginType = ptypes[i]
		if plugin.pluginType == plugExternal {
			// External plugins spit their default config to stdout when called with command="configure"
			plugin.pluginPath = eppaths[plug]
			cfg, err := getExtDefCfg(&plugin)
			if err != nil {
				Log(Error, err)
				continue
			}
			if err := yaml.Unmarshal(*cfg, &plugin); err != nil {
				Log(Error, fmt.Errorf("Problem unmarshalling plugin default config for \"%s\", skipping: %v", plug, err))
				continue
			}
		} else {
			if err := yaml.Unmarshal([]byte(pluginHandlers[plug].DefaultConfig), &plugin); err != nil {
				Log(Error, fmt.Errorf("Problem unmarshalling plugin default config for \"%s\", skipping: %v", plug, err))
				continue
			}
		}
		// getConfigFile overlays the default config with local config
		if err := getConfigFile("plugins/"+plug+".yaml", false, &plugin); err != nil {
			Log(Error, fmt.Errorf("Problem with local configuration for plugin \"%s\", skipping: %v", plug, err))
			continue
		}
		if plugin.Disabled {
			Log(Info, fmt.Sprintf("Plugin \"%s\" is disabled, skipping", plug))
			continue
		}
		Log(Info, "Loaded configuration for plugin", plug)
		// Use bot default plugin channels if none defined, unless AllChannels requested. Admin can override.
		if len(plugin.Channels) == 0 && len(pchan) > 0 && !plugin.AllChannels {
			plugin.Channels = pchan
		}
		Log(Info, fmt.Sprintf("Plugin \"%s\" will be active in channels %q; all channels: %t", plug, plugin.Channels, plugin.AllChannels))
		// Compile the regex's
		for i := range plugin.CommandMatches {
			command := &plugin.CommandMatches[i]
			regex := strings.Replace(command.Regex, " ?", `\s*`, -1)
			regex = strings.Replace(regex, " ", `\s+`, -1)
			re, err := regexp.Compile(`^\s*` + regex + `\s*$`)
			if err != nil {
				Log(Error, fmt.Errorf("Skipping %s, couldn't compile command regular expression \"%s\": %v", plug, regex, err))
				continue PlugLoop
			}
			command.re = re
		}
		for i := range plugin.ReplyMatchers {
			reply := &plugin.ReplyMatchers[i]
			regex := strings.Replace(reply.Regex, " ?", `\s*`, -1)
			regex = strings.Replace(regex, " ", `\s+`, -1)
			re, err := regexp.Compile(`^\s*` + regex + `\s*$`)
			if err != nil {
				Log(Error, fmt.Errorf("Skipping %s, couldn't compile reply regular expression \"%s\": %v", plug, regex, err))
				continue PlugLoop
			}
			reply.re = re
		}
		for i := range plugin.MessageMatches {
			// Note that full message regexes don't get the beginning and end anchors added - the individual plugin
			// will need to do this if necessary.
			message := &plugin.MessageMatches[i]
			regex := strings.Replace(message.Regex, " ?", `\s*`, -1)
			regex = strings.Replace(regex, " ", `\s+`, -1)
			re, err := regexp.Compile(regex)
			if err != nil {
				Log(Error, fmt.Errorf("Skipping %s, couldn't compile message regular expression \"%s\": %v", plug, regex, err))
				continue PlugLoop
			}
			message.re = re
		}
		plugin.name = plug
		// Copy the pointer to the empty config struct / empty struct (when no config)
		pt := reflect.ValueOf(pluginHandlers[plug].Config)
		if pt.Kind() == reflect.Ptr {
			if plugin.Config != nil {
				// reflect magic: create a pointer to a new empty config struct for the plugin
				plugin.config = reflect.New(reflect.Indirect(pt).Type()).Interface()
				if err := json.Unmarshal(plugin.Config, plugin.config); err != nil {
					Log(Error, fmt.Sprintf("Error unmarshalling plugin config json to config: %v", err))
				}
			} else {
				Log(Debug, fmt.Sprintf("Plugin \"%s\" has custom config, but no local custom config provided", plug))
			}
		} else {
			Log(Debug, "config interface isn't a pointer, skipping unmarshal for plugin:", plug)
		}
		// Generate the random id
		p := make([]byte, 16)
		random.Read(p)
		plugin.pluginID = fmt.Sprintf("%x", p)
		pfinder[plugin.pluginID] = plugIndex
		// Store this plugin's config in the temporary list
		Log(Info, fmt.Sprintf("Recorded plugin #%d, \"%s\"", plugIndex, plugin.name))
		plist = append(plist, &plugin)
		plugIndex++
	}

	reInitPlugins := false
	b.lock.Lock()
	plugins = plist
	plugIDmap = pfinder
	// loadPluginConfig is called in newBot, before the connector has started;
	// don't init plugins in that case.
	if b.Connector != nil {
		reInitPlugins = true
	}
	b.lock.Unlock()
	if reInitPlugins {
		initializePlugins()
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package peer_test

import (
	"fmt"
	"testing"

	"go.uber.org/yarpc/api/peer"
	"go.uber.org/yarpc/api/peer/peertest"

	"github.com/golang/mock/gomock"
	"github.com/stretchr/testify/assert"
)

func TestErrPeerHasNoReferenceToSubscriber(t *testing.T) {
	ctrl := gomock.NewController(t)
	identifier := peertest.NewMockIdentifier(ctrl)
	subscriber := peertest.NewMockSubscriber(ctrl)

	wantErr := fmt.Sprintf("peer (%v) has no reference to peer subscriber (%v)", identifier, subscriber)

	err := &peer.ErrPeerHasNoReferenceToSubscriber{identifier, subscriber}
	assert.Equal(t, wantErr, err.Error())
}

func TestErrTransportHasNoReferenceToPeer2(t *testing.T) {
	transportName := "test-transport"
	peerIdentifier := "test-peer-id"

	wantErr := fmt.Sprintf("transport %q has no reference to peer %q", transportName, peerIdentifier)

	err := &peer.ErrTransportHasNoReferenceToPeer{transportName, peerIdentifier}
	assert.Equal(t, wantErr, err.Error())
}

func TestErrInvalidPeerType(t *testing.T) {
	expectedType := "test-type"
	peerIdentifier := peertest.NewMockIdentifier(gomock.NewController(t))

	wantErr := fmt.Sprintf("expected peer type (%s) but got peer (%v)", expectedType, peerIdentifier)

	err := &peer.ErrInvalidPeerType{expectedType, peerIdentifier}
	assert.Equal(t, wantErr, err.Error())
}

func TestErrPeerListAlreadyStarted(t *testing.T) {
	peerList := "test-peer-list"
	wantErr := fmt.Sprintf("%s has already been started", peerList)

	err := peer.ErrPeerListAlreadyStarted(peerList)
	assert.Equal(t, wantErr, err.Error())
}

func TestErrPeerListNotStarted(t *testing.T) {
	peerList := "test-peer-list"
	wantErr := fmt.Sprintf("%s has not been started or was stopped", peerList)

	err := peer.ErrPeerListNotStarted(peerList)
	assert.Equal(t, wantErr, err.Error())
}

func TestErrInvalidPeerConversion(t *testing.T) {
	p := peertest.NewMockPeer(gomock.NewController(t))
	expectedType := "test-type"

	wantErr := fmt.Sprintf("cannot convert peer (%v) to type %s", p, expectedType)

	err := &peer.ErrInvalidPeerConversion{p, expectedType}
	assert.Equal(t, wantErr, err.Error())
}

func TestErrInvalidTransportConversion(t *testing.T) {
	transport := peertest.NewMockTransport(gomock.NewController(t))
	expectedType := "test-type"

	wantErr := fmt.Sprintf("cannot convert transport (%v) to type %s", transport, expectedType)

	err := &peer.ErrInvalidTransportConversion{transport, expectedType}
	assert.Equal(t, wantErr, err.Error())
}

func TestErrPeerAddAlreadyInList(t *testing.T) {
	p := "test-peer"
	wantErr := fmt.Sprintf("can't add peer %q because is already in peerlist", p)

	err := peer.ErrPeerAddAlreadyInList(p)
	assert.Equal(t, wantErr, err.Error())
}

func TestErrPeerRemoveNotInList(t *testing.T) {
	p := "test-peer"
	wantErr := fmt.Sprintf("can't remove peer (%s) because it is not in peerlist", p)

	err := peer.ErrPeerRemoveNotInList(p)
	assert.Equal(t, wantErr, err.Error())
}

func TestErrChooseContextHasNoDeadline(t *testing.T) {
	peerList := "test-peer"
	wantErr := fmt.Sprintf("can't wait for peer without a context deadline for peerlist %q", peerList)

	err := peer.ErrChooseContextHasNoDeadline(peerList)
	assert.Equal(t, wantErr, err.Error())
}
<commit_msg>Fix lint in testing (#1386)<commit_after>// Copyright (c) 2017 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package peer_test

import (
	"fmt"
	"testing"

	"go.uber.org/yarpc/api/peer"
	"go.uber.org/yarpc/api/peer/peertest"

	"github.com/golang/mock/gomock"
	"github.com/stretchr/testify/assert"
)

func TestErrPeerHasNoReferenceToSubscriber(t *testing.T) {
	ctrl := gomock.NewController(t)
	identifier := peertest.NewMockIdentifier(ctrl)
	subscriber := peertest.NewMockSubscriber(ctrl)

	wantErr := fmt.Sprintf("peer (%v) has no reference to peer subscriber (%v)", identifier, subscriber)

	err := &peer.ErrPeerHasNoReferenceToSubscriber{PeerIdentifier: identifier, PeerSubscriber: subscriber}
	assert.Equal(t, wantErr, err.Error())
}

func TestErrTransportHasNoReferenceToPeer2(t *testing.T) {
	transportName := "test-transport"
	peerIdentifier := "test-peer-id"

	wantErr := fmt.Sprintf("transport %q has no reference to peer %q", transportName, peerIdentifier)

	err := &peer.ErrTransportHasNoReferenceToPeer{TransportName: transportName, PeerIdentifier: peerIdentifier}
	assert.Equal(t, wantErr, err.Error())
}

func TestErrInvalidPeerType(t *testing.T) {
	expectedType := "test-type"
	peerIdentifier := peertest.NewMockIdentifier(gomock.NewController(t))

	wantErr := fmt.Sprintf("expected peer type (%s) but got peer (%v)", expectedType, peerIdentifier)

	err := &peer.ErrInvalidPeerType{ExpectedType: expectedType, PeerIdentifier: peerIdentifier}
	assert.Equal(t, wantErr, err.Error())
}

func TestErrPeerListAlreadyStarted(t *testing.T) {
	peerList := "test-peer-list"
	wantErr := fmt.Sprintf("%s has already been started", peerList)

	err := peer.ErrPeerListAlreadyStarted(peerList)
	assert.Equal(t, wantErr, err.Error())
}

func TestErrPeerListNotStarted(t *testing.T) {
	peerList := "test-peer-list"
	wantErr := fmt.Sprintf("%s has not been started or was stopped", peerList)

	err := peer.ErrPeerListNotStarted(peerList)
	assert.Equal(t, wantErr, err.Error())
}

func TestErrInvalidPeerConversion(t *testing.T) {
	p := peertest.NewMockPeer(gomock.NewController(t))
	expectedType := "test-type"

	wantErr := fmt.Sprintf("cannot convert peer (%v) to type %s", p, expectedType)

	err := &peer.ErrInvalidPeerConversion{Peer: p, ExpectedType: expectedType}
	assert.Equal(t, wantErr, err.Error())
}

func TestErrInvalidTransportConversion(t *testing.T) {
	transport := peertest.NewMockTransport(gomock.NewController(t))
	expectedType := "test-type"

	wantErr := fmt.Sprintf("cannot convert transport (%v) to type %s", transport, expectedType)

	err := &peer.ErrInvalidTransportConversion{Transport: transport, ExpectedType: expectedType}
	assert.Equal(t, wantErr, err.Error())
}

func TestErrPeerAddAlreadyInList(t *testing.T) {
	p := "test-peer"
	wantErr := fmt.Sprintf("can't add peer %q because is already in peerlist", p)

	err := peer.ErrPeerAddAlreadyInList(p)
	assert.Equal(t, wantErr, err.Error())
}

func TestErrPeerRemoveNotInList(t *testing.T) {
	p := "test-peer"
	wantErr := fmt.Sprintf("can't remove peer (%s) because it is not in peerlist", p)

	err := peer.ErrPeerRemoveNotInList(p)
	assert.Equal(t, wantErr, err.Error())
}

func TestErrChooseContextHasNoDeadline(t *testing.T) {
	peerList := "test-peer"
	wantErr := fmt.Sprintf("can't wait for peer without a context deadline for peerlist %q", peerList)

	err := peer.ErrChooseContextHasNoDeadline(peerList)
	assert.Equal(t, wantErr, err.Error())
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"runtime"

	"github.com/Jeffail/benthos/lib/util/service"
	"github.com/Jeffail/benthos/lib/util/service/log"
	"github.com/Jeffail/benthos/lib/util/service/metrics"
)

//------------------------------------------------------------------------------

var endpoints []string

func registerEndpoint(path string, handler http.HandlerFunc) {
	endpoints = append(endpoints, path)

	http.HandleFunc(path, handler)
	http.HandleFunc("/benthos"+path, handler)
}

func handleVersion(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte(fmt.Sprintf("{\"version\":\"%v\", \"built\":\"%v\"}", service.Version, service.DateBuilt)))
}

func handleEndpoints(w http.ResponseWriter, r *http.Request) {
	res := struct {
		Endpoints []string `json:"endpoints"`
	}{
		Endpoints: endpoints,
	}
	resBytes, err := json.Marshal(res)
	if err != nil {
		w.WriteHeader(http.StatusBadGateway)
	} else {
		w.Write(resBytes)
	}
}

func registerHTTPEndpoints(
	conf Config,
	log log.Modular,
	stats metrics.Type,
) {
	handlePing := func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("pong"))
	}

	handleStackTrace := func(w http.ResponseWriter, r *http.Request) {
		stackSlice := make([]byte, 1024*100)
		s := runtime.Stack(stackSlice, true)
		w.Write(stackSlice[:s])
	}

	registerEndpoint("/stack", handleStackTrace)
	registerEndpoint("/ping", handlePing)
	registerEndpoint("/version", handleVersion)
	registerEndpoint("/endpoints", handleEndpoints)

	// If we want to expose a JSON stats endpoint we register the endpoints.
	if conf.Metrics.Type == "http_server" {
		if httpStats, ok := stats.(*metrics.HTTP); ok {
			registerEndpoint("/stats", httpStats.JSONHandler())
		}
	}
}

//------------------------------------------------------------------------------
<commit_msg>Add /config endpoints<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"runtime"

	"github.com/Jeffail/benthos/lib/util/service"
	"github.com/Jeffail/benthos/lib/util/service/log"
	"github.com/Jeffail/benthos/lib/util/service/metrics"
	yaml "gopkg.in/yaml.v2"
)

//------------------------------------------------------------------------------

var endpoints []string

func registerEndpoint(path string, handler http.HandlerFunc) {
	endpoints = append(endpoints, path)

	http.HandleFunc(path, handler)
	http.HandleFunc("/benthos"+path, handler)
}

func handleVersion(w http.ResponseWriter, r *http.Request) {
	w.Write([]byte(fmt.Sprintf("{\"version\":\"%v\", \"built\":\"%v\"}", service.Version, service.DateBuilt)))
}

func handleEndpoints(w http.ResponseWriter, r *http.Request) {
	res := struct {
		Endpoints []string `json:"endpoints"`
	}{
		Endpoints: endpoints,
	}
	resBytes, err := json.Marshal(res)
	if err != nil {
		w.WriteHeader(http.StatusBadGateway)
	} else {
		w.Write(resBytes)
	}
}

func registerHTTPEndpoints(
	conf Config,
	log log.Modular,
	stats metrics.Type,
) {
	handlePing := func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("pong"))
	}

	handleStackTrace := func(w http.ResponseWriter, r *http.Request) {
		stackSlice := make([]byte, 1024*100)
		s := runtime.Stack(stackSlice, true)
		w.Write(stackSlice[:s])
	}

	handlePrintJSONConfig := func(w http.ResponseWriter, r *http.Request) {
		resBytes, err := json.Marshal(conf)
		if err != nil {
			w.WriteHeader(http.StatusBadGateway)
			return
		}
		w.Write(resBytes)
	}

	handlePrintYAMLConfig := func(w http.ResponseWriter, r *http.Request) {
		resBytes, err := yaml.Marshal(conf)
		if err != nil {
			w.WriteHeader(http.StatusBadGateway)
			return
		}
		w.Write(resBytes)
	}

	registerEndpoint("/config/json", handlePrintJSONConfig)
	registerEndpoint("/config/yaml", handlePrintYAMLConfig)
	registerEndpoint("/stack", handleStackTrace)
	registerEndpoint("/ping", handlePing)
	registerEndpoint("/version", handleVersion)
	registerEndpoint("/endpoints", handleEndpoints)

	// If we want to expose a JSON stats endpoint we register the endpoints.
	if conf.Metrics.Type == "http_server" {
		if httpStats, ok := stats.(*metrics.HTTP); ok {
			registerEndpoint("/stats", httpStats.JSONHandler())
		}
	}
}

//------------------------------------------------------------------------------
<|endoftext|>


<commit_before>package command

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"net"
	"os"
	"reflect"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/funkygao/gafka"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/registry"
	zkr "github.com/funkygao/gafka/registry/zk"
	"github.com/funkygao/gafka/telemetry"
	"github.com/funkygao/gafka/telemetry/influxdb"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	"github.com/funkygao/gocli"
	gio "github.com/funkygao/golib/io"
	"github.com/funkygao/golib/locking"
	"github.com/funkygao/golib/signal"
	"github.com/funkygao/golib/sync2"
	log "github.com/funkygao/log4go"
	zklib "github.com/samuel/go-zookeeper/zk"
)

type Start struct {
	Ui  cli.Ui
	Cmd string

	startedAt time.Time

	zone       string
	root       string
	debugMode  bool
	command    string
	logfile    string
	pubPort    int
	subPort    int
	manPort    int
	starting   bool
	forwardFor bool
	httpAddr   string

	haproxyStatsUrl string
	influxdbAddr    string
	influxdbDbName  string

	quitCh, closed chan struct{}
	zkzone         *zk.ZkZone
	lastServers    BackendServers

	withF5       bool
	safeShutdown chan struct{}
	quiting      sync2.AtomicBool
	deadN        sync2.AtomicInt32
}

func (this *Start) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("start", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.logfile, "log", defaultLogfile, "")
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.root, "p", defaultPrefix, "")
	cmdFlags.BoolVar(&this.debugMode, "d", false, "")
	cmdFlags.BoolVar(&this.forwardFor, "forwardfor", false, "")
	cmdFlags.IntVar(&this.pubPort, "pub", 10891, "")
	cmdFlags.IntVar(&this.subPort, "sub", 10892, "")
	cmdFlags.IntVar(&this.manPort, "man", 10893, "")
	cmdFlags.BoolVar(&this.withF5, "f5", false, "")
	cmdFlags.StringVar(&this.haproxyStatsUrl, "statsurl", "", "")
	cmdFlags.StringVar(&this.influxdbAddr, "influxaddr", "", "")
	cmdFlags.StringVar(&this.influxdbDbName, "influxdb", "", "")
	cmdFlags.StringVar(&this.httpAddr, "addr", ":10894", "monitor http server addr")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if this.debugMode {
		this.pubPort = 9191
		this.subPort = 9192
		this.manPort = 9193
	}

	lockFilename := fmt.Sprintf("%s/.lock", this.root)
	if locking.InstanceLocked(lockFilename) {
		panic(fmt.Sprintf("locked[%s] by another instance", lockFilename))
	}

	locking.LockInstance(lockFilename)

	err := os.Chdir(this.root)
	swalllow(err)

	this.command = fmt.Sprintf("%s/sbin/haproxy", this.root)
	if _, err := os.Stat(this.command); err != nil {
		panic(err)
	}

	this.setupLogging(this.logfile, "info", "panic")
	this.starting = true
	this.startedAt = time.Now()

	if this.haproxyStatsUrl != "" &&
		this.influxdbAddr != "" && this.influxdbDbName != "" {
		rc, err := influxdb.NewConfig(this.influxdbAddr, this.influxdbDbName, "", "", time.Minute)
		if err != nil {
			panic(err)
		}
		telemetry.Default = influxdb.New(metrics.DefaultRegistry, rc)
		go func() {
			log.Info("telemetry started: %s", telemetry.Default.Name())

			if err := telemetry.Default.Start(); err != nil {
				log.Error("telemetry: %v", err)
			}
		}()

		m := haproxyMetrics{
			ctx:      this,
			interval: time.Second * 30,
			uri:      this.haproxyStatsUrl,
		}
		go m.start()
	}

	this.quitCh = make(chan struct{})
	this.closed = make(chan struct{})
	this.safeShutdown = make(chan struct{})
	signal.RegisterHandler(func(sig os.Signal) {
		log.Info("ehaproxy[%s] got signal: %s", gafka.BuildId, strings.ToUpper(sig.String()))

		if this.withF5 {
			// work closely with F5 for graceful shutdown
			this.quiting.Set(true)
			<-this.safeShutdown

			// wait for F5 mark me down
			time.Sleep(time.Second * 2)
		}

		this.shutdown()

		log.Info("removing %s", configFile)
		os.Remove(configFile)

		log.Info("removing lock[%s]", lockFilename)
		locking.UnlockInstance(lockFilename)

		close(this.quitCh)

		if telemetry.Default != nil {
			log.Info("stopping telemetry and flush all metrics...")
			telemetry.Default.Stop()
		}

		log.Info("ehaproxy[%s] shutdown complete", gafka.BuildId)
		log.Info("ehaproxy[%s] %s bye!", gafka.BuildId, time.Since(this.startedAt))

		close(this.closed)
	}, syscall.SIGINT, syscall.SIGTERM)

	this.main()

	<-this.closed
	log.Close()

	return
}

func (this *Start) main() {
	ctx.LoadFromHome()
	this.zkzone = zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	zkConnEvt, ok := this.zkzone.SessionEvents()
	if !ok {
		panic("someone stealing my events")
	}

	registry.Default = zkr.New(this.zkzone)

	log.Info("ehaproxy[%s] starting...", gafka.BuildId)
	go this.runMonitorServer(this.httpAddr)

	zkConnected := false
	for {
		instances, instancesChange, err := registry.Default.WatchInstances()
		if err != nil {
			log.Error("zone[%s] %s", this.zkzone.Name(), err)
			time.Sleep(time.Second)
			continue
		}

		if zkConnected {
			if len(instances) > 0 {
				this.reload(instances)
			} else {
				// resilience to zk problem by local cache
				log.Warn("backend all shutdown? skip this change")
				time.Sleep(time.Second)
				continue
			}
		}

		select {
		case <-this.quitCh:
			return

		case evt := <-zkConnEvt:
			if evt.State == zklib.StateHasSession && !zkConnected {
				log.Info("zk connected")
				zkConnected = true
			} else if zkConnected && evt.Path == "" {
				log.Warn("zk jitter: %+v", evt)
			}

		case <-instancesChange:
			log.Info("instances changed!!")
		}
	}

}

func (this *Start) reload(kwInstances []string) {
	var servers = BackendServers{
		CpuNum:      ctx.NumCPU(),
		HaproxyRoot: this.root,
		ForwardFor:  this.forwardFor,
		PubPort:     this.pubPort,
		SubPort:     this.subPort,
		ManPort:     this.manPort,
	}
	servers.reset()
	for _, kwNode := range kwInstances {
		data, _, err := this.zkzone.Conn().Get(kwNode)
		if err != nil {
			log.Error("%s: %v", kwNode, err)
			continue
		}

		info := make(map[string]string)
		if err = json.Unmarshal([]byte(data), &info); err != nil {
			log.Error("%s: %v", data, err)
			continue
		}

		// pub
		if info["pub"] != "" {
			_, port, _ := net.SplitHostPort(info["pub"])
			be := Backend{
				Name: "p" + info["id"],
				Addr: info["pub"],
				Cpu:  info["cpu"],
				Port: port,
			}
			servers.Pub = append(servers.Pub, be)
		}

		// sub
		if info["sub"] != "" {
			_, port, _ := net.SplitHostPort(info["sub"])
			be := Backend{
				Name: "s" + info["id"],
				Addr: info["sub"],
				Cpu:  info["cpu"],
				Port: port,
			}
			servers.Sub = append(servers.Sub, be)
		}

		// man
		if info["man"] != "" {
			_, port, _ := net.SplitHostPort(info["man"])
			be := Backend{
				Name: "m" + info["id"],
				Addr: info["man"],
				Cpu:  info["cpu"],
				Port: port,
			}
			servers.Man = append(servers.Man, be)
		}
	}

	for i := 0; i < ctx.NumCPU(); i++ {
		servers.Dashboard = append(servers.Dashboard, Backend{
			Port: fmt.Sprintf("%d", dashboardPortHead+i),
			Name: fmt.Sprintf("%d", i+1), // process id starts from 1
		})
	}

	if servers.empty() {
		log.Warn("empty backend servers, all shutdown?")
		return
	}

	if reflect.DeepEqual(this.lastServers, servers) {
		log.Warn("backend servers stays unchanged")
		return
	}

	this.lastServers = servers
	if err := this.createConfigFile(servers); err != nil {
		log.Error(err)
		return
	}

	if err := this.reloadHAproxy(); err != nil {
		log.Error("reloading haproxy: %v", err)
		panic(err)
	}
}

func (this *Start) shutdown() {
	// kill haproxy
	log.Info("killling haproxy processes")

	f, e := os.Open(haproxyPidFile)
	if e != nil {
		log.Error("shutdown %v", e)
		return
	}

	reader := bufio.NewReader(f)
	for {
		l, e := gio.ReadLine(reader)
		if e != nil {
			// EOF
			break
		}

		pid, _ := strconv.Atoi(string(l))
		p := &os.Process{
			Pid: pid,
		}
		if err := p.Kill(); err != nil {
			log.Error(err)
		} else {
			log.Info("haproxy[%d] terminated", pid)
		}
	}

	log.Info("removing %s", haproxyPidFile)
	os.Remove(haproxyPidFile)
}

func (this *Start) Synopsis() string {
	return fmt.Sprintf("Start %s system on localhost", this.Cmd)
}

func (this *Start) Help() string {
	help := fmt.Sprintf(`
Usage: %s start [options]

    Start %s system on localhost

Options:

    -z zone
      Default %s

    -d
      Debug mode

    -statsurl url
      haproxy stats url

    -influxaddr addr

    -influxdb dbName

    -forwardfor
      Default false.
      If true, haproxy will add X-Forwarded-For http header.

    -pub pub server listen port

    -sub sub server listen port

    -man manager server listen port

    -p directory prefix
      Default %s

    -log log file
      Default %s

`, this.Cmd, this.Cmd, ctx.ZkDefaultZone(), defaultPrefix, defaultLogfile)
	return strings.TrimSpace(help)
}
<commit_msg>add timeout<commit_after>package command

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"net"
	"os"
	"reflect"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/funkygao/gafka"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/registry"
	zkr "github.com/funkygao/gafka/registry/zk"
	"github.com/funkygao/gafka/telemetry"
	"github.com/funkygao/gafka/telemetry/influxdb"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	"github.com/funkygao/gocli"
	gio "github.com/funkygao/golib/io"
	"github.com/funkygao/golib/locking"
	"github.com/funkygao/golib/signal"
	"github.com/funkygao/golib/sync2"
	log "github.com/funkygao/log4go"
	zklib "github.com/samuel/go-zookeeper/zk"
)

type Start struct {
	Ui  cli.Ui
	Cmd string

	startedAt time.Time

	zone       string
	root       string
	debugMode  bool
	command    string
	logfile    string
	pubPort    int
	subPort    int
	manPort    int
	starting   bool
	forwardFor bool
	httpAddr   string

	haproxyStatsUrl string
	influxdbAddr    string
	influxdbDbName  string

	quitCh, closed chan struct{}
	zkzone         *zk.ZkZone
	lastServers    BackendServers

	withF5       bool
	safeShutdown chan struct{}
	quiting      sync2.AtomicBool
	deadN        sync2.AtomicInt32
}

func (this *Start) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("start", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.logfile, "log", defaultLogfile, "")
	cmdFlags.StringVar(&this.zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&this.root, "p", defaultPrefix, "")
	cmdFlags.BoolVar(&this.debugMode, "d", false, "")
	cmdFlags.BoolVar(&this.forwardFor, "forwardfor", false, "")
	cmdFlags.IntVar(&this.pubPort, "pub", 10891, "")
	cmdFlags.IntVar(&this.subPort, "sub", 10892, "")
	cmdFlags.IntVar(&this.manPort, "man", 10893, "")
	cmdFlags.BoolVar(&this.withF5, "f5", false, "")
	cmdFlags.StringVar(&this.haproxyStatsUrl, "statsurl", "", "")
	cmdFlags.StringVar(&this.influxdbAddr, "influxaddr", "", "")
	cmdFlags.StringVar(&this.influxdbDbName, "influxdb", "", "")
	cmdFlags.StringVar(&this.httpAddr, "addr", ":10894", "monitor http server addr")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if this.debugMode {
		this.pubPort = 9191
		this.subPort = 9192
		this.manPort = 9193
	}

	lockFilename := fmt.Sprintf("%s/.lock", this.root)
	if locking.InstanceLocked(lockFilename) {
		panic(fmt.Sprintf("locked[%s] by another instance", lockFilename))
	}

	locking.LockInstance(lockFilename)

	err := os.Chdir(this.root)
	swalllow(err)

	this.command = fmt.Sprintf("%s/sbin/haproxy", this.root)
	if _, err := os.Stat(this.command); err != nil {
		panic(err)
	}

	this.setupLogging(this.logfile, "info", "panic")
	this.starting = true
	this.startedAt = time.Now()

	if this.haproxyStatsUrl != "" &&
		this.influxdbAddr != "" && this.influxdbDbName != "" {
		rc, err := influxdb.NewConfig(this.influxdbAddr, this.influxdbDbName, "", "", time.Minute)
		if err != nil {
			panic(err)
		}
		telemetry.Default = influxdb.New(metrics.DefaultRegistry, rc)
		go func() {
			log.Info("telemetry started: %s", telemetry.Default.Name())

			if err := telemetry.Default.Start(); err != nil {
				log.Error("telemetry: %v", err)
			}
		}()

		m := haproxyMetrics{
			ctx:      this,
			interval: time.Second * 30,
			uri:      this.haproxyStatsUrl,
		}
		go m.start()
	}

	this.quitCh = make(chan struct{})
	this.closed = make(chan struct{})
	this.safeShutdown = make(chan struct{})
	signal.RegisterHandler(func(sig os.Signal) {
		log.Info("ehaproxy[%s] got signal: %s", gafka.BuildId, strings.ToUpper(sig.String()))

		if this.withF5 {
			// work closely with F5 for graceful shutdown
			this.quiting.Set(true)
			select {
			case <-this.safeShutdown:
			case <-time.After(time.Second * 30):
			}

			// wait for F5 mark me down
			time.Sleep(time.Second * 2)
		}

		this.shutdown()

		log.Info("removing %s", configFile)
		os.Remove(configFile)

		log.Info("removing lock[%s]", lockFilename)
		locking.UnlockInstance(lockFilename)

		close(this.quitCh)

		if telemetry.Default != nil {
			log.Info("stopping telemetry and flush all metrics...")
			telemetry.Default.Stop()
		}

		log.Info("ehaproxy[%s] shutdown complete", gafka.BuildId)
		log.Info("ehaproxy[%s] %s bye!", gafka.BuildId, time.Since(this.startedAt))

		close(this.closed)
	}, syscall.SIGINT, syscall.SIGTERM)

	this.main()

	<-this.closed
	log.Close()

	return
}

func (this *Start) main() {
	ctx.LoadFromHome()
	this.zkzone = zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	zkConnEvt, ok := this.zkzone.SessionEvents()
	if !ok {
		panic("someone stealing my events")
	}

	registry.Default = zkr.New(this.zkzone)

	log.Info("ehaproxy[%s] starting...", gafka.BuildId)
	go this.runMonitorServer(this.httpAddr)

	zkConnected := false
	for {
		instances, instancesChange, err := registry.Default.WatchInstances()
		if err != nil {
			log.Error("zone[%s] %s", this.zkzone.Name(), err)
			time.Sleep(time.Second)
			continue
		}

		if zkConnected {
			if len(instances) > 0 {
				this.reload(instances)
			} else {
				// resilience to zk problem by local cache
				log.Warn("backend all shutdown? skip this change")
				time.Sleep(time.Second)
				continue
			}
		}

		select {
		case <-this.quitCh:
			return

		case evt := <-zkConnEvt:
			if evt.State == zklib.StateHasSession && !zkConnected {
				log.Info("zk connected")
				zkConnected = true
			} else if zkConnected && evt.Path == "" {
				log.Warn("zk jitter: %+v", evt)
			}

		case <-instancesChange:
			log.Info("instances changed!!")
		}
	}

}

func (this *Start) reload(kwInstances []string) {
	var servers = BackendServers{
		CpuNum:      ctx.NumCPU(),
		HaproxyRoot: this.root,
		ForwardFor:  this.forwardFor,
		PubPort:     this.pubPort,
		SubPort:     this.subPort,
		ManPort:     this.manPort,
	}
	servers.reset()
	for _, kwNode := range kwInstances {
		data, _, err := this.zkzone.Conn().Get(kwNode)
		if err != nil {
			log.Error("%s: %v", kwNode, err)
			continue
		}

		info := make(map[string]string)
		if err = json.Unmarshal([]byte(data), &info); err != nil {
			log.Error("%s: %v", data, err)
			continue
		}

		// pub
		if info["pub"] != "" {
			_, port, _ := net.SplitHostPort(info["pub"])
			be := Backend{
				Name: "p" + info["id"],
				Addr: info["pub"],
				Cpu:  info["cpu"],
				Port: port,
			}
			servers.Pub = append(servers.Pub, be)
		}

		// sub
		if info["sub"] != "" {
			_, port, _ := net.SplitHostPort(info["sub"])
			be := Backend{
				Name: "s" + info["id"],
				Addr: info["sub"],
				Cpu:  info["cpu"],
				Port: port,
			}
			servers.Sub = append(servers.Sub, be)
		}

		// man
		if info["man"] != "" {
			_, port, _ := net.SplitHostPort(info["man"])
			be := Backend{
				Name: "m" + info["id"],
				Addr: info["man"],
				Cpu:  info["cpu"],
				Port: port,
			}
			servers.Man = append(servers.Man, be)
		}
	}

	for i := 0; i < ctx.NumCPU(); i++ {
		servers.Dashboard = append(servers.Dashboard, Backend{
			Port: fmt.Sprintf("%d", dashboardPortHead+i),
			Name: fmt.Sprintf("%d", i+1), // process id starts from 1
		})
	}

	if servers.empty() {
		log.Warn("empty backend servers, all shutdown?")
		return
	}

	if reflect.DeepEqual(this.lastServers, servers) {
		log.Warn("backend servers stays unchanged")
		return
	}

	this.lastServers = servers
	if err := this.createConfigFile(servers); err != nil {
		log.Error(err)
		return
	}

	if err := this.reloadHAproxy(); err != nil {
		log.Error("reloading haproxy: %v", err)
		panic(err)
	}
}

func (this *Start) shutdown() {
	// kill haproxy
	log.Info("killling haproxy processes")

	f, e := os.Open(haproxyPidFile)
	if e != nil {
		log.Error("shutdown %v", e)
		return
	}

	reader := bufio.NewReader(f)
	for {
		l, e := gio.ReadLine(reader)
		if e != nil {
			// EOF
			break
		}

		pid, _ := strconv.Atoi(string(l))
		p := &os.Process{
			Pid: pid,
		}
		if err := p.Kill(); err != nil {
			log.Error(err)
		} else {
			log.Info("haproxy[%d] terminated", pid)
		}
	}

	log.Info("removing %s", haproxyPidFile)
	os.Remove(haproxyPidFile)
}

func (this *Start) Synopsis() string {
	return fmt.Sprintf("Start %s system on localhost", this.Cmd)
}

func (this *Start) Help() string {
	help := fmt.Sprintf(`
Usage: %s start [options]

    Start %s system on localhost

Options:

    -z zone
      Default %s

    -d
      Debug mode

    -statsurl url
      haproxy stats url

    -influxaddr addr

    -influxdb dbName

    -forwardfor
      Default false.
      If true, haproxy will add X-Forwarded-For http header.

    -pub pub server listen port

    -sub sub server listen port

    -man manager server listen port

    -p directory prefix
      Default %s

    -log log file
      Default %s

`, this.Cmd, this.Cmd, ctx.ZkDefaultZone(), defaultPrefix, defaultLogfile)
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>package mirror

import (
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/golib/gofmt"
	"github.com/funkygao/kafka-cg/consumergroup"
	log "github.com/funkygao/log4go"
)

func (this *Mirror) pump(sub *consumergroup.ConsumerGroup, pub sarama.AsyncProducer,
	stop, stopped chan struct{}) {
	defer func() {
		log.Info("closing sub...")
		sub.Close()

		stopped <- struct{}{} // notify others I'm done
	}()

	active := true
	backoff := time.Second * 8
	idle := time.Second * 10
	for {
		select {
		case <-this.quit:
			log.Trace("got signal quit")
			return

		case <-stop:
			// yes sir!
			log.Trace("got signal stop")
			return

		case <-time.After(idle):
			active = false
			log.Info("idle 10s waiting for new message")

		case msg, ok := <-sub.Messages():
			if !ok {
				log.Warn("sub encounters end of message stream")
				return
			}

			if !active || this.Debug {
				log.Info("<-[#%d] T:%s M:%s", this.transferN, msg.Topic, string(msg.Value))
			}
			active = true

			pub.Input() <- &sarama.ProducerMessage{
				Topic: msg.Topic,
				Key:   sarama.ByteEncoder(msg.Key),
				Value: sarama.ByteEncoder(msg.Value),
			}
			if this.AutoCommit {
				sub.CommitUpto(msg)
			}

			// rate limit, never overflood the limited bandwidth between IDCs
			// FIXME when compressed, the bandwidth calculation is wrong
			bytesN := len(msg.Topic) + len(msg.Key) + len(msg.Value) + 20 // payload overhead
			if this.bandwidthRateLimiter != nil && !this.bandwidthRateLimiter.Pour(bytesN) {
				time.Sleep(backoff)
				log.Warn("%s -> bandwidth reached, backoff %s", gofmt.ByteSize(this.transferBytes), backoff)
			}

			this.transferBytes += int64(bytesN)
			this.transferN++
			if this.transferN%this.ProgressStep == 0 {
				log.Info("%s %s %s", gofmt.Comma(this.transferN), gofmt.ByteSize(this.transferBytes), msg.Topic)
			}

		case err := <-sub.Errors():
			log.Error("sub %v", err)
		}
	}
}
<commit_msg>log before do sleep<commit_after>package mirror

import (
	"time"

	"github.com/Shopify/sarama"
	"github.com/funkygao/golib/gofmt"
	"github.com/funkygao/kafka-cg/consumergroup"
	log "github.com/funkygao/log4go"
)

func (this *Mirror) pump(sub *consumergroup.ConsumerGroup, pub sarama.AsyncProducer,
	stop, stopped chan struct{}) {
	defer func() {
		log.Info("closing sub...")
		sub.Close()

		stopped <- struct{}{} // notify others I'm done
	}()

	active := true
	backoff := time.Second * 8
	idle := time.Second * 10
	for {
		select {
		case <-this.quit:
			log.Trace("got signal quit")
			return

		case <-stop:
			// yes sir!
			log.Trace("got signal stop")
			return

		case <-time.After(idle):
			active = false
			log.Info("idle 10s waiting for new message")

		case msg, ok := <-sub.Messages():
			if !ok {
				log.Warn("sub encounters end of message stream")
				return
			}

			if !active || this.Debug {
				log.Info("<-[#%d] T:%s M:%s", this.transferN, msg.Topic, string(msg.Value))
			}
			active = true

			pub.Input() <- &sarama.ProducerMessage{
				Topic: msg.Topic,
				Key:   sarama.ByteEncoder(msg.Key),
				Value: sarama.ByteEncoder(msg.Value),
			}
			if this.AutoCommit {
				sub.CommitUpto(msg)
			}

			// rate limit, never overflood the limited bandwidth between IDCs
			// FIXME when compressed, the bandwidth calculation is wrong
			bytesN := len(msg.Topic) + len(msg.Key) + len(msg.Value) + 20 // payload overhead
			if this.bandwidthRateLimiter != nil && !this.bandwidthRateLimiter.Pour(bytesN) {
				log.Warn("%s -> bandwidth reached, backoff %s", gofmt.ByteSize(this.transferBytes), backoff)
				time.Sleep(backoff)
			}

			this.transferBytes += int64(bytesN)
			this.transferN++
			if this.transferN%this.ProgressStep == 0 {
				log.Info("%s %s %s", gofmt.Comma(this.transferN), gofmt.ByteSize(this.transferBytes), msg.Topic)
			}

		case err := <-sub.Errors():
			log.Error("sub %v", err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/gophergala/matching-snuggies/slicerjob"
)

func main() {
	server := flag.String("server", "localhost:8888", "snuggied server address")
	slicerBackend := flag.String("backend", "slic3r", "backend slicer")
	slicerPreset := flag.String("preset", "hiQ", "specify a configuration preset for the backend")
	gcodeDest := flag.String("o", "", "specify an output gcode filename")
	flag.Parse()

	if flag.NArg() < 1 {
		log.Fatalf("missing argument: mesh file")
	}
	meshpath := flag.Arg(0)

	client := &Client{
		ServerAddr: *server,
	}

	// send files to the slicer to be printed and poll the slicer until the job
	// has completed.
	log.Printf("sending file(s) to snuggied server at %v", *server)
	job, err := client.SliceFile(*slicerBackend, *slicerPreset, meshpath)
	if err != nil {
		log.Fatalf("sending files: %v", err)
	}
	tick := time.After(100 * time.Millisecond)
	for job.Status != "complete" {
		// TODO: retry with exponential backoff on network failure
		select {
		case <-tick:
			job, err = client.SlicerStatus(job)
			if err != nil {
				log.Fatalf("waiting: %v", err)
			}
		}
	}

	// download gcode from the slicer and write to the specified file.
	log.Printf("retreiving gcode file")
	r, err := client.GCode(job)
	if err != nil {
		log.Fatalf("gcode: %v", err)
	}
	defer r.Close()
	var f *os.File
	if *gcodeDest == "" {
		f = os.Stdout
	} else {
		f, err = os.Create(*gcodeDest)
		if err != nil {
			log.Panic(err)
		}
	}
	defer func() {
		err := f.Close()
		if err != nil {
			log.Panic(err)
		}
	}()
	_, err = io.Copy(f, r)
	if err != nil {
		log.Panic(err)
	}
}

type Client struct {
	Client     *http.Client
	ServerAddr string
	HTTPS      bool
}

// SliceFiles tells the server to slice the specified paths.
func (c *Client) SliceFile(backend, preset string, path string) (*slicerjob.Job, error) {
	// check that a mesh file is given as the first argument and open it
	// so it may to encode in the form.
	if !IsMeshFile(path) {
		log.Fatalf("path is not a mesh file: %v", path)
	}
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	// write the multipart form out to a temporary file.  the temporary
	// file is closed and unlinked when the function terminates.
	tmp, err := ioutil.TempFile("", "matching-snuggies-post-")
	if err != nil {
		return nil, fmt.Errorf("tempfile: %v", err)
	}
	defer os.Remove(tmp.Name())
	defer tmp.Close()
	bodyw := multipart.NewWriter(tmp)
	err = c.writeJobForm(bodyw, backend, preset, path, f)
	if err != nil {
		return nil, fmt.Errorf("tempfile: %v", err)
	}

	// seek back to the beginning of the form and POST it to the slicer
	// server.  decode a slicerjob.Job from successful responses.
	var job *slicerjob.Job
	_, err = tmp.Seek(0, 0)
	if err != nil {
		return nil, fmt.Errorf("tempfile: %v", err)
	}
	url := c.url("/slicer/jobs/")
	log.Printf("POST %v", url)
	resp, err := c.client().Post(url, bodyw.FormDataContentType(), tmp)
	if err != nil {
		return nil, fmt.Errorf("POST /slicer/jobs/: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		p, _ := ioutil.ReadAll(resp.Body)
		return nil, fmt.Errorf("response: %q (%v)", p, http.StatusText(resp.StatusCode))
	}
	err = json.NewDecoder(resp.Body).Decode(&job)
	if err != nil {
		return nil, fmt.Errorf("response: %v", err)
	}
	return job, nil
}

func (c *Client) writeJobForm(w *multipart.Writer, backend, preset, filename string, r io.Reader) error {
	err := w.WriteField("slicer", backend)
	if err != nil {
		return err
	}
	err = w.WriteField("preset", preset)
	if err != nil {
		return err
	}
	file, err := w.CreateFormFile("meshfile", filepath.Base(filename))
	if err != nil {
		return err
	}
	_, err = io.Copy(file, r)
	if err != nil {
		return err
	}
	return w.Close()
}

// SlicerStatus returns a current copy of the provided job.
func (c *Client) SlicerStatus(job *slicerjob.Job) (*slicerjob.Job, error) {
	if job.ID == "" {
		return nil, fmt.Errorf("job missing id")
	}
	var jobcurr *slicerjob.Job
	url := c.url("/slicer/jobs/" + job.ID)
	log.Printf("POST %v", url)
	resp, err := c.client().Get(url)
	if err != nil {
		return nil, fmt.Errorf("POST /slicer/jobs/: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		p, _ := ioutil.ReadAll(resp.Body)
		return nil, fmt.Errorf("response: %q (%v)", p, http.StatusText(resp.StatusCode))
	}
	err = json.NewDecoder(resp.Body).Decode(&jobcurr)
	if err != nil {
		return nil, fmt.Errorf("response: %v", err)
	}
	return jobcurr, nil
}

// GCode requests the gcode for job.
func (c *Client) GCode(job *slicerjob.Job) (io.ReadCloser, error) {
	var jobcurr *slicerjob.Job
	url := c.url("/slicer/gcodes/" + jobcurr.ID)
	log.Printf("POST %v", url)
	resp, err := c.client().Get(url)
	if err != nil {
		return nil, fmt.Errorf("POST /slicer/jobs/: %v", err)
	}
	if resp.StatusCode != 200 {
		p, _ := ioutil.ReadAll(resp.Body)
		return nil, fmt.Errorf("response: %q (%v)", p, http.StatusText(resp.StatusCode))
	}
	return resp.Body, nil
}

func (c *Client) client() *http.Client {
	if c.Client == nil {
		return http.DefaultClient
	}
	return c.Client
}

func (c *Client) url(pathquery string) string {
	pathquery = strings.TrimPrefix(pathquery, "/")
	scheme := "http"
	if c.HTTPS {
		scheme = "https"
	}
	return scheme + "://" + c.ServerAddr + "/" + pathquery
}

var meshExts = map[string]bool{
	".stl": true,
	".amf": true,
}

func IsMeshFile(path string) bool {
	return meshExts[filepath.Ext(path)]
}
<commit_msg>fixes in client to work with server<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/gophergala/matching-snuggies/slicerjob"
)

func main() {
	server := flag.String("server", "localhost:8888", "snuggied server address")
	slicerBackend := flag.String("backend", "slic3r", "backend slicer")
	slicerPreset := flag.String("preset", "hiQ", "specify a configuration preset for the backend")
	gcodeDest := flag.String("o", "", "specify an output gcode filename")
	flag.Parse()

	if flag.NArg() < 1 {
		log.Fatalf("missing argument: mesh file")
	}
	meshpath := flag.Arg(0)

	client := &Client{
		ServerAddr: *server,
	}

	// send files to the slicer to be printed and poll the slicer until the job
	// has completed.
	log.Printf("sending file(s) to snuggied server at %v", *server)
	job, err := client.SliceFile(*slicerBackend, *slicerPreset, meshpath)
	if err != nil {
		log.Fatalf("sending files: %v", err)
	}
	tick := time.After(100 * time.Millisecond)
	for job.Status != "complete" {
		// TODO: retry with exponential backoff on network failure
		select {
		case <-tick:
			job, err = client.SlicerStatus(job)
			if err != nil {
				log.Fatalf("waiting: %v", err)
			}
		}
	}

	// download gcode from the slicer and write to the specified file.
	log.Printf("retreiving gcode file")
	r, err := client.GCode(job)
	if err != nil {
		log.Fatalf("gcode: %v", err)
	}
	defer r.Close()
	var f *os.File
	if *gcodeDest == "" {
		f = os.Stdout
	} else {
		f, err = os.Create(*gcodeDest)
		if err != nil {
			log.Panic(err)
		}
	}
	defer func() {
		err := f.Close()
		if err != nil {
			log.Panic(err)
		}
	}()
	_, err = io.Copy(f, r)
	if err != nil {
		log.Panic(err)
	}
}

type Client struct {
	Client     *http.Client
	ServerAddr string
	HTTPS      bool
}

// SliceFiles tells the server to slice the specified paths.
func (c *Client) SliceFile(backend, preset string, path string) (*slicerjob.Job, error) {
	// check that a mesh file is given as the first argument and open it
	// so it may to encode in the form.
	if !IsMeshFile(path) {
		log.Fatalf("path is not a mesh file: %v", path)
	}
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	// write the multipart form out to a temporary file.  the temporary
	// file is closed and unlinked when the function terminates.
	tmp, err := ioutil.TempFile("", "matching-snuggies-post-")
	if err != nil {
		return nil, fmt.Errorf("tempfile: %v", err)
	}
	defer os.Remove(tmp.Name())
	defer tmp.Close()
	bodyw := multipart.NewWriter(tmp)
	err = c.writeJobForm(bodyw, backend, preset, path, f)
	if err != nil {
		return nil, fmt.Errorf("tempfile: %v", err)
	}

	// seek back to the beginning of the form and POST it to the slicer
	// server.  decode a slicerjob.Job from successful responses.
	var job *slicerjob.Job
	_, err = tmp.Seek(0, 0)
	if err != nil {
		return nil, fmt.Errorf("tempfile: %v", err)
	}
	url := c.url("/slicer/jobs")
	log.Printf("POST %v", url)
	resp, err := c.client().Post(url, bodyw.FormDataContentType(), tmp)
	if err != nil {
		return nil, fmt.Errorf("POST /slicer/jobs: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		p, _ := ioutil.ReadAll(resp.Body)
		return nil, fmt.Errorf("response: %q (%v)", p, http.StatusText(resp.StatusCode))
	}
	err = json.NewDecoder(resp.Body).Decode(&job)
	if err != nil {
		return nil, fmt.Errorf("response: %v", err)
	}
	return job, nil
}

func (c *Client) writeJobForm(w *multipart.Writer, backend, preset, filename string, r io.Reader) error {
	err := w.WriteField("slicer", backend)
	if err != nil {
		return err
	}
	err = w.WriteField("preset", preset)
	if err != nil {
		return err
	}
	file, err := w.CreateFormFile("meshfile", filepath.Base(filename))
	if err != nil {
		return err
	}
	_, err = io.Copy(file, r)
	if err != nil {
		return err
	}
	return w.Close()
}

// SlicerStatus returns a current copy of the provided job.
func (c *Client) SlicerStatus(job *slicerjob.Job) (*slicerjob.Job, error) {
	if job.ID == "" {
		return nil, fmt.Errorf("job missing id")
	}
	var jobcurr *slicerjob.Job
	url := c.url("/slicer/jobs/" + job.ID)
	log.Printf("POST %v", url)
	resp, err := c.client().Get(url)
	if err != nil {
		return nil, fmt.Errorf("POST /slicer/jobs/: %v", err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != 200 {
		p, _ := ioutil.ReadAll(resp.Body)
		return nil, fmt.Errorf("response: %q (%v)", p, http.StatusText(resp.StatusCode))
	}
	err = json.NewDecoder(resp.Body).Decode(&jobcurr)
	if err != nil {
		return nil, fmt.Errorf("response: %v", err)
	}
	return jobcurr, nil
}

// GCode requests the gcode for job.
func (c *Client) GCode(job *slicerjob.Job) (io.ReadCloser, error) {
	url := c.url("/slicer/gcodes/" + job.ID)
	log.Printf("POST %v", url)
	resp, err := c.client().Get(url)
	if err != nil {
		return nil, fmt.Errorf("POST /slicer/jobs/: %v", err)
	}
	if resp.StatusCode != 200 {
		p, _ := ioutil.ReadAll(resp.Body)
		return nil, fmt.Errorf("response: %q (%v)", p, http.StatusText(resp.StatusCode))
	}
	return resp.Body, nil
}

func (c *Client) client() *http.Client {
	if c.Client == nil {
		return http.DefaultClient
	}
	return c.Client
}

func (c *Client) url(pathquery string) string {
	pathquery = strings.TrimPrefix(pathquery, "/")
	scheme := "http"
	if c.HTTPS {
		scheme = "https"
	}
	return scheme + "://" + c.ServerAddr + "/" + pathquery
}

var meshExts = map[string]bool{
	".stl": true,
	".amf": true,
}

func IsMeshFile(path string) bool {
	return meshExts[filepath.Ext(path)]
}
<|endoftext|>


<commit_before>package secret

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/cmd/swarmctl/common"
	"github.com/spf13/cobra"
)

var createCmd = &cobra.Command{
	Use:   "create <secret name>",
	Short: "Create a secret",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) != 1 {
			return errors.New("create command takes a unique secret name as an argument, and accepts secret data via stdin")
		}

		secretData, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			return fmt.Errorf("Error reading content from STDIN: %v", err)
		}

		client, err := common.Dial(cmd)
		if err != nil {
			return err
		}

		spec := &api.SecretSpec{
			Annotations: api.Annotations{Name: args[0]},
			Data:        secretData,
		}

		resp, err := client.CreateSecret(common.Context(cmd), &api.CreateSecretRequest{Spec: spec})
		if err != nil {
			return err
		}
		fmt.Println(resp.Secret.ID)
		return nil
	},
}
<commit_msg>Add a -f option to let a user pass a filename instead of requiring STDIN.<commit_after>package secret

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/docker/swarmkit/api"
	"github.com/docker/swarmkit/cmd/swarmctl/common"
	"github.com/spf13/cobra"
)

var createCmd = &cobra.Command{
	Use:   "create <secret name>",
	Short: "Create a secret",
	RunE: func(cmd *cobra.Command, args []string) error {
		if len(args) != 1 {
			return errors.New(
				"create command takes a unique secret name as an argument, and accepts secret data via stdin or via a file")
		}

		flags := cmd.Flags()
		var (
			secretData []byte
			err        error
		)

		if flags.Changed("file") {
			filename, err := flags.GetString("file")
			if err != nil {
				return err
			}
			secretData, err = ioutil.ReadFile(filename)
			if err != nil {
				return fmt.Errorf("Error reading from file '%s': %s", filename, err.Error())
			}
		} else {
			secretData, err = ioutil.ReadAll(os.Stdin)
			if err != nil {
				return fmt.Errorf("Error reading content from STDIN: %s", err.Error())
			}
		}

		client, err := common.Dial(cmd)
		if err != nil {
			return err
		}

		spec := &api.SecretSpec{
			Annotations: api.Annotations{Name: args[0]},
			Data:        secretData,
		}

		resp, err := client.CreateSecret(common.Context(cmd), &api.CreateSecretRequest{Spec: spec})
		if err != nil {
			return err
		}
		fmt.Println(resp.Secret.ID)
		return nil
	},
}

func init() {
	createCmd.Flags().StringP("file", "f", "", "Rather than read the secret from STDIN, read from the given file")
}
<|endoftext|>


<commit_before><commit_msg>update component test case<commit_after><|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"sort"
	"strings"
	"time"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"github.com/awslabs/aws-sdk-go/aws"
	"github.com/awslabs/aws-sdk-go/service/route53"
)

func resourceAwsRoute53Zone() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsRoute53ZoneCreate,
		Read:   resourceAwsRoute53ZoneRead,
		Update: resourceAwsRoute53ZoneUpdate,
		Delete: resourceAwsRoute53ZoneDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"vpc_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},

			"vpc_region": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Computed: true,
			},

			"zone_id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"name_servers": &schema.Schema{
				Type:     schema.TypeList,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Computed: true,
			},

			"tags": tagsSchema(),
		},
	}
}

func resourceAwsRoute53ZoneCreate(d *schema.ResourceData, meta interface{}) error {
	r53 := meta.(*AWSClient).r53conn

	comment := &route53.HostedZoneConfig{Comment: aws.String("Managed by Terraform")}
	req := &route53.CreateHostedZoneInput{
		Name:             aws.String(d.Get("name").(string)),
		HostedZoneConfig: comment,
		CallerReference:  aws.String(time.Now().Format(time.RFC3339Nano)),
	}
	if v := d.Get("vpc_id"); v != "" {
		req.VPC = &route53.VPC{
			VPCID:     aws.String(v.(string)),
			VPCRegion: aws.String(meta.(*AWSClient).region),
		}
		if w := d.Get("vpc_region"); w != "" {
			req.VPC.VPCRegion = aws.String(w.(string))
		}
	}

	log.Printf("[DEBUG] Creating Route53 hosted zone: %s", *req.Name)
	resp, err := r53.CreateHostedZone(req)
	if err != nil {
		return err
	}

	// Store the zone_id
	zone := cleanZoneID(*resp.HostedZone.ID)
	d.Set("zone_id", zone)
	d.SetId(zone)
	d.Set("vpc_region", req.VPC.VPCRegion)

	// Wait until we are done initializing
	wait := resource.StateChangeConf{
		Delay:      30 * time.Second,
		Pending:    []string{"PENDING"},
		Target:     "INSYNC",
		Timeout:    10 * time.Minute,
		MinTimeout: 2 * time.Second,
		Refresh: func() (result interface{}, state string, err error) {
			changeRequest := &route53.GetChangeInput{
				ID: aws.String(cleanChangeID(*resp.ChangeInfo.ID)),
			}
			return resourceAwsGoRoute53Wait(r53, changeRequest)
		},
	}
	_, err = wait.WaitForState()
	if err != nil {
		return err
	}
	return resourceAwsRoute53ZoneUpdate(d, meta)
}

func resourceAwsRoute53ZoneRead(d *schema.ResourceData, meta interface{}) error {
	r53 := meta.(*AWSClient).r53conn
	zone, err := r53.GetHostedZone(&route53.GetHostedZoneInput{ID: aws.String(d.Id())})
	if err != nil {
		// Handle a deleted zone
		if r53err, ok := err.(aws.APIError); ok && r53err.Code == "NoSuchHostedZone" {
			d.SetId("")
			return nil
		}
		return err
	}

	if !*zone.HostedZone.Config.PrivateZone {
		ns := make([]string, len(zone.DelegationSet.NameServers))
		for i := range zone.DelegationSet.NameServers {
			ns[i] = *zone.DelegationSet.NameServers[i]
		}
		sort.Strings(ns)
		if err := d.Set("name_servers", ns); err != nil {
			return fmt.Errorf("[DEBUG] Error setting name servers for: %s, error: %#v", d.Id(), err)
		}
	} else {
		d.Set("name_servers", nil)
		var associatedVPC *route53.VPC
		for _, vpc := range zone.VPCs {
			if *vpc.VPCID == d.Get("vpc_id") {
				associatedVPC = vpc
			}
		}
		if associatedVPC == nil {
			return fmt.Errorf("[DEBUG] VPC: %v is not associated with Zone: %v", d.Get("vpc_id"), d.Id())
		}
	}

	// get tags
	req := &route53.ListTagsForResourceInput{
		ResourceID:   aws.String(d.Id()),
		ResourceType: aws.String("hostedzone"),
	}

	resp, err := r53.ListTagsForResource(req)
	if err != nil {
		return err
	}

	var tags []*route53.Tag
	if resp.ResourceTagSet != nil {
		tags = resp.ResourceTagSet.Tags
	}

	if err := d.Set("tags", tagsToMapR53(tags)); err != nil {
		return err
	}

	return nil
}

func resourceAwsRoute53ZoneUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).r53conn

	if err := setTagsR53(conn, d); err != nil {
		return err
	} else {
		d.SetPartial("tags")
	}

	return resourceAwsRoute53ZoneRead(d, meta)
}

func resourceAwsRoute53ZoneDelete(d *schema.ResourceData, meta interface{}) error {
	r53 := meta.(*AWSClient).r53conn

	log.Printf("[DEBUG] Deleting Route53 hosted zone: %s (ID: %s)",
		d.Get("name").(string), d.Id())
	_, err := r53.DeleteHostedZone(&route53.DeleteHostedZoneInput{ID: aws.String(d.Id())})
	if err != nil {
		return err
	}

	return nil
}

func resourceAwsGoRoute53Wait(r53 *route53.Route53, ref *route53.GetChangeInput) (result interface{}, state string, err error) {

	status, err := r53.GetChange(ref)
	if err != nil {
		return nil, "UNKNOWN", err
	}
	return true, *status.ChangeInfo.Status, nil
}

// cleanChangeID is used to remove the leading /change/
func cleanChangeID(ID string) string {
	return cleanPrefix(ID, "/change/")
}

// cleanZoneID is used to remove the leading /hostedzone/
func cleanZoneID(ID string) string {
	return cleanPrefix(ID, "/hostedzone/")
}

// cleanPrefix removes a string prefix from an ID
func cleanPrefix(ID, prefix string) string {
	if strings.HasPrefix(ID, prefix) {
		ID = strings.TrimPrefix(ID, prefix)
	}
	return ID
}
<commit_msg>Only d.Set("vpc_region") when creating a private zone.<commit_after>package aws

import (
	"fmt"
	"log"
	"sort"
	"strings"
	"time"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"

	"github.com/awslabs/aws-sdk-go/aws"
	"github.com/awslabs/aws-sdk-go/service/route53"
)

func resourceAwsRoute53Zone() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsRoute53ZoneCreate,
		Read:   resourceAwsRoute53ZoneRead,
		Update: resourceAwsRoute53ZoneUpdate,
		Delete: resourceAwsRoute53ZoneDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},

			"vpc_id": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
			},

			"vpc_region": &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				Computed: true,
			},

			"zone_id": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},

			"name_servers": &schema.Schema{
				Type:     schema.TypeList,
				Elem:     &schema.Schema{Type: schema.TypeString},
				Computed: true,
			},

			"tags": tagsSchema(),
		},
	}
}

func resourceAwsRoute53ZoneCreate(d *schema.ResourceData, meta interface{}) error {
	r53 := meta.(*AWSClient).r53conn

	comment := &route53.HostedZoneConfig{Comment: aws.String("Managed by Terraform")}
	req := &route53.CreateHostedZoneInput{
		Name:             aws.String(d.Get("name").(string)),
		HostedZoneConfig: comment,
		CallerReference:  aws.String(time.Now().Format(time.RFC3339Nano)),
	}
	if v := d.Get("vpc_id"); v != "" {
		req.VPC = &route53.VPC{
			VPCID:     aws.String(v.(string)),
			VPCRegion: aws.String(meta.(*AWSClient).region),
		}
		if w := d.Get("vpc_region"); w != "" {
			req.VPC.VPCRegion = aws.String(w.(string))
		}
		d.Set("vpc_region", req.VPC.VPCRegion)
	}

	log.Printf("[DEBUG] Creating Route53 hosted zone: %s", *req.Name)
	resp, err := r53.CreateHostedZone(req)
	if err != nil {
		return err
	}

	// Store the zone_id
	zone := cleanZoneID(*resp.HostedZone.ID)
	d.Set("zone_id", zone)
	d.SetId(zone)

	// Wait until we are done initializing
	wait := resource.StateChangeConf{
		Delay:      30 * time.Second,
		Pending:    []string{"PENDING"},
		Target:     "INSYNC",
		Timeout:    10 * time.Minute,
		MinTimeout: 2 * time.Second,
		Refresh: func() (result interface{}, state string, err error) {
			changeRequest := &route53.GetChangeInput{
				ID: aws.String(cleanChangeID(*resp.ChangeInfo.ID)),
			}
			return resourceAwsGoRoute53Wait(r53, changeRequest)
		},
	}
	_, err = wait.WaitForState()
	if err != nil {
		return err
	}
	return resourceAwsRoute53ZoneUpdate(d, meta)
}

func resourceAwsRoute53ZoneRead(d *schema.ResourceData, meta interface{}) error {
	r53 := meta.(*AWSClient).r53conn
	zone, err := r53.GetHostedZone(&route53.GetHostedZoneInput{ID: aws.String(d.Id())})
	if err != nil {
		// Handle a deleted zone
		if r53err, ok := err.(aws.APIError); ok && r53err.Code == "NoSuchHostedZone" {
			d.SetId("")
			return nil
		}
		return err
	}

	if !*zone.HostedZone.Config.PrivateZone {
		ns := make([]string, len(zone.DelegationSet.NameServers))
		for i := range zone.DelegationSet.NameServers {
			ns[i] = *zone.DelegationSet.NameServers[i]
		}
		sort.Strings(ns)
		if err := d.Set("name_servers", ns); err != nil {
			return fmt.Errorf("[DEBUG] Error setting name servers for: %s, error: %#v", d.Id(), err)
		}
	} else {
		d.Set("name_servers", nil)
		var associatedVPC *route53.VPC
		for _, vpc := range zone.VPCs {
			if *vpc.VPCID == d.Get("vpc_id") {
				associatedVPC = vpc
			}
		}
		if associatedVPC == nil {
			return fmt.Errorf("[DEBUG] VPC: %v is not associated with Zone: %v", d.Get("vpc_id"), d.Id())
		}
	}

	// get tags
	req := &route53.ListTagsForResourceInput{
		ResourceID:   aws.String(d.Id()),
		ResourceType: aws.String("hostedzone"),
	}

	resp, err := r53.ListTagsForResource(req)
	if err != nil {
		return err
	}

	var tags []*route53.Tag
	if resp.ResourceTagSet != nil {
		tags = resp.ResourceTagSet.Tags
	}

	if err := d.Set("tags", tagsToMapR53(tags)); err != nil {
		return err
	}

	return nil
}

func resourceAwsRoute53ZoneUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).r53conn

	if err := setTagsR53(conn, d); err != nil {
		return err
	} else {
		d.SetPartial("tags")
	}

	return resourceAwsRoute53ZoneRead(d, meta)
}

func resourceAwsRoute53ZoneDelete(d *schema.ResourceData, meta interface{}) error {
	r53 := meta.(*AWSClient).r53conn

	log.Printf("[DEBUG] Deleting Route53 hosted zone: %s (ID: %s)",
		d.Get("name").(string), d.Id())
	_, err := r53.DeleteHostedZone(&route53.DeleteHostedZoneInput{ID: aws.String(d.Id())})
	if err != nil {
		return err
	}

	return nil
}

func resourceAwsGoRoute53Wait(r53 *route53.Route53, ref *route53.GetChangeInput) (result interface{}, state string, err error) {

	status, err := r53.GetChange(ref)
	if err != nil {
		return nil, "UNKNOWN", err
	}
	return true, *status.ChangeInfo.Status, nil
}

// cleanChangeID is used to remove the leading /change/
func cleanChangeID(ID string) string {
	return cleanPrefix(ID, "/change/")
}

// cleanZoneID is used to remove the leading /hostedzone/
func cleanZoneID(ID string) string {
	return cleanPrefix(ID, "/hostedzone/")
}

// cleanPrefix removes a string prefix from an ID
func cleanPrefix(ID, prefix string) string {
	if strings.HasPrefix(ID, prefix) {
		ID = strings.TrimPrefix(ID, prefix)
	}
	return ID
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net"
	"strconv"
	"strings"
)

var hostname = flag.String("t", "google.com", "target domain")

func main() {
	flag.Parse()

	if isDomainName(*hostname) {
		addrs, err := net.LookupHost(*hostname)
		if err != nil {
			fmt.Println(err)
		}
		for _, a := range addrs {
			if isPrivate(a) {
				fmt.Print("[+] ")
			}
			fmt.Println(a)
		}

	} else {
		fmt.Println("Invalid domain name: " + *hostname)
	}

}

func isPrivate(ip string) bool {
	ip4 := net.ParseIP(ip)
	if ip4 != nil {
		splits := strings.SplitN(ip, ".", 4)
		if (splits[0] == "10") || (splits[0] == "192" && splits[1] == "168") {
			return true
		}
		b, err := strconv.Atoi(splits[1])
		if err != nil {
			return false
		}
		if splits[0] == "172" && (b >= 16 || b <= 31) {
			return true
		}
	}
	return false
}

func isDomainName(s string) bool {
	// See RFC 1035, RFC 3696.
	if len(s) == 0 {
		return false
	}
	if len(s) > 255 {
		return false
	}

	last := byte('.')
	ok := false // Ok once we've seen a letter.
	partlen := 0
	for i := 0; i < len(s); i++ {
		c := s[i]
		switch {
		default:
			return false
		case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '_':
			ok = true
			partlen++
		case '0' <= c && c <= '9':
			// fine
			partlen++
		case c == '-':
			// Byte before dash cannot be dot.
			if last == '.' {
				return false
			}
			partlen++
		case c == '.':
			// Byte before dot cannot be dot, dash.
			if last == '.' || last == '-' {
				return false
			}
			if partlen > 63 || partlen == 0 {
				return false
			}
			partlen = 0
		}
		last = c
	}
	if last == '-' || partlen > 63 {
		return false
	}

	return ok
}
<commit_msg>Fix bug for IPv6 addresses<commit_after>package main

import (
	"flag"
	"fmt"
	"net"
	"strconv"
	"strings"
)

var hostname = flag.String("t", "google.com", "target domain")

func main() {
	flag.Parse()

	if isDomainName(*hostname) {
		addrs, err := net.LookupHost(*hostname)
		if err != nil {
			fmt.Println(err)
		}
		for _, a := range addrs {
			if isPrivate(a) {
				fmt.Print("[+] ")
			}
			fmt.Println(a)
		}

	} else {
		fmt.Println("Invalid domain name: " + *hostname)
	}

}

func isPrivate(ip string) bool {
	ip4 := net.ParseIP(ip).To4()
	if ip4 != nil {
		splits := strings.SplitN(ip, ".", 4)
		if (splits[0] == "10") || (splits[0] == "192" && splits[1] == "168") {
			return true
		}
		b, err := strconv.Atoi(splits[1])
		if err != nil {
			return false
		}
		if splits[0] == "172" && (b >= 16 || b <= 31) {
			return true
		}
	}
	return false
}

// This function was copied straight out of dnsclient.go
// in http://golang.org/src/net/dnsclient.go
func isDomainName(s string) bool {
	// See RFC 1035, RFC 3696.
	if len(s) == 0 {
		return false
	}
	if len(s) > 255 {
		return false
	}

	last := byte('.')
	ok := false // Ok once we've seen a letter.
	partlen := 0
	for i := 0; i < len(s); i++ {
		c := s[i]
		switch {
		default:
			return false
		case 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '_':
			ok = true
			partlen++
		case '0' <= c && c <= '9':
			// fine
			partlen++
		case c == '-':
			// Byte before dash cannot be dot.
			if last == '.' {
				return false
			}
			partlen++
		case c == '.':
			// Byte before dot cannot be dot, dash.
			if last == '.' || last == '-' {
				return false
			}
			if partlen > 63 || partlen == 0 {
				return false
			}
			partlen = 0
		}
		last = c
	}
	if last == '-' || partlen > 63 {
		return false
	}

	return ok
}
<|endoftext|>


<commit_before><commit_msg>More fix to watchers<commit_after><|endoftext|>


<commit_before>// +build linux freebsd

package configs

var (
	// These are devices that are to be both allowed and created.
	DefaultSimpleDevices = []*Device{
		// /dev/null and zero
		{
			Path:        "/dev/null",
			Type:        'c',
			Major:       1,
			Minor:       3,
			Permissions: "rwm",
			FileMode:    0666,
		},
		{
			Path:        "/dev/zero",
			Type:        'c',
			Major:       1,
			Minor:       5,
			Permissions: "rwm",
			FileMode:    0666,
		},

		{
			Path:        "/dev/full",
			Type:        'c',
			Major:       1,
			Minor:       7,
			Permissions: "rwm",
			FileMode:    0666,
		},

		// consoles and ttys
		{
			Path:        "/dev/tty",
			Type:        'c',
			Major:       5,
			Minor:       0,
			Permissions: "rwm",
			FileMode:    0666,
		},

		// /dev/urandom,/dev/random
		{
			Path:        "/dev/urandom",
			Type:        'c',
			Major:       1,
			Minor:       9,
			Permissions: "rwm",
			FileMode:    0666,
		},
		{
			Path:        "/dev/random",
			Type:        'c',
			Major:       1,
			Minor:       8,
			Permissions: "rwm",
			FileMode:    0666,
		},
	}
	DefaultAllowedDevices = append([]*Device{
		// allow mknod for any device
		{
			Type:        'c',
			Major:       Wildcard,
			Minor:       Wildcard,
			Permissions: "m",
		},
		{
			Type:        'b',
			Major:       Wildcard,
			Minor:       Wildcard,
			Permissions: "m",
		},

		{
			Path:        "/dev/console",
			Type:        'c',
			Major:       5,
			Minor:       1,
			Permissions: "rwm",
		},
		{
			Path:        "/dev/tty0",
			Type:        'c',
			Major:       4,
			Minor:       0,
			Permissions: "rwm",
		},
		{
			Path:        "/dev/tty1",
			Type:        'c',
			Major:       4,
			Minor:       1,
			Permissions: "rwm",
		},
		// /dev/pts/ - pts namespaces are "coming soon"
		{
			Path:        "",
			Type:        'c',
			Major:       136,
			Minor:       Wildcard,
			Permissions: "rwm",
		},
		{
			Path:        "",
			Type:        'c',
			Major:       5,
			Minor:       2,
			Permissions: "rwm",
		},

		// tuntap
		{
			Path:        "",
			Type:        'c',
			Major:       10,
			Minor:       200,
			Permissions: "rwm",
		},
	}, DefaultSimpleDevices...)
	DefaultAutoCreatedDevices = append([]*Device{
		{
			// /dev/fuse is created but not allowed.
			// This is to allow java to work.  Because java
			// Insists on there being a /dev/fuse
			// https://github.com/docker/docker/issues/514
			// https://github.com/docker/docker/issues/2393
			//
			Path:        "/dev/fuse",
			Type:        'c',
			Major:       10,
			Minor:       229,
			Permissions: "rwm",
		},
	}, DefaultSimpleDevices...)
)
<commit_msg>Do not allow access to /dev/tty{0,1}<commit_after>// +build linux freebsd

package configs

var (
	// These are devices that are to be both allowed and created.
	DefaultSimpleDevices = []*Device{
		// /dev/null and zero
		{
			Path:        "/dev/null",
			Type:        'c',
			Major:       1,
			Minor:       3,
			Permissions: "rwm",
			FileMode:    0666,
		},
		{
			Path:        "/dev/zero",
			Type:        'c',
			Major:       1,
			Minor:       5,
			Permissions: "rwm",
			FileMode:    0666,
		},

		{
			Path:        "/dev/full",
			Type:        'c',
			Major:       1,
			Minor:       7,
			Permissions: "rwm",
			FileMode:    0666,
		},

		// consoles and ttys
		{
			Path:        "/dev/tty",
			Type:        'c',
			Major:       5,
			Minor:       0,
			Permissions: "rwm",
			FileMode:    0666,
		},

		// /dev/urandom,/dev/random
		{
			Path:        "/dev/urandom",
			Type:        'c',
			Major:       1,
			Minor:       9,
			Permissions: "rwm",
			FileMode:    0666,
		},
		{
			Path:        "/dev/random",
			Type:        'c',
			Major:       1,
			Minor:       8,
			Permissions: "rwm",
			FileMode:    0666,
		},
	}
	DefaultAllowedDevices = append([]*Device{
		// allow mknod for any device
		{
			Type:        'c',
			Major:       Wildcard,
			Minor:       Wildcard,
			Permissions: "m",
		},
		{
			Type:        'b',
			Major:       Wildcard,
			Minor:       Wildcard,
			Permissions: "m",
		},

		{
			Path:        "/dev/console",
			Type:        'c',
			Major:       5,
			Minor:       1,
			Permissions: "rwm",
		},
		// /dev/pts/ - pts namespaces are "coming soon"
		{
			Path:        "",
			Type:        'c',
			Major:       136,
			Minor:       Wildcard,
			Permissions: "rwm",
		},
		{
			Path:        "",
			Type:        'c',
			Major:       5,
			Minor:       2,
			Permissions: "rwm",
		},

		// tuntap
		{
			Path:        "",
			Type:        'c',
			Major:       10,
			Minor:       200,
			Permissions: "rwm",
		},
	}, DefaultSimpleDevices...)
	DefaultAutoCreatedDevices = append([]*Device{
		{
			// /dev/fuse is created but not allowed.
			// This is to allow java to work.  Because java
			// Insists on there being a /dev/fuse
			// https://github.com/docker/docker/issues/514
			// https://github.com/docker/docker/issues/2393
			//
			Path:        "/dev/fuse",
			Type:        'c',
			Major:       10,
			Minor:       229,
			Permissions: "rwm",
		},
	}, DefaultSimpleDevices...)
)
<|endoftext|>