<commit_before>package build

import (
	"fmt"
	"gopkg.in/yaml.v2"
	"neon/util"
	"os"
	"path/filepath"
)

const (
	// location of the repository root
	REPO_ROOT = "~/.neon"
)

// Build structure
type Build struct {
	File        string
	Dir         string
	Here        string
	Name        string
	Default     []string
	Doc         string
	Scripts     []string
	Properties  util.Object
	Environment map[string]string
	Targets     map[string]*Target
	Context     *Context
	Parents     []*Build
	Index       *Index
	Stack       *Stack
}

// Possible fields for a build file
var FIELDS = []string{"name", "doc", "default", "context", "extends",
	"singleton", "properties", "configuration", "environment", "targets"}

// Make a build from a build file
func NewBuild(file string) (*Build, error) {
	build := &Build{}
	path := util.ExpandUserHome(file)
	build.File = filepath.Base(path)
	base, err := filepath.Abs(filepath.Dir(path))
	if err != nil {
		return nil, fmt.Errorf("getting build file directory: %v", err)
	}
	build.Dir = base
	here, err := os.Getwd()
	if err != nil {
		return nil, fmt.Errorf("getting current directory: %v", err)
	}
	build.Here = here
	source, err := util.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("loading build file '%s': %v", path, err)
	}
	var object util.Object
	err = yaml.Unmarshal(source, &object)
	if err != nil {
		return nil, fmt.Errorf("build must be a map with string keys")
	}
	if err := object.CheckFields(FIELDS); err != nil {
		return nil, fmt.Errorf("parsing build file: %v", err)
	}
	if err := ParseSingleton(object); err != nil {
		return nil, err
	}
	if err := ParseName(object, build); err != nil {
		return nil, err
	}
	if err := ParseDefault(object, build); err != nil {
		return nil, err
	}
	if err := ParseDoc(object, build); err != nil {
		return nil, err
	}
	if err := ParseContext(object, build); err != nil {
		return nil, err
	}
	if err := ParseExtends(object, build); err != nil {
		return nil, err
	}
	if err := ParseProperties(object, build); err != nil {
		return nil, err
	}
	if err := ParseConfiguration(object, build); err != nil {
		return nil, err
	}
	if err := ParseEnvironment(object, build); err != nil {
		return nil, err
	}
	if err := ParseTargets(object, build); err != nil {
		return nil, err
	}
	return build, nil
}

// Parse singleton field of the build
func ParseSingleton(object util.Object) error {
	if object.HasField("singleton") {
		port, err := object.GetInteger("singleton")
		if err != nil {
			return fmt.Errorf("getting singleton port: %v", err)
		}
		if err := util.Singleton(port); err != nil {
			return fmt.Errorf("another instance of the build is already running")
		}
	}
	return nil
}

// Parse name field of the build
func ParseName(object util.Object, build *Build) error {
	if object.HasField("name") {
		name, err := object.GetString("name")
		if err != nil {
			return fmt.Errorf("getting build name: %v", err)
		}
		build.Name = name
	}
	return nil
}

// Parse default field of the build
func ParseDefault(object util.Object, build *Build) error {
	if object.HasField("default") {
		list, err := object.GetListStringsOrString("default")
		if err != nil {
			return fmt.Errorf("getting default targets: %v", err)
		}
		build.Default = list
	}
	return nil
}

// Parse doc field of the build
func ParseDoc(object util.Object, build *Build) error {
	if object.HasField("doc") {
		doc, err := object.GetString("doc")
		if err != nil {
			return fmt.Errorf("getting build doc: %v", err)
		}
		build.Doc = doc
	}
	return nil
}

// Parse context field of the build
func ParseContext(object util.Object, build *Build) error {
	if object.HasField("context") {
		scripts, err := object.GetListStringsOrString("context")
		if err != nil {
			return fmt.Errorf("getting context: %v", err)
		}
		build.Scripts = scripts
	}
	return nil
}

// Parse extends field of the build
func ParseExtends(object util.Object, build *Build) error {
	if object.HasField("extends") {
		parents, err := object.GetListStringsOrString("extends")
		if err != nil {
			return fmt.Errorf("parsing parents: %v", err)
		}
		var extends []*Build
		for _, parent := range parents {
			file := filepath.Join(REPO_ROOT, parent)
			extend, err := NewBuild(file)
			if err != nil {
				return fmt.Errorf("loading parent '%s': %v", parent, err)
			}
			extends = append(extends, extend)
		}
		build.Parents = extends
	}
	return nil
}

// Parse build properties
func ParseProperties(object util.Object, build *Build) error {
	properties := make(map[string]interface{})
	var err error
	if object.HasField("properties") {
		properties, err = object.GetObject("properties")
		if err != nil {
			return fmt.Errorf("parsing properties: %v", err)
		}
	}
	build.Properties = properties
	return nil
}

// Parse build configuration
func ParseConfiguration(object util.Object, build *Build) error {
	if object.HasField("configuration") {
		var config util.Object
		files, err := object.GetListStringsOrString("configuration")
		if err != nil {
			return fmt.Errorf("getting configuration file: %v", err)
		}
		for _, file := range files {
			file = util.ExpandAndJoinToRoot(build.Dir, file)
			source, err := util.ReadFile(file)
			if err != nil {
				return fmt.Errorf("reading configuration file: %v", err)
			}
			err = yaml.Unmarshal(source, &config)
			if err != nil {
				return fmt.Errorf("configuration must be a map with string keys")
			}
			for name, value := range config {
				build.Properties[name] = value
			}
		}
	}
	return nil
}

// Parse build environment
func ParseEnvironment(object util.Object, build *Build) error {
	environment := make(map[string]string)
	if object.HasField("environment") {
		env, err := object.GetObject("environment")
		if err != nil {
			return fmt.Errorf("parsing environmen: %v", err)
		}
		environment, err = env.ToMapStringString()
		if err != nil {
			return fmt.Errorf("getting environment: %v", err)
		}
	}
	build.Environment = environment
	return nil
}

// Parse build targets
func ParseTargets(object util.Object, build *Build) error {
	targets := util.Object(make(map[string]interface{}))
	var err error
	if object.HasField("targets") {
		targets, err = object.GetObject("targets")
		if err != nil {
			return fmt.Errorf("parsing targets: %v", err)
		}
	}
	build.Targets = make(map[string]*Target)
	for name := range targets {
		object, err := targets.GetObject(name)
		if err != nil {
			return fmt.Errorf("parsing target '%s': %v", name, err)
		}
		target, err := NewTarget(build, name, object)
		if err != nil {
			return fmt.Errorf("parsing target '%s': %v", name, err)
		}
		build.Targets[name] = target
	}
	return nil
}

// Return the build properties, including those inherited from parents
func (build *Build) GetProperties() util.Object {
	var properties = make(map[string]interface{})
	for _, parent := range build.Parents {
		for name, value := range parent.GetProperties() {
			properties[name] = value
		}
	}
	for name, value := range build.Properties {
		properties[name] = value
	}
	return properties
}

// Return the build environment, including those inherited from parents
func (build *Build) GetEnvironment() map[string]string {
	var environment = make(map[string]string)
	for _, parent := range build.Parents {
		for name, value := range parent.GetEnvironment() {
			environment[name] = value
		}
	}
	for name, value := range build.Environment {
		environment[name] = value
	}
	return environment
}

// Return the build targets, including those inherited from parents
func (build *Build) GetTargets() map[string]*Target {
	var targets = make(map[string]*Target)
	for _, parent := range build.Parents {
		for name, target := range parent.GetTargets() {
			targets[name] = target
		}
	}
	for name, target := range build.Targets {
		targets[name] = target
	}
	return targets
}

// Return target with given name. If not defined in build, return target
// inherited from parent
func (build *Build) GetTargetByName(name string) *Target {
	target, found := build.Targets[name]
	if found {
		return target
	} else {
		for _, parent := range build.Parents {
			target = parent.GetTargetByName(name)
			if target != nil {
				return target
			}
		}
	}
	return nil
}

// Initialize build:
// - Set build dir and change to it
// - Create context and set it for build
func (build *Build) Init() error {
	os.Chdir(build.Dir)
	context, err := NewContext(build)
	if err != nil {
		return fmt.Errorf("evaluating context: %v", err)
	}
	build.SetDir(build.Dir)
	build.SetContext(context)
	build.SetStack(NewStack())
	return nil
}

// Set the build directory, propagating to parents
func (build *Build) SetDir(dir string) {
	build.Dir = dir
	for _, parent := range build.Parents {
		parent.SetDir(dir)
	}
}

// Set the build context, propagating to parents
func (build *Build) SetContext(context *Context) {
	build.Context = context
	for _, parent := range build.Parents {
		parent.SetContext(context)
	}
}

// Set the build stack, propagating to parents
func (build *Build) SetStack(stack *Stack) {
	build.Stack = stack
	for _, parent := range build.Parents {
		parent.SetStack(stack)
	}
}

// Set command line properties, that overwrite build ones
func (build *Build) SetCommandLineProperties(props string) error {
	var object util.Object
	err := yaml.Unmarshal([]byte(props), &object)
	if err != nil {
		return fmt.Errorf("parsing command line properties: properties must be a map with string keys")
	}
	for name, value := range object {
		build.Properties[name] = value
	}
	return nil
}

// Return default targets. If none is defined in build, return those from
// parents
func (build *Build) GetDefault() []string {
	if len(build.Default) > 0 {
		return build.Default
	} else {
		for _, parent := range build.Parents {
			if len(parent.Default) > 0 {
				return parent.Default
			}
		}
		for _, parent := range build.Parents {
			parentDefault := parent.GetDefault()
			if len(parentDefault) > 0 {
				return parentDefault
			}
		}
	}
	return build.Default
}

// Run build given targets. If no target is given, run default one.
func (build *Build) Run(targets []string) error {
	if len(targets) == 0 {
		targets = build.GetDefault()
		if len(targets) == 0 {
			return fmt.Errorf("no default target")
		}
	}
	for _, target := range targets {
		err := build.RunTarget(target)
		if err != nil {
			return err
		}
	}
	return nil
}

// Run given target
func (build *Build) RunTarget(name string) error {
	target := build.GetTargetByName(name)
	if target == nil {
		return fmt.Errorf("target '%s' not found", name)
	}
	err := target.Run()
	if err != nil {
		return fmt.Errorf("running target '%s': %v", name, err)
	}
	return nil
}

// Run parent target
func RunParentTarget(build *Build, name string) (bool, error) {
	for _, parent := range build.Parents {
		target := parent.GetTargetByName(name)
		if target != nil {
			err := target.RunSteps()
			if err != nil {
				return true, fmt.Errorf("running target '%s': %v", name, err)
			}
			return true, nil
		} else {
			ok, err := RunParentTarget(parent, name)
			if ok || err != nil {
				return ok, err
			}
		}
	}
	return false, nil
}
<commit_msg>Made extends starting with './' relative to the build file<commit_after>package build

import (
	"fmt"
	"gopkg.in/yaml.v2"
	"neon/util"
	"os"
	"path/filepath"
	"path"
	"strings"
)

const (
	// location of the repository root
	REPO_ROOT = "~/.neon"
)

// Build structure
type Build struct {
	File        string
	Dir         string
	Here        string
	Name        string
	Default     []string
	Doc         string
	Scripts     []string
	Properties  util.Object
	Environment map[string]string
	Targets     map[string]*Target
	Context     *Context
	Parents     []*Build
	Index       *Index
	Stack       *Stack
}

// Possible fields for a build file
var FIELDS = []string{"name", "doc", "default", "context", "extends",
	"singleton", "properties", "configuration", "environment", "targets"}

// Make a build from a build file
func NewBuild(file string) (*Build, error) {
	build := &Build{}
	path := util.ExpandUserHome(file)
	build.File = filepath.Base(path)
	base, err := filepath.Abs(filepath.Dir(path))
	if err != nil {
		return nil, fmt.Errorf("getting build file directory: %v", err)
	}
	build.Dir = base
	here, err := os.Getwd()
	if err != nil {
		return nil, fmt.Errorf("getting current directory: %v", err)
	}
	build.Here = here
	source, err := util.ReadFile(path)
	if err != nil {
		return nil, fmt.Errorf("loading build file '%s': %v", path, err)
	}
	var object util.Object
	err = yaml.Unmarshal(source, &object)
	if err != nil {
		return nil, fmt.Errorf("build must be a map with string keys")
	}
	if err := object.CheckFields(FIELDS); err != nil {
		return nil, fmt.Errorf("parsing build file: %v", err)
	}
	if err := ParseSingleton(object); err != nil {
		return nil, err
	}
	if err := ParseName(object, build); err != nil {
		return nil, err
	}
	if err := ParseDefault(object, build); err != nil {
		return nil, err
	}
	if err := ParseDoc(object, build); err != nil {
		return nil, err
	}
	if err := ParseContext(object, build); err != nil {
		return nil, err
	}
	if err := ParseExtends(object, build); err != nil {
		return nil, err
	}
	if err := ParseProperties(object, build); err != nil {
		return nil, err
	}
	if err := ParseConfiguration(object, build); err != nil {
		return nil, err
	}
	if err := ParseEnvironment(object, build); err != nil {
		return nil, err
	}
	if err := ParseTargets(object, build); err != nil {
		return nil, err
	}
	return build, nil
}

// Parse singleton field of the build
func ParseSingleton(object util.Object) error {
	if object.HasField("singleton") {
		port, err := object.GetInteger("singleton")
		if err != nil {
			return fmt.Errorf("getting singleton port: %v", err)
		}
		if err := util.Singleton(port); err != nil {
			return fmt.Errorf("another instance of the build is already running")
		}
	}
	return nil
}

// Parse name field of the build
func ParseName(object util.Object, build *Build) error {
	if object.HasField("name") {
		name, err := object.GetString("name")
		if err != nil {
			return fmt.Errorf("getting build name: %v", err)
		}
		build.Name = name
	}
	return nil
}

// Parse default field of the build
func ParseDefault(object util.Object, build *Build) error {
	if object.HasField("default") {
		list, err := object.GetListStringsOrString("default")
		if err != nil {
			return fmt.Errorf("getting default targets: %v", err)
		}
		build.Default = list
	}
	return nil
}

// Parse doc field of the build
func ParseDoc(object util.Object, build *Build) error {
	if object.HasField("doc") {
		doc, err := object.GetString("doc")
		if err != nil {
			return fmt.Errorf("getting build doc: %v", err)
		}
		build.Doc = doc
	}
	return nil
}

// Parse context field of the build
func ParseContext(object util.Object, build *Build) error {
	if object.HasField("context") {
		scripts, err := object.GetListStringsOrString("context")
		if err != nil {
			return fmt.Errorf("getting context: %v", err)
		}
		build.Scripts = scripts
	}
	return nil
}

// Parse extends field of the build
func ParseExtends(object util.Object, build *Build) error {
	if object.HasField("extends") {
		parents, err := object.GetListStringsOrString("extends")
		if err != nil {
			return fmt.Errorf("parsing parents: %v", err)
		}
		var extends []*Build
		for _, parent := range parents {
			file := ParentPath(parent, build)
			extend, err := NewBuild(file)
			if err != nil {
				return fmt.Errorf("loading parent '%s': %v", parent, err)
			}
			extends = append(extends, extend)
		}
		build.Parents = extends
	}
	return nil
}

// Get parent build file path
func ParentPath(parent string, build *Build) string {
	if path.IsAbs(parent) {
		return parent
	} else if strings.HasPrefix(parent, "./") {
		return filepath.Join(build.Dir, parent)
	} else {
		return filepath.Join(REPO_ROOT, parent)
	}
}

// Parse build properties
func ParseProperties(object util.Object, build *Build) error {
	properties := make(map[string]interface{})
	var err error
	if object.HasField("properties") {
		properties, err = object.GetObject("properties")
		if err != nil {
			return fmt.Errorf("parsing properties: %v", err)
		}
	}
	build.Properties = properties
	return nil
}

// Parse build configuration
func ParseConfiguration(object util.Object, build *Build) error {
	if object.HasField("configuration") {
		var config util.Object
		files, err := object.GetListStringsOrString("configuration")
		if err != nil {
			return fmt.Errorf("getting configuration file: %v", err)
		}
		for _, file := range files {
			file = util.ExpandAndJoinToRoot(build.Dir, file)
			source, err := util.ReadFile(file)
			if err != nil {
				return fmt.Errorf("reading configuration file: %v", err)
			}
			err = yaml.Unmarshal(source, &config)
			if err != nil {
				return fmt.Errorf("configuration must be a map with string keys")
			}
			for name, value := range config {
				build.Properties[name] = value
			}
		}
	}
	return nil
}

// Parse build environment
func ParseEnvironment(object util.Object, build *Build) error {
	environment := make(map[string]string)
	if object.HasField("environment") {
		env, err := object.GetObject("environment")
		if err != nil {
			return fmt.Errorf("parsing environmen: %v", err)
		}
		environment, err = env.ToMapStringString()
		if err != nil {
			return fmt.Errorf("getting environment: %v", err)
		}
	}
	build.Environment = environment
	return nil
}

// Parse build targets
func ParseTargets(object util.Object, build *Build) error {
	targets := util.Object(make(map[string]interface{}))
	var err error
	if object.HasField("targets") {
		targets, err = object.GetObject("targets")
		if err != nil {
			return fmt.Errorf("parsing targets: %v", err)
		}
	}
	build.Targets = make(map[string]*Target)
	for name := range targets {
		object, err := targets.GetObject(name)
		if err != nil {
			return fmt.Errorf("parsing target '%s': %v", name, err)
		}
		target, err := NewTarget(build, name, object)
		if err != nil {
			return fmt.Errorf("parsing target '%s': %v", name, err)
		}
		build.Targets[name] = target
	}
	return nil
}

// Return the build properties, including those inherited from parents
func (build *Build) GetProperties() util.Object {
	var properties = make(map[string]interface{})
	for _, parent := range build.Parents {
		for name, value := range parent.GetProperties() {
			properties[name] = value
		}
	}
	for name, value := range build.Properties {
		properties[name] = value
	}
	return properties
}

// Return the build environment, including those inherited from parents
func (build *Build) GetEnvironment() map[string]string {
	var environment = make(map[string]string)
	for _, parent := range build.Parents {
		for name, value := range parent.GetEnvironment() {
			environment[name] = value
		}
	}
	for name, value := range build.Environment {
		environment[name] = value
	}
	return environment
}

// Return the build targets, including those inherited from parents
func (build *Build) GetTargets() map[string]*Target {
	var targets = make(map[string]*Target)
	for _, parent := range build.Parents {
		for name, target := range parent.GetTargets() {
			targets[name] = target
		}
	}
	for name, target := range build.Targets {
		targets[name] = target
	}
	return targets
}

// Return target with given name. If not defined in build, return target
// inherited from parent
func (build *Build) GetTargetByName(name string) *Target {
	target, found := build.Targets[name]
	if found {
		return target
	} else {
		for _, parent := range build.Parents {
			target = parent.GetTargetByName(name)
			if target != nil {
				return target
			}
		}
	}
	return nil
}

// Initialize build:
// - Set build dir and change to it
// - Create context and set it for build
func (build *Build) Init() error {
	os.Chdir(build.Dir)
	context, err := NewContext(build)
	if err != nil {
		return fmt.Errorf("evaluating context: %v", err)
	}
	build.SetDir(build.Dir)
	build.SetContext(context)
	build.SetStack(NewStack())
	return nil
}

// Set the build directory, propagating to parents
func (build *Build) SetDir(dir string) {
	build.Dir = dir
	for _, parent := range build.Parents {
		parent.SetDir(dir)
	}
}

// Set the build context, propagating to parents
func (build *Build) SetContext(context *Context) {
	build.Context = context
	for _, parent := range build.Parents {
		parent.SetContext(context)
	}
}

// Set the build stack, propagating to parents
func (build *Build) SetStack(stack *Stack) {
	build.Stack = stack
	for _, parent := range build.Parents {
		parent.SetStack(stack)
	}
}

// Set command line properties, that overwrite build ones
func (build *Build) SetCommandLineProperties(props string) error {
	var object util.Object
	err := yaml.Unmarshal([]byte(props), &object)
	if err != nil {
		return fmt.Errorf("parsing command line properties: properties must be a map with string keys")
	}
	for name, value := range object {
		build.Properties[name] = value
	}
	return nil
}

// Return default targets. If none is defined in build, return those from
// parents
func (build *Build) GetDefault() []string {
	if len(build.Default) > 0 {
		return build.Default
	} else {
		for _, parent := range build.Parents {
			if len(parent.Default) > 0 {
				return parent.Default
			}
		}
		for _, parent := range build.Parents {
			parentDefault := parent.GetDefault()
			if len(parentDefault) > 0 {
				return parentDefault
			}
		}
	}
	return build.Default
}

// Run build given targets. If no target is given, run default one.
func (build *Build) Run(targets []string) error {
	if len(targets) == 0 {
		targets = build.GetDefault()
		if len(targets) == 0 {
			return fmt.Errorf("no default target")
		}
	}
	for _, target := range targets {
		err := build.RunTarget(target)
		if err != nil {
			return err
		}
	}
	return nil
}

// Run given target
func (build *Build) RunTarget(name string) error {
	target := build.GetTargetByName(name)
	if target == nil {
		return fmt.Errorf("target '%s' not found", name)
	}
	err := target.Run()
	if err != nil {
		return fmt.Errorf("running target '%s': %v", name, err)
	}
	return nil
}

// Run parent target
func RunParentTarget(build *Build, name string) (bool, error) {
	for _, parent := range build.Parents {
		target := parent.GetTargetByName(name)
		if target != nil {
			err := target.RunSteps()
			if err != nil {
				return true, fmt.Errorf("running target '%s': %v", name, err)
			}
			return true, nil
		} else {
			ok, err := RunParentTarget(parent, name)
			if ok || err != nil {
				return ok, err
			}
		}
	}
	return false, nil
}
<|endoftext|>


<commit_before>// Package mysql parses the mysql slow query log
package mysql

import (
	"database/sql"
	"fmt"
	"log"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/honeycombio/honeytail/event"

	"github.com/Sirupsen/logrus"
	_ "github.com/go-sql-driver/mysql"
	"github.com/percona/go-mysql/query"
)

// 3 sample log entries
//
// # Time: 2016-04-01T00:31:09.817887Z
// # User@Host: root[root] @ localhost []  Id:   233
// # Query_time: 0.008393  Lock_time: 0.000154 Rows_sent: 1  Rows_examined: 357
// SET timestamp=1459470669;
// show status like 'Uptime';
//
// # Time: 2016-04-01T00:31:09.853523Z
// # User@Host: root[root] @ localhost []  Id:   233
// # Query_time: 0.020424  Lock_time: 0.000147 Rows_sent: 494  Rows_examined: 494
// SET timestamp=1459470669;
// SHOW /*innotop*/ GLOBAL VARIABLES;
//
// # Time: 2016-04-01T00:31:09.856726Z
// # User@Host: root[root] @ localhost []  Id:   233
// # Query_time: 0.000021  Lock_time: 0.000000 Rows_sent: 494  Rows_examined: 494
// SET timestamp=1459470669;
// # administrator command: Ping;
//
// We should ignore the administrator command entry; the stats it presents (eg rows_sent)
// are actually for the previous command

var (
	reTime       = myRegexp{regexp.MustCompile("^# Time: (?P<time>[^ ]+)Z *$")}
	reAdminPing  = myRegexp{regexp.MustCompile("^# administrator command: Ping; *$")}
	reUser       = myRegexp{regexp.MustCompile("^# User@Host: (?P<user>[^ ]+) @ (?P<host>[^ ]+).*$")}
	reQueryStats = myRegexp{regexp.MustCompile("^# Query_time: (?P<queryTime>[0-9.]+) *Lock_time: (?P<lockTime>[0-9.]+) *Rows_sent: (?P<rowsSent>[0-9]+) *Rows_examined: (?P<rowsExamined>[0-9]+) *$")}
	reSetTime    = myRegexp{regexp.MustCompile("^SET timestamp=(?P<unixTime>[0-9]+);$")}
	reQuery      = myRegexp{regexp.MustCompile("^(?P<query>[^#]*).*$")}
	reUse        = myRegexp{regexp.MustCompile("^(?i)use ")}

	rdsStr  = "rds"
	ec2Str  = "ec2"
	selfStr = "self"
)

const timeFormat = "2006-01-02T15:04:05.000000"

type Options struct {
	Host          string `long:"host" description:"MySQL host in the format (address:port)"`
	User          string `long:"user" description:"MySQL username"`
	Pass          string `long:"pass" description:"MySQL password"`
	QueryInterval uint   `long:"interval" description:"interval for querying the MySQL DB in seconds" default:"30"`
}

type Parser struct {
	conf     Options
	wg       sync.WaitGroup
	nower    Nower
	hostedOn string
	readOnly bool
}

type Nower interface {
	Now() time.Time
}

type RealNower struct{}

func (n *RealNower) Now() time.Time {
	return time.Now().UTC()
}

// rawEvent is the unparsed multi-line entry representing a single slow query
type rawEvent struct {
	lines []string
}

// slowQuery represents the structured form of a query from the slow query log
type SlowQuery struct {
	Timestamp       time.Time `json:"time"`
	UnixTime        int       `json:"unixtime"`
	User            string    `json:"user"`
	Host            string    `json:"host"`
	QueryTime       float64   `json:"query_time"`
	LockTime        float64   `json:"lock_time"`
	RowsSent        int       `json:"rows_sent"`
	RowsExamined    int       `json:"rows_examined"`
	Query           string    `json:"query,omitempty"`
	NormalizedQuery string    `json:"normalized_query,omitempty"`
	DB              string    `json:"db,omitempty"`
	HostedOn        string    `json:"hosted_on,omitempty"`
	ReadOnly        bool      `json:"read_only,omitempty"`
	skipQuery       bool
}

func (p *Parser) Init(options interface{}) error {
	p.conf = *options.(*Options)
	p.nower = &RealNower{}
	if p.conf.Host != "" {
		url := fmt.Sprintf("%s:%s@tcp(%s)/", p.conf.User, p.conf.Pass, p.conf.Host)
		db, err := sql.Open("mysql", url)
		if err != nil {
			log.Fatal(err)
		}
		// update hostedOn and readOnly every <n> seconds
		go func() {
			defer db.Close()
			ticker := time.NewTicker(time.Second * time.Duration(p.conf.QueryInterval))
			for _ = range ticker.C {
				p.hostedOn = getHostedOn(db)
				p.readOnly = getReadOnly(db)
			}
		}()
	}
	return nil
}

func getReadOnly(db *sql.DB) bool {
	rows, err := db.Query("SELECT @@global.read_only;")
	if err != nil {
		log.Println(err)
		return false
	}
	defer rows.Close()

	var value bool

	rows.Next()
	if err := rows.Scan(&value); err != nil {
		log.Println(err)
		return false
	}
	if err := rows.Err(); err != nil {
		log.Println(err)
		return false
	}
	return value
}

func getHostedOn(db *sql.DB) string {
	rows, err := db.Query("SHOW GLOBAL VARIABLES WHERE Variable_name = 'basedir';")
	if err != nil {
		log.Println(err)
		return ""
	}
	defer rows.Close()

	var varName, value string

	for rows.Next() {
		if err := rows.Scan(&varName, &value); err != nil {
			log.Println(err)
			return ""
		}
		if strings.HasPrefix(value, "/rdsdbbin/") {
			return rdsStr
		}
	}

	// TODO: implement ec2 detection

	if err := rows.Err(); err != nil {
		log.Println(err)
		return ""
	}
	return selfStr
}

func (p *Parser) ProcessLines(lines <-chan string, send chan<- event.Event) {
	// start up a goroutine to handle grouped sets of lines
	rawEvents := make(chan rawEvent)
	var wg sync.WaitGroup
	p.wg = wg
	defer p.wg.Wait()
	p.wg.Add(1)
	go p.handleEvents(rawEvents, send)

	// flag to indicate when we've got a complete event to send
	var sendEvent bool
	groupedLines := make([]string, 0, 5)
	for line := range lines {
		if strings.HasPrefix(line, "# Time: ") {
			// we've started a new event. Send the previous one.
			sendEvent = true
		}
		if sendEvent {
			sendEvent = false
			rawEvents <- rawEvent{lines: groupedLines}
			groupedLines = make([]string, 0, 5)
		}
		groupedLines = append(groupedLines, line)
	}
	if len(groupedLines) != 0 {
		rawEvents <- rawEvent{lines: groupedLines}
	}
	logrus.Debug("lines channel is closed, ending mysql processor")
	close(rawEvents)
}

func (p *Parser) handleEvents(rawEvents <-chan rawEvent, send chan<- event.Event) {
	defer p.wg.Done()
	for rawE := range rawEvents {
		sq := p.handleEvent(rawE)
		sq.HostedOn = p.hostedOn
		sq.ReadOnly = p.readOnly
		ev, err := p.processSlowQuery(sq)
		if err != nil {
			logrus.WithFields(logrus.Fields{
				"err":       err,
				"slowQuery": sq,
			}).Debug("skipping query")
			continue
		}
		send <- ev
	}
	logrus.Debug("done with mysql handleEvents")
}

// handle a single event
func (p *Parser) handleEvent(rawE rawEvent) SlowQuery {
	var err error
	sq := SlowQuery{}
	logrus.WithFields(logrus.Fields{
		"rawE": rawE,
	}).Debug("About to parse event")
	for _, line := range rawE.lines {
		// parse each line and populate the SlowQuery object
		switch {
		case reTime.MatchString(line):
			matchGroups := reTime.FindStringSubmatchMap(line)
			sq.Timestamp, err = time.Parse(timeFormat, matchGroups["time"])
			if err != nil {
				sq.Timestamp = p.nower.Now()
			}
		case reAdminPing.MatchString(line):
			// this evetn is an administrative ping and we should
			// ignore the entire event
			logrus.WithFields(logrus.Fields{
				"line":  line,
				"event": rawE,
			}).Debug("readmin ping detected; skipping this event")
			sq.skipQuery = true
		case reUser.MatchString(line):
			matchGroups := reUser.FindStringSubmatchMap(line)
			sq.User = matchGroups["user"]
			sq.Host = matchGroups["host"]
		case reQueryStats.MatchString(line):
			matchGroups := reQueryStats.FindStringSubmatchMap(line)
			sq.QueryTime, err = strconv.ParseFloat(matchGroups["queryTime"], 64)
			sq.LockTime, err = strconv.ParseFloat(matchGroups["lockTime"], 64)
			sq.RowsSent, err = strconv.Atoi(matchGroups["rowsSent"])
			sq.RowsExamined, err = strconv.Atoi(matchGroups["rowsExamined"])
		case reUse.FindString(line) != "":
			db := strings.TrimPrefix(line, reUse.FindString(line))
			db = strings.TrimRight(db, ";")
			db = strings.Trim(db, "`")
			sq.DB = db
			// Use this line as the query unless, if a real query follows it will be replaced.
			sq.Query = line
		case reSetTime.MatchString(line):
			matchGroups := reSetTime.FindStringSubmatchMap(line)
			sq.UnixTime, err = strconv.Atoi(matchGroups["unixTime"])
		case reQuery.MatchString(line):
			matchGroups := reQuery.FindStringSubmatchMap(line)
			sq.Query = matchGroups["query"]
			sq.NormalizedQuery = query.Fingerprint(sq.Query)
		default:
			// unknown row; log and skip
			logrus.WithFields(logrus.Fields{
				"line": line,
			}).Debug("No regex match for line in the middle of a query. skipping")
		}
	}
	// We always need a timestamp; if we never found the line, use Now()
	if sq.Timestamp.IsZero() {
		sq.Timestamp = p.nower.Now()
	}
	return sq
}

// custom error to indicate empty query
type emptyQueryError struct {
	err string
}

func (e *emptyQueryError) Error() string {
	e.err = "skipped slow query"
	return e.err
}

func (p *Parser) processSlowQuery(sq SlowQuery) (event.Event, error) {
	// if we didn't match any lines at all, skip the query
	if sq == (SlowQuery{}) {
		sq.skipQuery = true
	}
	// OK, we've collected all the lines, send in the event
	if !sq.skipQuery {
		return event.Event{
			Timestamp: sq.Timestamp,
			Data:      sq.mapify(),
		}, nil
	}
	// we're skipping this query
	return event.Event{}, &emptyQueryError{}
}

func (s SlowQuery) mapify() map[string]interface{} {
	return map[string]interface{}{
		"time":             s.Timestamp,
		"unixtime":         s.UnixTime,
		"user":             s.User,
		"host":             s.Host,
		"query_time":       s.QueryTime,
		"lock_time":        s.LockTime,
		"rows_sent":        s.RowsSent,
		"rows_examined":    s.RowsExamined,
		"query":            s.Query,
		"normalized_query": s.NormalizedQuery,
		"hosted_on":        s.HostedOn,
		"read_only":        s.ReadOnly,
	}
}
<commit_msg>get replica lag and role (primary/secondary)<commit_after>// Package mysql parses the mysql slow query log
package mysql

import (
	"database/sql"
	"errors"
	"fmt"
	"log"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/honeycombio/honeytail/event"

	"github.com/Sirupsen/logrus"
	_ "github.com/go-sql-driver/mysql"
	"github.com/percona/go-mysql/query"
)

// 3 sample log entries
//
// # Time: 2016-04-01T00:31:09.817887Z
// # User@Host: root[root] @ localhost []  Id:   233
// # Query_time: 0.008393  Lock_time: 0.000154 Rows_sent: 1  Rows_examined: 357
// SET timestamp=1459470669;
// show status like 'Uptime';
//
// # Time: 2016-04-01T00:31:09.853523Z
// # User@Host: root[root] @ localhost []  Id:   233
// # Query_time: 0.020424  Lock_time: 0.000147 Rows_sent: 494  Rows_examined: 494
// SET timestamp=1459470669;
// SHOW /*innotop*/ GLOBAL VARIABLES;
//
// # Time: 2016-04-01T00:31:09.856726Z
// # User@Host: root[root] @ localhost []  Id:   233
// # Query_time: 0.000021  Lock_time: 0.000000 Rows_sent: 494  Rows_examined: 494
// SET timestamp=1459470669;
// # administrator command: Ping;
//
// We should ignore the administrator command entry; the stats it presents (eg rows_sent)
// are actually for the previous command

var (
	reTime       = myRegexp{regexp.MustCompile("^# Time: (?P<time>[^ ]+)Z *$")}
	reAdminPing  = myRegexp{regexp.MustCompile("^# administrator command: Ping; *$")}
	reUser       = myRegexp{regexp.MustCompile("^# User@Host: (?P<user>[^ ]+) @ (?P<host>[^ ]+).*$")}
	reQueryStats = myRegexp{regexp.MustCompile("^# Query_time: (?P<queryTime>[0-9.]+) *Lock_time: (?P<lockTime>[0-9.]+) *Rows_sent: (?P<rowsSent>[0-9]+) *Rows_examined: (?P<rowsExamined>[0-9]+) *$")}
	reSetTime    = myRegexp{regexp.MustCompile("^SET timestamp=(?P<unixTime>[0-9]+);$")}
	reQuery      = myRegexp{regexp.MustCompile("^(?P<query>[^#]*).*$")}
	reUse        = myRegexp{regexp.MustCompile("^(?i)use ")}

	rdsStr  = "rds"
	ec2Str  = "ec2"
	selfStr = "self"
)

const timeFormat = "2006-01-02T15:04:05.000000"

type Options struct {
	Host          string `long:"host" description:"MySQL host in the format (address:port)"`
	User          string `long:"user" description:"MySQL username"`
	Pass          string `long:"pass" description:"MySQL password"`
	QueryInterval uint   `long:"interval" description:"interval for querying the MySQL DB in seconds" default:"30"`
}

type Parser struct {
	conf       Options
	wg         sync.WaitGroup
	nower      Nower
	hostedOn   string
	readOnly   bool
	replicaLag int64
	role       string
}

type Nower interface {
	Now() time.Time
}

type RealNower struct{}

func (n *RealNower) Now() time.Time {
	return time.Now().UTC()
}

// rawEvent is the unparsed multi-line entry representing a single slow query
type rawEvent struct {
	lines []string
}

// slowQuery represents the structured form of a query from the slow query log
type SlowQuery struct {
	Timestamp       time.Time `json:"time"`
	UnixTime        int       `json:"unixtime"`
	User            string    `json:"user"`
	Host            string    `json:"host"`
	QueryTime       float64   `json:"query_time"`
	LockTime        float64   `json:"lock_time"`
	RowsSent        int       `json:"rows_sent"`
	RowsExamined    int       `json:"rows_examined"`
	Query           string    `json:"query,omitempty"`
	NormalizedQuery string    `json:"normalized_query,omitempty"`
	DB              string    `json:"db,omitempty"`
	HostedOn        string    `json:"hosted_on,omitempty"`
	ReadOnly        bool      `json:"read_only,omitempty"`
	ReplicaLag      int64     `json:"replica_lag,omitempty"`
	Role            string    `json:"role,omitempty"`
	skipQuery       bool
}

func (p *Parser) Init(options interface{}) error {
	p.conf = *options.(*Options)
	p.nower = &RealNower{}
	if p.conf.Host != "" {
		url := fmt.Sprintf("%s:%s@tcp(%s)/", p.conf.User, p.conf.Pass, p.conf.Host)
		db, err := sql.Open("mysql", url)
		if err != nil {
			log.Fatal(err)
		}
		// update hostedOn and readOnly every <n> seconds
		go func() {
			defer db.Close()
			ticker := time.NewTicker(time.Second * time.Duration(p.conf.QueryInterval))
			for _ = range ticker.C {
				p.hostedOn = getHostedOn(db)
				p.readOnly = getReadOnly(db)
				p.replicaLag = getReplicaLag(db)
				p.role = getRole(db)
			}
		}()
	}
	return nil
}

func getReadOnly(db *sql.DB) bool {
	rows, err := db.Query("SELECT @@global.read_only;")
	if err != nil {
		log.Println(err)
		return false
	}
	defer rows.Close()

	var value bool

	rows.Next()
	if err := rows.Scan(&value); err != nil {
		log.Println(err)
		return false
	}
	if err := rows.Err(); err != nil {
		log.Println(err)
		return false
	}
	return value
}

func getHostedOn(db *sql.DB) string {
	rows, err := db.Query("SHOW GLOBAL VARIABLES WHERE Variable_name = 'basedir';")
	if err != nil {
		log.Println(err)
		return ""
	}
	defer rows.Close()

	var varName, value string

	for rows.Next() {
		if err := rows.Scan(&varName, &value); err != nil {
			log.Println(err)
			return ""
		}
		if strings.HasPrefix(value, "/rdsdbbin/") {
			return rdsStr
		}
	}

	// TODO: implement ec2 detection

	if err := rows.Err(); err != nil {
		log.Println(err)
		return ""
	}
	return selfStr
}

func getReplicaLag(db *sql.DB) int64 {
	rows, err := db.Query("SHOW SLAVE STATUS")
	if err != nil {
		log.Println(err)
		return -1
	}
	if !rows.Next() {
		log.Println(errors.New("No slave status"))
		return -1
	}
	defer rows.Close()

	columns, _ := rows.Columns()
	values := make([]interface{}, len(columns))
	for i := range values {
		var v sql.RawBytes
		values[i] = &v
	}

	err = rows.Scan(values...)
	if err != nil {
		log.Println(err)
		return -1
	}

	for i, name := range columns {
		bp := values[i].(*sql.RawBytes)
		vs := string(*bp)
		if name == "Seconds_Behind_Master" {
			vi, err := strconv.ParseInt(vs, 10, 64)
			if err != nil {
				return -1
			}
			return vi
		}
	}
	return -1
}

func getRole(db *sql.DB) string {
	rows, err := db.Query("SHOW MASTER STATUS")
	if err != nil {
		log.Println(err)
		return "unknown"
	}
	defer rows.Close()
	if !rows.Next() {
		return "secondary"
	}
	return "primary"
}

func (p *Parser) ProcessLines(lines <-chan string, send chan<- event.Event) {
	// start up a goroutine to handle grouped sets of lines
	rawEvents := make(chan rawEvent)
	var wg sync.WaitGroup
	p.wg = wg
	defer p.wg.Wait()
	p.wg.Add(1)
	go p.handleEvents(rawEvents, send)

	// flag to indicate when we've got a complete event to send
	var sendEvent bool
	groupedLines := make([]string, 0, 5)
	for line := range lines {
		if strings.HasPrefix(line, "# Time: ") {
			// we've started a new event. Send the previous one.
			sendEvent = true
		}
		if sendEvent {
			sendEvent = false
			rawEvents <- rawEvent{lines: groupedLines}
			groupedLines = make([]string, 0, 5)
		}
		groupedLines = append(groupedLines, line)
	}
	if len(groupedLines) != 0 {
		rawEvents <- rawEvent{lines: groupedLines}
	}
	logrus.Debug("lines channel is closed, ending mysql processor")
	close(rawEvents)
}

func (p *Parser) handleEvents(rawEvents <-chan rawEvent, send chan<- event.Event) {
	defer p.wg.Done()
	for rawE := range rawEvents {
		sq := p.handleEvent(rawE)
		sq.HostedOn = p.hostedOn
		sq.ReadOnly = p.readOnly
		sq.ReplicaLag = p.replicaLag
		sq.Role = p.role
		ev, err := p.processSlowQuery(sq)
		if err != nil {
			logrus.WithFields(logrus.Fields{
				"err":       err,
				"slowQuery": sq,
			}).Debug("skipping query")
			continue
		}
		send <- ev
	}
	logrus.Debug("done with mysql handleEvents")
}

// handle a single event
func (p *Parser) handleEvent(rawE rawEvent) SlowQuery {
	var err error
	sq := SlowQuery{}
	logrus.WithFields(logrus.Fields{
		"rawE": rawE,
	}).Debug("About to parse event")
	for _, line := range rawE.lines {
		// parse each line and populate the SlowQuery object
		switch {
		case reTime.MatchString(line):
			matchGroups := reTime.FindStringSubmatchMap(line)
			sq.Timestamp, err = time.Parse(timeFormat, matchGroups["time"])
			if err != nil {
				sq.Timestamp = p.nower.Now()
			}
		case reAdminPing.MatchString(line):
			// this evetn is an administrative ping and we should
			// ignore the entire event
			logrus.WithFields(logrus.Fields{
				"line":  line,
				"event": rawE,
			}).Debug("readmin ping detected; skipping this event")
			sq.skipQuery = true
		case reUser.MatchString(line):
			matchGroups := reUser.FindStringSubmatchMap(line)
			sq.User = matchGroups["user"]
			sq.Host = matchGroups["host"]
		case reQueryStats.MatchString(line):
			matchGroups := reQueryStats.FindStringSubmatchMap(line)
			sq.QueryTime, err = strconv.ParseFloat(matchGroups["queryTime"], 64)
			sq.LockTime, err = strconv.ParseFloat(matchGroups["lockTime"], 64)
			sq.RowsSent, err = strconv.Atoi(matchGroups["rowsSent"])
			sq.RowsExamined, err = strconv.Atoi(matchGroups["rowsExamined"])
		case reUse.FindString(line) != "":
			db := strings.TrimPrefix(line, reUse.FindString(line))
			db = strings.TrimRight(db, ";")
			db = strings.Trim(db, "`")
			sq.DB = db
			// Use this line as the query unless, if a real query follows it will be replaced.
			sq.Query = line
		case reSetTime.MatchString(line):
			matchGroups := reSetTime.FindStringSubmatchMap(line)
			sq.UnixTime, err = strconv.Atoi(matchGroups["unixTime"])
		case reQuery.MatchString(line):
			matchGroups := reQuery.FindStringSubmatchMap(line)
			sq.Query = matchGroups["query"]
			sq.NormalizedQuery = query.Fingerprint(sq.Query)
		default:
			// unknown row; log and skip
			logrus.WithFields(logrus.Fields{
				"line": line,
			}).Debug("No regex match for line in the middle of a query. skipping")
		}
	}
	// We always need a timestamp; if we never found the line, use Now()
	if sq.Timestamp.IsZero() {
		sq.Timestamp = p.nower.Now()
	}
	return sq
}

// custom error to indicate empty query
type emptyQueryError struct {
	err string
}

func (e *emptyQueryError) Error() string {
	e.err = "skipped slow query"
	return e.err
}

func (p *Parser) processSlowQuery(sq SlowQuery) (event.Event, error) {
	// if we didn't match any lines at all, skip the query
	if sq == (SlowQuery{}) {
		sq.skipQuery = true
	}
	// OK, we've collected all the lines, send in the event
	if !sq.skipQuery {
		return event.Event{
			Timestamp: sq.Timestamp,
			Data:      sq.mapify(),
		}, nil
	}
	// we're skipping this query
	return event.Event{}, &emptyQueryError{}
}

func (s SlowQuery) mapify() map[string]interface{} {
	return map[string]interface{}{
		"time":             s.Timestamp,
		"unixtime":         s.UnixTime,
		"user":             s.User,
		"host":             s.Host,
		"query_time":       s.QueryTime,
		"lock_time":        s.LockTime,
		"rows_sent":        s.RowsSent,
		"rows_examined":    s.RowsExamined,
		"query":            s.Query,
		"normalized_query": s.NormalizedQuery,
		"hosted_on":        s.HostedOn,
		"read_only":        s.ReadOnly,
		"replica_lag":      s.ReplicaLag,
		"role":             s.Role,
	}
}
<|endoftext|>


<commit_before>package pagerank

import "math"

type Float float32;

type Interface interface {
  Rank(followingProb, tolerance Float, resultFunc func(label int, rank Float))
}

type pageRank struct {
  inLinks [][]int
  numberOutLinks []int
  currentAvailableIndex int
  keyToIndex map[int]int
  indexToKey map[int]int
}

func New() *pageRank {
  pr := new(pageRank)
  pr.Clear()
  return pr;
}

func (pr *pageRank) keyAsArrayIndex(key int) int {
  index, ok := pr.keyToIndex[key]

  if !ok {
    pr.currentAvailableIndex++
    index = pr.currentAvailableIndex
    pr.keyToIndex[key] = index
    pr.indexToKey[index] = key
  }

  return index
}

func (pr *pageRank) updateInLinks(fromAsIndex, toAsIndex int) {
  missingSlots := len(pr.keyToIndex) - len(pr.inLinks)

  if missingSlots > 0 {
    pr.inLinks = append(pr.inLinks, make([][]int, missingSlots)...)
  }

  pr.inLinks[toAsIndex] = append(pr.inLinks[toAsIndex], fromAsIndex)
}

func (pr *pageRank) updateNumberOutLinks(fromAsIndex int) {
  missingSlots :=  len(pr.keyToIndex) - len(pr.numberOutLinks)

  if missingSlots > 0 {
    pr.numberOutLinks = append(pr.numberOutLinks, make([]int, missingSlots)...)
  }

  pr.numberOutLinks[fromAsIndex] += 1
}

func (pr *pageRank) linkWithIndices(fromAsIndex, toAsIndex int) {
  pr.updateInLinks(fromAsIndex, toAsIndex)
  pr.updateNumberOutLinks(fromAsIndex)
}

func (pr *pageRank) Link(from, to int) {
  fromAsIndex := pr.keyAsArrayIndex(from)
  toAsIndex := pr.keyAsArrayIndex(to)

  pr.linkWithIndices(fromAsIndex, toAsIndex)
}

func (pr *pageRank) calculateDanglingNodes() []int {
  danglingNodes := make([]int, 0, len(pr.numberOutLinks))

  for i := range pr.numberOutLinks {
    if pr.numberOutLinks[i] == 0 {
      danglingNodes = append(danglingNodes, i)
    }
  }

  return danglingNodes
}

func (pr *pageRank) step(followingProb, tOverSize Float, p []Float, danglingNodes []int) []Float {
  innerProduct := Float(0)

  for i := range danglingNodes {
    innerProduct += p[danglingNodes[i]]
  }

  innerProductOverSize := innerProduct / Float(len(p))
  vsum := Float(0)
  v := make([]Float, len(p))

  for i := range pr.inLinks {
    ksum := Float(0)
    inLinksForI := pr.inLinks[i]

    for j := range inLinksForI {
      index := inLinksForI[j]
      ksum += p[index] / Float(pr.numberOutLinks[index])
    }

    v[i] = followingProb * (ksum + innerProductOverSize) + tOverSize
    vsum += v[i]
  }

  inverseOfSum := 1.0 / vsum

  for i := range v {
    v[i] *= inverseOfSum
  }

  return v
}

func calculateChange(p , new_p []Float) Float {
  acc := Float(0)

  for i := range p {
    acc += Float(math.Abs(float64(p[i] - new_p[i])))
  }

  return acc
}

func (pr *pageRank) Rank(followingProb, tolerance Float, resultFunc func(label int, rank Float)) {
  size := len(pr.keyToIndex)
  inverseOfSize := 1.0 / Float(size)
  tOverSize := (1.0 - followingProb) / Float(size)
  danglingNodes := pr.calculateDanglingNodes()

  p := make([]Float, size)
  for i := range p {
    p[i] = inverseOfSize
  }

  change := Float(2.0)

  for change > tolerance {
    new_p := pr.step(followingProb, tOverSize, p, danglingNodes)
    change = calculateChange(p, new_p)
    p = new_p
  }

  for i := range p {
    resultFunc(pr.indexToKey[i], p[i])
  }
}

func (pr *pageRank) Clear() {
  pr.inLinks = [][]int{}
  pr.numberOutLinks = []int{}
  pr.currentAvailableIndex = -1
  pr.keyToIndex = make(map[int]int)
  pr.indexToKey = make(map[int]int)
}
<commit_msg>Add Link to Interface<commit_after>package pagerank

import "math"

type Float float32;

type Interface interface {
  Rank(followingProb, tolerance Float, resultFunc func(label int, rank Float))
  Link(from, to int)
}

type pageRank struct {
  inLinks [][]int
  numberOutLinks []int
  currentAvailableIndex int
  keyToIndex map[int]int
  indexToKey map[int]int
}

func New() *pageRank {
  pr := new(pageRank)
  pr.Clear()
  return pr;
}

func (pr *pageRank) keyAsArrayIndex(key int) int {
  index, ok := pr.keyToIndex[key]

  if !ok {
    pr.currentAvailableIndex++
    index = pr.currentAvailableIndex
    pr.keyToIndex[key] = index
    pr.indexToKey[index] = key
  }

  return index
}

func (pr *pageRank) updateInLinks(fromAsIndex, toAsIndex int) {
  missingSlots := len(pr.keyToIndex) - len(pr.inLinks)

  if missingSlots > 0 {
    pr.inLinks = append(pr.inLinks, make([][]int, missingSlots)...)
  }

  pr.inLinks[toAsIndex] = append(pr.inLinks[toAsIndex], fromAsIndex)
}

func (pr *pageRank) updateNumberOutLinks(fromAsIndex int) {
  missingSlots :=  len(pr.keyToIndex) - len(pr.numberOutLinks)

  if missingSlots > 0 {
    pr.numberOutLinks = append(pr.numberOutLinks, make([]int, missingSlots)...)
  }

  pr.numberOutLinks[fromAsIndex] += 1
}

func (pr *pageRank) linkWithIndices(fromAsIndex, toAsIndex int) {
  pr.updateInLinks(fromAsIndex, toAsIndex)
  pr.updateNumberOutLinks(fromAsIndex)
}

func (pr *pageRank) Link(from, to int) {
  fromAsIndex := pr.keyAsArrayIndex(from)
  toAsIndex := pr.keyAsArrayIndex(to)

  pr.linkWithIndices(fromAsIndex, toAsIndex)
}

func (pr *pageRank) calculateDanglingNodes() []int {
  danglingNodes := make([]int, 0, len(pr.numberOutLinks))

  for i := range pr.numberOutLinks {
    if pr.numberOutLinks[i] == 0 {
      danglingNodes = append(danglingNodes, i)
    }
  }

  return danglingNodes
}

func (pr *pageRank) step(followingProb, tOverSize Float, p []Float, danglingNodes []int) []Float {
  innerProduct := Float(0)

  for i := range danglingNodes {
    innerProduct += p[danglingNodes[i]]
  }

  innerProductOverSize := innerProduct / Float(len(p))
  vsum := Float(0)
  v := make([]Float, len(p))

  for i := range pr.inLinks {
    ksum := Float(0)
    inLinksForI := pr.inLinks[i]

    for j := range inLinksForI {
      index := inLinksForI[j]
      ksum += p[index] / Float(pr.numberOutLinks[index])
    }

    v[i] = followingProb * (ksum + innerProductOverSize) + tOverSize
    vsum += v[i]
  }

  inverseOfSum := 1.0 / vsum

  for i := range v {
    v[i] *= inverseOfSum
  }

  return v
}

func calculateChange(p , new_p []Float) Float {
  acc := Float(0)

  for i := range p {
    acc += Float(math.Abs(float64(p[i] - new_p[i])))
  }

  return acc
}

func (pr *pageRank) Rank(followingProb, tolerance Float, resultFunc func(label int, rank Float)) {
  size := len(pr.keyToIndex)
  inverseOfSize := 1.0 / Float(size)
  tOverSize := (1.0 - followingProb) / Float(size)
  danglingNodes := pr.calculateDanglingNodes()

  p := make([]Float, size)
  for i := range p {
    p[i] = inverseOfSize
  }

  change := Float(2.0)

  for change > tolerance {
    new_p := pr.step(followingProb, tOverSize, p, danglingNodes)
    change = calculateChange(p, new_p)
    p = new_p
  }

  for i := range p {
    resultFunc(pr.indexToKey[i], p[i])
  }
}

func (pr *pageRank) Clear() {
  pr.inLinks = [][]int{}
  pr.numberOutLinks = []int{}
  pr.currentAvailableIndex = -1
  pr.keyToIndex = make(map[int]int)
  pr.indexToKey = make(map[int]int)
}
<|endoftext|>


<commit_before>package grok

var patterns = map[string]string{
	"USERNAME":           `[a-zA-Z0-9._-]+`,
	"USER":               `%{USERNAME}`,
	"EMAILLOCALPART":     `[a-zA-Z][a-zA-Z0-9_.+-=:]+`,
	"EMAILADDRESS":       `%{EMAILLOCALPART}@%{HOSTNAME}`,
	"HTTPDUSER":          `%{EMAILADDRESS}|%{USER}`,
	"INT":                `(?:[+-]?(?:[0-9]+))`,
	"BASE10NUM":          `([+-]?(?:[0-9]+(?:\.[0-9]+)?)|\.[0-9]+)`,
	"NUMBER":             `(?:%{BASE10NUM})`,
	"BASE16NUM":          `(0[xX]?[0-9a-fA-F]+)`,
	"POSINT":             `\b(?:[1-9][0-9]*)\b`,
	"NONNEGINT":          `\b(?:[0-9]+)\b`,
	"WORD":               `\b\w+\b`,
	"NOTSPACE":           `\S+`,
	"SPACE":              `\s*`,
	"DATA":               `.*?`,
	"GREEDYDATA":         `.*`,
	"QUOTEDSTRING":       `"([^"\\]*(\\.[^"\\]*)*)"|\'([^\'\\]*(\\.[^\'\\]*)*)\'`,
	"UUID":               `[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}`,
	"MAC":                `(?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})`,
	"CISCOMAC":           `(?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})`,
	"WINDOWSMAC":         `(?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})`,
	"COMMONMAC":          `(?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})`,
	"IPV6":               `((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?`,
	"IPV4":               `(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)`,
	"IP":                 `(?:%{IPV6}|%{IPV4})`,
	"HOSTNAME":           `\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)`,
	"HOST":               `%{HOSTNAME}`,
	"IPORHOST":           `(?:%{IP}|%{HOSTNAME})`,
	"HOSTPORT":           `%{IPORHOST}:%{POSINT}`,
	"PATH":               `(?:%{UNIXPATH}|%{WINPATH})`,
	"UNIXPATH":           `(/[\w_%!$@:.,-]?/?)(\S+)?`,
	"TTY":                `(?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))`,
	"WINPATH":            `([A-Za-z]:|\\)(?:\\[^\\?*]*)+`,
	"URIPROTO":           `[A-Za-z]+(\+[A-Za-z+]+)?`,
	"URIHOST":            `%{IPORHOST}(?::%{POSINT:port})?`,
	"URIPATH":            `(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+`,
	"URIPARAM":           `\?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]*`,
	"URIPATHPARAM":       `%{URIPATH}(?:%{URIPARAM})?`,
	"URI":                `%{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?`,
	"MONTH":              `\b(?:Jan(?:uary|uar)?|Feb(?:ruary|ruar)?|M(?:a|ä)?r(?:ch|z)?|Apr(?:il)?|Ma(?:y|i)?|Jun(?:e|i)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|O(?:c|k)?t(?:ober)?|Nov(?:ember)?|De(?:c|z)(?:ember)?)\b`,
	"MONTHNUM":           `(?:0?[1-9]|1[0-2])`,
	"MONTHNUM2":          `(?:0[1-9]|1[0-2])`,
	"MONTHDAY":           `(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])`,
	"DAY":                `(?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)`,
	"YEAR":               `(\d\d){1,2}`,
	"HOUR":               `(?:2[0123]|[01]?[0-9])`,
	"MINUTE":             `(?:[0-5][0-9])`,
	"SECOND":             `(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)`,
	"TIME":               `([^0-9]?)%{HOUR}:%{MINUTE}(?::%{SECOND})([^0-9]?)`,
	"DATE_US":            `%{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}`,
	"DATE_EU":            `%{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}`,
	"ISO8601_TIMEZONE":   `(?:Z|[+-]%{HOUR}(?::?%{MINUTE}))`,
	"ISO8601_SECOND":     `(?:%{SECOND}|60)`,
	"TIMESTAMP_ISO8601":  `%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?`,
	"DATE":               `%{DATE_US}|%{DATE_EU}`,
	"DATESTAMP":          `%{DATE}[- ]%{TIME}`,
	"TZ":                 `(?:[PMCE][SD]T|UTC)`,
	"DATESTAMP_RFC822":   `%{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}`,
	"DATESTAMP_RFC2822":  `%{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}`,
	"DATESTAMP_OTHER":    `%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}`,
	"DATESTAMP_EVENTLOG": `%{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}`,
	"HTTPDERROR_DATE":    `%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}`,
	"SYSLOGTIMESTAMP":    `%{MONTH} +%{MONTHDAY} %{TIME}`,
	"PROG":               `[\x21-\x5a\x5c\x5e-\x7e]+`,
	"SYSLOGPROG":         `%{PROG:program}(?:\[%{POSINT:pid}\])?`,
	"SYSLOGHOST":         `%{IPORHOST}`,
	"SYSLOGFACILITY":     `<%{NONNEGINT:facility}.%{NONNEGINT:priority}>`,
	"HTTPDATE":           `%{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}`,
	"QS":                 `%{QUOTEDSTRING}`,
	"SYSLOGBASE":         `%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:`,
	"COMMONAPACHELOG":    `%{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)`,
	"COMBINEDAPACHELOG":  `%{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}`,
	"HTTPD20_ERRORLOG":   `\[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg}`,
	"HTTPD24_ERRORLOG":   `\[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}`,
	"HTTPD_ERRORLOG":     `%{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}`,
	"LOGLEVEL":           `([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)`,
}
<commit_msg>Add support for envoy common access logs<commit_after>package grok

var patterns = map[string]string{
	"USERNAME":             `[a-zA-Z0-9._-]+`,
	"USER":                 `%{USERNAME}`,
	"EMAILLOCALPART":       `[a-zA-Z][a-zA-Z0-9_.+-=:]+`,
	"EMAILADDRESS":         `%{EMAILLOCALPART}@%{HOSTNAME}`,
	"HTTPDUSER":            `%{EMAILADDRESS}|%{USER}`,
	"INT":                  `(?:[+-]?(?:[0-9]+))`,
	"BASE10NUM":            `([+-]?(?:[0-9]+(?:\.[0-9]+)?)|\.[0-9]+)`,
	"NUMBER":               `(?:%{BASE10NUM})`,
	"BASE16NUM":            `(0[xX]?[0-9a-fA-F]+)`,
	"POSINT":               `\b(?:[1-9][0-9]*)\b`,
	"NONNEGINT":            `\b(?:[0-9]+)\b`,
	"WORD":                 `\b\w+\b`,
	"NOTSPACE":             `\S+`,
	"SPACE":                `\s*`,
	"DATA":                 `.*?`,
	"GREEDYDATA":           `.*`,
	"QUOTEDSTRING":         `"([^"\\]*(\\.[^"\\]*)*)"|\'([^\'\\]*(\\.[^\'\\]*)*)\'`,
	"UUID":                 `[A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12}`,
	"MAC":                  `(?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC})`,
	"CISCOMAC":             `(?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4})`,
	"WINDOWSMAC":           `(?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2})`,
	"COMMONMAC":            `(?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2})`,
	"IPV6":                 `((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?`,
	"IPV4":                 `(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)`,
	"IP":                   `(?:%{IPV6}|%{IPV4})`,
	"HOSTNAME":             `\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)`,
	"HOST":                 `%{HOSTNAME}`,
	"IPORHOST":             `(?:%{IP}|%{HOSTNAME})`,
	"HOSTPORT":             `%{IPORHOST}:%{POSINT}`,
	"PATH":                 `(?:%{UNIXPATH}|%{WINPATH})`,
	"UNIXPATH":             `(/[\w_%!$@:.,-]?/?)(\S+)?`,
	"TTY":                  `(?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+))`,
	"WINPATH":              `([A-Za-z]:|\\)(?:\\[^\\?*]*)+`,
	"URIPROTO":             `[A-Za-z]+(\+[A-Za-z+]+)?`,
	"URIHOST":              `%{IPORHOST}(?::%{POSINT:port})?`,
	"URIPATH":              `(?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+`,
	"URIPARAM":             `\?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]<>]*`,
	"URIPATHPARAM":         `%{URIPATH}(?:%{URIPARAM})?`,
	"URI":                  `%{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})?`,
	"MONTH":                `\b(?:Jan(?:uary|uar)?|Feb(?:ruary|ruar)?|M(?:a|ä)?r(?:ch|z)?|Apr(?:il)?|Ma(?:y|i)?|Jun(?:e|i)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|O(?:c|k)?t(?:ober)?|Nov(?:ember)?|De(?:c|z)(?:ember)?)\b`,
	"MONTHNUM":             `(?:0?[1-9]|1[0-2])`,
	"MONTHNUM2":            `(?:0[1-9]|1[0-2])`,
	"MONTHDAY":             `(?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9])`,
	"DAY":                  `(?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?)`,
	"YEAR":                 `(\d\d){1,2}`,
	"HOUR":                 `(?:2[0123]|[01]?[0-9])`,
	"MINUTE":               `(?:[0-5][0-9])`,
	"SECOND":               `(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)`,
	"TIME":                 `([^0-9]?)%{HOUR}:%{MINUTE}(?::%{SECOND})([^0-9]?)`,
	"DATE_US":              `%{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR}`,
	"DATE_EU":              `%{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR}`,
	"ISO8601_TIMEZONE":     `(?:Z|[+-]%{HOUR}(?::?%{MINUTE}))`,
	"ISO8601_SECOND":       `(?:%{SECOND}|60)`,
	"TIMESTAMP_ISO8601":    `%{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}?`,
	"DATE":                 `%{DATE_US}|%{DATE_EU}`,
	"DATESTAMP":            `%{DATE}[- ]%{TIME}`,
	"TZ":                   `(?:[PMCE][SD]T|UTC)`,
	"DATESTAMP_RFC822":     `%{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ}`,
	"DATESTAMP_RFC2822":    `%{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE}`,
	"DATESTAMP_OTHER":      `%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR}`,
	"DATESTAMP_EVENTLOG":   `%{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND}`,
	"HTTPDERROR_DATE":      `%{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{YEAR}`,
	"SYSLOGTIMESTAMP":      `%{MONTH} +%{MONTHDAY} %{TIME}`,
	"PROG":                 `[\x21-\x5a\x5c\x5e-\x7e]+`,
	"SYSLOGPROG":           `%{PROG:program}(?:\[%{POSINT:pid}\])?`,
	"SYSLOGHOST":           `%{IPORHOST}`,
	"SYSLOGFACILITY":       `<%{NONNEGINT:facility}.%{NONNEGINT:priority}>`,
	"HTTPDATE":             `%{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT}`,
	"QS":                   `%{QUOTEDSTRING}`,
	"SYSLOGBASE":           `%{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}:`,
	"COMMONAPACHELOG":      `%{IPORHOST:clientip} %{HTTPDUSER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-)`,
	"COMBINEDAPACHELOG":    `%{COMMONAPACHELOG} %{QS:referrer} %{QS:agent}`,
	"HTTPD20_ERRORLOG":     `\[%{HTTPDERROR_DATE:timestamp}\] \[%{LOGLEVEL:loglevel}\] (?:\[client %{IPORHOST:clientip}\] ){0,1}%{GREEDYDATA:errormsg}`,
	"HTTPD24_ERRORLOG":     `\[%{HTTPDERROR_DATE:timestamp}\] \[%{WORD:module}:%{LOGLEVEL:loglevel}\] \[pid %{POSINT:pid}:tid %{NUMBER:tid}\]( \(%{POSINT:proxy_errorcode}\)%{DATA:proxy_errormessage}:)?( \[client %{IPORHOST:client}:%{POSINT:clientport}\])? %{DATA:errorcode}: %{GREEDYDATA:message}`,
	"HTTPD_ERRORLOG":       `%{HTTPD20_ERRORLOG}|%{HTTPD24_ERRORLOG}`,
	"LOGLEVEL":             `([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?)`,
	"COMMONENVOYACCESSLOG": `\[%{TIMESTAMP_ISO8601:timestamp}\] \"%{DATA:method} (?:%{URIPATH:uri_path}(?:%{URIPARAM:uri_param})?|%{DATA:}) %{DATA:protocol}\" %{NUMBER:status_code} %{DATA:response_flags} %{NUMBER:bytes_received} %{NUMBER:bytes_sent} %{NUMBER:duration} (?:%{NUMBER:upstream_service_time}|%{DATA:tcp_service_time}) \"%{DATA:forwarded_for}\" \"%{DATA:user_agent}\" \"%{DATA:request_id}\" \"%{DATA:authority}\" \"%{DATA:upstream_service}\"`,
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"strconv"
	"strings"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/pagination"
	"github.com/rackspace/gophercloud/rackspace"
	"github.com/rackspace/gophercloud/rackspace/lb/v1/lbs"
	"github.com/rackspace/gophercloud/rackspace/lb/v1/nodes"
)

func waitForReady(client *gophercloud.ServiceClient, loadBalancerID int) {

	// Ensure the load balancer is ready
	gophercloud.WaitFor(60, func() (bool, error) {
		lb, err := lbs.Get(client, loadBalancerID).Extract()
		if err != nil {
			return false, err
		}

		if lb.Status != lbs.ACTIVE {
			// It has not yet reached ACTIVE
			return false, nil
		}

		// It has reached ACTIVE
		return true, nil
	})
}

func findNodeByIPPort(
	client *gophercloud.ServiceClient,
	loadBalancerID int,
	address string,
	port int,
) *nodes.Node {

	// nil until found
	var found *nodes.Node

	pager := nodes.List(client, loadBalancerID, nil)

	pager.EachPage(func(page pagination.Page) (bool, error) {
		lbNodes, err := nodes.ExtractNodes(page)
		if err != nil {
			log.Fatalf("Error during paging load balancer: %v", err)
		}

		for _, trialNode := range lbNodes {
			if trialNode.Address == address && trialNode.Port == port {
				found = &trialNode
				return false, nil
			}

		}

		return true, nil
	})

	return found
}

func getIP(ipPtr *string) (string, error) {

	// IP overriding order will be:
	//  - The -ip flag takes primary precedence
	//  - Service Net Environment variable
	//  - Public Net Environment variable
	//  - Gleaning a 10 dot out of the network interfaces (typically eth1)
	//  - eth0

	serviceNetIPv4 := os.Getenv("RAX_SERVICENET_IPV4")
	publicNetIPv4 := os.Getenv("RAX_PUBLICNET_IPV4")

	switch {
	case *ipPtr != "":
		return *ipPtr, nil
	case serviceNetIPv4 != "":
		return serviceNetIPv4, nil
	case publicNetIPv4 != "":
		return publicNetIPv4, nil
	}

	addrs, err := net.InterfaceAddrs()

	if err != nil {
		return "", err
	}

	for _, addr := range addrs {
		cidr := addr.String()
		ip := strings.Split(cidr, "/")[0]

		if strings.HasPrefix(ip, "10.") {
			return ip, nil
		}
	}

	// Find eth0
	eth0, err := net.InterfaceByName("eth0")
	if err != nil {
		return "", fmt.Errorf("trouble finding eth0: %v", err)
	}

	addrs, err = eth0.Addrs()
	if err != nil {
		return "", err
	}

	for _, addr := range addrs {
		cidr := addr.String()
		ip := strings.Split(cidr, "/")[0]

		fmt.Println(ip)
	}

	return "", errors.New("unable to determine an IP for load balancing")

}

func main() {
	var err error

	disabledPtr := flag.Bool("disable", false, "Disable the node on the load balancer")

	//NOTE: peekaboo allows setting the IP by using
	//        - environment variables: RAX_SERVICENET_IPV4 or RAX_PUBLICNET_IPV4
	//        - finding an ip on the system starting with 10. (service net)
	//        - locating the eth0 interface
	//        - providing an ip as a flag is fine too and will take precedence
	ipPtr := flag.String("ip", "", "IP address to register/deregister on the load balancer")
	flag.Parse()

	username := os.Getenv("OS_USERNAME")
	APIKey := os.Getenv("OS_PASSWORD")
	region := os.Getenv("OS_REGION_NAME")

	if username == "" || APIKey == "" || region == "" {
		fmt.Fatalf("One or more of $OS_USERNAME, $OS_PASSWORD, and $OS_REGION_NAME not set")
	}

	// These get converted into integers later
	strLoadBalancerID := os.Getenv("LOAD_BALANCER_ID")
	strAppPort := os.Getenv("APP_PORT")

	/**
	 * Retrieve port for load balancer's node, defaulting to 80
	 */
	var nodePort = 80

	if strAppPort == "" {
		log.Printf("$APP_PORT not set, defaulting to 80")
	} else {
		nodePort, err = strconv.Atoi(strAppPort)
		if err != nil {
			log.Fatalf("Unable to parse integer from $APP_PORT: %v\n", strAppPort)
		}
	}

	/**
	 * Determine the IP Address for the load balancer's node
	 */
	nodeAddress, err := getIP(ipPtr)
	if err != nil {
		log.Fatalln(err)
	}

	log.Printf("Determined IP: %v", nodeAddress)
	log.Printf("Determined Port: %v", nodePort)

	/**
	 * Retrive Load Balancer ID
	 */
	if strLoadBalancerID == "" {
		log.Fatalln("$LOAD_BALANCER_ID must be set")
	}
	loadBalancerID, err := strconv.Atoi(strLoadBalancerID)
	if err != nil {
		log.Fatalf("$LOAD_BALANCER_ID not an integer: %v\n", loadBalancerID)
	}

	provider, err := rackspace.AuthenticatedClient(gophercloud.AuthOptions{
		Username: username,
		APIKey:   APIKey,
	})

	if err != nil {
		log.Fatalf("Trouble authenticating to Rackspace: %v\n", err)
	}

	client, err := rackspace.NewLBV1(provider, gophercloud.EndpointOpts{
		Region: region,
	})

	if err != nil {
		log.Fatalf("Creating load balancer client in %v failed: %v\n", region, err)
	}

	log.Println("Client ready")

	waitForReady(client, loadBalancerID)

	nodePtr := findNodeByIPPort(client, loadBalancerID, nodeAddress, nodePort)

	condition := nodes.ENABLED
	if *disabledPtr {
		//TODO: Watch the interface on the right process/container to determine
		//      when connections have dropped, and set to DISABLED
		condition = nodes.DRAINING
	}

	log.Printf("Telling %v on port %v to be %v\n", nodeAddress, nodePort, condition)

	if nodePtr != nil {
		log.Printf("Found existing node %v", *nodePtr)

		opts := nodes.UpdateOpts{
			Condition: condition,
		}

		updateResult := nodes.Update(client, loadBalancerID, nodePtr.ID, opts)
		err = updateResult.ExtractErr()
		if err != nil {
			log.Panicf("Updating node failed: %v", err)
		}

	} else {
		log.Printf("Creating new node")
		opts := nodes.CreateOpts{
			nodes.CreateOpt{
				Address:   nodeAddress,
				Port:      nodePort,
				Condition: condition,
			},
		}

		nodePager := nodes.Create(client, loadBalancerID, opts)
		nodeList, err := nodePager.ExtractNodes()
		if err != nil || len(nodeList) != 1 {
			log.Panicf("Something went terribly wrong on node creation: %v\n", nodeList)
		}
		nodePtr = &nodeList[0]
	}

	waitForReady(client, loadBalancerID)

	// After update, get the version of the node Rackspace has
	result := nodes.Get(client, loadBalancerID, nodePtr.ID)
	nodePtr, err = result.Extract()
	if err != nil {
		log.Panicln(err)
	}

	log.Printf("Updated state for node: %v\n", *nodePtr)

}
<commit_msg>godoc or go home<commit_after>// peekaboo registers the current machine (or provided IP) to a Rackspace load
// balancer, enabling (or disabling) itself.

package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"strconv"
	"strings"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/pagination"
	"github.com/rackspace/gophercloud/rackspace"
	"github.com/rackspace/gophercloud/rackspace/lb/v1/lbs"
	"github.com/rackspace/gophercloud/rackspace/lb/v1/nodes"
)

// waitForReady waits for the load balancer with id loadBalancerID to become
// ACTIVE. It times out after 60 seconds and streamrolls on ahead.
func waitForReady(client *gophercloud.ServiceClient, loadBalancerID int) {

	// Ensure the load balancer is ready
	gophercloud.WaitFor(60, func() (bool, error) {
		lb, err := lbs.Get(client, loadBalancerID).Extract()
		if err != nil {
			return false, err
		}

		if lb.Status != lbs.ACTIVE {
			// It has not yet reached ACTIVE
			return false, nil
		}

		// It has reached ACTIVE
		return true, nil
	})

}

// findNodeByIPPort gets a load balancer node by IP and Port
func findNodeByIPPort(
	client *gophercloud.ServiceClient,
	loadBalancerID int,
	address string,
	port int,
) *nodes.Node {

	// nil until found
	var found *nodes.Node

	pager := nodes.List(client, loadBalancerID, nil)

	pager.EachPage(func(page pagination.Page) (bool, error) {
		lbNodes, err := nodes.ExtractNodes(page)
		if err != nil {
			log.Fatalf("Error during paging load balancer: %v", err)
		}

		for _, trialNode := range lbNodes {
			if trialNode.Address == address && trialNode.Port == port {
				found = &trialNode
				return false, nil
			}

		}

		return true, nil
	})

	return found
}

// getIP tries to best guess what IP to work with
//
// Precedence for IP is determined by
//  - ipPtr being non-nil (peekaboo's -ip flag)
//  - Service Net Environment variable $RAX_SERVICENET_IPV4
//  - Public Net Environment variable $RAX_PUBLICNET_IPV4
//  - Gleaning a 10 dot out of the network interfaces (likely service net)
//  - eth0
func getIP(ipPtr *string) (string, error) {

	serviceNetIPv4 := os.Getenv("RAX_SERVICENET_IPV4")
	publicNetIPv4 := os.Getenv("RAX_PUBLICNET_IPV4")

	switch {
	case *ipPtr != "":
		return *ipPtr, nil
	case serviceNetIPv4 != "":
		return serviceNetIPv4, nil
	case publicNetIPv4 != "":
		return publicNetIPv4, nil
	}

	addrs, err := net.InterfaceAddrs()

	if err != nil {
		return "", err
	}

	for _, addr := range addrs {
		cidr := addr.String()
		ip := strings.Split(cidr, "/")[0]

		if strings.HasPrefix(ip, "10.") {
			return ip, nil
		}
	}

	// Find eth0
	eth0, err := net.InterfaceByName("eth0")
	if err != nil {
		return "", fmt.Errorf("trouble finding eth0: %v", err)
	}

	addrs, err = eth0.Addrs()
	if err != nil {
		return "", err
	}

	for _, addr := range addrs {
		cidr := addr.String()
		ip := strings.Split(cidr, "/")[0]

		fmt.Println(ip)
	}

	return "", errors.New("unable to determine an IP for load balancing")

}

func main() {
	var err error

	disabledPtr := flag.Bool("disable", false, "Disable the node on the load balancer")

	//NOTE: peekaboo allows setting the IP by using
	//        - environment variables: RAX_SERVICENET_IPV4 or RAX_PUBLICNET_IPV4
	//        - finding an ip on the system starting with 10. (service net)
	//        - locating the eth0 interface
	//        - providing an ip as a flag is fine too and will take precedence
	ipPtr := flag.String("ip", "", "IP address to register/deregister on the load balancer")
	flag.Parse()

	username := os.Getenv("OS_USERNAME")
	APIKey := os.Getenv("OS_PASSWORD")
	region := os.Getenv("OS_REGION_NAME")

	if username == "" || APIKey == "" || region == "" {
		fmt.Fatalf("One or more of $OS_USERNAME, $OS_PASSWORD, and $OS_REGION_NAME not set")
	}

	// These get converted into integers later
	strLoadBalancerID := os.Getenv("LOAD_BALANCER_ID")
	strAppPort := os.Getenv("APP_PORT")

	/**
	 * Retrieve port for load balancer's node, defaulting to 80
	 */
	var nodePort = 80

	if strAppPort == "" {
		log.Printf("$APP_PORT not set, defaulting to 80")
	} else {
		nodePort, err = strconv.Atoi(strAppPort)
		if err != nil {
			log.Fatalf("Unable to parse integer from $APP_PORT: %v\n", strAppPort)
		}
	}

	/**
	 * Determine the IP Address for the load balancer's node
	 */
	nodeAddress, err := getIP(ipPtr)
	if err != nil {
		log.Fatalln(err)
	}

	log.Printf("Determined IP: %v", nodeAddress)
	log.Printf("Determined Port: %v", nodePort)

	/**
	 * Retrive Load Balancer ID
	 */
	if strLoadBalancerID == "" {
		log.Fatalln("$LOAD_BALANCER_ID must be set")
	}
	loadBalancerID, err := strconv.Atoi(strLoadBalancerID)
	if err != nil {
		log.Fatalf("$LOAD_BALANCER_ID not an integer: %v\n", loadBalancerID)
	}

	provider, err := rackspace.AuthenticatedClient(gophercloud.AuthOptions{
		Username: username,
		APIKey:   APIKey,
	})

	if err != nil {
		log.Fatalf("Trouble authenticating to Rackspace: %v\n", err)
	}

	client, err := rackspace.NewLBV1(provider, gophercloud.EndpointOpts{
		Region: region,
	})

	if err != nil {
		log.Fatalf("Creating load balancer client in %v failed: %v\n", region, err)
	}

	log.Println("Client ready")

	waitForReady(client, loadBalancerID)

	nodePtr := findNodeByIPPort(client, loadBalancerID, nodeAddress, nodePort)

	condition := nodes.ENABLED
	if *disabledPtr {
		//TODO: Watch the interface on the right process/container to determine
		//      when connections have dropped, and set to DISABLED
		condition = nodes.DRAINING
	}

	log.Printf("Telling %v on port %v to be %v\n", nodeAddress, nodePort, condition)

	if nodePtr != nil {
		log.Printf("Found existing node %v", *nodePtr)

		opts := nodes.UpdateOpts{
			Condition: condition,
		}

		updateResult := nodes.Update(client, loadBalancerID, nodePtr.ID, opts)
		err = updateResult.ExtractErr()
		if err != nil {
			log.Panicf("Updating node failed: %v", err)
		}

	} else {
		log.Printf("Creating new node")
		opts := nodes.CreateOpts{
			nodes.CreateOpt{
				Address:   nodeAddress,
				Port:      nodePort,
				Condition: condition,
			},
		}

		nodePager := nodes.Create(client, loadBalancerID, opts)
		nodeList, err := nodePager.ExtractNodes()
		if err != nil || len(nodeList) != 1 {
			log.Panicf("Something went terribly wrong on node creation: %v\n", nodeList)
		}
		nodePtr = &nodeList[0]
	}

	waitForReady(client, loadBalancerID)

	// After update, get the version of the node Rackspace has
	result := nodes.Get(client, loadBalancerID, nodePtr.ID)
	nodePtr, err = result.Extract()
	if err != nil {
		log.Panicln(err)
	}

	log.Printf("Updated state for node: %v\n", *nodePtr)

}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"strconv"
	"strings"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/pagination"
	"github.com/rackspace/gophercloud/rackspace"
	"github.com/rackspace/gophercloud/rackspace/lb/v1/lbs"
	"github.com/rackspace/gophercloud/rackspace/lb/v1/nodes"
)

func waitForReady(client *gophercloud.ServiceClient, loadBalancerID int) {

	// Ensure the load balancer is ready
	gophercloud.WaitFor(60, func() (bool, error) {
		lb, err := lbs.Get(client, loadBalancerID).Extract()
		if err != nil {
			return false, err
		}

		if lb.Status != lbs.ACTIVE {
			// It has not yet reached ACTIVE
			return false, nil
		}

		// It has reached ACTIVE
		return true, nil
	})
}

func findNodeByIPPort(
	client *gophercloud.ServiceClient,
	loadBalancerID int,
	address string,
	port int,
) *nodes.Node {

	// nil until found
	var found *nodes.Node

	pager := nodes.List(client, loadBalancerID, nil)

	pager.EachPage(func(page pagination.Page) (bool, error) {
		lbNodes, err := nodes.ExtractNodes(page)
		if err != nil {
			log.Fatalf("Error during paging load balancer: %v", err)
		}

		for _, trialNode := range lbNodes {
			if trialNode.Address == address && trialNode.Port == port {
				found = &trialNode
				return false, nil
			}

		}

		return true, nil
	})

	return found
}

func getIP(ipPtr *string) (string, error) {

	// IP overriding order will be:
	//  - The -ip flag takes primary precedence
	//  - Service Net Environment variable
	//  - Public Net Environment variable
	//  - Gleaning a 10 dot out of the network interfaces (typically eth1)
	//  - eth0

	serviceNetIPv4 := os.Getenv("RAX_SERVICENET_IPV4")
	publicNetIPv4 := os.Getenv("RAX_PUBLICNET_IPV4")

	switch {
	case *ipPtr != "":
		return *ipPtr, nil
	case serviceNetIPv4 != "":
		return serviceNetIPv4, nil
	case publicNetIPv4 != "":
		return publicNetIPv4, nil
	}

	addrs, err := net.InterfaceAddrs()

	if err != nil {
		return "", err
	}

	for _, addr := range addrs {
		cidr := addr.String()
		ip := strings.Split(cidr, "/")[0]

		if strings.HasPrefix(ip, "10.") {
			return ip, nil
		}
	}

	// Find eth0
	eth0, err := net.InterfaceByName("eth0")
	if err != nil {
		return "", fmt.Errorf("Trouble finding eth0: %v", err)
	}

	addrs, err = eth0.Addrs()
	if err != nil {
		return "", err
	}

	for _, addr := range addrs {
		cidr := addr.String()
		ip := strings.Split(cidr, "/")[0]

		fmt.Println(ip)
	}

	return "", errors.New("Unable to determine an IP for load balancing")

}

func main() {
	var err error

	disabledPtr := flag.Bool("disable", false, "Disable the node on the load balancer")

	//NOTE: peekaboo allows setting the IP by using
	//        - environment variables: RAX_SERVICENET_IPV4 or RAX_PUBLICNET_IPV4
	//        - finding an ip on the system starting with 10. (service net)
	//        - locating the eth0 interface
	//        - providing an ip as a flag is fine too and will take precedence
	ipPtr := flag.String("ip", "", "IP address to register/deregister on the load balancer")
	flag.Parse()

	username := os.Getenv("OS_USERNAME")
	APIKey := os.Getenv("OS_PASSWORD")
	region := os.Getenv("OS_REGION_NAME")

	if username == "" || APIKey == "" || region == "" {
		fmt.Fatalf("One or more of $OS_USERNAME, $OS_PASSWORD, and $OS_REGION_NAME not set")
	}

	// These get converted into integers later
	strLoadBalancerID := os.Getenv("LOAD_BALANCER_ID")
	strAppPort := os.Getenv("APP_PORT")

	/**
	 * Retrieve port for load balancer's node, defaulting to 80
	 */
	var nodePort = 80

	if strAppPort == "" {
		log.Printf("$APP_PORT not set, defaulting to 80")
	} else {
		nodePort, err = strconv.Atoi(strAppPort)
		if err != nil {
			log.Fatalf("Unable to parse integer from $APP_PORT: %v\n", strAppPort)
		}
	}

	/**
	 * Determine the IP Address for the load balancer's node
	 */
	nodeAddress, err := getIP(ipPtr)
	if err != nil {
		log.Fatalln(err)
	}

	log.Printf("Determined IP: %v", nodeAddress)
	log.Printf("Determined Port: %v", nodePort)

	/**
	 * Retrive Load Balancer ID
	 */
	if strLoadBalancerID == "" {
		log.Fatalln("$LOAD_BALANCER_ID must be set")
	}
	loadBalancerID, err := strconv.Atoi(strLoadBalancerID)
	if err != nil {
		log.Fatalf("$LOAD_BALANCER_ID not an integer: %v\n", loadBalancerID)
	}

	provider, err := rackspace.AuthenticatedClient(gophercloud.AuthOptions{
		Username: username,
		APIKey:   APIKey,
	})

	if err != nil {
		log.Fatalf("Trouble authenticating to Rackspace: %v\n", err)
	}

	client, err := rackspace.NewLBV1(provider, gophercloud.EndpointOpts{
		Region: region,
	})

	if err != nil {
		log.Fatalf("Creating load balancer client in %v failed: %v\n", region, err)
	}

	log.Println("Client ready")

	waitForReady(client, loadBalancerID)

	nodePtr := findNodeByIPPort(client, loadBalancerID, nodeAddress, nodePort)

	condition := nodes.ENABLED
	if *disabledPtr {
		//TODO: Watch the interface on the right process/container to determine
		//      when connections have dropped, and set to DISABLED
		condition = nodes.DRAINING
	}

	log.Printf("Telling %v on port %v to be %v\n", nodeAddress, nodePort, condition)

	if nodePtr != nil {
		log.Printf("Found existing node %v", *nodePtr)

		opts := nodes.UpdateOpts{
			Condition: condition,
		}

		updateResult := nodes.Update(client, loadBalancerID, nodePtr.ID, opts)
		err = updateResult.ExtractErr()
		if err != nil {
			log.Panicf("Updating node failed: %v", err)
		}

	} else {
		log.Printf("Creating new node")
		opts := nodes.CreateOpts{
			nodes.CreateOpt{
				Address:   nodeAddress,
				Port:      nodePort,
				Condition: condition,
			},
		}

		nodePager := nodes.Create(client, loadBalancerID, opts)
		nodeList, err := nodePager.ExtractNodes()
		if err != nil || len(nodeList) != 1 {
			log.Panicf("Something went terribly wrong on node creation: %v\n", nodeList)
		}
		nodePtr = &nodeList[0]
	}

	waitForReady(client, loadBalancerID)

	// After update, get the version of the node Rackspace has
	result := nodes.Get(client, loadBalancerID, nodePtr.ID)
	nodePtr, err = result.Extract()
	if err != nil {
		log.Panicln(err)
	}

	log.Printf("Updated state for node: %v\n", *nodePtr)

}
<commit_msg>Linter told me not to capitalize created errors<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"strconv"
	"strings"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/pagination"
	"github.com/rackspace/gophercloud/rackspace"
	"github.com/rackspace/gophercloud/rackspace/lb/v1/lbs"
	"github.com/rackspace/gophercloud/rackspace/lb/v1/nodes"
)

func waitForReady(client *gophercloud.ServiceClient, loadBalancerID int) {

	// Ensure the load balancer is ready
	gophercloud.WaitFor(60, func() (bool, error) {
		lb, err := lbs.Get(client, loadBalancerID).Extract()
		if err != nil {
			return false, err
		}

		if lb.Status != lbs.ACTIVE {
			// It has not yet reached ACTIVE
			return false, nil
		}

		// It has reached ACTIVE
		return true, nil
	})
}

func findNodeByIPPort(
	client *gophercloud.ServiceClient,
	loadBalancerID int,
	address string,
	port int,
) *nodes.Node {

	// nil until found
	var found *nodes.Node

	pager := nodes.List(client, loadBalancerID, nil)

	pager.EachPage(func(page pagination.Page) (bool, error) {
		lbNodes, err := nodes.ExtractNodes(page)
		if err != nil {
			log.Fatalf("Error during paging load balancer: %v", err)
		}

		for _, trialNode := range lbNodes {
			if trialNode.Address == address && trialNode.Port == port {
				found = &trialNode
				return false, nil
			}

		}

		return true, nil
	})

	return found
}

func getIP(ipPtr *string) (string, error) {

	// IP overriding order will be:
	//  - The -ip flag takes primary precedence
	//  - Service Net Environment variable
	//  - Public Net Environment variable
	//  - Gleaning a 10 dot out of the network interfaces (typically eth1)
	//  - eth0

	serviceNetIPv4 := os.Getenv("RAX_SERVICENET_IPV4")
	publicNetIPv4 := os.Getenv("RAX_PUBLICNET_IPV4")

	switch {
	case *ipPtr != "":
		return *ipPtr, nil
	case serviceNetIPv4 != "":
		return serviceNetIPv4, nil
	case publicNetIPv4 != "":
		return publicNetIPv4, nil
	}

	addrs, err := net.InterfaceAddrs()

	if err != nil {
		return "", err
	}

	for _, addr := range addrs {
		cidr := addr.String()
		ip := strings.Split(cidr, "/")[0]

		if strings.HasPrefix(ip, "10.") {
			return ip, nil
		}
	}

	// Find eth0
	eth0, err := net.InterfaceByName("eth0")
	if err != nil {
		return "", fmt.Errorf("trouble finding eth0: %v", err)
	}

	addrs, err = eth0.Addrs()
	if err != nil {
		return "", err
	}

	for _, addr := range addrs {
		cidr := addr.String()
		ip := strings.Split(cidr, "/")[0]

		fmt.Println(ip)
	}

	return "", errors.New("unable to determine an IP for load balancing")

}

func main() {
	var err error

	disabledPtr := flag.Bool("disable", false, "Disable the node on the load balancer")

	//NOTE: peekaboo allows setting the IP by using
	//        - environment variables: RAX_SERVICENET_IPV4 or RAX_PUBLICNET_IPV4
	//        - finding an ip on the system starting with 10. (service net)
	//        - locating the eth0 interface
	//        - providing an ip as a flag is fine too and will take precedence
	ipPtr := flag.String("ip", "", "IP address to register/deregister on the load balancer")
	flag.Parse()

	username := os.Getenv("OS_USERNAME")
	APIKey := os.Getenv("OS_PASSWORD")
	region := os.Getenv("OS_REGION_NAME")

	if username == "" || APIKey == "" || region == "" {
		fmt.Fatalf("One or more of $OS_USERNAME, $OS_PASSWORD, and $OS_REGION_NAME not set")
	}

	// These get converted into integers later
	strLoadBalancerID := os.Getenv("LOAD_BALANCER_ID")
	strAppPort := os.Getenv("APP_PORT")

	/**
	 * Retrieve port for load balancer's node, defaulting to 80
	 */
	var nodePort = 80

	if strAppPort == "" {
		log.Printf("$APP_PORT not set, defaulting to 80")
	} else {
		nodePort, err = strconv.Atoi(strAppPort)
		if err != nil {
			log.Fatalf("Unable to parse integer from $APP_PORT: %v\n", strAppPort)
		}
	}

	/**
	 * Determine the IP Address for the load balancer's node
	 */
	nodeAddress, err := getIP(ipPtr)
	if err != nil {
		log.Fatalln(err)
	}

	log.Printf("Determined IP: %v", nodeAddress)
	log.Printf("Determined Port: %v", nodePort)

	/**
	 * Retrive Load Balancer ID
	 */
	if strLoadBalancerID == "" {
		log.Fatalln("$LOAD_BALANCER_ID must be set")
	}
	loadBalancerID, err := strconv.Atoi(strLoadBalancerID)
	if err != nil {
		log.Fatalf("$LOAD_BALANCER_ID not an integer: %v\n", loadBalancerID)
	}

	provider, err := rackspace.AuthenticatedClient(gophercloud.AuthOptions{
		Username: username,
		APIKey:   APIKey,
	})

	if err != nil {
		log.Fatalf("Trouble authenticating to Rackspace: %v\n", err)
	}

	client, err := rackspace.NewLBV1(provider, gophercloud.EndpointOpts{
		Region: region,
	})

	if err != nil {
		log.Fatalf("Creating load balancer client in %v failed: %v\n", region, err)
	}

	log.Println("Client ready")

	waitForReady(client, loadBalancerID)

	nodePtr := findNodeByIPPort(client, loadBalancerID, nodeAddress, nodePort)

	condition := nodes.ENABLED
	if *disabledPtr {
		//TODO: Watch the interface on the right process/container to determine
		//      when connections have dropped, and set to DISABLED
		condition = nodes.DRAINING
	}

	log.Printf("Telling %v on port %v to be %v\n", nodeAddress, nodePort, condition)

	if nodePtr != nil {
		log.Printf("Found existing node %v", *nodePtr)

		opts := nodes.UpdateOpts{
			Condition: condition,
		}

		updateResult := nodes.Update(client, loadBalancerID, nodePtr.ID, opts)
		err = updateResult.ExtractErr()
		if err != nil {
			log.Panicf("Updating node failed: %v", err)
		}

	} else {
		log.Printf("Creating new node")
		opts := nodes.CreateOpts{
			nodes.CreateOpt{
				Address:   nodeAddress,
				Port:      nodePort,
				Condition: condition,
			},
		}

		nodePager := nodes.Create(client, loadBalancerID, opts)
		nodeList, err := nodePager.ExtractNodes()
		if err != nil || len(nodeList) != 1 {
			log.Panicf("Something went terribly wrong on node creation: %v\n", nodeList)
		}
		nodePtr = &nodeList[0]
	}

	waitForReady(client, loadBalancerID)

	// After update, get the version of the node Rackspace has
	result := nodes.Get(client, loadBalancerID, nodePtr.ID)
	nodePtr, err = result.Extract()
	if err != nil {
		log.Panicln(err)
	}

	log.Printf("Updated state for node: %v\n", *nodePtr)

}
<|endoftext|>