<commit_before>package users

import (
	"errors"

	"github.com/racker/perigee"
	"github.com/rackspace/gophercloud"
	os "github.com/rackspace/gophercloud/openstack/identity/v2/users"
	"github.com/rackspace/gophercloud/pagination"
)

// List returns a pager that allows traversal over a collection of users.
func List(client *gophercloud.ServiceClient) pagination.Pager {
	return os.List(client)
}

type commonOpts struct {
	// Required. The username to assign to the user. When provided, the username
	// must:
	// - start with an alphabetical (A-Za-z) character
	// - have a minimum length of 1 character
	//
	// The username may contain upper and lowercase characters, as well as any of
	// the following special character: . - @ _
	Username string

	// Required. Email address for the user account.
	Email string

	// Required. Indicates whether the user can authenticate after the user
	// account is created. If no value is specified, the default value is true.
	Enabled os.EnabledState

	// Optional. The password to assign to the user. If provided, the password
	// must:
	// - start with an alphabetical (A-Za-z) character
	// - have a minimum length of 8 characters
	// - contain at least one uppercase character, one lowercase character, and
	//   one numeric character.
	//
	// The password may contain any of the following special characters: . - @ _
	Password string
}

// CreateOpts represents the options needed when creating new users.
type CreateOpts commonOpts

// ToUserCreateMap assembles a request body based on the contents of a CreateOpts.
func (opts CreateOpts) ToUserCreateMap() (map[string]interface{}, error) {
	m := make(map[string]interface{})

	if opts.Username == "" {
		return m, errors.New("Username is a required field")
	}
	if opts.Enabled == nil {
		return m, errors.New("Enabled is a required field")
	}
	if opts.Email == "" {
		return m, errors.New("Email is a required field")
	}

	if opts.Username != "" {
		m["username"] = opts.Username
	}
	if opts.Email != "" {
		m["email"] = opts.Email
	}
	if opts.Enabled != nil {
		m["enabled"] = opts.Enabled
	}
	if opts.Password != "" {
		m["OS-KSADM:password"] = opts.Password
	}

	return map[string]interface{}{"user": m}, nil
}

// Create is the operation responsible for creating new users.
func Create(client *gophercloud.ServiceClient, opts os.CreateOptsBuilder) CreateResult {
	return CreateResult{os.Create(client, opts)}
}

// Get requests details on a single user, either by ID.
func Get(client *gophercloud.ServiceClient, id string) GetResult {
	return GetResult{os.Get(client, id)}
}

// UpdateOptsBuilder allows extentions to add additional attributes to the Update request.
type UpdateOptsBuilder interface {
	ToUserUpdateMap() map[string]interface{}
}

// UpdateOpts specifies the base attributes that may be updated on an existing server.
type UpdateOpts commonOpts

// ToUserUpdateMap formats an UpdateOpts structure into a request body.
func (opts UpdateOpts) ToUserUpdateMap() map[string]interface{} {
	m := make(map[string]interface{})

	if opts.Username != "" {
		m["username"] = opts.Username
	}
	if opts.Enabled != nil {
		m["enabled"] = &opts.Enabled
	}
	if opts.Email != "" {
		m["email"] = opts.Email
	}

	return map[string]interface{}{"user": m}
}

// Update is the operation responsible for updating exist users by their UUID.
func Update(client *gophercloud.ServiceClient, id string, opts UpdateOptsBuilder) UpdateResult {
	var result UpdateResult

	_, result.Err = perigee.Request("POST", os.ResourceURL(client, id), perigee.Options{
		Results:     &result.Body,
		ReqBody:     opts.ToUserUpdateMap(),
		MoreHeaders: client.AuthenticatedHeaders(),
		OkCodes:     []int{200},
	})

	return result
}

// Delete is the operation responsible for permanently deleting an API user.
func Delete(client *gophercloud.ServiceClient, id string) os.DeleteResult {
	return os.Delete(client, id)
}
<commit_msg>Expose this commonOpts as well<commit_after>package users

import (
	"errors"

	"github.com/racker/perigee"
	"github.com/rackspace/gophercloud"
	os "github.com/rackspace/gophercloud/openstack/identity/v2/users"
	"github.com/rackspace/gophercloud/pagination"
)

// List returns a pager that allows traversal over a collection of users.
func List(client *gophercloud.ServiceClient) pagination.Pager {
	return os.List(client)
}

// CommonOpts are the options which are shared between CreateOpts and
// UpdateOpts
type CommonOpts struct {
	// Required. The username to assign to the user. When provided, the username
	// must:
	// - start with an alphabetical (A-Za-z) character
	// - have a minimum length of 1 character
	//
	// The username may contain upper and lowercase characters, as well as any of
	// the following special character: . - @ _
	Username string

	// Required. Email address for the user account.
	Email string

	// Required. Indicates whether the user can authenticate after the user
	// account is created. If no value is specified, the default value is true.
	Enabled os.EnabledState

	// Optional. The password to assign to the user. If provided, the password
	// must:
	// - start with an alphabetical (A-Za-z) character
	// - have a minimum length of 8 characters
	// - contain at least one uppercase character, one lowercase character, and
	//   one numeric character.
	//
	// The password may contain any of the following special characters: . - @ _
	Password string
}

// CreateOpts represents the options needed when creating new users.
type CreateOpts CommonOpts

// ToUserCreateMap assembles a request body based on the contents of a CreateOpts.
func (opts CreateOpts) ToUserCreateMap() (map[string]interface{}, error) {
	m := make(map[string]interface{})

	if opts.Username == "" {
		return m, errors.New("Username is a required field")
	}
	if opts.Enabled == nil {
		return m, errors.New("Enabled is a required field")
	}
	if opts.Email == "" {
		return m, errors.New("Email is a required field")
	}

	if opts.Username != "" {
		m["username"] = opts.Username
	}
	if opts.Email != "" {
		m["email"] = opts.Email
	}
	if opts.Enabled != nil {
		m["enabled"] = opts.Enabled
	}
	if opts.Password != "" {
		m["OS-KSADM:password"] = opts.Password
	}

	return map[string]interface{}{"user": m}, nil
}

// Create is the operation responsible for creating new users.
func Create(client *gophercloud.ServiceClient, opts os.CreateOptsBuilder) CreateResult {
	return CreateResult{os.Create(client, opts)}
}

// Get requests details on a single user, either by ID.
func Get(client *gophercloud.ServiceClient, id string) GetResult {
	return GetResult{os.Get(client, id)}
}

// UpdateOptsBuilder allows extentions to add additional attributes to the Update request.
type UpdateOptsBuilder interface {
	ToUserUpdateMap() map[string]interface{}
}

// UpdateOpts specifies the base attributes that may be updated on an existing server.
type UpdateOpts CommonOpts

// ToUserUpdateMap formats an UpdateOpts structure into a request body.
func (opts UpdateOpts) ToUserUpdateMap() map[string]interface{} {
	m := make(map[string]interface{})

	if opts.Username != "" {
		m["username"] = opts.Username
	}
	if opts.Enabled != nil {
		m["enabled"] = &opts.Enabled
	}
	if opts.Email != "" {
		m["email"] = opts.Email
	}

	return map[string]interface{}{"user": m}
}

// Update is the operation responsible for updating exist users by their UUID.
func Update(client *gophercloud.ServiceClient, id string, opts UpdateOptsBuilder) UpdateResult {
	var result UpdateResult

	_, result.Err = perigee.Request("POST", os.ResourceURL(client, id), perigee.Options{
		Results:     &result.Body,
		ReqBody:     opts.ToUserUpdateMap(),
		MoreHeaders: client.AuthenticatedHeaders(),
		OkCodes:     []int{200},
	})

	return result
}

// Delete is the operation responsible for permanently deleting an API user.
func Delete(client *gophercloud.ServiceClient, id string) os.DeleteResult {
	return os.Delete(client, id)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"fmt"
)

type Synchronizer struct {
	Storage *Storage
}

func NewSynchronizer(storage *Storage) *Synchronizer {
	s := new(Synchronizer)
	s.Storage = storage

	return s
}

func (synchronizer *Synchronizer) Start() {
	provider := NewMattKetmoProvider()
	domains, _ := provider.Get()

	for _, domain := range domains {
		err := synchronizer.Storage.Add(domain)
		fmt.Println(domain)

		if err != nil {
			log.Fatal(err)
		}
	}

	fmt.Println("done")
}<commit_msg>Add function for adding domains<commit_after>package main

import (
	"log"
	"fmt"
)

type Synchronizer struct {
	Storage *Storage
}

func NewSynchronizer(storage *Storage) *Synchronizer {
	synchronizer := new(Synchronizer)
	synchronizer.Storage = storage

	return synchronizer
}

func (synchronizer *Synchronizer) Start() {
	provider := NewMattKetmoProvider()
	domains, _ := provider.Get()
	synchronizer.addDomains(domains)

	fmt.Println("done")
}

func (synchronizer *Synchronizer) addDomains(domains []string) {
	for _, domain := range domains {
		err := synchronizer.Storage.Add(domain)
		fmt.Println(domain)

		if err != nil {
			log.Fatal(err)
		}
	}
}<|endoftext|>


<commit_before><commit_msg>base32: remove unnecessary "if" condition<commit_after><|endoftext|>


<commit_before>package scroll

import (
	"net/http"
	"regexp"
	"strconv"
	"time"

	"github.com/mailgun/log"
)

// Retrieve a POST request field as a string.
// Returns `MissingFieldError` if requested field is missing.
func GetStringField(r *http.Request, fieldName string) (string, error) {
	if _, ok := r.Form[fieldName]; !ok {
		return "", MissingFieldError{fieldName}
	}
	return r.FormValue(fieldName), nil
}

// Retrieves requested field as a string, allowSet provides input sanitization. If an
// error occurs, returns either a `MissingFieldError` or an `UnsafeFieldError`.
func GetStringFieldSafe(r *http.Request, fieldName string, allowSet AllowSet) (string, error) {
	if _, ok := r.Form[fieldName]; !ok {
		return "", MissingFieldError{fieldName}
	}

	fieldValue := r.FormValue(fieldName)
	err := allowSet.IsSafe(fieldValue)
	if err != nil {
		return "", UnsafeFieldError{fieldName, err.Error()}
	}

	return fieldValue, nil
}

// Retrieve a POST request field as a string.
// If the requested field is missing, returns provided default value.
func GetStringFieldWithDefault(r *http.Request, fieldName, defaultValue string) string {
	if fieldValue, err := GetStringField(r, fieldName); err == nil {
		return fieldValue
	}
	return defaultValue
}

// A multiParamRegex is used to convert Ruby and PHP style array params.
// PHP uses ["param[0]", "param[1]",..] instead of ["param", "param",..]
// Ruby uses ["param[]", "param[]",..]
var multiParamRegex *regexp.Regexp

func init() {
	multiParamRegex = regexp.MustCompile(`^([a-z:]*)\[\d*\]$`)
}

// Retrieve fields with the same name as an array of strings.
func GetMultipleFields(r *http.Request, fieldName string) ([]string, error) {
	var values = []string{}

	for field, value := range r.Form {
		// Strip the square brackets.
		if multiParamRegex.ReplaceAllString(field, "$1") == fieldName {
			values = append(values, value...)
		}
	}

	if len(values) == 0 {
		return []string{}, MissingFieldError{fieldName}
	}

	return values, nil
}

// Retrieve a POST request field as an integer.
// Returns `MissingFieldError` if requested field is missing.
func GetIntField(r *http.Request, fieldName string) (int, error) {
	stringField, err := GetStringField(r, fieldName)
	if err != nil {
		return 0, err
	}
	intField, err := strconv.Atoi(stringField)
	if err != nil {
		return 0, InvalidFormatError{fieldName, stringField}
	}
	return intField, nil
}

// Retrieve a request field as a float.
// Returns `MissingFieldError` if requested field is missing.
func GetFloatField(r *http.Request, fieldName string) (float64, error) {
	stringField, err := GetStringField(r, fieldName)
	if err != nil {
		return 0, err
	}
	floatField, err := strconv.ParseFloat(stringField, 64)
	if err != nil {
		return 0, InvalidFormatError{fieldName, stringField}
	}
	return floatField, nil
}

// Helper method to retrieve an optional timestamp from POST request field.
// If no timestamp provided, returns current time.
// Returns `InvalidFormatError` if provided timestamp can't be parsed.
func GetTimestampField(r *http.Request, fieldName string) (time.Time, error) {
	if _, ok := r.Form[fieldName]; !ok {
		return time.Now(), MissingFieldError{fieldName}
	}
	parsedTime, err := time.Parse(time.RFC1123, r.FormValue(fieldName))
	if err != nil {
		log.Infof("Failed to convert timestamp %v: %v", r.FormValue(fieldName), err)
		return time.Now(), InvalidFormatError{fieldName, r.FormValue(fieldName)}
	}
	return parsedTime, nil
}

// GetDurationField retrieves a request field as a time.Duration, which is not allowed to be negative.
// Returns `MissingFieldError` if requested field is missing.
func GetDurationField(r *http.Request, fieldName string) (time.Duration, error) {
	s, err := GetStringField(r, fieldName)
	if err != nil {
		return 0, err
	}
	d, err := time.ParseDuration(s)
	if err != nil || d < 0 {
		return 0, InvalidFormatError{fieldName, s}
	}
	return d, nil
}

func HasField(r *http.Request, fieldName string) bool {
	if _, ok := r.Form[fieldName]; !ok {
		return false
	}
	return true
}
<commit_msg>Added support for timestamps with an numeric offset<commit_after>package scroll

import (
	"net/http"
	"regexp"
	"strconv"
	"time"

	"github.com/mailgun/log"
)

// Retrieve a POST request field as a string.
// Returns `MissingFieldError` if requested field is missing.
func GetStringField(r *http.Request, fieldName string) (string, error) {
	if _, ok := r.Form[fieldName]; !ok {
		return "", MissingFieldError{fieldName}
	}
	return r.FormValue(fieldName), nil
}

// Retrieves requested field as a string, allowSet provides input sanitization. If an
// error occurs, returns either a `MissingFieldError` or an `UnsafeFieldError`.
func GetStringFieldSafe(r *http.Request, fieldName string, allowSet AllowSet) (string, error) {
	if _, ok := r.Form[fieldName]; !ok {
		return "", MissingFieldError{fieldName}
	}

	fieldValue := r.FormValue(fieldName)
	err := allowSet.IsSafe(fieldValue)
	if err != nil {
		return "", UnsafeFieldError{fieldName, err.Error()}
	}

	return fieldValue, nil
}

// Retrieve a POST request field as a string.
// If the requested field is missing, returns provided default value.
func GetStringFieldWithDefault(r *http.Request, fieldName, defaultValue string) string {
	if fieldValue, err := GetStringField(r, fieldName); err == nil {
		return fieldValue
	}
	return defaultValue
}

// A multiParamRegex is used to convert Ruby and PHP style array params.
// PHP uses ["param[0]", "param[1]",..] instead of ["param", "param",..]
// Ruby uses ["param[]", "param[]",..]
var multiParamRegex *regexp.Regexp

func init() {
	multiParamRegex = regexp.MustCompile(`^([a-z:]*)\[\d*\]$`)
}

// Retrieve fields with the same name as an array of strings.
func GetMultipleFields(r *http.Request, fieldName string) ([]string, error) {
	var values = []string{}

	for field, value := range r.Form {
		// Strip the square brackets.
		if multiParamRegex.ReplaceAllString(field, "$1") == fieldName {
			values = append(values, value...)
		}
	}

	if len(values) == 0 {
		return []string{}, MissingFieldError{fieldName}
	}

	return values, nil
}

// Retrieve a POST request field as an integer.
// Returns `MissingFieldError` if requested field is missing.
func GetIntField(r *http.Request, fieldName string) (int, error) {
	stringField, err := GetStringField(r, fieldName)
	if err != nil {
		return 0, err
	}
	intField, err := strconv.Atoi(stringField)
	if err != nil {
		return 0, InvalidFormatError{fieldName, stringField}
	}
	return intField, nil
}

// Retrieve a request field as a float.
// Returns `MissingFieldError` if requested field is missing.
func GetFloatField(r *http.Request, fieldName string) (float64, error) {
	stringField, err := GetStringField(r, fieldName)
	if err != nil {
		return 0, err
	}
	floatField, err := strconv.ParseFloat(stringField, 64)
	if err != nil {
		return 0, InvalidFormatError{fieldName, stringField}
	}
	return floatField, nil
}

// Helper method to retrieve an optional timestamp from POST request field.
// If no timestamp provided, returns current time.
// Returns `InvalidFormatError` if provided timestamp can't be parsed.
func GetTimestampField(r *http.Request, fieldName string) (time.Time, error) {
	if _, ok := r.Form[fieldName]; !ok {
		return time.Now(), MissingFieldError{fieldName}
	}
	parsedTime, err := time.Parse(time.RFC1123, r.FormValue(fieldName))
	if err != nil {
		// Attempt to parse with offset timezone before giving up
		parsedTime, err = time.Parse(time.RFC1123Z, r.FormValue(fieldName))
		if err != nil {
			log.Infof("Failed to convert timestamp %v: %v", r.FormValue(fieldName), err)
			return time.Now(), InvalidFormatError{fieldName, r.FormValue(fieldName)}
		}
	}
	return parsedTime, nil
}

// GetDurationField retrieves a request field as a time.Duration, which is not allowed to be negative.
// Returns `MissingFieldError` if requested field is missing.
func GetDurationField(r *http.Request, fieldName string) (time.Duration, error) {
	s, err := GetStringField(r, fieldName)
	if err != nil {
		return 0, err
	}
	d, err := time.ParseDuration(s)
	if err != nil || d < 0 {
		return 0, InvalidFormatError{fieldName, s}
	}
	return d, nil
}

func HasField(r *http.Request, fieldName string) bool {
	if _, ok := r.Form[fieldName]; !ok {
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package controllers

import (
	"net/http"

	"github.com/anonx/sunplate/action"

	"github.com/revel/revel/testing"
)

// App is a sample controller.
type App struct {
	*Controller
	*NotController
	*NotController1
}

// NotController is not a controller as it doesn't have methods.
type NotController struct {
}

// NotController1 does have methods but it is still not a controller
// as there are not actions among those methods.
type NotController1 struct {
}

func (c NotController1) test() {
}

// HelloWorld is a sample action.
func (c App) HelloWorld(page int) action.Result {
	return nil
}

// Initially is a magic method that is executed before every request.
func (c *Controller) Initially(w http.ResponseWriter, r *http.Request) bool {
	return false
}

// Finally is a magic method that is executed after every request.
func (c *Controller) Finally(w http.ResponseWriter, r *http.Request) bool {
	return false
}

// UnsupportedAction is not an action as it requires argument that is not
// of builtin type.
func (c Controller) UnsupportedAction(t testing.TestSuite) action.Result {
	return nil
}

// Smth is not an action as it returns nothing.
func (c App) Smth() {
}

func init() {
}
<commit_msg>Added an action to tests that uses reserved word<commit_after>package controllers

import (
	"net/http"

	"github.com/anonx/sunplate/action"

	"github.com/revel/revel/testing"
)

// App is a sample controller.
type App struct {
	*Controller
	*NotController
	*NotController1
}

// NotController is not a controller as it doesn't have methods.
type NotController struct {
}

// NotController1 does have methods but it is still not a controller
// as there are not actions among those methods.
type NotController1 struct {
}

func (c NotController1) test() {
}

// HelloWorld is a sample action.
func (c App) HelloWorld(page int) action.Result {
	return nil
}

// Initially is a magic method that is executed before every request.
func (c *Controller) Initially(w http.ResponseWriter, r *http.Request) bool {
	return false
}

// Finally is a magic method that is executed after every request.
func (c *Controller) Finally(w http.ResponseWriter, r *http.Request) bool {
	return false
}

// UnsupportedAction is not an action as it requires argument that is not
// of builtin type.
func (c Controller) UnsupportedAction(t testing.TestSuite) action.Result {
	return nil
}

// Smth is not an action as it returns nothing.
func (c App) Smth() {
}

// Finally should be ignored as it is using a reserved word for its name.
// The method signature is expected to be
// (c App) Finally(http.ResponseWriter, *http.Request) bool.
func (c App) Finally(smth string) action.Result {
	return nil
}

func init() {
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"io"
	"log"
	"regexp"
	"strings"
)

var fillinPattern = regexp.MustCompile(`{{[A-Za-z][-0-9A-Za-z_]*(:[A-Za-z][-0-9A-Za-z_]*)?}}`)

func collectIdentifiers(args []string) []*Identifier {
	var identifiers []*Identifier
	for _, arg := range args {
		matches := fillinPattern.FindAllString(arg, -1)
		for _, match := range matches {
			identifiers = append(identifiers, identifierFromMatch(match))
		}
	}
	return identifiers
}

// Fillin fills in the command arguments
func Fillin(args []string, r io.Reader, w io.Writer, in *bufio.Reader, out *bufio.Writer) []string {
	ret := make([]string, len(args))
	config, err := ReadConfig(r)
	if err != nil {
		log.Fatal(err)
	}
	if config.Scopes == nil {
		config.Scopes = make(map[string]*Scope)
	}
	values := Resolve(collectIdentifiers(args), config, in, out)
	if !empty(values) {
		insertValues(config.Scopes, values)
	}
	if err := WriteConfig(w, config); err != nil {
		log.Fatal(err)
	}
	for i, arg := range args {
		ret[i] = fillinPattern.ReplaceAllStringFunc(arg, func(match string) string {
			return lookup(values, identifierFromMatch(match))
		})
	}
	return ret
}

func identifierFromMatch(match string) *Identifier {
	match = match[2 : len(match)-2]
	var scope, key string
	if strings.ContainsRune(match, ':') {
		xs := strings.Split(match, ":")
		scope = strings.TrimSpace(xs[0])
		key = strings.TrimSpace(xs[1])
	} else {
		key = strings.TrimSpace(match)
	}
	return &Identifier{scope, key}
}

func insertValues(scopes map[string]*Scope, values map[string]map[string]string) {
	for scope := range values {
		if _, ok := scopes[scope]; !ok {
			scopes[scope] = &Scope{}
		}
		newValues := make([]map[string]string, 0)
		strs := make(map[string]bool)
		insert := func(v map[string]string) {
			s := stringifyValue(v)
			if _, ok := strs[s]; !ok {
				strs[s] = true
				newValues = append(newValues, v)
			}
		}
		insert(values[scope])
		for _, v := range scopes[scope].Values {
			insert(v)
		}
		scopes[scope].Values = newValues
	}
}
<commit_msg>update identifier pattern to support square brackets<commit_after>package main

import (
	"bufio"
	"io"
	"log"
	"regexp"
	"strings"
)

var fillinPattern = regexp.MustCompile(`{{[A-Za-z][-0-9A-Za-z_]*(:[A-Za-z][-0-9A-Za-z_]*)?}}|\[\[[A-Za-z][-0-9A-Za-z_]*(:[A-Za-z][-0-9A-Za-z_]*)?\]\]`)

func collectIdentifiers(args []string) []*Identifier {
	var identifiers []*Identifier
	for _, arg := range args {
		matches := fillinPattern.FindAllString(arg, -1)
		for _, match := range matches {
			identifiers = append(identifiers, identifierFromMatch(match))
		}
	}
	return identifiers
}

// Fillin fills in the command arguments
func Fillin(args []string, r io.Reader, w io.Writer, in *bufio.Reader, out *bufio.Writer) []string {
	ret := make([]string, len(args))
	config, err := ReadConfig(r)
	if err != nil {
		log.Fatal(err)
	}
	if config.Scopes == nil {
		config.Scopes = make(map[string]*Scope)
	}
	values := Resolve(collectIdentifiers(args), config, in, out)
	if !empty(values) {
		insertValues(config.Scopes, values)
	}
	if err := WriteConfig(w, config); err != nil {
		log.Fatal(err)
	}
	for i, arg := range args {
		ret[i] = fillinPattern.ReplaceAllStringFunc(arg, func(match string) string {
			return lookup(values, identifierFromMatch(match))
		})
	}
	return ret
}

func identifierFromMatch(match string) *Identifier {
	match = match[2 : len(match)-2]
	var scope, key string
	if strings.ContainsRune(match, ':') {
		xs := strings.Split(match, ":")
		scope = strings.TrimSpace(xs[0])
		key = strings.TrimSpace(xs[1])
	} else {
		key = strings.TrimSpace(match)
	}
	return &Identifier{scope, key}
}

func insertValues(scopes map[string]*Scope, values map[string]map[string]string) {
	for scope := range values {
		if _, ok := scopes[scope]; !ok {
			scopes[scope] = &Scope{}
		}
		newValues := make([]map[string]string, 0)
		strs := make(map[string]bool)
		insert := func(v map[string]string) {
			s := stringifyValue(v)
			if _, ok := strs[s]; !ok {
				strs[s] = true
				newValues = append(newValues, v)
			}
		}
		insert(values[scope])
		for _, v := range scopes[scope].Values {
			insert(v)
		}
		scopes[scope].Values = newValues
	}
}
<|endoftext|>


<commit_before>package consensus

import (
	"math"
	"math/big"
)

// A Currency represents a number of siacoins or siafunds. Internally, a
// Currency value is unbounded; however, Currency values sent over the wire
// protocol are subject to a maximum size of 255 bytes (approximately 10^614).
// Unlike the math/big library, whose methods modify their receiver, all
// arithmetic Currency methods return a new value. This is necessary to
// preserve the immutability of types containing Currency fields.
type Currency struct {
	i big.Int
}

func NewCurrency(b *big.Int) (c Currency) {
	c.i = *b
	return
}

func NewCurrency64(x uint64) (c Currency) {
	c.i.SetUint64(x)
	return
}

func (c Currency) Big() *big.Int {
	return &c.i
}

func (c Currency) Cmp(y Currency) int {
	return c.i.Cmp(&y.i)
}

func (c Currency) IsZero() bool {
	return c.i.Sign() == 0
}

func (c Currency) Add(x Currency) (y Currency) {
	y.i.Add(&c.i, &x.i)
	return
}

func (c Currency) Sub(x Currency) (y Currency) {
	y.i.Sub(&c.i, &x.i)
	return
}

func (c Currency) Mul(x Currency) (y Currency) {
	y.i.Mul(&c.i, &x.i)
	return
}

func (c Currency) Div(x Currency) (y Currency) {
	y.i.Div(&c.i, &x.i)
	return
}

func (c Currency) MulFloat(x float64) (y Currency) {
	yRat := new(big.Rat).Mul(
		new(big.Rat).SetInt(&c.i),
		new(big.Rat).SetFloat64(x),
	)
	y.i.Div(yRat.Num(), yRat.Denom())
	return
}

func (c Currency) Sqrt() (y Currency) {
	f, _ := new(big.Rat).SetInt(&c.i).Float64()
	sqrt := new(big.Rat).SetFloat64(math.Sqrt(f))
	y.i.Div(sqrt.Num(), sqrt.Denom())
	return
}

func (c Currency) RoundDown(nearest int64) error {
	round := big.NewInt(nearest)
	c.i.Div(&c.i, round)
	c.i.Mul(&c.i, round)
	return nil
}

// MarshalSia implements the encoding.SiaMarshaler interface. It returns the
// byte-slice representation of the Currency's internal big.Int, prepended
// with a single byte indicating the length of the slice. This implies a
// maximum encodable value of 2^(255 * 8), or approximately 10^614.
//
// Note that as the bytes of the big.Int correspond to the absolute value of
// the integer, there is no way to marshal a negative Currency.
func (c Currency) MarshalSia() []byte {
	b := c.i.Bytes()
	return append(
		[]byte{byte(len(b))},
		b...,
	)
}

// UnmarshalSia implements the encoding.SiaUnmarshaler interface. See
// MarshalSia for a description of how Currency values are marshalled.
func (c *Currency) UnmarshalSia(b []byte) int {
	var n int
	n, b = int(b[0]), b[1:]
	c.i.SetBytes(b[:n])
	return 1 + n
}
<commit_msg>implement RoundDown<commit_after>package consensus

import (
	"math"
	"math/big"
)

// A Currency represents a number of siacoins or siafunds. Internally, a
// Currency value is unbounded; however, Currency values sent over the wire
// protocol are subject to a maximum size of 255 bytes (approximately 10^614).
// Unlike the math/big library, whose methods modify their receiver, all
// arithmetic Currency methods return a new value. This is necessary to
// preserve the immutability of types containing Currency fields.
type Currency struct {
	i big.Int
}

func NewCurrency(b *big.Int) (c Currency) {
	c.i = *b
	return
}

func NewCurrency64(x uint64) (c Currency) {
	c.i.SetUint64(x)
	return
}

func (c Currency) Big() *big.Int {
	return &c.i
}

func (c Currency) Cmp(y Currency) int {
	return c.i.Cmp(&y.i)
}

func (c Currency) IsZero() bool {
	return c.i.Sign() == 0
}

func (c Currency) Add(x Currency) (y Currency) {
	y.i.Add(&c.i, &x.i)
	return
}

func (c Currency) Sub(x Currency) (y Currency) {
	y.i.Sub(&c.i, &x.i)
	return
}

func (c Currency) Mul(x Currency) (y Currency) {
	y.i.Mul(&c.i, &x.i)
	return
}

func (c Currency) Div(x Currency) (y Currency) {
	y.i.Div(&c.i, &x.i)
	return
}

func (c Currency) MulFloat(x float64) (y Currency) {
	yRat := new(big.Rat).Mul(
		new(big.Rat).SetInt(&c.i),
		new(big.Rat).SetFloat64(x),
	)
	y.i.Div(yRat.Num(), yRat.Denom())
	return
}

func (c Currency) Sqrt() (y Currency) {
	f, _ := new(big.Rat).SetInt(&c.i).Float64()
	sqrt := new(big.Rat).SetFloat64(math.Sqrt(f))
	y.i.Div(sqrt.Num(), sqrt.Denom())
	return
}

// RoundDown returns the largest multiple of n <= c.
func (c Currency) RoundDown(n uint64) (y Currency) {
	diff := new(big.Int).Mod(&c.i, new(big.Int).SetUint64(n))
	y.i.Sub(&c.i, diff)
	return
}

// MarshalSia implements the encoding.SiaMarshaler interface. It returns the
// byte-slice representation of the Currency's internal big.Int, prepended
// with a single byte indicating the length of the slice. This implies a
// maximum encodable value of 2^(255 * 8), or approximately 10^614.
//
// Note that as the bytes of the big.Int correspond to the absolute value of
// the integer, there is no way to marshal a negative Currency.
func (c Currency) MarshalSia() []byte {
	b := c.i.Bytes()
	return append(
		[]byte{byte(len(b))},
		b...,
	)
}

// UnmarshalSia implements the encoding.SiaUnmarshaler interface. See
// MarshalSia for a description of how Currency values are marshalled.
func (c *Currency) UnmarshalSia(b []byte) int {
	var n int
	n, b = int(b[0]), b[1:]
	c.i.SetBytes(b[:n])
	return 1 + n
}
<|endoftext|>


<commit_before>package consul

import (
	"fmt"
	"github.com/hashicorp/consul/consul/structs"
	"github.com/hashicorp/consul/testutil"
	"net"
	"os"
	"testing"
	"time"
)

func testClientConfig(t *testing.T, NodeName string) (string, *Config) {
	dir := tmpDir(t)
	config := DefaultConfig()
	config.Datacenter = "dc1"
	config.DataDir = dir
	config.NodeName = NodeName
	config.RPCAddr = &net.TCPAddr{
		IP:   []byte{127, 0, 0, 1},
		Port: getPort(),
	}
	config.SerfLANConfig.MemberlistConfig.BindAddr = "127.0.0.1"
	config.SerfLANConfig.MemberlistConfig.BindPort = getPort()
	config.SerfLANConfig.MemberlistConfig.ProbeTimeout = 200 * time.Millisecond
	config.SerfLANConfig.MemberlistConfig.ProbeInterval = time.Second
	config.SerfLANConfig.MemberlistConfig.GossipInterval = 100 * time.Millisecond

	return dir, config
}

func testClient(t *testing.T) (string, *Client) {
	return testClientDC(t, "dc1")
}

func testClientDC(t *testing.T, dc string) (string, *Client) {
	dir, config := testClientConfig(t, "testco.internal")
	config.Datacenter = dc

	client, err := NewClient(config)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	return dir, client
}

func TestClient_StartStop(t *testing.T) {
	dir, client := testClient(t)
	defer os.RemoveAll(dir)

	if err := client.Shutdown(); err != nil {
		t.Fatalf("err: %v", err)
	}
}

func TestClient_JoinLAN(t *testing.T) {
	dir1, s1 := testServer(t)
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	dir2, c1 := testClient(t)
	defer os.RemoveAll(dir2)
	defer c1.Shutdown()

	// Try to join
	addr := fmt.Sprintf("127.0.0.1:%d",
		s1.config.SerfLANConfig.MemberlistConfig.BindPort)
	if _, err := c1.JoinLAN([]string{addr}); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Check the members
	if len(s1.LANMembers()) != 2 {
		t.Fatalf("bad len")
	}

	if len(c1.LANMembers()) != 2 {
		t.Fatalf("bad len")
	}

	// Check we have a new consul
	testutil.WaitForResult(func() (bool, error) {
		return len(c1.consuls) == 1, nil
	}, func(err error) {
		t.Fatalf("expected consul server")
	})
}

func TestClient_RPC(t *testing.T) {
	dir1, s1 := testServer(t)
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	dir2, c1 := testClient(t)
	defer os.RemoveAll(dir2)
	defer c1.Shutdown()

	// Try an RPC
	var out struct{}
	if err := c1.RPC("Status.Ping", struct{}{}, &out); err != structs.ErrNoServers {
		t.Fatalf("err: %v", err)
	}

	// Try to join
	addr := fmt.Sprintf("127.0.0.1:%d",
		s1.config.SerfLANConfig.MemberlistConfig.BindPort)
	if _, err := c1.JoinLAN([]string{addr}); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Check the members
	if len(s1.LANMembers()) != 2 {
		t.Fatalf("bad len")
	}

	if len(c1.LANMembers()) != 2 {
		t.Fatalf("bad len")
	}

	// RPC should succeed
	testutil.WaitForResult(func() (bool, error) {
		err := c1.RPC("Status.Ping", struct{}{}, &out)
		return err == nil, err
	}, func(err error) {
		t.Fatalf("err: %v", err)
	})
}

func TestClient_RPC_TLS(t *testing.T) {
	dir1, conf1 := testServerConfig(t, "a.testco.internal")
	conf1.VerifyIncoming = true
	conf1.VerifyOutgoing = true
	configureTLS(conf1)
	s1, err := NewServer(conf1)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	dir2, conf2 := testClientConfig(t, "b.testco.internal")
	conf2.VerifyOutgoing = true
	configureTLS(conf2)
	c1, err := NewClient(conf2)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	defer os.RemoveAll(dir2)
	defer c1.Shutdown()

	// Try an RPC
	var out struct{}
	if err := c1.RPC("Status.Ping", struct{}{}, &out); err != structs.ErrNoServers {
		t.Fatalf("err: %v", err)
	}

	// Try to join
	addr := fmt.Sprintf("127.0.0.1:%d",
		s1.config.SerfLANConfig.MemberlistConfig.BindPort)
	if _, err := c1.JoinLAN([]string{addr}); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Check the members
	if len(s1.LANMembers()) != 2 {
		t.Fatalf("bad len")
	}

	if len(c1.LANMembers()) != 2 {
		t.Fatalf("bad len")
	}

	// RPC should succeed
	testutil.WaitForResult(func() (bool, error) {
		err := c1.RPC("Status.Ping", struct{}{}, &out)
		return err == nil, err
	}, func(err error) {
		t.Fatalf("err: %v", err)
	})
}
<commit_msg>Wait for joining member in `TestClient_JoinLAN`<commit_after>package consul

import (
	"fmt"
	"github.com/hashicorp/consul/consul/structs"
	"github.com/hashicorp/consul/testutil"
	"net"
	"os"
	"testing"
	"time"
)

func testClientConfig(t *testing.T, NodeName string) (string, *Config) {
	dir := tmpDir(t)
	config := DefaultConfig()
	config.Datacenter = "dc1"
	config.DataDir = dir
	config.NodeName = NodeName
	config.RPCAddr = &net.TCPAddr{
		IP:   []byte{127, 0, 0, 1},
		Port: getPort(),
	}
	config.SerfLANConfig.MemberlistConfig.BindAddr = "127.0.0.1"
	config.SerfLANConfig.MemberlistConfig.BindPort = getPort()
	config.SerfLANConfig.MemberlistConfig.ProbeTimeout = 200 * time.Millisecond
	config.SerfLANConfig.MemberlistConfig.ProbeInterval = time.Second
	config.SerfLANConfig.MemberlistConfig.GossipInterval = 100 * time.Millisecond

	return dir, config
}

func testClient(t *testing.T) (string, *Client) {
	return testClientDC(t, "dc1")
}

func testClientDC(t *testing.T, dc string) (string, *Client) {
	dir, config := testClientConfig(t, "testco.internal")
	config.Datacenter = dc

	client, err := NewClient(config)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	return dir, client
}

func TestClient_StartStop(t *testing.T) {
	dir, client := testClient(t)
	defer os.RemoveAll(dir)

	if err := client.Shutdown(); err != nil {
		t.Fatalf("err: %v", err)
	}
}

func TestClient_JoinLAN(t *testing.T) {
	dir1, s1 := testServer(t)
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	dir2, c1 := testClient(t)
	defer os.RemoveAll(dir2)
	defer c1.Shutdown()

	// Try to join
	addr := fmt.Sprintf("127.0.0.1:%d",
		s1.config.SerfLANConfig.MemberlistConfig.BindPort)
	if _, err := c1.JoinLAN([]string{addr}); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Check the members
	testutil.WaitForResult(func() (bool, error) {
		server_check := len(s1.LANMembers()) == 2
		client_check := len(c1.LANMembers()) == 2
		return server_check && client_check, nil
	}, func(err error) {
		t.Fatalf("bad len")
	})

	// Check we have a new consul
	testutil.WaitForResult(func() (bool, error) {
		return len(c1.consuls) == 1, nil
	}, func(err error) {
		t.Fatalf("expected consul server")
	})
}

func TestClient_RPC(t *testing.T) {
	dir1, s1 := testServer(t)
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	dir2, c1 := testClient(t)
	defer os.RemoveAll(dir2)
	defer c1.Shutdown()

	// Try an RPC
	var out struct{}
	if err := c1.RPC("Status.Ping", struct{}{}, &out); err != structs.ErrNoServers {
		t.Fatalf("err: %v", err)
	}

	// Try to join
	addr := fmt.Sprintf("127.0.0.1:%d",
		s1.config.SerfLANConfig.MemberlistConfig.BindPort)
	if _, err := c1.JoinLAN([]string{addr}); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Check the members
	if len(s1.LANMembers()) != 2 {
		t.Fatalf("bad len")
	}

	if len(c1.LANMembers()) != 2 {
		t.Fatalf("bad len")
	}

	// RPC should succeed
	testutil.WaitForResult(func() (bool, error) {
		err := c1.RPC("Status.Ping", struct{}{}, &out)
		return err == nil, err
	}, func(err error) {
		t.Fatalf("err: %v", err)
	})
}

func TestClient_RPC_TLS(t *testing.T) {
	dir1, conf1 := testServerConfig(t, "a.testco.internal")
	conf1.VerifyIncoming = true
	conf1.VerifyOutgoing = true
	configureTLS(conf1)
	s1, err := NewServer(conf1)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	defer os.RemoveAll(dir1)
	defer s1.Shutdown()

	dir2, conf2 := testClientConfig(t, "b.testco.internal")
	conf2.VerifyOutgoing = true
	configureTLS(conf2)
	c1, err := NewClient(conf2)
	if err != nil {
		t.Fatalf("err: %v", err)
	}
	defer os.RemoveAll(dir2)
	defer c1.Shutdown()

	// Try an RPC
	var out struct{}
	if err := c1.RPC("Status.Ping", struct{}{}, &out); err != structs.ErrNoServers {
		t.Fatalf("err: %v", err)
	}

	// Try to join
	addr := fmt.Sprintf("127.0.0.1:%d",
		s1.config.SerfLANConfig.MemberlistConfig.BindPort)
	if _, err := c1.JoinLAN([]string{addr}); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Check the members
	if len(s1.LANMembers()) != 2 {
		t.Fatalf("bad len")
	}

	if len(c1.LANMembers()) != 2 {
		t.Fatalf("bad len")
	}

	// RPC should succeed
	testutil.WaitForResult(func() (bool, error) {
		err := c1.RPC("Status.Ping", struct{}{}, &out)
		return err == nil, err
	}, func(err error) {
		t.Fatalf("err: %v", err)
	})
}
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package opencensusreceiver

import (
	"context"
	"errors"
	"fmt"
	"net"
	"net/http"
	"sync"

	agentmetricspb "github.com/census-instrumentation/opencensus-proto/gen-go/agent/metrics/v1"
	agenttracepb "github.com/census-instrumentation/opencensus-proto/gen-go/agent/trace/v1"
	gatewayruntime "github.com/grpc-ecosystem/grpc-gateway/runtime"
	"github.com/rs/cors"
	"github.com/soheilhy/cmux"
	"google.golang.org/grpc"

	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/config"
	"go.opentelemetry.io/collector/config/configgrpc"
	"go.opentelemetry.io/collector/consumer"
	"go.opentelemetry.io/collector/receiver/opencensusreceiver/internal/ocmetrics"
	"go.opentelemetry.io/collector/receiver/opencensusreceiver/internal/octrace"
)

// ocReceiver is the type that exposes Trace and Metrics reception.
type ocReceiver struct {
	mu                 sync.Mutex
	ln                 net.Listener
	serverGRPC         *grpc.Server
	serverHTTP         *http.Server
	gatewayMux         *gatewayruntime.ServeMux
	corsOrigins        []string
	grpcServerSettings configgrpc.GRPCServerSettings

	traceReceiver   *octrace.Receiver
	metricsReceiver *ocmetrics.Receiver

	traceConsumer   consumer.Traces
	metricsConsumer consumer.Metrics

	startTracesReceiverOnce  sync.Once
	startMetricsReceiverOnce sync.Once

	id config.ComponentID
}

// newOpenCensusReceiver just creates the OpenCensus receiver services. It is the caller's
// responsibility to invoke the respective Start*Reception methods as well
// as the various Stop*Reception methods to end it.
func newOpenCensusReceiver(
	id config.ComponentID,
	transport string,
	addr string,
	tc consumer.Traces,
	mc consumer.Metrics,
	opts ...ocOption,
) (*ocReceiver, error) {
	// TODO: (@odeke-em) use options to enable address binding changes.
	ln, err := net.Listen(transport, addr)
	if err != nil {
		return nil, fmt.Errorf("failed to bind to address %q: %v", addr, err)
	}

	ocr := &ocReceiver{
		id:              id,
		ln:              ln,
		corsOrigins:     []string{}, // Disable CORS by default.
		gatewayMux:      gatewayruntime.NewServeMux(),
		traceConsumer:   tc,
		metricsConsumer: mc,
	}

	for _, opt := range opts {
		opt.withReceiver(ocr)
	}

	return ocr, nil
}

// Start runs the trace receiver on the gRPC server. Currently
// it also enables the metrics receiver too.
func (ocr *ocReceiver) Start(_ context.Context, host component.Host) error {
	hasConsumer := false
	if ocr.traceConsumer != nil {
		hasConsumer = true
		if err := ocr.registerTraceConsumer(host); err != nil {
			return err
		}
	}

	if ocr.metricsConsumer != nil {
		hasConsumer = true
		if err := ocr.registerMetricsConsumer(host); err != nil {
			return err
		}
	}

	if !hasConsumer {
		return errors.New("cannot start receiver: no consumers were specified")
	}

	if err := ocr.startServer(host); err != nil {
		return err
	}

	// At this point we've successfully started all the services/receivers.
	// Add other start routines here.
	return nil
}

func (ocr *ocReceiver) registerTraceConsumer(host component.Host) error {
	var err error

	ocr.startTracesReceiverOnce.Do(func() {
		ocr.traceReceiver, err = octrace.New(ocr.id, ocr.traceConsumer)
		if err != nil {
			return
		}

		var srv *grpc.Server
		srv, err = ocr.grpcServer(host)
		if err != nil {
			return
		}

		agenttracepb.RegisterTraceServiceServer(srv, ocr.traceReceiver)

	})

	return err
}

func (ocr *ocReceiver) registerMetricsConsumer(host component.Host) error {
	var err error

	ocr.startMetricsReceiverOnce.Do(func() {
		ocr.metricsReceiver, err = ocmetrics.New(ocr.id, ocr.metricsConsumer)
		if err != nil {
			return
		}

		var srv *grpc.Server
		srv, err = ocr.grpcServer(host)
		if err != nil {
			return
		}

		agentmetricspb.RegisterMetricsServiceServer(srv, ocr.metricsReceiver)
	})
	return err
}

func (ocr *ocReceiver) grpcServer(host component.Host) (*grpc.Server, error) {
	ocr.mu.Lock()
	defer ocr.mu.Unlock()

	if ocr.serverGRPC == nil {
		opts, err := ocr.grpcServerSettings.ToServerOption(host.GetExtensions())
		if err != nil {
			return nil, err
		}
		ocr.serverGRPC = grpc.NewServer(opts...)
	}

	return ocr.serverGRPC, nil
}

// Shutdown is a method to turn off receiving.
func (ocr *ocReceiver) Shutdown(context.Context) error {
	ocr.mu.Lock()
	defer ocr.mu.Unlock()

	var err error
	if ocr.serverHTTP != nil {
		err = ocr.serverHTTP.Close()
	}

	if ocr.ln != nil {
		_ = ocr.ln.Close()
	}

	// TODO: @(odeke-em) investigate what utility invoking (*grpc.Server).Stop()
	// gives us yet we invoke (net.Listener).Close().
	// Sure (*grpc.Server).Stop() enables proper shutdown but imposes
	// a painful and artificial wait time that goes into 20+seconds yet most of our
	// tests and code should be reactive in less than even 1second.
	// ocr.serverGRPC.Stop()

	return err
}

func (ocr *ocReceiver) httpServer() *http.Server {
	ocr.mu.Lock()
	defer ocr.mu.Unlock()

	if ocr.serverHTTP == nil {
		var mux http.Handler = ocr.gatewayMux
		if len(ocr.corsOrigins) > 0 {
			co := cors.Options{AllowedOrigins: ocr.corsOrigins}
			mux = cors.New(co).Handler(mux)
		}
		ocr.serverHTTP = &http.Server{Handler: mux}
	}

	return ocr.serverHTTP
}

func (ocr *ocReceiver) startServer(host component.Host) error {
	// Register the grpc-gateway on the HTTP server mux
	c := context.Background()
	opts := []grpc.DialOption{grpc.WithInsecure()}
	endpoint := ocr.ln.Addr().String()

	_, ok := ocr.ln.(*net.UnixListener)
	if ok {
		endpoint = "unix:" + endpoint
	}

	if err := agenttracepb.RegisterTraceServiceHandlerFromEndpoint(c, ocr.gatewayMux, endpoint, opts); err != nil {
		return err
	}

	if err := agentmetricspb.RegisterMetricsServiceHandlerFromEndpoint(c, ocr.gatewayMux, endpoint, opts); err != nil {
		return err
	}

	// Start the gRPC and HTTP/JSON (grpc-gateway) servers on the same port.
	m := cmux.New(ocr.ln)
	grpcL := m.MatchWithWriters(
		cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc"),
		cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc+proto"))

	httpL := m.Match(cmux.Any())
	go func() {
		if errGrpc := ocr.serverGRPC.Serve(grpcL); errGrpc != nil {
			host.ReportFatalError(errGrpc)
		}
	}()
	go func() {
		if errHTTP := ocr.httpServer().Serve(httpL); errHTTP != http.ErrServerClosed {
			host.ReportFatalError(errHTTP)
		}
	}()
	go func() {
		if errServe := m.Serve(); errServe != nil {
			host.ReportFatalError(errServe)
		}
	}()
	return nil
}
<commit_msg>Do not report fatal error when cmux.ErrServerClosed (#3703)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package opencensusreceiver

import (
	"context"
	"errors"
	"fmt"
	"net"
	"net/http"
	"sync"

	agentmetricspb "github.com/census-instrumentation/opencensus-proto/gen-go/agent/metrics/v1"
	agenttracepb "github.com/census-instrumentation/opencensus-proto/gen-go/agent/trace/v1"
	gatewayruntime "github.com/grpc-ecosystem/grpc-gateway/runtime"
	"github.com/rs/cors"
	"github.com/soheilhy/cmux"
	"google.golang.org/grpc"

	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/config"
	"go.opentelemetry.io/collector/config/configgrpc"
	"go.opentelemetry.io/collector/consumer"
	"go.opentelemetry.io/collector/receiver/opencensusreceiver/internal/ocmetrics"
	"go.opentelemetry.io/collector/receiver/opencensusreceiver/internal/octrace"
)

// ocReceiver is the type that exposes Trace and Metrics reception.
type ocReceiver struct {
	mu                 sync.Mutex
	ln                 net.Listener
	serverGRPC         *grpc.Server
	serverHTTP         *http.Server
	gatewayMux         *gatewayruntime.ServeMux
	corsOrigins        []string
	grpcServerSettings configgrpc.GRPCServerSettings

	traceReceiver   *octrace.Receiver
	metricsReceiver *ocmetrics.Receiver

	traceConsumer   consumer.Traces
	metricsConsumer consumer.Metrics

	startTracesReceiverOnce  sync.Once
	startMetricsReceiverOnce sync.Once

	id config.ComponentID
}

// newOpenCensusReceiver just creates the OpenCensus receiver services. It is the caller's
// responsibility to invoke the respective Start*Reception methods as well
// as the various Stop*Reception methods to end it.
func newOpenCensusReceiver(
	id config.ComponentID,
	transport string,
	addr string,
	tc consumer.Traces,
	mc consumer.Metrics,
	opts ...ocOption,
) (*ocReceiver, error) {
	// TODO: (@odeke-em) use options to enable address binding changes.
	ln, err := net.Listen(transport, addr)
	if err != nil {
		return nil, fmt.Errorf("failed to bind to address %q: %v", addr, err)
	}

	ocr := &ocReceiver{
		id:              id,
		ln:              ln,
		corsOrigins:     []string{}, // Disable CORS by default.
		gatewayMux:      gatewayruntime.NewServeMux(),
		traceConsumer:   tc,
		metricsConsumer: mc,
	}

	for _, opt := range opts {
		opt.withReceiver(ocr)
	}

	return ocr, nil
}

// Start runs the trace receiver on the gRPC server. Currently
// it also enables the metrics receiver too.
func (ocr *ocReceiver) Start(_ context.Context, host component.Host) error {
	hasConsumer := false
	if ocr.traceConsumer != nil {
		hasConsumer = true
		if err := ocr.registerTraceConsumer(host); err != nil {
			return err
		}
	}

	if ocr.metricsConsumer != nil {
		hasConsumer = true
		if err := ocr.registerMetricsConsumer(host); err != nil {
			return err
		}
	}

	if !hasConsumer {
		return errors.New("cannot start receiver: no consumers were specified")
	}

	if err := ocr.startServer(host); err != nil {
		return err
	}

	// At this point we've successfully started all the services/receivers.
	// Add other start routines here.
	return nil
}

func (ocr *ocReceiver) registerTraceConsumer(host component.Host) error {
	var err error

	ocr.startTracesReceiverOnce.Do(func() {
		ocr.traceReceiver, err = octrace.New(ocr.id, ocr.traceConsumer)
		if err != nil {
			return
		}

		var srv *grpc.Server
		srv, err = ocr.grpcServer(host)
		if err != nil {
			return
		}

		agenttracepb.RegisterTraceServiceServer(srv, ocr.traceReceiver)

	})

	return err
}

func (ocr *ocReceiver) registerMetricsConsumer(host component.Host) error {
	var err error

	ocr.startMetricsReceiverOnce.Do(func() {
		ocr.metricsReceiver, err = ocmetrics.New(ocr.id, ocr.metricsConsumer)
		if err != nil {
			return
		}

		var srv *grpc.Server
		srv, err = ocr.grpcServer(host)
		if err != nil {
			return
		}

		agentmetricspb.RegisterMetricsServiceServer(srv, ocr.metricsReceiver)
	})
	return err
}

func (ocr *ocReceiver) grpcServer(host component.Host) (*grpc.Server, error) {
	ocr.mu.Lock()
	defer ocr.mu.Unlock()

	if ocr.serverGRPC == nil {
		opts, err := ocr.grpcServerSettings.ToServerOption(host.GetExtensions())
		if err != nil {
			return nil, err
		}
		ocr.serverGRPC = grpc.NewServer(opts...)
	}

	return ocr.serverGRPC, nil
}

// Shutdown is a method to turn off receiving.
func (ocr *ocReceiver) Shutdown(context.Context) error {
	ocr.mu.Lock()
	defer ocr.mu.Unlock()

	var err error
	if ocr.serverHTTP != nil {
		err = ocr.serverHTTP.Close()
	}

	if ocr.ln != nil {
		_ = ocr.ln.Close()
	}

	// TODO: @(odeke-em) investigate what utility invoking (*grpc.Server).Stop()
	// gives us yet we invoke (net.Listener).Close().
	// Sure (*grpc.Server).Stop() enables proper shutdown but imposes
	// a painful and artificial wait time that goes into 20+seconds yet most of our
	// tests and code should be reactive in less than even 1second.
	// ocr.serverGRPC.Stop()

	return err
}

func (ocr *ocReceiver) httpServer() *http.Server {
	ocr.mu.Lock()
	defer ocr.mu.Unlock()

	if ocr.serverHTTP == nil {
		var mux http.Handler = ocr.gatewayMux
		if len(ocr.corsOrigins) > 0 {
			co := cors.Options{AllowedOrigins: ocr.corsOrigins}
			mux = cors.New(co).Handler(mux)
		}
		ocr.serverHTTP = &http.Server{Handler: mux}
	}

	return ocr.serverHTTP
}

func (ocr *ocReceiver) startServer(host component.Host) error {
	// Register the grpc-gateway on the HTTP server mux
	c := context.Background()
	opts := []grpc.DialOption{grpc.WithInsecure()}
	endpoint := ocr.ln.Addr().String()

	_, ok := ocr.ln.(*net.UnixListener)
	if ok {
		endpoint = "unix:" + endpoint
	}

	if err := agenttracepb.RegisterTraceServiceHandlerFromEndpoint(c, ocr.gatewayMux, endpoint, opts); err != nil {
		return err
	}

	if err := agentmetricspb.RegisterMetricsServiceHandlerFromEndpoint(c, ocr.gatewayMux, endpoint, opts); err != nil {
		return err
	}

	// Start the gRPC and HTTP/JSON (grpc-gateway) servers on the same port.
	m := cmux.New(ocr.ln)
	grpcL := m.MatchWithWriters(
		cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc"),
		cmux.HTTP2MatchHeaderFieldSendSettings("content-type", "application/grpc+proto"))

	httpL := m.Match(cmux.Any())
	go func() {
		if errGrpc := ocr.serverGRPC.Serve(grpcL); errGrpc != nil {
			host.ReportFatalError(errGrpc)
		}
	}()
	go func() {
		if errHTTP := ocr.httpServer().Serve(httpL); errHTTP != nil && errHTTP != http.ErrServerClosed {
			host.ReportFatalError(errHTTP)
		}
	}()
	go func() {
		if errServe := m.Serve(); errServe != nil && errServe != cmux.ErrServerClosed {
			host.ReportFatalError(errServe)
		}
	}()
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/tomsteele/blacksheepwall/bsw"
	"log"
	"net"
	"os"
	"sort"
	"text/tabwriter"
)

var usage = `
  Usage: blacksheepwall [options] <ip address or CIDR>

  Options:
  -h, --help            Show Usage and exit.
  -version              Show version and exit.
  -debug                Enable debugging and show errors returned from tasks.
  -concurrency <int>    Max amount of concurrent tasks.    [default: 100]
  -server <string>      DNS server address.    [default: "8.8.8.8"]
  -input <string>       Line separated file of networks (CIDR) or
                        IP Addresses.
  -ipv6	                Look for additional AAAA records where applicable.
  -domain <string>      Target domain to use for certain tasks.
  -dictionary <string>  Attempt to retrieve the CNAME and A record for
                        each subdomain in the line separated file.
  -yandex <string>      Provided a Yandex search XML API url. Use the Yandex
                        search 'rhost:' operator to find subdomains of a
                        provided domain..
  -bing	<string>        Provided a base64 encoded API key. Use the Bing search
                        API's 'ip:' operator to lookup hostnames for each host.
  -headers              Perform HTTP(s) requests to each host and look for
                        hostnames in a possible Location header.
  -reverse              Retrieve the PTR for each host.
  -tls                  Attempt to retrieve names from TLS certificates
                        (CommonName and Subject Alternative Name).
  -viewdns              Lookup each host using viewdns.info's Reverse IP
                        Lookup function.
  -fcrdns               Verify results by attempting to retrieve the A or AAAA record for
                        each result previously identified hostname.
  -clean                Print results as unique hostnames for each host.
  -csv                  Print results in csv format.
  -json                 Print results as JSON.

`

// Returns all ip addresses from each CIDR range in a list.
func linesToIpList(lines []string) ([]string, error) {
	ipList := []string{}
	for _, line := range lines {
		if net.ParseIP(line) != nil {
			ipList = append(ipList, line)
		} else if ip, network, err := net.ParseCIDR(line); err == nil {
			for ip := ip.Mask(network.Mask); network.Contains(ip); increaseIp(ip) {
				ipList = append(ipList, ip.String())
			}
		} else {
			return ipList, errors.New("\"" + line + "\" is not an IP Address or CIDR Network")
		}
	}
	return ipList, nil
}

// Increases IP by a single address
func increaseIp(ip net.IP) {
	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}
}

// Reads lines from a file and returns as a slice.
func readFileLines(path string) ([]string, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	lines := []string{}
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	return lines, scanner.Err()
}

type task func() (bsw.Results, error)
type empty struct{}

func main() {
	// Command line options. For usage information see the
	// usage variable above.
	var (
		flVersion     = flag.Bool("version", false, "")
		flConcurrency = flag.Int("concurrency", 100, "")
		flDebug       = flag.Bool("debug", false, "")
		flipv6        = flag.Bool("ipv6", false, "")
		flServerAddr  = flag.String("server", "8.8.8.8", "")
		flIpFile      = flag.String("input", "", "")
		flReverse     = flag.Bool("reverse", false, "")
		flHeader      = flag.Bool("headers", false, "")
		flTLS         = flag.Bool("tls", false, "")
		flViewDnsInfo = flag.Bool("viewdns", false, "")
		flBing        = flag.String("bing", "", "")
		flYandex      = flag.String("yandex", "", "")
		flDomain      = flag.String("domain", "", "")
		flDictFile    = flag.String("dictionary", "", "")
		flFcrdns      = flag.Bool("fcrdns", false, "")
		flClean       = flag.Bool("clean", false, "")
		flCsv         = flag.Bool("csv", false, "")
		flJson        = flag.Bool("json", false, "")
	)
	flag.Usage = func() { fmt.Print(usage) }
	flag.Parse()

	if *flVersion {
		fmt.Println("blacksheepwall version ", bsw.VERSION)
		os.Exit(0)
	}

	// Holds all ip addresses for testing
	ipAddrList := []string{}

	// Used to hold a ip or CIDR range passed as fl.Arg(0)
	var flNetwork string

	// Verify that some sort of work load was given in commands
	if *flIpFile == "" && *flDomain == "" && len(flag.Args()) < 1 {
		log.Fatal("You didn't provide any work for me to do")
	}
	if *flYandex != "" && *flDomain == "" {
		log.Fatal("Yandex API requires domain set with -domain")
	}
	if *flDictFile != "" && *flDomain == "" {
		log.Fatal("Dictionary lookup requires domain set with -domain")
	}
	if *flDomain != "" && *flYandex == "" && *flDictFile == "" {
		log.Fatal("-domain provided but no methods provided that use it")
	}

	// Get first argument that is not an option and turn it into a list of ips
	if len(flag.Args()) > 0 {
		flNetwork = flag.Arg(0)
		list, err := linesToIpList([]string{flNetwork})
		if err != nil {
			log.Fatal(err.Error())
		}
		ipAddrList = append(ipAddrList, list...)
	}

	// If file given as -input, read lines and turn each possible ip or network into
	// a list of ips. Appends list to ipAddrList. Will fail fatally if line in file
	// is not a valid ip or CIDR range.
	if *flIpFile != "" {
		lines, err := readFileLines(*flIpFile)
		if err != nil {
			log.Fatal("Error reading " + *flIpFile + " " + err.Error())
		}
		list, err := linesToIpList(lines)
		if err != nil {
			log.Fatal(err.Error())
		}
		ipAddrList = append(ipAddrList, list...)
	}

	// tracker: Chanel uses an empty struct to track when all goroutines in the pool
	//          have completed as well as a single call from the gatherer.
	//
	// tasks:   Chanel used in the goroutine pool to manage incoming work. A task is
	//          a function wrapper that returns a slice of results and a possible error.
	//
	// res:     When each task is called in the pool, it will send valid results to
	//          the res channel.
	tracker := make(chan empty)
	tasks := make(chan task, *flConcurrency)
	res := make(chan bsw.Results, *flConcurrency)
	// Use a map that acts like a set to store only unique results
	resMap:= make(map[bsw.Result]bool)

	// Start up *flConcurrency amount of goroutines
	log.Printf("Spreading tasks across %d goroutines", *flConcurrency)
	for i := 0; i < *flConcurrency; i++ {
		go func() {
			var c = 0
			for def := range tasks {
				result, err := def()
				if m := c % 2; m == 0 {
					c = 3
					os.Stderr.WriteString("\rWorking \\")
				} else {
					c = 2
					os.Stderr.WriteString("\rWorking /")
				}
				if err != nil && *flDebug {
					log.Println(err.Error())
				}
				if err == nil {
					res <- result
				}
			}
			tracker <- empty{}
		}()
	}

	// Ingest incoming results
	go func() {
		for result := range res {
			if len(result) < 1 {
				continue
			}
			if *flFcrdns {
				for _, r := range result {
					ip, err := bsw.LookupName(r.Hostname, *flServerAddr)
					if err == nil && len(ip) > 0 {
						resMap[bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname}] = true
					}
					ip, err = bsw.LookupName6(r.Hostname, *flServerAddr)
					if err == nil && len(ip) > 0 {
						resMap[bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname}] = true
					}
				}
			} else {
				for _, r := range result {
					resMap[r] = true
				}
			}
		}
		tracker <- empty{}
	}()

	// Bing has two possible search paths. We need to find which one is valid.
	var bingPath string
	if *flBing != "" {
		p, err := bsw.FindBingSearchPath(*flBing)
		if err != nil {
			log.Fatal(err.Error())
		}
		bingPath = p
	}

	// ip based functionality should be added to the pool here
	for _, h := range ipAddrList {
		host := h
		if *flReverse {
			tasks <- func() (bsw.Results, error) { return bsw.Reverse(host, *flServerAddr) }
		}
		if *flTLS {
			tasks <- func() (bsw.Results, error) { return bsw.TLS(host) }
		}
		if *flViewDnsInfo {
			tasks <- func() (bsw.Results, error) { return bsw.ViewDnsInfo(host) }
		}
		if *flBing != "" && bingPath != "" {
			tasks <- func() (bsw.Results, error) { return bsw.BingAPI(host, *flBing, bingPath) }
		}
		if *flHeader {
			tasks <- func() (bsw.Results, error) { return bsw.Headers(host) }
		}

	}

	// Domain based functions will likely require separate blocks

	// Subdomain dictionary guessing
	if *flDictFile != "" && *flDomain != "" {
		nameList, err := readFileLines(*flDictFile)
		if err != nil {
			log.Fatal("Error reading " + *flDictFile + " " + err.Error())
		}
		// Get an ip for a possible wildcard domain and use it as a blacklist
		blacklist := bsw.GetWildCard(*flDomain, *flServerAddr)
		var blacklist6 string
		if *flipv6 {
			blacklist6 = bsw.GetWildCard6(*flDomain, *flServerAddr)
		}
		for _, n := range nameList {
			sub := n
			tasks <- func() (bsw.Results, error) { return bsw.Dictionary(*flDomain, sub, blacklist, *flServerAddr) }
			if *flipv6 {
				tasks <- func() (bsw.Results, error) { return bsw.Dictionary6(*flDomain, sub, blacklist6, *flServerAddr) }
			}
		}
	}

	if *flYandex != "" && *flDomain != "" {
		tasks <- func() (bsw.Results, error) { return bsw.YandexAPI(*flDomain, *flYandex, *flServerAddr) }
	}

	// Close the tasks channel after all jobs have completed and for each
	// goroutine in the pool receive an empty message from  tracker.
	close(tasks)
	for i := 0; i < *flConcurrency; i++ {
		<-tracker
	}
	close(res)
	// Receive and empty message from the result gatherer
	<-tracker
	os.Stderr.WriteString("\r")
	log.Println("All tasks completed\n")

	// Create a results slice from the unique set in resMap. Allows for sorting.
	results := bsw.Results{}
	for k, _ := range resMap {
		results = append(results, k)
	}
	sort.Sort(results)

	// Output options
	switch {
	case *flJson:
		j, _ := json.MarshalIndent(results, "", "    ")
		fmt.Println(string(j))
	case *flCsv:
		for _, r := range results {
			fmt.Printf("%s,%s,%s\n", r.Hostname, r.IP, r.Source)
		}
	case *flClean:
		cleanSet := make(map[string][]string)
		for _, r := range results {
			cleanSet[r.IP] = append(cleanSet[r.IP], r.Hostname)
		}
		for k, v := range cleanSet {
			fmt.Printf("%s:\n", k)
			for _, h := range v {
				fmt.Printf("\t%s\n", h)
			}
		}
	default:
		w := tabwriter.NewWriter(os.Stdout, 0, 8, 4, ' ', 0)
		fmt.Fprintln(w, "IP\tHostname\tSource")
		for _, r := range results {
			fmt.Fprintf(w, "%s\t%s\t%s\n", r.IP, r.Hostname, r.Source)
		}
		w.Flush()
	}
}
<commit_msg>Improves comments<commit_after>/* Blacksheepwall is a hostname reconnaissance tool, it is similar to other
tools, but has a focus on speed.*/

package main

import (
	"bufio"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/tomsteele/blacksheepwall/bsw"
	"log"
	"net"
	"os"
	"sort"
	"text/tabwriter"
)

const usage = `
  Usage: blacksheepwall [options] <ip address or CIDR>

  Options:
  -h, --help            Show Usage and exit.
  -version              Show version and exit.
  -debug                Enable debugging and show errors returned from tasks.
  -concurrency <int>    Max amount of concurrent tasks.    [default: 100]
  -server <string>      DNS server address.    [default: "8.8.8.8"]
  -input <string>       Line separated file of networks (CIDR) or
                        IP Addresses.
  -ipv6	                Look for additional AAAA records where applicable.
  -domain <string>      Target domain to use for certain tasks.
  -dictionary <string>  Attempt to retrieve the CNAME and A record for
                        each subdomain in the line separated file.
  -yandex <string>      Provided a Yandex search XML API url. Use the Yandex
                        search 'rhost:' operator to find subdomains of a
                        provided domain..
  -bing	<string>        Provided a base64 encoded API key. Use the Bing search
                        API's 'ip:' operator to lookup hostnames for each host.
  -headers              Perform HTTP(s) requests to each host and look for
                        hostnames in a possible Location header.
  -reverse              Retrieve the PTR for each host.
  -tls                  Attempt to retrieve names from TLS certificates
                        (CommonName and Subject Alternative Name).
  -viewdns              Lookup each host using viewdns.info's Reverse IP
                        Lookup function.
  -fcrdns               Verify results by attempting to retrieve the A or AAAA record for
                        each result previously identified hostname.
  -clean                Print results as unique hostnames for each host.
  -csv                  Print results in csv format.
  -json                 Print results as JSON.

`

// Processes a list of IP addresses or networks in CIDR format.
// Returning a list of all possible IP addresses.
func linesToIpList(lines []string) ([]string, error) {
	ipList := []string{}
	for _, line := range lines {
		if net.ParseIP(line) != nil {
			ipList = append(ipList, line)
		} else if ip, network, err := net.ParseCIDR(line); err == nil {
			for ip := ip.Mask(network.Mask); network.Contains(ip); increaseIp(ip) {
				ipList = append(ipList, ip.String())
			}
		} else {
			return ipList, errors.New("\"" + line + "\" is not an IP Address or CIDR Network")
		}
	}
	return ipList, nil
}

// Increases an IP by a single address.
func increaseIp(ip net.IP) {
	for j := len(ip) - 1; j >= 0; j-- {
		ip[j]++
		if ip[j] > 0 {
			break
		}
	}
}

// Reads lines from a file and return as a slice.
func readFileLines(path string) ([]string, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()
	lines := []string{}
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}
	return lines, scanner.Err()
}

type task func() (bsw.Results, error)
type empty struct{}

func main() {
	// Command line options. For usage information see the
	// usage variable above.
	var (
		flVersion     = flag.Bool("version", false, "")
		flConcurrency = flag.Int("concurrency", 100, "")
		flDebug       = flag.Bool("debug", false, "")
		flipv6        = flag.Bool("ipv6", false, "")
		flServerAddr  = flag.String("server", "8.8.8.8", "")
		flIpFile      = flag.String("input", "", "")
		flReverse     = flag.Bool("reverse", false, "")
		flHeader      = flag.Bool("headers", false, "")
		flTLS         = flag.Bool("tls", false, "")
		flViewDnsInfo = flag.Bool("viewdns", false, "")
		flBing        = flag.String("bing", "", "")
		flYandex      = flag.String("yandex", "", "")
		flDomain      = flag.String("domain", "", "")
		flDictFile    = flag.String("dictionary", "", "")
		flFcrdns      = flag.Bool("fcrdns", false, "")
		flClean       = flag.Bool("clean", false, "")
		flCsv         = flag.Bool("csv", false, "")
		flJson        = flag.Bool("json", false, "")
	)
	flag.Usage = func() { fmt.Print(usage) }
	flag.Parse()

	if *flVersion {
		fmt.Println("blacksheepwall version ", bsw.VERSION)
		os.Exit(0)
	}

	// Holds all IP addresses for testing.
	ipAddrList := []string{}

	// Used to hold a ip or CIDR range passed as fl.Arg(0).
	var flNetwork string

	// Verify that some sort of work load was given in commands.
	if *flIpFile == "" && *flDomain == "" && len(flag.Args()) < 1 {
		log.Fatal("You didn't provide any work for me to do")
	}
	if *flYandex != "" && *flDomain == "" {
		log.Fatal("Yandex API requires domain set with -domain")
	}
	if *flDictFile != "" && *flDomain == "" {
		log.Fatal("Dictionary lookup requires domain set with -domain")
	}
	if *flDomain != "" && *flYandex == "" && *flDictFile == "" {
		log.Fatal("-domain provided but no methods provided that use it")
	}

	// Get first argument that is not an option and turn it into a list of IPs.
	if len(flag.Args()) > 0 {
		flNetwork = flag.Arg(0)
		list, err := linesToIpList([]string{flNetwork})
		if err != nil {
			log.Fatal(err.Error())
		}
		ipAddrList = append(ipAddrList, list...)
	}

	// If file given as -input, read lines and turn each possible IP or network into
	// a list of IPs. Appends list to ipAddrList. Will fail fatally if line in file
	// is not a valid IP or CIDR range.
	if *flIpFile != "" {
		lines, err := readFileLines(*flIpFile)
		if err != nil {
			log.Fatal("Error reading " + *flIpFile + " " + err.Error())
		}
		list, err := linesToIpList(lines)
		if err != nil {
			log.Fatal(err.Error())
		}
		ipAddrList = append(ipAddrList, list...)
	}

	// tracker: Chanel uses an empty struct to track when all goroutines in the pool
	//          have completed as well as a single call from the gatherer.
	//
	// tasks:   Chanel used in the goroutine pool to manage incoming work. A task is
	//          a function wrapper that returns a slice of results and a possible error.
	//
	// res:     When each task is called in the pool, it will send valid results to
	//          the res channel.
	tracker := make(chan empty)
	tasks := make(chan task, *flConcurrency)
	res := make(chan bsw.Results, *flConcurrency)
	// Use a map that acts like a set to store only unique results.
	resMap := make(map[bsw.Result]bool)

	// Start up *flConcurrency amount of goroutines.
	log.Printf("Spreading tasks across %d goroutines", *flConcurrency)
	for i := 0; i < *flConcurrency; i++ {
		go func() {
			var c = 0
			for def := range tasks {
				result, err := def()
				if m := c % 2; m == 0 {
					c = 3
					os.Stderr.WriteString("\rWorking \\")
				} else {
					c = 2
					os.Stderr.WriteString("\rWorking /")
				}
				if err != nil && *flDebug {
					log.Println(err.Error())
				}
				if err == nil {
					res <- result
				}
			}
			tracker <- empty{}
		}()
	}

	// Ingest incoming results.
	go func() {
		for result := range res {
			if len(result) < 1 {
				continue
			}
			if *flFcrdns {
				for _, r := range result {
					ip, err := bsw.LookupName(r.Hostname, *flServerAddr)
					if err == nil && len(ip) > 0 {
						resMap[bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname}] = true
					}
					ip, err = bsw.LookupName6(r.Hostname, *flServerAddr)
					if err == nil && len(ip) > 0 {
						resMap[bsw.Result{Source: "fcrdns", IP: ip, Hostname: r.Hostname}] = true
					}
				}
			} else {
				for _, r := range result {
					resMap[r] = true
				}
			}
		}
		tracker <- empty{}
	}()

	// Bing has two possible search paths. We need to find which one is valid.
	var bingPath string
	if *flBing != "" {
		p, err := bsw.FindBingSearchPath(*flBing)
		if err != nil {
			log.Fatal(err.Error())
		}
		bingPath = p
	}

	// IP based functionality should be added to the pool here.
	for _, h := range ipAddrList {
		host := h
		if *flReverse {
			tasks <- func() (bsw.Results, error) { return bsw.Reverse(host, *flServerAddr) }
		}
		if *flTLS {
			tasks <- func() (bsw.Results, error) { return bsw.TLS(host) }
		}
		if *flViewDnsInfo {
			tasks <- func() (bsw.Results, error) { return bsw.ViewDnsInfo(host) }
		}
		if *flBing != "" && bingPath != "" {
			tasks <- func() (bsw.Results, error) { return bsw.BingAPI(host, *flBing, bingPath) }
		}
		if *flHeader {
			tasks <- func() (bsw.Results, error) { return bsw.Headers(host) }
		}

	}

	// Domain based functions will likely require separate blocks and should be added below.

	// Subdomain dictionary guessing.
	if *flDictFile != "" && *flDomain != "" {
		nameList, err := readFileLines(*flDictFile)
		if err != nil {
			log.Fatal("Error reading " + *flDictFile + " " + err.Error())
		}
		// Get an IP for a possible wildcard domain and use it as a blacklist.
		blacklist := bsw.GetWildCard(*flDomain, *flServerAddr)
		var blacklist6 string
		if *flipv6 {
			blacklist6 = bsw.GetWildCard6(*flDomain, *flServerAddr)
		}
		for _, n := range nameList {
			sub := n
			tasks <- func() (bsw.Results, error) { return bsw.Dictionary(*flDomain, sub, blacklist, *flServerAddr) }
			if *flipv6 {
				tasks <- func() (bsw.Results, error) { return bsw.Dictionary6(*flDomain, sub, blacklist6, *flServerAddr) }
			}
		}
	}

	if *flYandex != "" && *flDomain != "" {
		tasks <- func() (bsw.Results, error) { return bsw.YandexAPI(*flDomain, *flYandex, *flServerAddr) }
	}

	// Close the tasks channel after all jobs have completed and for each
	// goroutine in the pool receive an empty message from  tracker.
	close(tasks)
	for i := 0; i < *flConcurrency; i++ {
		<-tracker
	}
	close(res)
	// Receive and empty message from the result gatherer.
	<-tracker
	os.Stderr.WriteString("\r")
	log.Println("All tasks completed\n")

	// Create a results slice from the unique set in resMap. Allows for sorting.
	results := bsw.Results{}
	for k, _ := range resMap {
		results = append(results, k)
	}
	sort.Sort(results)

	switch {
	case *flJson:
		j, _ := json.MarshalIndent(results, "", "    ")
		fmt.Println(string(j))
	case *flCsv:
		for _, r := range results {
			fmt.Printf("%s,%s,%s\n", r.Hostname, r.IP, r.Source)
		}
	case *flClean:
		cleanSet := make(map[string][]string)
		for _, r := range results {
			cleanSet[r.IP] = append(cleanSet[r.IP], r.Hostname)
		}
		for k, v := range cleanSet {
			fmt.Printf("%s:\n", k)
			for _, h := range v {
				fmt.Printf("\t%s\n", h)
			}
		}
	default:
		w := tabwriter.NewWriter(os.Stdout, 0, 8, 4, ' ', 0)
		fmt.Fprintln(w, "IP\tHostname\tSource")
		for _, r := range results {
			fmt.Fprintf(w, "%s\t%s\t%s\n", r.IP, r.Hostname, r.Source)
		}
		w.Flush()
	}
}
<|endoftext|>


<commit_before>package ppogdatspec

import (
	"flag"
	"github.com/the42/ogdat"
	htmltpl "html/template"
	"io"
	"log"
	"os"
	texttpl "text/template"
)

var inputfile = flag.String("if", "", "Eingabe mit OGD-Spezifikation (Standard: stdin)")
var outputfile = flag.String("if", "", "Ausgabe der Spezifikation nach (Standard: stdout)")
var templateset = flag.String("ts", "render.html.tpl", "(Satz von) Template-Dateien, die die Transformation der Spezifikation ins Ausgabeformat beschreibt")
var html = flag.Bool("html", true, "Anwendung von HTML-Escaping in der Ausgabe")
var help = flag.Bool("help", false, "Hilfe zur Verwendung")

type Templater interface {
	Execute(io.Writer, interface{}) error
}

func main() {
	flag.Parse()
	if *help {
		flag.PrintDefaults()
		return
	}

	if *inputfile == "" {
		*inputfile = os.Stdin.Name()
	}
	spec, err := ogdat.Loadogdatspec("unknown", *inputfile)
	if err != nil {
		log.Panicf("Could not load specification file %s, the error was %s\n", *inputfile, err)
	}

	var ofile *os.File
	if *outputfile == "" {
		ofile = os.Stdout
	} else {
		var err error
		ofile, err = os.OpenFile(*outputfile, os.O_RDWR|os.O_CREATE|os.O_EXCL, os.FileMode(0666))
		if err != nil {
			log.Panicf("Can't open file %s for writing, the error was: %s\n", inputfile, err)
		}
		defer ofile.Close()
	}

	var tpl Templater
	if *html {
		tpl = htmltpl.Must(htmltpl.ParseFiles(*templateset))
	} else {
		tpl = texttpl.Must(texttpl.ParseFiles(*templateset))
	}
	tpl.Execute(ofile, spec)
}
<commit_msg>this is an executable, so name package must be "main"<commit_after>package main

import (
	"flag"
	"github.com/the42/ogdat"
	htmltpl "html/template"
	"io"
	"log"
	"os"
	texttpl "text/template"
)

var inputfile = flag.String("if", "", "Eingabe mit OGD-Spezifikation (Standard: stdin)")
var outputfile = flag.String("if", "", "Ausgabe der Spezifikation nach (Standard: stdout)")
var templateset = flag.String("ts", "render.html.tpl", "(Satz von) Template-Dateien, die die Transformation der Spezifikation ins Ausgabeformat beschreibt")
var html = flag.Bool("html", true, "Anwendung von HTML-Escaping in der Ausgabe")
var help = flag.Bool("help", false, "Hilfe zur Verwendung")

type Templater interface {
	Execute(io.Writer, interface{}) error
}

func main() {
	flag.Parse()
	if *help {
		flag.PrintDefaults()
		return
	}

	if *inputfile == "" {
		*inputfile = os.Stdin.Name()
	}
	spec, err := ogdat.Loadogdatspec("unknown", *inputfile)
	if err != nil {
		log.Panicf("Could not load specification file %s, the error was %s\n", *inputfile, err)
	}

	var ofile *os.File
	if *outputfile == "" {
		ofile = os.Stdout
	} else {
		var err error
		ofile, err = os.OpenFile(*outputfile, os.O_RDWR|os.O_CREATE|os.O_EXCL, os.FileMode(0666))
		if err != nil {
			log.Panicf("Can't open file %s for writing, the error was: %s\n", inputfile, err)
		}
		defer ofile.Close()
	}

	var tpl Templater
	if *html {
		tpl = htmltpl.Must(htmltpl.ParseFiles(*templateset))
	} else {
		tpl = texttpl.Must(texttpl.ParseFiles(*templateset))
	}
	tpl.Execute(ofile, spec)
}
<|endoftext|>