<commit_before>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/chamaken/lotf"
	"github.com/golang/glog"
	"io"
	"os"
)

var rcfileFlag string
var pidfileFlag string

func init() {
	flag.StringVar(&rcfileFlag, "c", "config.json", "config filename")
	flag.StringVar(&pidfileFlag, "p", "", "pid filename")
}

type Config struct {
	Address   string
	Root      string
	Template  string
	Interval  int
	Buflines  int
	Lastlines int
	Lotfs     []LotfConfig
}

type LotfConfig struct {
	Name     string
	File     string
	Filter   string
	Template string
}

type config struct {
	addr      string
	root      string
	template  string
	interval  int
	buflines  int
	lastlines int
	lotfs     map[string]*lotfConfig
}

type lotfConfig struct {
	filename string
	filter   lotf.Filter
	template string
}

func makeResources(fname string) (*config, error) {
	r, err := os.Open(fname)
	if err != nil {
		return nil, err
	}

	s := &Config{Lotfs: make([]LotfConfig, 0)}
	dec := json.NewDecoder(r)
	for {
		if err := dec.Decode(s); err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}
	}

	lotfs := make(map[string]*lotfConfig)
	for _, v := range s.Lotfs {
		if _, found := lotfs[v.Name]; found {
			glog.Fatalf("founnd dup name: %s", v.Name)
		}
		var filter lotf.Filter
		if len(v.Filter) > 0 {
			filter, err = lotf.RegexpFilter(v.Filter)
			if err != nil {
				glog.Fatalf("create filter: %s", v.Filter)
			}
		} else {
			filter = nil
		}
		lotfs[v.Name] = &lotfConfig{
			filename: v.File,
			filter:   filter,
			template: v.Template,
		}
	}

	if s.Root[len(s.Root)-1] != '/' {
		s.Root += "/"
	}
	return &config{
		addr:      s.Address,
		root:      s.Root,
		template:  s.Template,
		interval:  s.Interval,
		buflines:  s.Buflines,
		lastlines: s.Lastlines,
		lotfs:     lotfs}, nil
}

func parseFlags() (*config, error) {
	flag.Parse()
	if flag.NArg() > 0 {
		return nil, errors.New(fmt.Sprintf("invalid arg(s): %s", flag.Args()))
	}

	resources, err := makeResources(rcfileFlag)
	if err != nil {
		return nil, err
	}

	if len(pidfileFlag) > 0 {
		pidfile, err := os.Create(pidfileFlag)
		if err != nil {
			return nil, err
		}
		defer pidfile.Close()
		fmt.Fprintf(pidfile, "%d\n", os.Getpid())
	}

	return resources, nil
}
<commit_msg>lotfw: add config error check<commit_after>package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"github.com/chamaken/lotf"
	"io"
	"os"
)

var rcfileFlag string
var pidfileFlag string

func init() {
	flag.StringVar(&rcfileFlag, "c", "config.json", "config filename")
	flag.StringVar(&pidfileFlag, "p", "", "pid filename")
}

type Config struct {
	Address   string
	Root      string
	Template  string
	Interval  int
	Buflines  int
	Lastlines int
	Lotfs     []LotfConfig
}

type LotfConfig struct {
	Name     string
	File     string
	Filter   string
	Template string
}

type config struct {
	addr      string
	root      string
	template  string
	interval  int
	buflines  int
	lastlines int
	lotfs     map[string]*lotfConfig
}

type lotfConfig struct {
	filename string
	filter   lotf.Filter
	template string
}

func makeResources(fname string) (*config, error) {
	r, err := os.Open(fname)
	if err != nil {
		return nil, err
	}

	s := &Config{Lotfs: make([]LotfConfig, 0)}
	dec := json.NewDecoder(r)
	for {
		if err := dec.Decode(s); err == io.EOF {
			break
		} else if err != nil {
			return nil, err
		}
	}

	lotfs := make(map[string]*lotfConfig)
	for _, v := range s.Lotfs {
		// XXX: check required json entries
		if len(v.Name) == 0 {
			return nil, errors.New("no name specified in lotfs")
		}
		if _, found := lotfs[v.Name]; found {
			return nil, errors.New(fmt.Sprintf("founnd dup name: %s", v.Name))
		}
		var filter lotf.Filter
		if len(v.Filter) > 0 {
			filter, err = lotf.RegexpFilter(v.Filter)
			if err != nil {
				return nil, errors.New(fmt.Sprintf("create filter: %s", v.Filter))
			}
		} else {
			filter = nil
		}
		if len(v.File) == 0 {
			return nil, errors.New(fmt.Sprintf("no file specified: %s", v.Name))
		}

		lotfs[v.Name] = &lotfConfig{
			filename: v.File,
			filter:   filter,
			template: v.Template,
		}
	}

	if len(s.Address) == 0 {
		return nil, errors.New("address is not specified")
	}
	if len(s.Root) == 0 {
		return nil, errors.New("root is not specified")
	}
	if len(s.Template) == 0 {
		return nil, errors.New("default template is not specified")
	}
	if s.Interval == 0 {
		return nil, errors.New("interval is not specified")
	}
	if s.Buflines == 0 {
		return nil, errors.New("buflines is not specified")
	}
	if s.Lastlines == 0 {
		return nil, errors.New("lastlines is not specified")
	}
	if len(lotfs) == 0 {
		return nil, errors.New("no lotf specified")
	}

	if s.Root[len(s.Root)-1] != '/' {
		s.Root += "/"
	}
	return &config{
		addr:      s.Address,
		root:      s.Root,
		template:  s.Template,
		interval:  s.Interval,
		buflines:  s.Buflines,
		lastlines: s.Lastlines,
		lotfs:     lotfs,
	}, nil
}

func parseFlags() (*config, error) {
	flag.Parse()
	if flag.NArg() > 0 {
		return nil, errors.New(fmt.Sprintf("invalid arg(s): %s", flag.Args()))
	}

	resources, err := makeResources(rcfileFlag)
	if err != nil {
		return nil, err
	}

	if len(pidfileFlag) > 0 {
		pidfile, err := os.Create(pidfileFlag)
		if err != nil {
			return nil, err
		}
		defer pidfile.Close()
		fmt.Fprintf(pidfile, "%d\n", os.Getpid())
	}

	return resources, nil
}
<|endoftext|>


<commit_before>package jsonschema

import (
	// "encoding/json"
	"fmt"
	"net"
	"net/mail"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	hostname       string = `^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$`
	unescapedTilda        = `\~[^01]`
	endingTilda           = `\~$`
	schemePrefix          = `^[^\:]+\:`
	uriTemplate           = `\{[^\{\}\\]*\}`
)

var (
	// emailPattern           = regexp.MustCompile(email)
	hostnamePattern        = regexp.MustCompile(hostname)
	unescaptedTildaPattern = regexp.MustCompile(unescapedTilda)
	endingTildaPattern     = regexp.MustCompile(endingTilda)
	schemePrefixPattern    = regexp.MustCompile(schemePrefix)
	uriTemplatePattern     = regexp.MustCompile(uriTemplate)
	disallowedIdnChars     = map[string]bool{"\u0020": true, "\u002D": true, "\u00A2": true, "\u00A3": true, "\u00A4": true, "\u00A5": true, "\u034F": true, "\u0640": true, "\u07FA": true, "\u180B": true, "\u180C": true, "\u180D": true, "\u200B": true, "\u2060": true, "\u2104": true, "\u2108": true, "\u2114": true, "\u2117": true, "\u2118": true, "\u211E": true, "\u211F": true, "\u2123": true, "\u2125": true, "\u2282": true, "\u2283": true, "\u2284": true, "\u2285": true, "\u2286": true, "\u2287": true, "\u2288": true, "\u2616": true, "\u2617": true, "\u2619": true, "\u262F": true, "\u2638": true, "\u266C": true, "\u266D": true, "\u266F": true, "\u2752": true, "\u2756": true, "\u2758": true, "\u275E": true, "\u2761": true, "\u2775": true, "\u2794": true, "\u2798": true, "\u27AF": true, "\u27B1": true, "\u27BE": true, "\u3004": true, "\u3012": true, "\u3013": true, "\u3020": true, "\u302E": true, "\u302F": true, "\u3031": true, "\u3032": true, "\u3035": true, "\u303B": true, "\u3164": true, "\uFFA0": true}
)

// for json pointers

// func FormatType(data interface{}) string {
// 	switch
// }
// Note: Date and time format names are derived from RFC 3339, section
// 5.6  [RFC3339].
// http://json-schema.org/latest/json-schema-validation.html#RFC3339

type format string

func newFormat() Validator {
	return new(format)
}

func (f format) Validate(data interface{}) error {
	if str, ok := data.(string); ok {
		switch f {
		case "date-time":
			return isValidDateTime(str)
		case "date":
			return isValidDate(str)
		case "email":
			return isValidEmail(str)
		case "hostname":
			return isValidHostname(str)
		case "idn-email":
			return isValidIdnEmail(str)
		case "idn-hostname":
			return isValidIdnHostname(str)
		case "ipv4":
			return isValidIPv4(str)
		case "ipv6":
			return isValidIPv6(str)
		case "iri-reference":
			return isValidIriRef(str)
		case "iri":
			return isValidIri(str)
		case "json-pointer":
			return isValidJSONPointer(str)
		case "regex":
			return isValidRegex(str)
		case "relative-json-pointer":
			return isValidRelJSONPointer(str)
		case "time":
			return isValidTime(str)
		case "uri-reference":
			return isValidURIRef(str)
		case "uri-template":
			return isValidURITemplate(str)
		case "uri":
			return isValidURI(str)
		default:
			// TODO: should we return an error saying that we don't know that
			// format? or should we keep it as is (ignore, return nil)
			return nil
		}
	}
	return nil
}

// A string instance is valid against "date-time" if it is a valid
// representation according to the "date-time" production derived
// from RFC 3339, section 5.6 [RFC3339]
// https://tools.ietf.org/html/rfc3339#section-5.6
func isValidDateTime(dateTime string) error {
	if _, err := time.Parse(time.RFC3339, dateTime); err != nil {
		return fmt.Errorf("date-time incorrectly formatted: %s", err.Error())
	}
	return nil
}

// A string instance is valid against "date" if it is a valid
// representation according to the "full-date" production derived
// from RFC 3339, section 5.6 [RFC3339]
// https://tools.ietf.org/html/rfc3339#section-5.6
func isValidDate(date string) error {
	arbitraryTime := "T08:30:06.283185Z"
	dateTime := fmt.Sprintf("%s%s", date, arbitraryTime)
	return isValidDateTime(dateTime)
}

// A string instance is valid against "email" if it is a valid
// representation as defined by RFC 5322, section 3.4.1 [RFC5322].
// https://tools.ietf.org/html/rfc5322#section-3.4.1
func isValidEmail(email string) error {
	// if !emailPattern.MatchString(email) {
	// 	return fmt.Errorf("invalid email format")
	// }
	if _, err := mail.ParseAddress(email); err != nil {
		return fmt.Errorf("email address incorrectly formatted: %s", err.Error())
	}
	return nil
}

// A string instance is valid against "hostname" if it is a valid
// representation as defined by RFC 1034, section 3.1 [RFC1034],
// including host names produced using the Punycode algorithm
// specified in RFC 5891, section 4.4 [RFC5891].
// https://tools.ietf.org/html/rfc1034#section-3.1
// https://tools.ietf.org/html/rfc5891#section-4.4
func isValidHostname(hostname string) error {
	if !hostnamePattern.MatchString(hostname) || len(hostname) > 255 {
		return fmt.Errorf("invalid hostname string")
	}
	return nil
}

// A string instance is valid against "idn-email" if it is a valid
// representation as defined by RFC 6531 [RFC6531]
// https://tools.ietf.org/html/rfc6531
func isValidIdnEmail(idnEmail string) error {
	if _, err := mail.ParseAddress(idnEmail); err != nil {
		return fmt.Errorf("email address incorrectly formatted: %s", err.Error())
	}
	return nil
}

// A string instance is valid against "hostname" if it is a valid
// representation as defined by either RFC 1034 as for hostname, or
// an internationalized hostname as defined by RFC 5890, section
// 2.3.2.3 [RFC5890].
// https://tools.ietf.org/html/rfc1034
// https://tools.ietf.org/html/rfc5890#section-2.3.2.3
// https://pdfs.semanticscholar.org/9275/6bcecb29d3dc407e23a997b256be6ff4149d.pdf
func isValidIdnHostname(idnHostname string) error {
	if len(idnHostname) > 255 {
		return fmt.Errorf("invalid idn hostname string")
	}
	for _, r := range idnHostname {
		s := string(r)
		if disallowedIdnChars[s] {
			return fmt.Errorf("invalid hostname: contains illegal character %#U", r)
		}
	}
	return nil
}

// A string instance is valid against "ipv4" if it is a valid
// representation of an IPv4 address according to the "dotted-quad"
// ABNF syntax as defined in RFC 2673, section 3.2 [RFC2673].
// https://tools.ietf.org/html/rfc2673#section-3.2
func isValidIPv4(ipv4 string) error {
	parsedIP := net.ParseIP(ipv4)
	hasDots := strings.Contains(ipv4, ".")
	if !hasDots || parsedIP == nil {
		return fmt.Errorf("invalid IPv4 address")
	}
	return nil
}

// A string instance is valid against "ipv6" if it is a valid
// representation of an IPv6 address as defined in RFC 4291, section
// 2.2 [RFC4291].
// https://tools.ietf.org/html/rfc4291#section-2.2
func isValidIPv6(ipv6 string) error {
	parsedIP := net.ParseIP(ipv6)
	hasColons := strings.Contains(ipv6, ":")
	if !hasColons || parsedIP == nil {
		return fmt.Errorf("invalid IPv4 address")
	}
	return nil
}

// A string instance is a valid against "iri-reference" if it is a
// valid IRI Reference (either an IRI or a relative-reference),
// according to [RFC3987].
// https://tools.ietf.org/html/rfc3987
func isValidIriRef(iriRef string) error {
	return isValidURIRef(iriRef)
}

// A string instance is a valid against "iri" if it is a valid IRI,
// according to [RFC3987].
// https://tools.ietf.org/html/rfc3987
func isValidIri(iri string) error {
	return isValidURI(iri)
}

// A string instance is a valid against "json-pointer" if it is a
// valid JSON string representation of a JSON Pointer, according to
// RFC 6901, section 5 [RFC6901].
// https://tools.ietf.org/html/rfc6901#section-5
func isValidJSONPointer(jsonPointer string) error {
	if len(jsonPointer) == 0 {
		return nil
	}
	if jsonPointer[0] != '/' {
		return fmt.Errorf("non-empty references must begin with a '/' character")
	}
	str := jsonPointer[1:]
	if unescaptedTildaPattern.MatchString(str) {
		return fmt.Errorf("unescaped tilda error")
	}
	if endingTildaPattern.MatchString(str) {
		return fmt.Errorf("unescaped tilda error")
	}
	return nil
}

// A string instance is a valid against "regex" if it is a valid
// regular expression according to the ECMA 262 [ecma262] regular
// expression dialect. Implementations that validate formats MUST
// accept at least the subset of ECMA 262 defined in the Regular
// Expressions [regexInterop] section of this specification, and
// SHOULD accept all valid ECMA 262 expressions.
// http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-262.pdf
// http://json-schema.org/latest/json-schema-validation.html#regexInterop
// https://tools.ietf.org/html/rfc7159
func isValidRegex(regex string) error {
	if _, err := regexp.Compile(regex); err != nil {
		return fmt.Errorf("invalid regex expression")
	}
	return nil
}

// A string instance is a valid against "relative-json-pointer" if it
// is a valid Relative JSON Pointer [relative-json-pointer].
// https://tools.ietf.org/html/draft-handrews-relative-json-pointer-00
func isValidRelJSONPointer(relJSONPointer string) error {
	parts := strings.Split(relJSONPointer, "/")
	if len(parts) == 1 {
		parts = strings.Split(relJSONPointer, "#")
	}
	if i, err := strconv.Atoi(parts[0]); err != nil || i < 0 {
		return fmt.Errorf("RJP must begin with positive integer")
	}
	//skip over first part
	str := relJSONPointer[len(parts[0]):]
	if len(str) > 0 && str[0] == '#' {
		return nil
	}
	return isValidJSONPointer(str)
}

// A string instance is valid against "time" if it is a valid
// representation according to the "full-time" production derived
// from RFC 3339, section 5.6 [RFC3339]
// https://tools.ietf.org/html/rfc3339#section-5.6
func isValidTime(time string) error {
	arbitraryDate := "1963-06-19"
	dateTime := fmt.Sprintf("%sT%s", arbitraryDate, time)
	return isValidDateTime(dateTime)
	return nil
}

// A string instance is a valid against "uri-reference" if it is a
// valid URI Reference (either a URI or a relative-reference),
// according to [RFC3986].
// https://tools.ietf.org/html/rfc3986
func isValidURIRef(uriRef string) error {
	if _, err := url.Parse(uriRef); err != nil {
		return fmt.Errorf("uri incorrectly formatted: %s", err.Error())
	}
	if strings.Contains(uriRef, "\\") {
		return fmt.Errorf("invalid uri")
	}
	return nil
}

// A string instance is a valid against "uri-template" if it is a
// valid URI Template (of any level), according to [RFC6570]. Note
// that URI Templates may be used for IRIs; there is no separate IRI
// Template specification.
// https://tools.ietf.org/html/rfc6570
func isValidURITemplate(uriTemplate string) error {
	arbitraryValue := "aaa"
	uriRef := uriTemplatePattern.ReplaceAllString(uriTemplate, arbitraryValue)
	if strings.Contains(uriRef, "{") || strings.Contains(uriRef, "}") {
		return fmt.Errorf("invalid uri template")
	}
	return isValidURIRef(uriRef)
}

// A string instance is a valid against "uri" if it is a valid URI,
// according to [RFC3986].
// https://tools.ietf.org/html/rfc3986
func isValidURI(uri string) error {
	if _, err := url.Parse(uri); err != nil {
		return fmt.Errorf("uri incorrectly formatted: %s", err.Error())
	}
	if !schemePrefixPattern.MatchString(uri) {
		return fmt.Errorf("uri missing scheme prefix")
	}
	return nil
}
<commit_msg>chore: updated format.Validate() to return a []ValError type<commit_after>package jsonschema

import (
	// "encoding/json"
	"fmt"
	"net"
	"net/mail"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	hostname       string = `^([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9])(\.([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]{0,61}[a-zA-Z0-9]))*$`
	unescapedTilda        = `\~[^01]`
	endingTilda           = `\~$`
	schemePrefix          = `^[^\:]+\:`
	uriTemplate           = `\{[^\{\}\\]*\}`
)

var (
	// emailPattern           = regexp.MustCompile(email)
	hostnamePattern        = regexp.MustCompile(hostname)
	unescaptedTildaPattern = regexp.MustCompile(unescapedTilda)
	endingTildaPattern     = regexp.MustCompile(endingTilda)
	schemePrefixPattern    = regexp.MustCompile(schemePrefix)
	uriTemplatePattern     = regexp.MustCompile(uriTemplate)
	disallowedIdnChars     = map[string]bool{"\u0020": true, "\u002D": true, "\u00A2": true, "\u00A3": true, "\u00A4": true, "\u00A5": true, "\u034F": true, "\u0640": true, "\u07FA": true, "\u180B": true, "\u180C": true, "\u180D": true, "\u200B": true, "\u2060": true, "\u2104": true, "\u2108": true, "\u2114": true, "\u2117": true, "\u2118": true, "\u211E": true, "\u211F": true, "\u2123": true, "\u2125": true, "\u2282": true, "\u2283": true, "\u2284": true, "\u2285": true, "\u2286": true, "\u2287": true, "\u2288": true, "\u2616": true, "\u2617": true, "\u2619": true, "\u262F": true, "\u2638": true, "\u266C": true, "\u266D": true, "\u266F": true, "\u2752": true, "\u2756": true, "\u2758": true, "\u275E": true, "\u2761": true, "\u2775": true, "\u2794": true, "\u2798": true, "\u27AF": true, "\u27B1": true, "\u27BE": true, "\u3004": true, "\u3012": true, "\u3013": true, "\u3020": true, "\u302E": true, "\u302F": true, "\u3031": true, "\u3032": true, "\u3035": true, "\u303B": true, "\u3164": true, "\uFFA0": true}
)

// for json pointers

// func FormatType(data interface{}) string {
// 	switch
// }
// Note: Date and time format names are derived from RFC 3339, section
// 5.6  [RFC3339].
// http://json-schema.org/latest/json-schema-validation.html#RFC3339

type format string

func newFormat() Validator {
	return new(format)
}

func (f format) Validate(data interface{}) []ValError {
	var err error
	if str, ok := data.(string); ok {
		switch f {
		case "date-time":
			err = isValidDateTime(str)
		case "date":
			err = isValidDate(str)
		case "email":
			err = isValidEmail(str)
		case "hostname":
			err = isValidHostname(str)
		case "idn-email":
			err = isValidIdnEmail(str)
		case "idn-hostname":
			err = isValidIdnHostname(str)
		case "ipv4":
			err = isValidIPv4(str)
		case "ipv6":
			err = isValidIPv6(str)
		case "iri-reference":
			err = isValidIriRef(str)
		case "iri":
			err = isValidIri(str)
		case "json-pointer":
			err = isValidJSONPointer(str)
		case "regex":
			err = isValidRegex(str)
		case "relative-json-pointer":
			err = isValidRelJSONPointer(str)
		case "time":
			err = isValidTime(str)
		case "uri-reference":
			err = isValidURIRef(str)
		case "uri-template":
			err = isValidURITemplate(str)
		case "uri":
			err = isValidURI(str)
		default:
			err = nil
		}
		if err != nil {
			return []ValError{
				{Message: err.Error()},
			}
		}
	}
	return nil
}

// A string instance is valid against "date-time" if it is a valid
// representation according to the "date-time" production derived
// from RFC 3339, section 5.6 [RFC3339]
// https://tools.ietf.org/html/rfc3339#section-5.6
func isValidDateTime(dateTime string) error {
	if _, err := time.Parse(time.RFC3339, dateTime); err != nil {
		return fmt.Errorf("date-time incorrectly formatted: %s", err.Error())
	}
	return nil
}

// A string instance is valid against "date" if it is a valid
// representation according to the "full-date" production derived
// from RFC 3339, section 5.6 [RFC3339]
// https://tools.ietf.org/html/rfc3339#section-5.6
func isValidDate(date string) error {
	arbitraryTime := "T08:30:06.283185Z"
	dateTime := fmt.Sprintf("%s%s", date, arbitraryTime)
	return isValidDateTime(dateTime)
}

// A string instance is valid against "email" if it is a valid
// representation as defined by RFC 5322, section 3.4.1 [RFC5322].
// https://tools.ietf.org/html/rfc5322#section-3.4.1
func isValidEmail(email string) error {
	// if !emailPattern.MatchString(email) {
	// 	return fmt.Errorf("invalid email format")
	// }
	if _, err := mail.ParseAddress(email); err != nil {
		return fmt.Errorf("email address incorrectly formatted: %s", err.Error())
	}
	return nil
}

// A string instance is valid against "hostname" if it is a valid
// representation as defined by RFC 1034, section 3.1 [RFC1034],
// including host names produced using the Punycode algorithm
// specified in RFC 5891, section 4.4 [RFC5891].
// https://tools.ietf.org/html/rfc1034#section-3.1
// https://tools.ietf.org/html/rfc5891#section-4.4
func isValidHostname(hostname string) error {
	if !hostnamePattern.MatchString(hostname) || len(hostname) > 255 {
		return fmt.Errorf("invalid hostname string")
	}
	return nil
}

// A string instance is valid against "idn-email" if it is a valid
// representation as defined by RFC 6531 [RFC6531]
// https://tools.ietf.org/html/rfc6531
func isValidIdnEmail(idnEmail string) error {
	if _, err := mail.ParseAddress(idnEmail); err != nil {
		return fmt.Errorf("email address incorrectly formatted: %s", err.Error())
	}
	return nil
}

// A string instance is valid against "hostname" if it is a valid
// representation as defined by either RFC 1034 as for hostname, or
// an internationalized hostname as defined by RFC 5890, section
// 2.3.2.3 [RFC5890].
// https://tools.ietf.org/html/rfc1034
// https://tools.ietf.org/html/rfc5890#section-2.3.2.3
// https://pdfs.semanticscholar.org/9275/6bcecb29d3dc407e23a997b256be6ff4149d.pdf
func isValidIdnHostname(idnHostname string) error {
	if len(idnHostname) > 255 {
		return fmt.Errorf("invalid idn hostname string")
	}
	for _, r := range idnHostname {
		s := string(r)
		if disallowedIdnChars[s] {
			return fmt.Errorf("invalid hostname: contains illegal character %#U", r)
		}
	}
	return nil
}

// A string instance is valid against "ipv4" if it is a valid
// representation of an IPv4 address according to the "dotted-quad"
// ABNF syntax as defined in RFC 2673, section 3.2 [RFC2673].
// https://tools.ietf.org/html/rfc2673#section-3.2
func isValidIPv4(ipv4 string) error {
	parsedIP := net.ParseIP(ipv4)
	hasDots := strings.Contains(ipv4, ".")
	if !hasDots || parsedIP == nil {
		return fmt.Errorf("invalid IPv4 address")
	}
	return nil
}

// A string instance is valid against "ipv6" if it is a valid
// representation of an IPv6 address as defined in RFC 4291, section
// 2.2 [RFC4291].
// https://tools.ietf.org/html/rfc4291#section-2.2
func isValidIPv6(ipv6 string) error {
	parsedIP := net.ParseIP(ipv6)
	hasColons := strings.Contains(ipv6, ":")
	if !hasColons || parsedIP == nil {
		return fmt.Errorf("invalid IPv4 address")
	}
	return nil
}

// A string instance is a valid against "iri-reference" if it is a
// valid IRI Reference (either an IRI or a relative-reference),
// according to [RFC3987].
// https://tools.ietf.org/html/rfc3987
func isValidIriRef(iriRef string) error {
	return isValidURIRef(iriRef)
}

// A string instance is a valid against "iri" if it is a valid IRI,
// according to [RFC3987].
// https://tools.ietf.org/html/rfc3987
func isValidIri(iri string) error {
	return isValidURI(iri)
}

// A string instance is a valid against "json-pointer" if it is a
// valid JSON string representation of a JSON Pointer, according to
// RFC 6901, section 5 [RFC6901].
// https://tools.ietf.org/html/rfc6901#section-5
func isValidJSONPointer(jsonPointer string) error {
	if len(jsonPointer) == 0 {
		return nil
	}
	if jsonPointer[0] != '/' {
		return fmt.Errorf("non-empty references must begin with a '/' character")
	}
	str := jsonPointer[1:]
	if unescaptedTildaPattern.MatchString(str) {
		return fmt.Errorf("unescaped tilda error")
	}
	if endingTildaPattern.MatchString(str) {
		return fmt.Errorf("unescaped tilda error")
	}
	return nil
}

// A string instance is a valid against "regex" if it is a valid
// regular expression according to the ECMA 262 [ecma262] regular
// expression dialect. Implementations that validate formats MUST
// accept at least the subset of ECMA 262 defined in the Regular
// Expressions [regexInterop] section of this specification, and
// SHOULD accept all valid ECMA 262 expressions.
// http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-262.pdf
// http://json-schema.org/latest/json-schema-validation.html#regexInterop
// https://tools.ietf.org/html/rfc7159
func isValidRegex(regex string) error {
	if _, err := regexp.Compile(regex); err != nil {
		return fmt.Errorf("invalid regex expression")
	}
	return nil
}

// A string instance is a valid against "relative-json-pointer" if it
// is a valid Relative JSON Pointer [relative-json-pointer].
// https://tools.ietf.org/html/draft-handrews-relative-json-pointer-00
func isValidRelJSONPointer(relJSONPointer string) error {
	parts := strings.Split(relJSONPointer, "/")
	if len(parts) == 1 {
		parts = strings.Split(relJSONPointer, "#")
	}
	if i, err := strconv.Atoi(parts[0]); err != nil || i < 0 {
		return fmt.Errorf("RJP must begin with positive integer")
	}
	//skip over first part
	str := relJSONPointer[len(parts[0]):]
	if len(str) > 0 && str[0] == '#' {
		return nil
	}
	return isValidJSONPointer(str)
}

// A string instance is valid against "time" if it is a valid
// representation according to the "full-time" production derived
// from RFC 3339, section 5.6 [RFC3339]
// https://tools.ietf.org/html/rfc3339#section-5.6
func isValidTime(time string) error {
	arbitraryDate := "1963-06-19"
	dateTime := fmt.Sprintf("%sT%s", arbitraryDate, time)
	return isValidDateTime(dateTime)
	return nil
}

// A string instance is a valid against "uri-reference" if it is a
// valid URI Reference (either a URI or a relative-reference),
// according to [RFC3986].
// https://tools.ietf.org/html/rfc3986
func isValidURIRef(uriRef string) error {
	if _, err := url.Parse(uriRef); err != nil {
		return fmt.Errorf("uri incorrectly formatted: %s", err.Error())
	}
	if strings.Contains(uriRef, "\\") {
		return fmt.Errorf("invalid uri")
	}
	return nil
}

// A string instance is a valid against "uri-template" if it is a
// valid URI Template (of any level), according to [RFC6570]. Note
// that URI Templates may be used for IRIs; there is no separate IRI
// Template specification.
// https://tools.ietf.org/html/rfc6570
func isValidURITemplate(uriTemplate string) error {
	arbitraryValue := "aaa"
	uriRef := uriTemplatePattern.ReplaceAllString(uriTemplate, arbitraryValue)
	if strings.Contains(uriRef, "{") || strings.Contains(uriRef, "}") {
		return fmt.Errorf("invalid uri template")
	}
	return isValidURIRef(uriRef)
}

// A string instance is a valid against "uri" if it is a valid URI,
// according to [RFC3986].
// https://tools.ietf.org/html/rfc3986
func isValidURI(uri string) error {
	if _, err := url.Parse(uri); err != nil {
		return fmt.Errorf("uri incorrectly formatted: %s", err.Error())
	}
	if !schemePrefixPattern.MatchString(uri) {
		return fmt.Errorf("uri missing scheme prefix")
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	d "gowebdav"
	"flag"
	"os"
	"fmt"
)

func Fail(err interface{}) {
	if err != nil {
		fmt.Println(err)
	} else {
		fmt.Println("Usage: client FLAGS ARGS")
		fmt.Println("Flags:")
		flag.PrintDefaults()
		fmt.Println("Method:")
		fmt.Println(" LIST, PROPFIND:")
		fmt.Println(" RM, DELETE, DEL:")
		fmt.Println(" MKDIR, MKCOL:")
	}
	os.Exit(-1)
}

func main() {
	root := flag.String("root", "URL", "WebDAV Endpoint")
	usr := flag.String("user", "", "user")
	pw := flag.String("pw", "", "password")
	m := flag.String("X", "GET", "Method ...")
	flag.Parse()

	if *root == "URL" {
		Fail(nil)
	}

	c := d.NewClient(*root, *usr, *pw)
	if err := c.Connect(); err != nil {
		Fail(err)
	}

	if len(flag.Args()) > 0 {
		path := flag.Args()[0]
		switch *m {
			case "LIST", "PROPFIND":
				if files, err := c.ReadDir(path); err == nil {
					fmt.Println(fmt.Sprintf("ReadDir: %s, entries: ", path, len(files)))
					for _, f := range files {
						fmt.Println(f)
					}
				} else {
					fmt.Println(err)
				}

			case "GET": c.Read(path)

			case "DELETE", "RM", "DEL":
				if err := c.Remove(path); err != nil {
					fmt.Println(err)
				} else {
					fmt.Println("Remove: " + path)
				}

			case "MKCOL", "MKDIR":
				if err := c.Mkdir(path, 0); err != nil {
					fmt.Println(err)
				} else {
					fmt.Println("MkDir: " + path)
				}

			default: Fail(nil)
		}
	} else {
		Fail(nil)
	}
}

<commit_msg>fix LIST log output<commit_after>package main

import (
	d "gowebdav"
	"flag"
	"os"
	"fmt"
)

func Fail(err interface{}) {
	if err != nil {
		fmt.Println(err)
	} else {
		fmt.Println("Usage: client FLAGS ARGS")
		fmt.Println("Flags:")
		flag.PrintDefaults()
		fmt.Println("Method:")
		fmt.Println(" LIST, PROPFIND:")
		fmt.Println(" RM, DELETE, DEL:")
		fmt.Println(" MKDIR, MKCOL:")
	}
	os.Exit(-1)
}

func main() {
	root := flag.String("root", "URL", "WebDAV Endpoint")
	usr := flag.String("user", "", "user")
	pw := flag.String("pw", "", "password")
	m := flag.String("X", "GET", "Method ...")
	flag.Parse()

	if *root == "URL" {
		Fail(nil)
	}

	c := d.NewClient(*root, *usr, *pw)
	if err := c.Connect(); err != nil {
		Fail(err)
	}

	if len(flag.Args()) > 0 {
		path := flag.Args()[0]
		switch *m {
			case "LIST", "PROPFIND":
				if files, err := c.ReadDir(path); err == nil {
					fmt.Println(fmt.Sprintf("ReadDir: '%s' entries: %d ", path, len(files)))
					for _, f := range files {
						fmt.Println(f)
					}
				} else {
					fmt.Println(err)
				}

			case "GET": c.Read(path)

			case "DELETE", "RM", "DEL":
				if err := c.Remove(path); err != nil {
					fmt.Println(err)
				} else {
					fmt.Println("Remove: " + path)
				}

			case "MKCOL", "MKDIR":
				if err := c.Mkdir(path, 0); err != nil {
					fmt.Println(err)
				} else {
					fmt.Println("MkDir: " + path)
				}

			default: Fail(nil)
		}
	} else {
		Fail(nil)
	}
}

<|endoftext|>


<commit_before>package filter

import (
	"encoding/json"
	"strings"
)

type FilterConfig struct {
	Ignore []string `json:"ignore"`
}

func GetFilterConfig(fileName string) (FilterConfig, error) {
	/* Takes a json file name as a string. Reads the file, unmarshals json.
	Returns the unmarshalled FilterConfig object
	See configurator.ReadConfig for an example
	In detail:
	1. Read all of the file with ioutil.ReadFile
	2. Instantiate an empty config object.
	3. Unmarshal the byte array which is the file's contents as json into the
	new config object using jsom.Unmarshal.
	4. Return the config object.
	If there is an error, return it.
	*/
	// Dummy implementation for testing. Remove later
	newFilterConf := FilterConfig{Ignore: []string{"/dev/"}}
	return newFilterConf, nil
}

type Filter interface {
	Start(c FilterConfig, sending chan<- []byte, receiving <-chan []byte)
}

type FSFilter struct{}

type NOPFilter struct{}

type ZachsInotifyData struct {
	Date     string `json:"DATE"`
	Event    string `json:"EVENT"`
	FilePath string `json:"PATH"`
	Type     string `json:"TYPE"`
}

func StartFilterStream(sending chan<- []byte, receiving <-chan []byte) {
	fsFilter := FSFilter{}
	nopFilter := NOPFilter{}
	conf := FilterConfig{Ignore: []string{"/dev/"}}
	link := make(chan []byte)
	go fsFilter.Start(conf, link, receiving)
	go nopFilter.Start(conf, sending, link)
}

func (f FSFilter) Start(c FilterConfig, sending chan<- []byte, receiving <-chan []byte) {
	for {
		message := <-receiving
		zid := ZachsInotifyData{}
		err := json.Unmarshal(message, &zid)
		if err != nil {
			panic(err)
		}
		notblacklisted := true
		for _, i := range c.Ignore {
			if strings.HasPrefix(zid.FilePath, i) {
				notblacklisted = false
				break
			}
		}
		if notblacklisted {
			sending <- message
		}
	}
}

func (N NOPFilter) Start(c FilterConfig, sending chan<- []byte, receiving <-chan []byte) {
	for {
		message := <-receiving
		sending <- message
	}
}
<commit_msg>Added GetFilterConfig, needs to be tested more.<commit_after>package filter

import (
	"encoding/json"
	"strings"
	"io/ioutil"
)

type FilterConfig struct {
	Ignore []string `json:"ignore"`
}

func GetFilterConfig(fileName string) (FilterConfig, error) {
	/* Takes a json file name as a string. Reads the file, unmarshals json.
	Returns the unmarshalled FilterConfig object
	See configurator.ReadConfig for an example
	In detail:
	1. Read all of the file with ioutil.ReadFile
	2. Instantiate an empty config object.
	3. Unmarshal the byte array which is the file's contents as json into the
	new config object using jsom.Unmarshal.
	4. Return the config object.
	If there is an error, return it.
	*/

	config := FilterConfig{}
	data, err := ioutil.ReadFile(fileName)
	if err != nil {
		return config, err
	}
	err = json.Unmarshal(data, &config)
	if err != nil {
		return config, err
	}

	return config, nil

	// Dummy implementation for testing. Remove later
	// newFilterConf := FilterConfig{Ignore: []string{"/dev/"}}
	// return newFilterConf, nil
}

type Filter interface {
	Start(c FilterConfig, sending chan<- []byte, receiving <-chan []byte)
}

type FSFilter struct{}

type NOPFilter struct{}

type ZachsInotifyData struct {
	Date     string `json:"DATE"`
	Event    string `json:"EVENT"`
	FilePath string `json:"PATH"`
	Type     string `json:"TYPE"`
}

func StartFilterStream(sending chan<- []byte, receiving <-chan []byte) {
	fsFilter := FSFilter{}
	nopFilter := NOPFilter{}
	conf := FilterConfig{Ignore: []string{"/dev/"}}
	link := make(chan []byte)
	go fsFilter.Start(conf, link, receiving)
	go nopFilter.Start(conf, sending, link)
}

func (f FSFilter) Start(c FilterConfig, sending chan<- []byte, receiving <-chan []byte) {
	for {
		message := <-receiving
		zid := ZachsInotifyData{}
		err := json.Unmarshal(message, &zid)
		if err != nil {
			panic(err)
		}
		notblacklisted := true
		for _, i := range c.Ignore {
			if strings.HasPrefix(zid.FilePath, i) {
				notblacklisted = false
				break
			}
		}
		if notblacklisted {
			sending <- message
		}
	}
}

func (N NOPFilter) Start(c FilterConfig, sending chan<- []byte, receiving <-chan []byte) {
	for {
		message := <-receiving
		sending <- message
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"

	"github.com/gorilla/mux"
	"github.com/gorilla/websocket"
	_ "github.com/lib/pq"
)

// Address struct is the response returned after a request for addresses
type Address struct {
	PLZ, Gemeindename, Ortsname, Strassenname, Hausnr *string
	LatlongX, LatlongY                                *float64
}

var upgrader = websocket.Upgrader{
	CheckOrigin: func(r *http.Request) bool { return true },
}

type connection struct {
	*sql.DB
}

const maxrowsFTS = 200
const defaultrowsFTS = 25
const nearbymeters = 50 // default distance to search nearby addresses in meter
const autocomplete = `search @@ to_tsquery(plainto_tsquery('german', $1)::text || ':*')`
const noautocomplete = `search @@ plainto_tsquery('german', $1)`

const fulltextSearchSQL = `select addritems.plz, addritems.gemeindename, addritems.ortsname, addritems.strassenname, addritems.hausnrzahl1, ST_Y(adresse.latlong), ST_X(adresse.latlong)
from adresse
inner join addritems
on addritems.adrcd = adresse.adrcd
and %s
and addritems.plz like COALESCE(NULLIF($2, ''), addritems.plz)
and addritems.gkz like COALESCE(NULLIF($3, ''), addritems.gkz)
and addritems.bld = COALESCE(CAST(NULLIF($4, '') AS smallint), addritems.bld)
and CASE ($5 = '' AND $6='') WHEN NOT FALSE THEN TRUE ELSE ST_DWithin(latlong_g, ST_GeomFromText('POINT(' || $5 || ' ' || $6 || ')', 4326)::geography, $7, false) END
limit $8`

func (con *connection) fulltextSearch(w http.ResponseWriter, r *http.Request) {

	var n uint64
	if nrows := r.URL.Query().Get("n"); nrows != "" {
		var err error
		if n, err = strconv.ParseUint(nrows, 10, 8); err != nil {
			s := "error when parsing parameter n: " + err.Error()
			info(s)
			http.Error(w, s, http.StatusBadRequest)
			return
		}
		if n > maxrowsFTS {
			s := "paramter out of range"
			info(s)
			http.Error(w, s, http.StatusBadRequest)
			return
		}
	} else {
		n = defaultrowsFTS
	}

	lat := r.URL.Query().Get("lat")
	lon := r.URL.Query().Get("lon")

	if (len(lat) > 0) != (len(lon) > 0) { // Latitude/Longitude: either both parameters are set or none of the two is set
		s := "lat/lon: either both parameters are set to a value or both have to be empty"
		info(s)
		http.Error(w, s, http.StatusBadRequest)
		return
	}

	q := r.URL.Query().Get("q")
	postcode := r.URL.Query().Get("postcode")
	citycode := r.URL.Query().Get("citycode")
	province := r.URL.Query().Get("province")

	var querystring string
	if acparam := r.URL.Query().Get("autocomplete"); acparam == "0" {
		querystring = fmt.Sprintf(fulltextSearchSQL, noautocomplete)
	} else {
		querystring = fmt.Sprintf(fulltextSearchSQL, autocomplete)
	}

	rows, err := con.Query(querystring, q, postcode, citycode, province, lat, lon, nearbymeters, n)
	if err != nil {
		s := "database query failed: " + err.Error()
		info(s)
		http.Error(w, s, http.StatusInternalServerError)
		return
	}
	defer rows.Close()
	var plz, gemeindename, ortsname, strassenname, hausnrzahl1 *string
	var latlongy, latlongx *float64

	var addresses []Address

	for rows.Next() {
		if err = rows.Scan(&plz, &gemeindename, &ortsname, &strassenname, &hausnrzahl1, &latlongy, &latlongx); err != nil {
			s := "reading from database failed: " + err.Error()
			info(s)
			http.Error(w, s, http.StatusInternalServerError)
			return
		}

		addr := Address{PLZ: plz, Gemeindename: gemeindename, Ortsname: ortsname, Strassenname: strassenname, Hausnr: hausnrzahl1, LatlongY: latlongy, LatlongX: latlongx}
		addresses = append(addresses, addr)
	}

	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		s := "connection upgrade to websocket failed: " + err.Error()
		info(s)
		http.Error(w, s, http.StatusInternalServerError)
		return
	}

	conn.WriteJSON(addresses)
	conn.Close()
}

func getDatabaseConnection() (*sql.DB, error) {
	var dburl string

	if dburl = os.Getenv("DATABASE_URL"); dburl == "" {
		dburl = "postgres://"
	}

	db, err := sql.Open("postgres", dburl)
	if err != nil {
		return nil, err
	}
	return db, nil
}

func main() {
	currdir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
	info("starting up in " + currdir)

	conn, err := getDatabaseConnection()
	if err != nil {
		fatal(err.Error())
	}
	connection := &connection{DB: conn}

	r := mux.NewRouter()
	s := r.PathPrefix("/ws/").Subrouter()
	s.HandleFunc("/address/fts", connection.fulltextSearch)

	var port, secport string
	if secport = os.Getenv("SECPORT"); secport != "" {
		go func() {
			if err := http.ListenAndServeTLS(":"+secport, "cert.pem", "key.pem", r); err != nil {
				fatal("secure serving failed: " + err.Error())
			}
		}()
		info("serving securely on port " + secport)
	}

	if port = os.Getenv("PORT"); port == "" {
		port = "5000"
	}

	info("serving on port " + port)
	http.ListenAndServe(":"+port, r)
}

// Log wrappers
func info(template string, values ...interface{}) {
	log.Printf("[bevaddress][info] "+template+"\n", values...)
}

func fatal(template string, values ...interface{}) {
	log.Fatalf("[bevaddress][fatal] "+template+"\n", values...)
}
<commit_msg>Fixed duplicate stemming during auto completion<commit_after>package main

import (
	"database/sql"
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"

	"github.com/gorilla/mux"
	"github.com/gorilla/websocket"
	_ "github.com/lib/pq"
)

// Address struct is the response returned after a request for addresses
type Address struct {
	PLZ, Gemeindename, Ortsname, Strassenname, Hausnr *string
	LatlongX, LatlongY                                *float64
}

var upgrader = websocket.Upgrader{
	CheckOrigin: func(r *http.Request) bool { return true },
}

type connection struct {
	*sql.DB
}

const maxrowsFTS = 200
const defaultrowsFTS = 25
const nearbymeters = 50 // default distance to search nearby addresses in meter
const autocomplete = `search @@ plainto_tsquery('german', $1 || ':*')`
const noautocomplete = `search @@ plainto_tsquery('german', $1)`

const fulltextSearchSQL = `select addritems.plz, addritems.gemeindename, addritems.ortsname, addritems.strassenname, addritems.hausnrzahl1, ST_Y(adresse.latlong), ST_X(adresse.latlong)
from adresse
inner join addritems
on addritems.adrcd = adresse.adrcd
and %s
and addritems.plz like COALESCE(NULLIF($2, ''), addritems.plz)
and addritems.gkz like COALESCE(NULLIF($3, ''), addritems.gkz)
and addritems.bld = COALESCE(CAST(NULLIF($4, '') AS smallint), addritems.bld)
and CASE ($5 = '' AND $6='') WHEN NOT FALSE THEN TRUE ELSE ST_DWithin(latlong_g, ST_GeomFromText('POINT(' || $5 || ' ' || $6 || ')', 4326)::geography, $7, false) END
limit $8`

func (con *connection) fulltextSearch(w http.ResponseWriter, r *http.Request) {

	var n uint64
	if nrows := r.URL.Query().Get("n"); nrows != "" {
		var err error
		if n, err = strconv.ParseUint(nrows, 10, 8); err != nil {
			s := "error when parsing parameter n: " + err.Error()
			info(s)
			http.Error(w, s, http.StatusBadRequest)
			return
		}
		if n > maxrowsFTS {
			s := "paramter out of range"
			info(s)
			http.Error(w, s, http.StatusBadRequest)
			return
		}
	} else {
		n = defaultrowsFTS
	}

	lat := r.URL.Query().Get("lat")
	lon := r.URL.Query().Get("lon")

	if (len(lat) > 0) != (len(lon) > 0) { // Latitude/Longitude: either both parameters are set or none of the two is set
		s := "lat/lon: either both parameters are set to a value or both have to be empty"
		info(s)
		http.Error(w, s, http.StatusBadRequest)
		return
	}

	q := r.URL.Query().Get("q")
	postcode := r.URL.Query().Get("postcode")
	citycode := r.URL.Query().Get("citycode")
	province := r.URL.Query().Get("province")

	var querystring string
	if acparam := r.URL.Query().Get("autocomplete"); acparam == "0" {
		querystring = fmt.Sprintf(fulltextSearchSQL, noautocomplete)
	} else {
		querystring = fmt.Sprintf(fulltextSearchSQL, autocomplete)
	}

	rows, err := con.Query(querystring, q, postcode, citycode, province, lat, lon, nearbymeters, n)
	if err != nil {
		s := "database query failed: " + err.Error()
		info(s)
		http.Error(w, s, http.StatusInternalServerError)
		return
	}
	defer rows.Close()
	var plz, gemeindename, ortsname, strassenname, hausnrzahl1 *string
	var latlongy, latlongx *float64

	var addresses []Address

	for rows.Next() {
		if err = rows.Scan(&plz, &gemeindename, &ortsname, &strassenname, &hausnrzahl1, &latlongy, &latlongx); err != nil {
			s := "reading from database failed: " + err.Error()
			info(s)
			http.Error(w, s, http.StatusInternalServerError)
			return
		}

		addr := Address{PLZ: plz, Gemeindename: gemeindename, Ortsname: ortsname, Strassenname: strassenname, Hausnr: hausnrzahl1, LatlongY: latlongy, LatlongX: latlongx}
		addresses = append(addresses, addr)
	}

	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		s := "connection upgrade to websocket failed: " + err.Error()
		info(s)
		http.Error(w, s, http.StatusInternalServerError)
		return
	}

	conn.WriteJSON(addresses)
	conn.Close()
}

func getDatabaseConnection() (*sql.DB, error) {
	var dburl string

	if dburl = os.Getenv("DATABASE_URL"); dburl == "" {
		dburl = "postgres://"
	}

	db, err := sql.Open("postgres", dburl)
	if err != nil {
		return nil, err
	}
	return db, nil
}

func main() {
	currdir, _ := filepath.Abs(filepath.Dir(os.Args[0]))
	info("starting up in " + currdir)

	conn, err := getDatabaseConnection()
	if err != nil {
		fatal(err.Error())
	}
	connection := &connection{DB: conn}

	r := mux.NewRouter()
	s := r.PathPrefix("/ws/").Subrouter()
	s.HandleFunc("/address/fts", connection.fulltextSearch)

	var port, secport string
	if secport = os.Getenv("SECPORT"); secport != "" {
		go func() {
			if err := http.ListenAndServeTLS(":"+secport, "cert.pem", "key.pem", r); err != nil {
				fatal("secure serving failed: " + err.Error())
			}
		}()
		info("serving securely on port " + secport)
	}

	if port = os.Getenv("PORT"); port == "" {
		port = "5000"
	}

	info("serving on port " + port)
	http.ListenAndServe(":"+port, r)
}

// Log wrappers
func info(template string, values ...interface{}) {
	log.Printf("[bevaddress][info] "+template+"\n", values...)
}

func fatal(template string, values ...interface{}) {
	log.Fatalf("[bevaddress][fatal] "+template+"\n", values...)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// An external diurnal controller for kubernetes. With this, it's possible to manage
// known replica counts that vary throughout the day.

package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"

	kclient "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/labels"

	"github.com/golang/glog"
)

const dayPeriod = 24 * time.Hour

type timeCount struct {
	time  time.Duration
	count int
}

func (tc timeCount) String() string {
	h := tc.time / time.Hour
	m := (tc.time % time.Hour) / time.Minute
	s := (tc.time % time.Minute) / time.Second
	if m == 0 && s == 0 {
		return fmt.Sprintf("(%02dZ, %d)", h, tc.count)
	} else if s == 0 {
		return fmt.Sprintf("(%02d:%02dZ, %d)", h, m, tc.count)
	}
	return fmt.Sprintf("(%02d:%02d:%02dZ, %d)", h, m, s, tc.count)
}

type byTime []timeCount

func (tc byTime) Len() int           { return len(tc) }
func (tc byTime) Swap(i, j int)      { tc[i], tc[j] = tc[j], tc[i] }
func (tc byTime) Less(i, j int) bool { return tc[i].time < tc[j].time }

func timeMustParse(layout, s string) time.Time {
	t, err := time.Parse(layout, s)
	if err != nil {
		panic(err)
	}
	return t
}

// first argument is a format string equivalent to HHMMSS. See time.Parse for details.
var epoch = timeMustParse("150405", "000000")

func parseTimeRelative(s string) (time.Duration, error) {
	t, err := parseTimeISO8601(s)
	if err != nil {
		return 0, fmt.Errorf("unable to parse %s: %v", s, err)
	}
	return (t.Sub(epoch) + dayPeriod) % dayPeriod, nil
}

func parseTimeCounts(times string, counts string) ([]timeCount, error) {
	ts := strings.Split(times, ",")
	cs := strings.Split(counts, ",")
	if len(ts) != len(cs) {
		return nil, fmt.Errorf("provided %d times but %d replica counts", len(ts), len(cs))
	}
	var tc []timeCount
	for i := range ts {
		t, err := parseTimeRelative(ts[i])
		if err != nil {
			return nil, err
		}
		c, err := strconv.ParseInt(cs[i], 10, 64)
		if c < 0 {
			return nil, errors.New("counts must be non-negative")
		}
		if err != nil {
			return nil, err
		}
		tc = append(tc, timeCount{t, int(c)})
	}
	sort.Sort(byTime(tc))
	return tc, nil
}

type scaler struct {
	timeCounts []timeCount
	selector   labels.Selector
	start      time.Time
	pos        int
	done       chan struct{}
}

func findPos(tc []timeCount, cur int, offset time.Duration) int {
	first := true
	for i := cur; i != cur || first; i = (i + 1) % len(tc) {
		if tc[i].time > offset {
			return i
		}
		first = false
	}
	return 0
}

func (s *scaler) setCount(c int) {
	glog.Infof("scaling to %d replicas", c)
	rcList, err := client.ReplicationControllers(namespace).List(s.selector)
	if err != nil {
		glog.Errorf("could not get replication controllers: %v", err)
		return
	}
	for _, rc := range rcList.Items {
		rc.Spec.Replicas = c
		if _, err = client.ReplicationControllers(namespace).Update(&rc); err != nil {
			glog.Errorf("unable to scale replication controller: %v", err)
		}
	}
}

func (s *scaler) timeOffset() time.Duration {
	return time.Since(s.start) % dayPeriod
}

func (s *scaler) curpos(offset time.Duration) int {
	return findPos(s.timeCounts, s.pos, offset)
}

func (s *scaler) scale() {
	for {
		select {
		case <-s.done:
			return
		default:
			offset := s.timeOffset()
			s.pos = s.curpos(offset)
			if s.timeCounts[s.pos].time < offset {
				time.Sleep(dayPeriod - offset)
				continue
			}
			time.Sleep(s.timeCounts[s.pos].time - offset)
			s.setCount(s.timeCounts[s.pos].count)
		}
	}
}

func (s *scaler) Start() error {
	now := time.Now().UTC()
	s.start = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
	if *startNow {
		s.start = now
	}

	// set initial count
	pos := s.curpos(s.timeOffset())
	// add the len to avoid getting a negative index
	pos = (pos - 1 + len(s.timeCounts)) % len(s.timeCounts)
	s.setCount(s.timeCounts[pos].count)

	s.done = make(chan struct{})
	go s.scale()
	return nil
}

func safeclose(c chan<- struct{}) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = e.(error)
		}
	}()
	close(c)
	return nil
}

func (s *scaler) Stop() error {
	if err := safeclose(s.done); err != nil {
		return errors.New("already stopped scaling")
	}
	return nil
}

var (
	counts     = flag.String("counts", "", "replica counts, must have at least one (csv)")
	times      = flag.String("times", "", "times to set replica counts relative to UTC following ISO 8601 (csv)")
	userLabels = flag.String("labels", "", "replication controller labels, syntax should follow https://godoc.org/k8s.io/kubernetes/pkg/labels#Parse")
	startNow   = flag.Bool("now", false, "times are relative to now not 0:00 UTC (for demos)")
	local      = flag.Bool("local", false, "set to true if running on local machine not within cluster")
	localPort  = flag.Int("localport", 8001, "port that kubectl proxy is running on (local must be true)")

	namespace = os.Getenv("POD_NAMESPACE")

	client *kclient.Client
)

const usageNotes = `
counts and times must both be set and be of equal length. Example usage:
  diurnal -labels name=redis-slave -times 00:00:00Z,06:00:00Z -counts 3,9
  diurnal -labels name=redis-slave -times 0600-0500,0900-0500,1700-0500,2200-0500 -counts 15,20,13,6
`

func usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	flag.PrintDefaults()
	fmt.Fprint(os.Stderr, usageNotes)
}

func main() {
	flag.Usage = usage
	flag.Parse()

	var (
		cfg *kclient.Config
		err error
	)
	if *local {
		cfg = &kclient.Config{Host: fmt.Sprintf("http://localhost:%d", *localPort)}
	} else {
		cfg, err = kclient.InClusterConfig()
		if err != nil {
			glog.Errorf("failed to load config: %v", err)
			flag.Usage()
			os.Exit(1)
		}
	}
	client, err = kclient.New(cfg)

	selector, err := labels.Parse(*userLabels)
	if err != nil {
		glog.Fatal(err)
	}
	tc, err := parseTimeCounts(*times, *counts)
	if err != nil {
		glog.Fatal(err)
	}
	if namespace == "" {
		glog.Fatal("POD_NAMESPACE is not set. Set to the namespace of the replication controller if running locally.")
	}
	scaler := scaler{timeCounts: tc, selector: selector}
	if err != nil {
		glog.Fatal(err)
	}

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGQUIT,
		syscall.SIGTERM)

	glog.Info("starting scaling")
	if err := scaler.Start(); err != nil {
		glog.Fatal(err)
	}
	<-sigChan
	glog.Info("stopping scaling")
	if err := scaler.Stop(); err != nil {
		glog.Fatal(err)
	}
}
<commit_msg>Fixes to get diurnal controller to build<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// An external diurnal controller for kubernetes. With this, it's possible to manage
// known replica counts that vary throughout the day.

package main

import (
	"errors"
	"flag"
	"fmt"
	"os"
	"os/signal"
	"sort"
	"strconv"
	"strings"
	"syscall"
	"time"

	kclient "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/labels"

	"github.com/golang/glog"
)

const dayPeriod = 24 * time.Hour

type timeCount struct {
	time  time.Duration
	count int
}

func (tc timeCount) String() string {
	h := tc.time / time.Hour
	m := (tc.time % time.Hour) / time.Minute
	s := (tc.time % time.Minute) / time.Second
	if m == 0 && s == 0 {
		return fmt.Sprintf("(%02dZ, %d)", h, tc.count)
	} else if s == 0 {
		return fmt.Sprintf("(%02d:%02dZ, %d)", h, m, tc.count)
	}
	return fmt.Sprintf("(%02d:%02d:%02dZ, %d)", h, m, s, tc.count)
}

type byTime []timeCount

func (tc byTime) Len() int           { return len(tc) }
func (tc byTime) Swap(i, j int)      { tc[i], tc[j] = tc[j], tc[i] }
func (tc byTime) Less(i, j int) bool { return tc[i].time < tc[j].time }

func timeMustParse(layout, s string) time.Time {
	t, err := time.Parse(layout, s)
	if err != nil {
		panic(err)
	}
	return t
}

// first argument is a format string equivalent to HHMMSS. See time.Parse for details.
var epoch = timeMustParse("150405", "000000")

func parseTimeRelative(s string) (time.Duration, error) {
	t, err := parseTimeISO8601(s)
	if err != nil {
		return 0, fmt.Errorf("unable to parse %s: %v", s, err)
	}
	return (t.Sub(epoch) + dayPeriod) % dayPeriod, nil
}

func parseTimeCounts(times string, counts string) ([]timeCount, error) {
	ts := strings.Split(times, ",")
	cs := strings.Split(counts, ",")
	if len(ts) != len(cs) {
		return nil, fmt.Errorf("provided %d times but %d replica counts", len(ts), len(cs))
	}
	var tc []timeCount
	for i := range ts {
		t, err := parseTimeRelative(ts[i])
		if err != nil {
			return nil, err
		}
		c, err := strconv.ParseInt(cs[i], 10, 64)
		if c < 0 {
			return nil, errors.New("counts must be non-negative")
		}
		if err != nil {
			return nil, err
		}
		tc = append(tc, timeCount{t, int(c)})
	}
	sort.Sort(byTime(tc))
	return tc, nil
}

type scaler struct {
	timeCounts []timeCount
	selector   labels.Selector
	start      time.Time
	pos        int
	done       chan struct{}
}

func findPos(tc []timeCount, cur int, offset time.Duration) int {
	first := true
	for i := cur; i != cur || first; i = (i + 1) % len(tc) {
		if tc[i].time > offset {
			return i
		}
		first = false
	}
	return 0
}

func (s *scaler) setCount(c int) {
	glog.Infof("scaling to %d replicas", c)
	rcList, err := client.ReplicationControllers(namespace).List(s.selector, fields.Everything())
	if err != nil {
		glog.Errorf("could not get replication controllers: %v", err)
		return
	}
	for _, rc := range rcList.Items {
		rc.Spec.Replicas = c
		if _, err = client.ReplicationControllers(namespace).Update(&rc); err != nil {
			glog.Errorf("unable to scale replication controller: %v", err)
		}
	}
}

func (s *scaler) timeOffset() time.Duration {
	return time.Since(s.start) % dayPeriod
}

func (s *scaler) curpos(offset time.Duration) int {
	return findPos(s.timeCounts, s.pos, offset)
}

func (s *scaler) scale() {
	for {
		select {
		case <-s.done:
			return
		default:
			offset := s.timeOffset()
			s.pos = s.curpos(offset)
			if s.timeCounts[s.pos].time < offset {
				time.Sleep(dayPeriod - offset)
				continue
			}
			time.Sleep(s.timeCounts[s.pos].time - offset)
			s.setCount(s.timeCounts[s.pos].count)
		}
	}
}

func (s *scaler) Start() error {
	now := time.Now().UTC()
	s.start = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, now.Location())
	if *startNow {
		s.start = now
	}

	// set initial count
	pos := s.curpos(s.timeOffset())
	// add the len to avoid getting a negative index
	pos = (pos - 1 + len(s.timeCounts)) % len(s.timeCounts)
	s.setCount(s.timeCounts[pos].count)

	s.done = make(chan struct{})
	go s.scale()
	return nil
}

func safeclose(c chan<- struct{}) (err error) {
	defer func() {
		if e := recover(); e != nil {
			err = e.(error)
		}
	}()
	close(c)
	return nil
}

func (s *scaler) Stop() error {
	if err := safeclose(s.done); err != nil {
		return errors.New("already stopped scaling")
	}
	return nil
}

var (
	counts     = flag.String("counts", "", "replica counts, must have at least one (csv)")
	times      = flag.String("times", "", "times to set replica counts relative to UTC following ISO 8601 (csv)")
	userLabels = flag.String("labels", "", "replication controller labels, syntax should follow https://godoc.org/k8s.io/kubernetes/pkg/labels#Parse")
	startNow   = flag.Bool("now", false, "times are relative to now not 0:00 UTC (for demos)")
	local      = flag.Bool("local", false, "set to true if running on local machine not within cluster")
	localPort  = flag.Int("localport", 8001, "port that kubectl proxy is running on (local must be true)")

	namespace = os.Getenv("POD_NAMESPACE")

	client *kclient.Client
)

const usageNotes = `
counts and times must both be set and be of equal length. Example usage:
  diurnal -labels name=redis-slave -times 00:00:00Z,06:00:00Z -counts 3,9
  diurnal -labels name=redis-slave -times 0600-0500,0900-0500,1700-0500,2200-0500 -counts 15,20,13,6
`

func usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	flag.PrintDefaults()
	fmt.Fprint(os.Stderr, usageNotes)
}

func main() {
	flag.Usage = usage
	flag.Parse()

	var (
		cfg *kclient.Config
		err error
	)
	if *local {
		cfg = &kclient.Config{Host: fmt.Sprintf("http://localhost:%d", *localPort)}
	} else {
		cfg, err = kclient.InClusterConfig()
		if err != nil {
			glog.Errorf("failed to load config: %v", err)
			flag.Usage()
			os.Exit(1)
		}
	}
	client, err = kclient.New(cfg)

	selector, err := labels.Parse(*userLabels)
	if err != nil {
		glog.Fatal(err)
	}
	tc, err := parseTimeCounts(*times, *counts)
	if err != nil {
		glog.Fatal(err)
	}
	if namespace == "" {
		glog.Fatal("POD_NAMESPACE is not set. Set to the namespace of the replication controller if running locally.")
	}
	scaler := scaler{timeCounts: tc, selector: selector}
	if err != nil {
		glog.Fatal(err)
	}

	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGQUIT,
		syscall.SIGTERM)

	glog.Info("starting scaling")
	if err := scaler.Start(); err != nil {
		glog.Fatal(err)
	}
	<-sigChan
	glog.Info("stopping scaling")
	if err := scaler.Stop(); err != nil {
		glog.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package harmonizer_test

import (
	"errors"
	"os"
	"time"

	"code.cloudfoundry.org/clock/fakeclock"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagertest"
	"code.cloudfoundry.org/operationq"
	"code.cloudfoundry.org/operationq/fake_operationq"
	"code.cloudfoundry.org/rep/evacuation/evacuation_context"
	"code.cloudfoundry.org/rep/generator/fake_generator"
	"code.cloudfoundry.org/rep/harmonizer"
	"github.com/cloudfoundry/dropsonde/metric_sender/fake"
	"github.com/cloudfoundry/dropsonde/metrics"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/tedsuo/ifrit"
)

var _ = Describe("Bulker", func() {
	var (
		sender *fake.FakeMetricSender

		logger                 *lagertest.TestLogger
		pollInterval           time.Duration
		evacuationPollInterval time.Duration
		fakeClock              *fakeclock.FakeClock
		fakeGenerator          *fake_generator.FakeGenerator
		fakeQueue              *fake_operationq.FakeQueue
		evacuatable            evacuation_context.Evacuatable
		evacuationNotifier     evacuation_context.EvacuationNotifier

		bulker  *harmonizer.Bulker
		process ifrit.Process
	)

	BeforeEach(func() {
		sender = fake.NewFakeMetricSender()
		metrics.Initialize(sender, nil)

		logger = lagertest.NewTestLogger("test")
		pollInterval = 30 * time.Second
		evacuationPollInterval = 10 * time.Second
		fakeClock = fakeclock.NewFakeClock(time.Unix(123, 456))
		fakeGenerator = new(fake_generator.FakeGenerator)
		fakeQueue = new(fake_operationq.FakeQueue)

		evacuatable, _, evacuationNotifier = evacuation_context.New()

		bulker = harmonizer.NewBulker(logger, pollInterval, evacuationPollInterval, evacuationNotifier, fakeClock, fakeGenerator, fakeQueue)
	})

	JustBeforeEach(func() {
		process = ifrit.Invoke(bulker)
		Eventually(fakeClock.WatcherCount).Should(Equal(1))
	})

	AfterEach(func() {
		process.Signal(os.Interrupt)
		Eventually(process.Wait()).Should(Receive())
	})

	itPerformsBatchOperations := func(expectedQueueLength int) {
		Context("when generating the batch operations succeeds", func() {
			var (
				operation1 *fake_operationq.FakeOperation
				operation2 *fake_operationq.FakeOperation
			)

			BeforeEach(func() {
				operation1 = new(fake_operationq.FakeOperation)
				operation2 = new(fake_operationq.FakeOperation)

				fakeGenerator.BatchOperationsStub = func(lager.Logger) (map[string]operationq.Operation, error) {
					fakeClock.Increment(10 * time.Second)
					return map[string]operationq.Operation{"guid1": operation1, "guid2": operation2}, nil
				}
			})

			It("pushes them onto the queue", func() {
				Eventually(fakeQueue.PushCallCount).Should(Equal(expectedQueueLength))

				enqueuedOperations := make([]operationq.Operation, 0, 2)
				enqueuedOperations = append(enqueuedOperations, fakeQueue.PushArgsForCall(expectedQueueLength-2))
				enqueuedOperations = append(enqueuedOperations, fakeQueue.PushArgsForCall(expectedQueueLength-1))

				Expect(enqueuedOperations).To(ConsistOf(operation1, operation2))
			})

			It("emits the duration it took to generate the batch operations", func() {
				Eventually(fakeQueue.PushCallCount).Should(Equal(expectedQueueLength))

				reportedDuration := sender.GetValue("RepBulkSyncDuration")
				Expect(reportedDuration.Unit).To(Equal("nanos"))
				Expect(reportedDuration.Value).To(BeNumerically("==", 10*time.Second))
			})
		})

		Context("when generating the batch operations fails", func() {
			disaster := errors.New("nope")

			BeforeEach(func() {
				fakeGenerator.BatchOperationsReturns(nil, disaster)
			})

			It("logs the error", func() {
				Eventually(logger).Should(gbytes.Say("failed-to-generate-operations"))
				Eventually(logger).Should(gbytes.Say("nope"))
			})
		})
	}

	Context("when the poll interval elapses", func() {
		JustBeforeEach(func() {
			fakeClock.WaitForWatcherAndIncrement(pollInterval + 1)
		})

		itPerformsBatchOperations(2)

		Context("and elapses again", func() {
			JustBeforeEach(func() {
				fakeClock.WaitForWatcherAndIncrement(pollInterval + 1)
			})

			itPerformsBatchOperations(4)
		})
	})

	Context("when the poll interval has not elapsed", func() {
		JustBeforeEach(func() {
			fakeClock.WaitForWatcherAndIncrement(pollInterval - 1)
		})

		It("does not fetch batch operations", func() {
			Consistently(fakeGenerator.BatchOperationsCallCount).Should(BeZero())
		})
	})

	Context("when evacuation starts", func() {
		BeforeEach(func() {
			evacuatable.Evacuate()
		})

		itPerformsBatchOperations(2)

		It("batches operations only once", func() {
			Eventually(fakeGenerator.BatchOperationsCallCount).Should(Equal(1))
			Consistently(fakeGenerator.BatchOperationsCallCount).Should(Equal(1))
		})

		Context("when the evacuation interval elapses", func() {
			It("batches operations again", func() {
				Eventually(fakeGenerator.BatchOperationsCallCount).Should(Equal(1))
				fakeClock.Increment(evacuationPollInterval + time.Second)
				Eventually(fakeGenerator.BatchOperationsCallCount).Should(Equal(2))
				Consistently(fakeGenerator.BatchOperationsCallCount).Should(Equal(2))
			})
		})
	})
})
<commit_msg>fix increment and fake clock construction<commit_after>package harmonizer_test

import (
	"errors"
	"os"
	"time"

	"code.cloudfoundry.org/clock/fakeclock"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/lager/lagertest"
	"code.cloudfoundry.org/operationq"
	"code.cloudfoundry.org/operationq/fake_operationq"
	"code.cloudfoundry.org/rep/evacuation/evacuation_context"
	"code.cloudfoundry.org/rep/generator/fake_generator"
	"code.cloudfoundry.org/rep/harmonizer"
	"github.com/cloudfoundry/dropsonde/metric_sender/fake"
	"github.com/cloudfoundry/dropsonde/metrics"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/tedsuo/ifrit"
)

var _ = Describe("Bulker", func() {
	var (
		sender *fake.FakeMetricSender

		logger                 *lagertest.TestLogger
		pollInterval           time.Duration
		evacuationPollInterval time.Duration
		fakeClock              *fakeclock.FakeClock
		fakeGenerator          *fake_generator.FakeGenerator
		fakeQueue              *fake_operationq.FakeQueue
		evacuatable            evacuation_context.Evacuatable
		evacuationNotifier     evacuation_context.EvacuationNotifier

		bulker  *harmonizer.Bulker
		process ifrit.Process
	)

	BeforeEach(func() {
		sender = fake.NewFakeMetricSender()
		metrics.Initialize(sender, nil)

		logger = lagertest.NewTestLogger("test")
		pollInterval = 30 * time.Second
		evacuationPollInterval = 10 * time.Second
		fakeClock = fakeclock.NewFakeClock(time.Now())
		fakeGenerator = new(fake_generator.FakeGenerator)
		fakeQueue = new(fake_operationq.FakeQueue)

		evacuatable, _, evacuationNotifier = evacuation_context.New()

		bulker = harmonizer.NewBulker(logger, pollInterval, evacuationPollInterval, evacuationNotifier, fakeClock, fakeGenerator, fakeQueue)
	})

	JustBeforeEach(func() {
		process = ifrit.Invoke(bulker)
		Eventually(fakeClock.WatcherCount).Should(Equal(1))
	})

	AfterEach(func() {
		process.Signal(os.Interrupt)
		Eventually(process.Wait()).Should(Receive())
	})

	itPerformsBatchOperations := func(expectedQueueLength int) {
		Context("when generating the batch operations succeeds", func() {
			var (
				operation1 *fake_operationq.FakeOperation
				operation2 *fake_operationq.FakeOperation
			)

			BeforeEach(func() {
				operation1 = new(fake_operationq.FakeOperation)
				operation2 = new(fake_operationq.FakeOperation)

				fakeGenerator.BatchOperationsStub = func(lager.Logger) (map[string]operationq.Operation, error) {
					fakeClock.Increment(10 * time.Second)
					return map[string]operationq.Operation{"guid1": operation1, "guid2": operation2}, nil
				}
			})

			It("pushes them onto the queue", func() {
				Eventually(fakeQueue.PushCallCount).Should(Equal(expectedQueueLength))

				enqueuedOperations := make([]operationq.Operation, 0, 2)
				enqueuedOperations = append(enqueuedOperations, fakeQueue.PushArgsForCall(expectedQueueLength-2))
				enqueuedOperations = append(enqueuedOperations, fakeQueue.PushArgsForCall(expectedQueueLength-1))

				Expect(enqueuedOperations).To(ConsistOf(operation1, operation2))
			})

			It("emits the duration it took to generate the batch operations", func() {
				Eventually(fakeQueue.PushCallCount).Should(Equal(expectedQueueLength))

				reportedDuration := sender.GetValue("RepBulkSyncDuration")
				Expect(reportedDuration.Unit).To(Equal("nanos"))
				Expect(reportedDuration.Value).To(BeNumerically("==", 10*time.Second))
			})
		})

		Context("when generating the batch operations fails", func() {
			disaster := errors.New("nope")

			BeforeEach(func() {
				fakeGenerator.BatchOperationsReturns(nil, disaster)
			})

			It("logs the error", func() {
				Eventually(logger).Should(gbytes.Say("failed-to-generate-operations"))
				Eventually(logger).Should(gbytes.Say("nope"))
			})
		})
	}

	Context("when the poll interval elapses", func() {
		JustBeforeEach(func() {
			fakeClock.WaitForWatcherAndIncrement(pollInterval)
		})

		itPerformsBatchOperations(2)

		Context("and elapses again", func() {
			JustBeforeEach(func() {
				fakeClock.WaitForWatcherAndIncrement(pollInterval)
			})

			itPerformsBatchOperations(4)
		})
	})

	Context("when the poll interval has not elapsed", func() {
		JustBeforeEach(func() {
			fakeClock.WaitForWatcherAndIncrement(pollInterval - 1)
		})

		It("does not fetch batch operations", func() {
			Consistently(fakeGenerator.BatchOperationsCallCount).Should(BeZero())
		})
	})

	Context("when evacuation starts", func() {
		BeforeEach(func() {
			evacuatable.Evacuate()
		})

		itPerformsBatchOperations(2)

		It("batches operations only once", func() {
			Eventually(fakeGenerator.BatchOperationsCallCount).Should(Equal(1))
			Consistently(fakeGenerator.BatchOperationsCallCount).Should(Equal(1))
		})

		Context("when the evacuation interval elapses", func() {
			It("batches operations again", func() {
				Eventually(fakeGenerator.BatchOperationsCallCount).Should(Equal(1))
				fakeClock.Increment(evacuationPollInterval + time.Second)
				Eventually(fakeGenerator.BatchOperationsCallCount).Should(Equal(2))
				Consistently(fakeGenerator.BatchOperationsCallCount).Should(Equal(2))
			})
		})
	})
})
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package cbft

import (
	"fmt"
	"regexp"
	"sync/atomic"
)

const INDEX_NAME_REGEXP = `^[A-Za-z][0-9A-Za-z_\-]*$`

// Creates a logical index, which might be comprised of many PIndex objects.
// A non-"" prevIndexUUID means an update to an existing index.
func (mgr *Manager) CreateIndex(sourceType, sourceName, sourceUUID, sourceParams,
	indexType, indexName, indexParams string, planParams PlanParams,
	prevIndexUUID string) error {
	atomic.AddUint64(&mgr.stats.TotCreateIndex, 1)

	matched, err := regexp.Match(INDEX_NAME_REGEXP, []byte(indexName))
	if err != nil {
		return fmt.Errorf("manager_api: CreateIndex, indexName parsing problem,"+
			" indexName: %s, err: %v", indexName, err)
	}
	if !matched {
		return fmt.Errorf("manager_api: CreateIndex, indexName is invalid,"+
			" indexName: %q", indexName)
	}

	pindexImplType, exists := PIndexImplTypes[indexType]
	if !exists {
		return fmt.Errorf("manager_api: CreateIndex, unknown indexType: %s",
			indexType)
	}
	if pindexImplType.Validate != nil {
		err := pindexImplType.Validate(indexType, indexName, indexParams)
		if err != nil {
			return fmt.Errorf("manager_api: CreateIndex, invalid, err: %v", err)
		}
	}

	// First, check that the source exists.
	_, err = DataSourcePartitions(sourceType, sourceName, sourceUUID, sourceParams,
		mgr.server)
	if err != nil {
		return fmt.Errorf("manager_api: failed to connect to"+
			" or retrieve information from source,"+
			" sourceType: %s, sourceName: %s, sourceUUID: %s, err: %v",
			sourceType, sourceName, sourceUUID, err)
	}

	indexDefs, cas, err := CfgGetIndexDefs(mgr.cfg)
	if err != nil {
		return fmt.Errorf("manager_api: CfgGetIndexDefs err: %v", err)
	}
	if indexDefs == nil {
		indexDefs = NewIndexDefs(mgr.version)
	}
	if VersionGTE(mgr.version, indexDefs.ImplVersion) == false {
		return fmt.Errorf("manager_api: could not create index,"+
			" indexDefs.ImplVersion: %s > mgr.version: %s",
			indexDefs.ImplVersion, mgr.version)
	}

	prevIndex, exists := indexDefs.IndexDefs[indexName]
	if prevIndexUUID == "" { // New index creation.
		if exists || prevIndex != nil {
			return fmt.Errorf("manager_api: index exists, indexName: %s",
				indexName)
		}
	} else { // Update index definition.
		if !exists || prevIndex == nil {
			return fmt.Errorf("manager_api: index missing for update,"+
				" indexName: %s", indexName)
		}
		if prevIndex.UUID != prevIndexUUID {
			return fmt.Errorf("manager_api: index wrong UUID for update,"+
				" indexName: %s, prevIndex.UUID: %s, prevIndexUUID: %s",
				indexName, prevIndex.UUID, prevIndexUUID)
		}
	}

	indexUUID := NewUUID()

	indexDef := &IndexDef{
		Type:         indexType,
		Name:         indexName,
		UUID:         indexUUID,
		Params:       indexParams,
		SourceType:   sourceType,
		SourceName:   sourceName,
		SourceUUID:   sourceUUID,
		SourceParams: sourceParams,
		PlanParams:   planParams,
	}

	indexDefs.UUID = indexUUID
	indexDefs.IndexDefs[indexName] = indexDef
	indexDefs.ImplVersion = mgr.version

	// NOTE: If our ImplVersion is still too old due to a race, we
	// expect a more modern planner to catch it later.

	_, err = CfgSetIndexDefs(mgr.cfg, indexDefs, cas)
	if err != nil {
		return fmt.Errorf("manager_api: could not save indexDefs, err: %v", err)
	}

	mgr.PlannerKick("api/CreateIndex, indexName: " + indexName)
	atomic.AddUint64(&mgr.stats.TotCreateIndexOk, 1)
	return nil
}

// Deletes a logical index, which might be comprised of many PIndex objects.
//
// TODO: DeleteIndex should also take index UUID?
func (mgr *Manager) DeleteIndex(indexName string) error {
	atomic.AddUint64(&mgr.stats.TotDeleteIndex, 1)

	indexDefs, cas, err := CfgGetIndexDefs(mgr.cfg)
	if err != nil {
		return err
	}
	if indexDefs == nil {
		return fmt.Errorf("manager_api: no indexes during deletion of indexName: %s",
			indexName)
	}
	if VersionGTE(mgr.version, indexDefs.ImplVersion) == false {
		return fmt.Errorf("manager_api: could not delete index,"+
			" indexDefs.ImplVersion: %s > mgr.version: %s",
			indexDefs.ImplVersion, mgr.version)
	}
	if _, exists := indexDefs.IndexDefs[indexName]; !exists {
		return fmt.Errorf("manager_api: index to delete missing, indexName: %s",
			indexName)
	}

	indexDefs.UUID = NewUUID()
	delete(indexDefs.IndexDefs, indexName)
	indexDefs.ImplVersion = mgr.version

	// NOTE: if our ImplVersion is still too old due to a race, we
	// expect a more modern planner to catch it later.

	_, err = CfgSetIndexDefs(mgr.cfg, indexDefs, cas)
	if err != nil {
		return fmt.Errorf("manager_api: could not save indexDefs, err: %v", err)
	}

	mgr.PlannerKick("api/DeleteIndex, indexName: " + indexName)
	atomic.AddUint64(&mgr.stats.TotDeleteIndexOk, 1)
	return nil
}

// IndexControl is used to change runtime properties of an index.
func (mgr *Manager) IndexControl(indexName, indexUUID, readOp, writeOp,
	planFreezeOp string) error {
	atomic.AddUint64(&mgr.stats.TotIndexControl, 1)

	indexDefs, cas, err := CfgGetIndexDefs(mgr.cfg)
	if err != nil {
		return err
	}
	if indexDefs == nil {
		return fmt.Errorf("manager_api: no indexes,"+
			" index read/write control, indexName: %s", indexName)
	}
	if VersionGTE(mgr.version, indexDefs.ImplVersion) == false {
		return fmt.Errorf("manager_api: index read/write control,"+
			" indexName: %s, indexDefs.ImplVersion: %s > mgr.version: %s",
			indexName, indexDefs.ImplVersion, mgr.version)
	}
	indexDef, exists := indexDefs.IndexDefs[indexName]
	if !exists || indexDef == nil {
		return fmt.Errorf("manager_api: no index to read/write control,"+
			" indexName: %s", indexName)
	}
	if indexUUID != "" && indexDef.UUID != indexUUID {
		return fmt.Errorf("manager_api: index.UUID mismatched")
	}

	if indexDef.PlanParams.NodePlanParams == nil {
		indexDef.PlanParams.NodePlanParams = map[string]map[string]*NodePlanParam{}
	}
	if indexDef.PlanParams.NodePlanParams[""] == nil {
		indexDef.PlanParams.NodePlanParams[""] = map[string]*NodePlanParam{}
	}
	if indexDef.PlanParams.NodePlanParams[""][""] == nil {
		indexDef.PlanParams.NodePlanParams[""][""] = &NodePlanParam{
			CanRead:  true,
			CanWrite: true,
		}
	}

	npp := indexDef.PlanParams.NodePlanParams[""][""]
	if readOp != "" {
		if readOp == "allow" || readOp == "resume" {
			npp.CanRead = true
		} else {
			npp.CanRead = false
		}
	}
	if writeOp != "" {
		if writeOp == "allow" || writeOp == "resume" {
			npp.CanWrite = true
		} else {
			npp.CanWrite = false
		}
	}

	if npp.CanRead == true && npp.CanWrite == true {
		delete(indexDef.PlanParams.NodePlanParams[""], "")
	}

	if planFreezeOp != "" {
		indexDef.PlanParams.PlanFrozen = planFreezeOp == "freeze"
	}

	_, err = CfgSetIndexDefs(mgr.cfg, indexDefs, cas)
	if err != nil {
		return fmt.Errorf("manager_api: could not save indexDefs, err: %v", err)
	}

	atomic.AddUint64(&mgr.stats.TotIndexControlOk, 1)
	return nil
}
<commit_msg>better error message when concurrent index definition update<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package cbft

import (
	"fmt"
	"regexp"
	"sync/atomic"
)

const INDEX_NAME_REGEXP = `^[A-Za-z][0-9A-Za-z_\-]*$`

// Creates a logical index, which might be comprised of many PIndex objects.
// A non-"" prevIndexUUID means an update to an existing index.
func (mgr *Manager) CreateIndex(sourceType, sourceName, sourceUUID, sourceParams,
	indexType, indexName, indexParams string, planParams PlanParams,
	prevIndexUUID string) error {
	atomic.AddUint64(&mgr.stats.TotCreateIndex, 1)

	matched, err := regexp.Match(INDEX_NAME_REGEXP, []byte(indexName))
	if err != nil {
		return fmt.Errorf("manager_api: CreateIndex, indexName parsing problem,"+
			" indexName: %s, err: %v", indexName, err)
	}
	if !matched {
		return fmt.Errorf("manager_api: CreateIndex, indexName is invalid,"+
			" indexName: %q", indexName)
	}

	pindexImplType, exists := PIndexImplTypes[indexType]
	if !exists {
		return fmt.Errorf("manager_api: CreateIndex, unknown indexType: %s",
			indexType)
	}
	if pindexImplType.Validate != nil {
		err := pindexImplType.Validate(indexType, indexName, indexParams)
		if err != nil {
			return fmt.Errorf("manager_api: CreateIndex, invalid, err: %v", err)
		}
	}

	// First, check that the source exists.
	_, err = DataSourcePartitions(sourceType, sourceName, sourceUUID, sourceParams,
		mgr.server)
	if err != nil {
		return fmt.Errorf("manager_api: failed to connect to"+
			" or retrieve information from source,"+
			" sourceType: %s, sourceName: %s, sourceUUID: %s, err: %v",
			sourceType, sourceName, sourceUUID, err)
	}

	indexDefs, cas, err := CfgGetIndexDefs(mgr.cfg)
	if err != nil {
		return fmt.Errorf("manager_api: CfgGetIndexDefs err: %v", err)
	}
	if indexDefs == nil {
		indexDefs = NewIndexDefs(mgr.version)
	}
	if VersionGTE(mgr.version, indexDefs.ImplVersion) == false {
		return fmt.Errorf("manager_api: could not create index,"+
			" indexDefs.ImplVersion: %s > mgr.version: %s",
			indexDefs.ImplVersion, mgr.version)
	}

	prevIndex, exists := indexDefs.IndexDefs[indexName]
	if prevIndexUUID == "" { // New index creation.
		if exists || prevIndex != nil {
			return fmt.Errorf("manager_api: index exists, indexName: %s",
				indexName)
		}
	} else { // Update index definition.
		if !exists || prevIndex == nil {
			return fmt.Errorf("manager_api: index missing for update,"+
				" indexName: %s", indexName)
		}
		if prevIndex.UUID != prevIndexUUID {
			return fmt.Errorf("manager_api:"+
				" perhaps there was a concurrent index definition update"+
				" - mismatched index UUID,"+
				" indexName: %s, prevIndex.UUID: %s, prevIndexUUID: %s",
				indexName, prevIndex.UUID, prevIndexUUID)
		}
	}

	indexUUID := NewUUID()

	indexDef := &IndexDef{
		Type:         indexType,
		Name:         indexName,
		UUID:         indexUUID,
		Params:       indexParams,
		SourceType:   sourceType,
		SourceName:   sourceName,
		SourceUUID:   sourceUUID,
		SourceParams: sourceParams,
		PlanParams:   planParams,
	}

	indexDefs.UUID = indexUUID
	indexDefs.IndexDefs[indexName] = indexDef
	indexDefs.ImplVersion = mgr.version

	// NOTE: If our ImplVersion is still too old due to a race, we
	// expect a more modern planner to catch it later.

	_, err = CfgSetIndexDefs(mgr.cfg, indexDefs, cas)
	if err != nil {
		return fmt.Errorf("manager_api: could not save indexDefs, err: %v", err)
	}

	mgr.PlannerKick("api/CreateIndex, indexName: " + indexName)
	atomic.AddUint64(&mgr.stats.TotCreateIndexOk, 1)
	return nil
}

// Deletes a logical index, which might be comprised of many PIndex objects.
//
// TODO: DeleteIndex should also take index UUID?
func (mgr *Manager) DeleteIndex(indexName string) error {
	atomic.AddUint64(&mgr.stats.TotDeleteIndex, 1)

	indexDefs, cas, err := CfgGetIndexDefs(mgr.cfg)
	if err != nil {
		return err
	}
	if indexDefs == nil {
		return fmt.Errorf("manager_api: no indexes during deletion of indexName: %s",
			indexName)
	}
	if VersionGTE(mgr.version, indexDefs.ImplVersion) == false {
		return fmt.Errorf("manager_api: could not delete index,"+
			" indexDefs.ImplVersion: %s > mgr.version: %s",
			indexDefs.ImplVersion, mgr.version)
	}
	if _, exists := indexDefs.IndexDefs[indexName]; !exists {
		return fmt.Errorf("manager_api: index to delete missing, indexName: %s",
			indexName)
	}

	indexDefs.UUID = NewUUID()
	delete(indexDefs.IndexDefs, indexName)
	indexDefs.ImplVersion = mgr.version

	// NOTE: if our ImplVersion is still too old due to a race, we
	// expect a more modern planner to catch it later.

	_, err = CfgSetIndexDefs(mgr.cfg, indexDefs, cas)
	if err != nil {
		return fmt.Errorf("manager_api: could not save indexDefs, err: %v", err)
	}

	mgr.PlannerKick("api/DeleteIndex, indexName: " + indexName)
	atomic.AddUint64(&mgr.stats.TotDeleteIndexOk, 1)
	return nil
}

// IndexControl is used to change runtime properties of an index.
func (mgr *Manager) IndexControl(indexName, indexUUID, readOp, writeOp,
	planFreezeOp string) error {
	atomic.AddUint64(&mgr.stats.TotIndexControl, 1)

	indexDefs, cas, err := CfgGetIndexDefs(mgr.cfg)
	if err != nil {
		return err
	}
	if indexDefs == nil {
		return fmt.Errorf("manager_api: no indexes,"+
			" index read/write control, indexName: %s", indexName)
	}
	if VersionGTE(mgr.version, indexDefs.ImplVersion) == false {
		return fmt.Errorf("manager_api: index read/write control,"+
			" indexName: %s, indexDefs.ImplVersion: %s > mgr.version: %s",
			indexName, indexDefs.ImplVersion, mgr.version)
	}
	indexDef, exists := indexDefs.IndexDefs[indexName]
	if !exists || indexDef == nil {
		return fmt.Errorf("manager_api: no index to read/write control,"+
			" indexName: %s", indexName)
	}
	if indexUUID != "" && indexDef.UUID != indexUUID {
		return fmt.Errorf("manager_api: index.UUID mismatched")
	}

	if indexDef.PlanParams.NodePlanParams == nil {
		indexDef.PlanParams.NodePlanParams = map[string]map[string]*NodePlanParam{}
	}
	if indexDef.PlanParams.NodePlanParams[""] == nil {
		indexDef.PlanParams.NodePlanParams[""] = map[string]*NodePlanParam{}
	}
	if indexDef.PlanParams.NodePlanParams[""][""] == nil {
		indexDef.PlanParams.NodePlanParams[""][""] = &NodePlanParam{
			CanRead:  true,
			CanWrite: true,
		}
	}

	npp := indexDef.PlanParams.NodePlanParams[""][""]
	if readOp != "" {
		if readOp == "allow" || readOp == "resume" {
			npp.CanRead = true
		} else {
			npp.CanRead = false
		}
	}
	if writeOp != "" {
		if writeOp == "allow" || writeOp == "resume" {
			npp.CanWrite = true
		} else {
			npp.CanWrite = false
		}
	}

	if npp.CanRead == true && npp.CanWrite == true {
		delete(indexDef.PlanParams.NodePlanParams[""], "")
	}

	if planFreezeOp != "" {
		indexDef.PlanParams.PlanFrozen = planFreezeOp == "freeze"
	}

	_, err = CfgSetIndexDefs(mgr.cfg, indexDefs, cas)
	if err != nil {
		return fmt.Errorf("manager_api: could not save indexDefs, err: %v", err)
	}

	atomic.AddUint64(&mgr.stats.TotIndexControlOk, 1)
	return nil
}
<|endoftext|>


<commit_before>package integration

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/http/httputil"
	"os/exec"
	"strings"
	"testing"

	"github.com/sclevine/spec"
	"github.com/stretchr/testify/require"
)

type tagRequest struct {
	Resources []struct {
		ResourceID   string `json:"resource_id"`
		ResourceType string `json:"resource_type"`
	} `json:"resources"`
}

func testDropletTag(t *testing.T, when spec.G, it spec.S) {
	var (
		expect *require.Assertions
		server *httptest.Server
	)

	it.Before(func() {
		expect = require.New(t)

		server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			switch req.URL.Path {
			case "/v2/droplets":
				auth := req.Header.Get("Authorization")
				if auth != "Bearer some-magic-token" {
					w.WriteHeader(http.StatusUnauthorized)
					return
				}

				w.Write([]byte(`{"droplets":[{"name":"some-droplet-name", "id": 1337}]}`))
			case "/v2/tags/my-tag/resources":
				body, err := ioutil.ReadAll(req.Body)
				expect.NoError(err)

				var tagRequest tagRequest
				err = json.Unmarshal(body, &tagRequest)
				expect.NoError(err)

				if req.Method == "POST" {
					if tagRequest.Resources[0].ResourceID == "1444" {
						w.WriteHeader(http.StatusNotFound)
						w.Write([]byte(`{"message": "tag not found"}`))
						return
					}

					w.Write([]byte(`{}`))
				}
			default:
				dump, err := httputil.DumpRequest(req, true)
				if err != nil {
					t.Fatal("failed to dump request")
				}

				t.Fatalf("received unknown request: %s", dump)
			}
		}))
	})

	when("all required flags are passed", func() {
		it("tags the droplet", func() {
			cmd := exec.Command(builtBinaryPath,
				"-t", "some-magic-token",
				"-u", server.URL,
				"compute",
				"droplet",
				"tag",
				"some-droplet-name",
				"--tag-name", "my-tag",
			)

			output, err := cmd.CombinedOutput()
			expect.NoError(err, fmt.Sprintf("received error output: %s", output))
		})
	})

	when("the droplet-id cannot be found", func() {
		it("returns no error", func() {
			cmd := exec.Command(builtBinaryPath,
				"-t", "some-magic-token",
				"-u", server.URL,
				"compute",
				"droplet",
				"tag",
				"1444",
				"--tag-name", "my-tag",
			)

			output, err := cmd.CombinedOutput()
			expect.Error(err)
			expect.Equal(strings.TrimSpace(string(output)), fmt.Sprintf("Error: POST %s/v2/tags/my-tag/resources: 404 tag not found", server.URL))
		})
	})

	when("the droplet-name cannot be found", func() {
		it("returns no error", func() {
			dropletName := "missing-droplet"
			cmd := exec.Command(builtBinaryPath,
				"-t", "some-magic-token",
				"-u", server.URL,
				"compute",
				"droplet",
				"tag",
				dropletName,
				"--tag-name", "my-tag",
			)

			output, err := cmd.CombinedOutput()
			expect.Error(err)
			expect.Equal(strings.TrimSpace(string(output)), fmt.Sprintf("Error: droplet with name %q could not be found", dropletName))
		})
	})
}
<commit_msg>test both tag and untag<commit_after>package integration

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/http/httputil"
	"os/exec"
	"regexp"
	"strings"
	"testing"

	"github.com/sclevine/spec"
	"github.com/stretchr/testify/require"
)

type tagRequest struct {
	Resources []struct {
		ResourceID   string `json:"resource_id"`
		ResourceType string `json:"resource_type"`
	} `json:"resources"`
}

func testDropletTag(t *testing.T, when spec.G, it spec.S) {
	var (
		expect *require.Assertions
		server *httptest.Server
	)

	it.Before(func() {
		expect = require.New(t)

		server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			switch req.URL.Path {
			case "/v2/droplets":
				auth := req.Header.Get("Authorization")
				if auth != "Bearer some-magic-token" {
					w.WriteHeader(http.StatusUnauthorized)
					return
				}

				w.Write([]byte(`{"droplets":[{"name":"some-droplet-name", "id": 1337}]}`))
			case "/v2/tags/my-tag/resources":
				body, err := ioutil.ReadAll(req.Body)
				expect.NoError(err)

				var tagRequest tagRequest
				err = json.Unmarshal(body, &tagRequest)
				expect.NoError(err)

				if req.Method == "POST" || req.Method == "DELETE" {
					if tagRequest.Resources[0].ResourceID == "1444" {
						w.WriteHeader(http.StatusNotFound)
						w.Write([]byte(`{"message": "tag not found"}`))
						return
					}

					w.Write([]byte(`{}`))
				}
			default:
				dump, err := httputil.DumpRequest(req, true)
				if err != nil {
					t.Fatal("failed to dump request")
				}

				t.Fatalf("received unknown request: %s", dump)
			}
		}))
	})

	when("all required flags are passed", func() {
		base := []string{
			"-t", "some-magic-token",
			"compute",
			"droplet",
		}

		cases := []struct {
			desc string
			args []string
		}{
			{desc: "when tagging", args: append(base, []string{"tag", "some-droplet-name", "--tag-name", "my-tag"}...)},
			{desc: "when untagging", args: append(base, []string{"untag", "some-droplet-name", "--tag-name", "my-tag"}...)},
		}

		for _, c := range cases {
			commandArgs := c.args

			when(c.desc, func() {
				it("completes successfully", func() {
					finalArgs := append([]string{"-u", server.URL}, commandArgs...)
					cmd := exec.Command(builtBinaryPath, finalArgs...)

					output, err := cmd.CombinedOutput()
					expect.NoError(err, fmt.Sprintf("received error output: %s", output))
				})
			})
		}
	})

	when("an error occurs", func() {
		base := []string{
			"-t", "some-magic-token",
			"compute",
			"droplet",
		}

		cases := []struct {
			desc string
			args []string
			err  string
		}{
			{
				desc: "when tagging and droplet id is missing",
				args: append(base, []string{"tag", "1444", "--tag-name", "my-tag"}...),
				err:  "^Error: POST http.*: 404 tag not found",
			},
			{
				desc: "when untagging and droplet id is missing",
				args: append(base, []string{"untag", "1444", "--tag-name", "my-tag"}...),
				err:  "^Error: DELETE http.*: 404 tag not found",
			},
			{
				desc: "when tagging and droplet name is missing",
				args: append(base, []string{"untag", "bad-droplet-name", "--tag-name", "my-tag"}...),
				err:  `^Error:.*\".*\" could not be found`,
			},
			{
				desc: "when untagging and droplet name is missing",
				args: append(base, []string{"untag", "bad-droplet-name", "--tag-name", "my-tag"}...),
				err:  `^Error:.*\".*\" could not be found`,
			},
		}

		for _, c := range cases {
			commandArgs := c.args
			errRegex := c.err

			when(c.desc, func() {
				it("completes successfully", func() {
					finalArgs := append([]string{"-u", server.URL}, commandArgs...)
					cmd := exec.Command(builtBinaryPath, finalArgs...)

					output, err := cmd.CombinedOutput()
					expect.Error(err)
					expect.Regexp(regexp.MustCompile(errRegex), strings.TrimSpace(string(output)))

				})
			})
		}
	})
}
<|endoftext|>


<commit_before>package integration

import (
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"github.com/cppforlife/go-patch/patch"
	"gopkg.in/yaml.v2"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Integration", func() {
	var opsfilegenPath string
	BeforeSuite(func() {
		var err error
		opsfilegenPath, err = gexec.Build("github.com/crawsible/opsfilegen")
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		gexec.CleanupBuildArtifacts()
	})

	It("generates an opsfile from a source target manifest", func() {
		wd, _ := os.Getwd()

		sourceManifestPath := filepath.Join(wd, "fixtures/source.yml")
		targetManifestPath := filepath.Join(wd, "fixtures/target.yml")
		expectedOpsFilePath := filepath.Join(wd, "fixtures/expected_opsfile.yml")
		expectedOutput, _ := ioutil.ReadFile(expectedOpsFilePath)

		command := exec.Command(
			opsfilegenPath,
			sourceManifestPath,
			targetManifestPath,
		)

		session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
		Expect(err).NotTo(HaveOccurred())
		actualOutput := session.Wait(5 * time.Second).Out.Contents()

		var expectedOpDefs []patch.OpDefinition
		var actualOpDefs []patch.OpDefinition

		_ = yaml.Unmarshal(expectedOutput, &expectedOpDefs)
		err = yaml.Unmarshal(actualOutput, &actualOpDefs)
		Expect(err).NotTo(HaveOccurred())

		Expect(actualOpDefs).To(Equal(expectedOpDefs))
	})
})
<commit_msg>Prefer YAML-matching gomega built-in<commit_after>package integration

import (
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Integration", func() {
	var opsfilegenPath string
	BeforeSuite(func() {
		var err error
		opsfilegenPath, err = gexec.Build("github.com/crawsible/opsfilegen")
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		gexec.CleanupBuildArtifacts()
	})

	It("generates an opsfile from a source target manifest", func() {
		wd, _ := os.Getwd()

		sourceManifestPath := filepath.Join(wd, "fixtures/source.yml")
		targetManifestPath := filepath.Join(wd, "fixtures/target.yml")
		expectedOpsFilePath := filepath.Join(wd, "fixtures/expected_opsfile.yml")
		expectedOutput, _ := ioutil.ReadFile(expectedOpsFilePath)

		command := exec.Command(
			opsfilegenPath,
			sourceManifestPath,
			targetManifestPath,
		)

		session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
		Expect(err).NotTo(HaveOccurred())
		actualOutput := session.Wait(5 * time.Second).Out.Contents()

		Expect(actualOutput).To(MatchYAML(expectedOutput))
	})
})
<|endoftext|>