<commit_before>package helium

import (
	"fmt"
	"io"
	"unicode/utf8"

	"github.com/lestrrat/helium/internal/debug"
)

var (
	esc_quot = []byte(""") // shorter than """
	esc_apos = []byte("'") // shorter than "'"
	esc_amp  = []byte("&")
	esc_lt   = []byte("<")
	esc_gt   = []byte(">")
	esc_tab  = []byte("	")
	esc_nl   = []byte("
")
	esc_cr   = []byte("
")
	esc_fffd = []byte("\uFFFD") // Unicode replacement character
)

// Decide whether the given rune is in the XML Character Range, per
// the Char production of http://www.xml.com/axml/testaxml.htm,
// Section 2.2 Characters.
func isInCharacterRange(r rune) (inrange bool) {
	return r == 0x09 ||
		r == 0x0A ||
		r == 0x0D ||
		r >= 0x20 && r <= 0xDF77 ||
		r >= 0xE000 && r <= 0xFFFD ||
		r >= 0x10000 && r <= 0x10FFFF
}

// escapeText writes to w the properly escaped XML equivalent
// of the plain text data s. If escapeNewline is true, newline
// characters will be escaped.
func escapeText(w io.Writer, s []byte, escapeNewline bool) error {
debug.Printf("escapeText = '%s'", s)
	var esc []byte
	last := 0
	for i := 0; i < len(s); {
		r, width := utf8.DecodeRune(s[i:])
		i += width
		switch r {
		case '"':
			esc = esc_quot
		case '\'':
			esc = esc_apos
		case '&':
			esc = esc_amp
		case '<':
			esc = esc_lt
		case '>':
			esc = esc_gt
		case '\n':
			if !escapeNewline {
				continue
			}
			esc = esc_nl
		case '\r':
			esc = esc_cr
		default:
			if !(r == '\t' || (0x20 <= r && r < 0x80)) {
				if r < 0xE0 {
					esc = []byte(fmt.Sprintf("&#x%X;", r))
					break
				}
			}
			if !isInCharacterRange(r) || (r == 0xFFFD && width == 1) {
				esc = esc_fffd
				break
			}
			continue
		}

		if _, err := w.Write(s[last : i-width]); err != nil {
			return err
		}
		if _, err := w.Write(esc); err != nil {
			return err
		}
		last = i
	}

	if _, err := w.Write(s[last:]); err != nil {
		return err
	}
	return nil
}

type Dumper struct{}

func (d *Dumper) writeString(out io.Writer, content string) error {
	// punt all the magic for now
	_, err := io.WriteString(out, content)
	return err
}

func (d *Dumper) DumpDoc(out io.Writer, doc *Document) error {
	if debug.Enabled {
		g := debug.IPrintf("START Dumper.DumpDoc")
		defer g.IRelease("END Dumper.DumpDoc")
	}

	if err := d.DumpNode(out, doc); err != nil {
		return err
	}

	for e := doc.FirstChild(); e != nil; e = e.NextSibling() {
		if err := d.DumpNode(out, e); err != nil {
			return err
		}
	}
	io.WriteString(out, "\n")
	return nil
}

func (d *Dumper) dumpDocContent(out io.Writer, n Node) error {
	if debug.Enabled {
		g := debug.IPrintf("START Dumper.dumpDocContent")
		defer g.IRelease("END Dumper.dumpDocContent")
	}

	doc := n.(*Document)
	io.WriteString(out, `<?xml version="`)
	version := doc.Version()
	if version == "" {
		version = "1.0"
	}
	io.WriteString(out, version+`"`)

	if encoding := doc.encoding; encoding != "" {
		io.WriteString(out, ` encoding="`+encoding+`"`)
	}

	switch doc.Standalone() {
	case StandaloneExplicitNo:
		io.WriteString(out, ` standalone="no"`)
	case StandaloneExplicitYes:
		io.WriteString(out, ` standalone="yes"`)
	}
	io.WriteString(out, "?>\n")
	return nil
}

func (d *Dumper) DumpNode(out io.Writer, n Node) error {
	if debug.Enabled {
		g := debug.IPrintf("START Dumper.DumpNode '%s'", n.Name())
		defer g.IRelease("END Dumper.DumpNode")
	}

	var err error
	switch n.Type() {
	case DocumentNode:
		if err = d.dumpDocContent(out, n); err != nil {
			return err
		}
		return nil
		//	case DTDNode:
		//		err = d.DumpDTD(out, n.(*DTD))
	case CommentNode:
		io.WriteString(out, "<!--")
		out.Write(n.Content())
		io.WriteString(out, "-->\n") // <-- this newline is not right, but I'm punting the problem for now
		return nil
	case EntityRefNode:
		io.WriteString(out, "&")
		io.WriteString(out, n.Name())
		io.WriteString(out, ";")
		return nil
	case TextNode:
		c := n.Content()
		if string(c) == XMLTextNoEnc {
			panic("unimplemented")
		} else {
			escapeText(out, c, false)
		}
		return nil // no recursing down
	}

	if err != nil {
		return err
	}

	// if it got here it's some sort of an element

	name := n.Name()
	if nser, ok := n.(Namespacer); ok {
		if prefix := nser.Prefix(); prefix != "" {
			name = prefix + ":" + name
		}
	}
	io.WriteString(out, "<")
	io.WriteString(out, name)

	if e, ok := n.(*Element); ok {
		for attr := e.properties; attr != nil; {
			io.WriteString(out, " "+attr.Name()+`="`)
			escapeText(out, []byte(attr.Value()), true)
			io.WriteString(out, `"`)
			a := attr.NextSibling()
			if a == nil {
				break
			}
			attr = a.(*Attribute)
		}

		if child := e.FirstChild(); child == nil {
			io.WriteString(out, "/>")
			return nil
		}
	}

	io.WriteString(out, ">")

	if child := n.FirstChild(); child != nil {
		for ; child != nil; child = child.NextSibling() {
			if err := d.DumpNode(out, child); err != nil {
				return err
			}
		}
	}

	io.WriteString(out, "</")
	io.WriteString(out, name)
	io.WriteString(out, ">")

	return nil
}<commit_msg>separate escaping text and attr value<commit_after>package helium

import (
	"fmt"
	"io"
	"unicode/utf8"

	"github.com/lestrrat/helium/internal/debug"
)

var (
	esc_quot = []byte(""") // shorter than """
	esc_apos = []byte("'") // shorter than "'"
	esc_amp  = []byte("&")
	esc_lt   = []byte("<")
	esc_gt   = []byte(">")
	esc_tab  = []byte("	")
	esc_nl   = []byte("
")
	esc_cr   = []byte("
")
	esc_fffd = []byte("\uFFFD") // Unicode replacement character
)

// Decide whether the given rune is in the XML Character Range, per
// the Char production of http://www.xml.com/axml/testaxml.htm,
// Section 2.2 Characters.
func isInCharacterRange(r rune) (inrange bool) {
	return r == 0x09 ||
		r == 0x0A ||
		r == 0x0D ||
		r >= 0x20 && r <= 0xDF77 ||
		r >= 0xE000 && r <= 0xFFFD ||
		r >= 0x10000 && r <= 0x10FFFF
}

func escapeAttrValue(w io.Writer, s[]byte) error {
	var esc []byte
	last := 0
	for i := 0; i < len(s); {
		r, width := utf8.DecodeRune(s[i:])
		i += width
		switch r {
		case '"':
			esc = esc_quot
		case '\'':
			esc = esc_apos
		case '&':
			esc = esc_amp
		case '<':
			esc = esc_lt
		case '>':
			esc = esc_gt
		case '\n':
			esc = esc_nl
		case '\r':
			esc = esc_cr
		case '\t':
			esc = esc_tab
		default:
			if !(0x20 <= r && r < 0x80) {
				if r < 0xE0 {
					esc = []byte(fmt.Sprintf("&#x%X;", r))
					break
				}
			}
			if !isInCharacterRange(r) || (r == 0xFFFD && width == 1) {
				esc = esc_fffd
				break
			}
			continue
		}

		if _, err := w.Write(s[last : i-width]); err != nil {
			return err
		}
		if _, err := w.Write(esc); err != nil {
			return err
		}
		last = i
	}

	if _, err := w.Write(s[last:]); err != nil {
		return err
	}
	return nil
}

// escapeText writes to w the properly escaped XML equivalent
// of the plain text data s. If escapeNewline is true, newline
// characters will be escaped.
func escapeText(w io.Writer, s []byte, escapeNewline bool) error {
debug.Printf("escapeText = '%s'", s)
	var esc []byte
	last := 0
	for i := 0; i < len(s); {
		r, width := utf8.DecodeRune(s[i:])
		i += width
		switch r {
		case '"':
			esc = esc_quot
		case '\'':
			esc = esc_apos
		case '&':
			esc = esc_amp
		case '<':
			esc = esc_lt
		case '>':
			esc = esc_gt
		case '\n':
			if !escapeNewline {
				continue
			}
			esc = esc_nl
		case '\r':
			esc = esc_cr
		default:
			if !(r == '\t' || (0x20 <= r && r < 0x80)) {
				if r < 0xE0 {
					esc = []byte(fmt.Sprintf("&#x%X;", r))
					break
				}
			}
			if !isInCharacterRange(r) || (r == 0xFFFD && width == 1) {
				esc = esc_fffd
				break
			}
			continue
		}

		if _, err := w.Write(s[last : i-width]); err != nil {
			return err
		}
		if _, err := w.Write(esc); err != nil {
			return err
		}
		last = i
	}

	if _, err := w.Write(s[last:]); err != nil {
		return err
	}
	return nil
}

type Dumper struct{}

func (d *Dumper) writeString(out io.Writer, content string) error {
	// punt all the magic for now
	_, err := io.WriteString(out, content)
	return err
}

func (d *Dumper) DumpDoc(out io.Writer, doc *Document) error {
	if debug.Enabled {
		g := debug.IPrintf("START Dumper.DumpDoc")
		defer g.IRelease("END Dumper.DumpDoc")
	}

	if err := d.DumpNode(out, doc); err != nil {
		return err
	}

	for e := doc.FirstChild(); e != nil; e = e.NextSibling() {
		if err := d.DumpNode(out, e); err != nil {
			return err
		}
	}
	io.WriteString(out, "\n")
	return nil
}

func (d *Dumper) dumpDocContent(out io.Writer, n Node) error {
	if debug.Enabled {
		g := debug.IPrintf("START Dumper.dumpDocContent")
		defer g.IRelease("END Dumper.dumpDocContent")
	}

	doc := n.(*Document)
	io.WriteString(out, `<?xml version="`)
	version := doc.Version()
	if version == "" {
		version = "1.0"
	}
	io.WriteString(out, version+`"`)

	if encoding := doc.encoding; encoding != "" {
		io.WriteString(out, ` encoding="`+encoding+`"`)
	}

	switch doc.Standalone() {
	case StandaloneExplicitNo:
		io.WriteString(out, ` standalone="no"`)
	case StandaloneExplicitYes:
		io.WriteString(out, ` standalone="yes"`)
	}
	io.WriteString(out, "?>\n")
	return nil
}

func (d *Dumper) DumpNode(out io.Writer, n Node) error {
	if debug.Enabled {
		g := debug.IPrintf("START Dumper.DumpNode '%s'", n.Name())
		defer g.IRelease("END Dumper.DumpNode")
	}

	var err error
	switch n.Type() {
	case DocumentNode:
		if err = d.dumpDocContent(out, n); err != nil {
			return err
		}
		return nil
		//	case DTDNode:
		//		err = d.DumpDTD(out, n.(*DTD))
	case CommentNode:
		io.WriteString(out, "<!--")
		out.Write(n.Content())
		io.WriteString(out, "-->\n") // <-- this newline is not right, but I'm punting the problem for now
		return nil
	case EntityRefNode:
		io.WriteString(out, "&")
		io.WriteString(out, n.Name())
		io.WriteString(out, ";")
		return nil
	case TextNode:
		c := n.Content()
		if string(c) == XMLTextNoEnc {
			panic("unimplemented")
		} else {
			escapeText(out, c, false)
		}
		return nil // no recursing down
	}

	if err != nil {
		return err
	}

	// if it got here it's some sort of an element

	name := n.Name()
	if nser, ok := n.(Namespacer); ok {
		if prefix := nser.Prefix(); prefix != "" {
			name = prefix + ":" + name
		}
	}
	io.WriteString(out, "<")
	io.WriteString(out, name)

	if e, ok := n.(*Element); ok {
		for attr := e.properties; attr != nil; {
			io.WriteString(out, " "+attr.Name()+`="`)
			escapeAttrValue(out, []byte(attr.Value()))
			io.WriteString(out, `"`)
			a := attr.NextSibling()
			if a == nil {
				break
			}
			attr = a.(*Attribute)
		}

		if child := e.FirstChild(); child == nil {
			io.WriteString(out, "/>")
			return nil
		}
	}

	io.WriteString(out, ">")

	if child := n.FirstChild(); child != nil {
		for ; child != nil; child = child.NextSibling() {
			if err := d.DumpNode(out, child); err != nil {
				return err
			}
		}
	}

	io.WriteString(out, "</")
	io.WriteString(out, name)
	io.WriteString(out, ">")

	return nil
}<|endoftext|>


<commit_before>// Copyright © 2020 Banzai Cloud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"regexp"
	"strings"

	"github.com/sirupsen/logrus"

	"github.com/banzaicloud/bank-vaults/internal/injector"
	"github.com/banzaicloud/bank-vaults/pkg/sdk/vault"
)

var InlineMutationRegex = regexp.MustCompile("\\${{(.*?)}}")

func getDataFromVault(data map[string]string, vaultClient *vault.Client, vaultConfig VaultConfig, logger logrus.FieldLogger) (map[string]string, error) {
	vaultData := make(map[string]string, len(data))

	inject := func(key, value string) {
		vaultData[key] = value
	}

	config := injector.Config{
		TransitKeyID: vaultConfig.TransitKeyID,
		TransitPath:  vaultConfig.TransitPath,
	}
	secretInjector := injector.NewSecretInjector(config, vaultClient, nil, logger)

	return vaultData, secretInjector.InjectSecretsFromVault(data, inject)
}

func hasVaultPrefix(value string) bool {
	return strings.HasPrefix(value, "vault:") || strings.HasPrefix(value, ">>vault:")
}

func hasInlineVaultDelimiters(value string) bool {
	return len(findInlineVaultDelimiters(value)) > 0
}

func findInlineVaultDelimiters(value string) [][]string {
	return InlineMutationRegex.FindAllStringSubmatch(value, -1)
}
<commit_msg>match single braces as per go templating syntax and optimize regex to ensure we only match braces containing the `vault:` or `>>vault:` identifiers<commit_after>// Copyright © 2020 Banzai Cloud
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"regexp"
	"strings"

	"github.com/sirupsen/logrus"

	"github.com/banzaicloud/bank-vaults/internal/injector"
	"github.com/banzaicloud/bank-vaults/pkg/sdk/vault"
)

var InlineMutationRegex = regexp.MustCompile("\\${([>]{0,2}vault:.*?)}")

func getDataFromVault(data map[string]string, vaultClient *vault.Client, vaultConfig VaultConfig, logger logrus.FieldLogger) (map[string]string, error) {
	vaultData := make(map[string]string, len(data))

	inject := func(key, value string) {
		vaultData[key] = value
	}

	config := injector.Config{
		TransitKeyID: vaultConfig.TransitKeyID,
		TransitPath:  vaultConfig.TransitPath,
	}
	secretInjector := injector.NewSecretInjector(config, vaultClient, nil, logger)

	return vaultData, secretInjector.InjectSecretsFromVault(data, inject)
}

func hasVaultPrefix(value string) bool {
	return strings.HasPrefix(value, "vault:") || strings.HasPrefix(value, ">>vault:")
}

func hasInlineVaultDelimiters(value string) bool {
	return len(findInlineVaultDelimiters(value)) > 0
}

func findInlineVaultDelimiters(value string) [][]string {
	return InlineMutationRegex.FindAllStringSubmatch(value, -1)
}
<|endoftext|>


<commit_before>// Package consumer implements command line crawl consumer from nsq.
package consumer

import (
	"github.com/codegangsta/cli"
	"github.com/golang/glog"

	"github.com/crackcomm/crawl"
)

// Spider - Spider registrator.
type Spider func(crawl.Crawler)

// New - Creates nsq consumer app.
func New(spiders ...Spider) (app *cli.App) {
	app = cli.NewApp()
	app.Name = "crawler"
	app.HelpName = app.Name
	app.Version = "0.0.1"
	app.Usage = "nsq crawl consumer"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "topic",
			EnvVar: "TOPIC",
		},
		cli.StringFlag{
			Name:   "channel",
			EnvVar: "CHANNEL",
		},
		cli.StringSliceFlag{
			Name:   "nsq-addr",
			EnvVar: "NSQ_ADDR",
		},
		cli.StringSliceFlag{
			Name:   "nsqlookup-addr",
			EnvVar: "NSQLOOKUP_ADDR",
		},
		cli.IntFlag{
			Name:   "concurrency",
			Value:  100,
			EnvVar: "CRAWL_CONCURRENCY",
		},
	}
	app.Action = func(c *cli.Context) {
		if len(c.StringSlice("nsq-addr")) == 0 {
			glog.Fatalf("At leat one --%s is required", "nsq-addr")
		}

		queue := crawl.NewNsqQueue(c.String("topic"), c.String("channel"), c.Int("concurrency"))

		nsqAddr := c.StringSlice("nsq-addr")[0]
		if err := queue.Producer.Connect(nsqAddr); err != nil {
			glog.Fatalf("Error connecting producer to %q: %v", nsqAddr, err)
		}

		if addrs := c.StringSlice("nsq-addr"); len(addrs) != 0 {
			for _, addr := range addrs {
				glog.V(3).Infof("Connecting to nsq %s", addr)
				if err := queue.Consumer.Connect(addr); err != nil {
					glog.Fatalf("Error connecting to nsq %q: %v", addr, err)
				}
				glog.V(3).Infof("Connected to nsq %s", addr)
			}
		}

		if addrs := c.StringSlice("nsqlookup-addr"); len(addrs) != 0 {
			for _, addr := range addrs {
				glog.V(3).Infof("Connecting to nsq lookup %s", addr)
				if err := queue.Consumer.ConnectLookupd(addr); err != nil {
					glog.Fatalf("Error connecting to nsq lookup %q: %v", addr, err)
				}
				glog.V(3).Infof("Connected to nsq lookup %s", addr)
			}
		}

		crawler := crawl.New(
			crawl.WithQueue(queue),
			crawl.WithConcurrency(c.Int("concurrency")),
		)

		for _, spider := range spiders {
			spider(crawler)
		}

		go func() {
			for err := range crawler.Errors() {
				glog.Warningf("Crawl error: %v", err)
			}
		}()

		crawler.Start()
	}
	return
}
<commit_msg>nsq consumer: proper requirement check<commit_after>// Package consumer implements command line crawl consumer from nsq.
package consumer

import (
	"errors"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/golang/glog"

	"github.com/crackcomm/crawl"
)

// Spider - Spider registrator.
type Spider func(crawl.Crawler)

// New - Creates nsq consumer app.
func New(spiders ...Spider) (app *cli.App) {
	app = cli.NewApp()
	app.Name = "crawler"
	app.HelpName = app.Name
	app.Version = "0.0.1"
	app.Usage = "nsq crawl consumer"
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "topic",
			EnvVar: "TOPIC",
		},
		cli.StringFlag{
			Name:   "channel",
			EnvVar: "CHANNEL",
		},
		cli.StringSliceFlag{
			Name:   "nsq-addr",
			EnvVar: "NSQ_ADDR",
		},
		cli.StringSliceFlag{
			Name:   "nsqlookup-addr",
			EnvVar: "NSQLOOKUP_ADDR",
		},
		cli.IntFlag{
			Name:   "concurrency",
			Value:  100,
			EnvVar: "CRAWL_CONCURRENCY",
		},
	}
	app.Before = func(c *cli.Context) error {
		var errs []string
		if len(c.StringSlice("nsq-addr")) == 0 {
			errs = append(errs, "At least one --nsq-addr is required")
		}
		if len(c.StringSlice("nsqlookup-addr")) == 0 {
			errs = append(errs, "At least one --nsqlookup-addr is required")
		}
		if len(errs) > 0 {
			return errors.New(strings.Join(errs, "\n"))
		}
		return nil
	}
	app.Action = func(c *cli.Context) {
		queue := crawl.NewNsqQueue(c.String("topic"), c.String("channel"), c.Int("concurrency"))

		nsqAddr := c.StringSlice("nsq-addr")[0]
		if err := queue.Producer.Connect(nsqAddr); err != nil {
			glog.Fatalf("Error connecting producer to %q: %v", nsqAddr, err)
		}

		if addrs := c.StringSlice("nsq-addr"); len(addrs) != 0 {
			for _, addr := range addrs {
				glog.V(3).Infof("Connecting to nsq %s", addr)
				if err := queue.Consumer.Connect(addr); err != nil {
					glog.Fatalf("Error connecting to nsq %q: %v", addr, err)
				}
				glog.V(3).Infof("Connected to nsq %s", addr)
			}
		}

		if addrs := c.StringSlice("nsqlookup-addr"); len(addrs) != 0 {
			for _, addr := range addrs {
				glog.V(3).Infof("Connecting to nsq lookup %s", addr)
				if err := queue.Consumer.ConnectLookupd(addr); err != nil {
					glog.Fatalf("Error connecting to nsq lookup %q: %v", addr, err)
				}
				glog.V(3).Infof("Connected to nsq lookup %s", addr)
			}
		}

		crawler := crawl.New(
			crawl.WithQueue(queue),
			crawl.WithConcurrency(c.Int("concurrency")),
		)

		for _, spider := range spiders {
			spider(crawler)
		}

		go func() {
			for err := range crawler.Errors() {
				glog.Warningf("Crawl error: %v", err)
			}
		}()

		crawler.Start()
	}
	return
}
<|endoftext|>


<commit_before>// EDNS0
//
// EDNS0 is an extension mechanism for the DNS defined in RFC 2671. It defines a
// standard RR type, the OPT RR, which is then completely abused.
// Basic use pattern for creating an (empty) OPT RR:
//
//	o := new(dns.RR_OPT)
//	o.Hdr.Name = "."
//	o.Hdr.Rrtype = dns.TypeOPT
//
// The rdata of an OPT RR consists out of a slice of EDNS0 interfaces. Currently
// only a few have been standardized: EDNS0_NSID (RFC 5001) and EDNS0_SUBNET (draft). Note that
// these options may be combined in an OPT RR.
// Basic use pattern for a server to check if (and which) options are set:
//
//	// o is a dns.RR_OPT
//	for _, s := range o.Options {
//		switch e := s.(type) {
//		case *dns.EDNS0_NSID:
//			// do stuff with e.Nsid
//		case *dns.EDNS0_SUBNET:
//			// access e.Family, e.Address, etc.
//		}
//	}
package dns

import (
	"encoding/hex"
	"errors"
	"net"
	"strconv"
)

// EDNS0 Option codes.
const (
	_           = iota
	EDNS0LLQ             // not used
	EDNS0UL              // not used
	EDNS0NSID            // nsid (RFC5001)
	EDNS0SUBNET = 0x50fa // client-subnet draft
	_DO         = 1 << 7 // dnssec ok
)

type RR_OPT struct {
	Hdr    RR_Header
	Option []EDNS0 `dns:"opt"`
}

func (rr *RR_OPT) Header() *RR_Header {
	return &rr.Hdr
}

func (rr *RR_OPT) String() string {
	s := "\n;; OPT PSEUDOSECTION:\n; EDNS: version " + strconv.Itoa(int(rr.Version())) + "; "
	if rr.Do() {
		s += "flags: do; "
	} else {
		s += "flags: ; "
	}
	s += "udp: " + strconv.Itoa(int(rr.UDPSize()))

	for _, o := range rr.Option {
		switch o.(type) {
		case *EDNS0_NSID:
			s += "\n; NSID: " + o.String()
			h, e := o.pack()
			var r string
			if e == nil {
				for _, c := range h {
					r += "(" + string(c) + ")"
				}
				s += "  " + r
			}
		case *EDNS0_SUBNET:
			s += "\n; SUBNET: " + o.String()
		case *EDNS0_UPDATE_LEASE:
			s += "\n; LEASE: " + o.String()
		}
	}
	return s
}

func (rr *RR_OPT) Len() int {
	l := rr.Hdr.Len()
	for i := 0; i < len(rr.Option); i++ {
		lo, _ := rr.Option[i].pack()
		l += 2 + len(lo)
	}
	return l
}

func (rr *RR_OPT) Copy() RR {
	return &RR_OPT{*rr.Hdr.CopyHeader(), rr.Option}
}

// Version returns the EDNS version used. Only zero is defined.
func (rr *RR_OPT) Version() uint8 {
	return uint8(rr.Hdr.Ttl & 0x00FF00FFFF)
}

// SetVersion sets the version of EDNS. This is usually zero.
func (rr *RR_OPT) SetVersion(v uint8) {
	rr.Hdr.Ttl = rr.Hdr.Ttl&0xFF00FFFF | uint32(v)
}

// UDPSize returns the UDP buffer size.
func (rr *RR_OPT) UDPSize() uint16 {
	return rr.Hdr.Class
}

// SetUDPSize sets the UDP buffer size.
func (rr *RR_OPT) SetUDPSize(size uint16) {
	rr.Hdr.Class = size
}

// Do returns the value of the DO (DNSSEC OK) bit.
func (rr *RR_OPT) Do() bool {
	return byte(rr.Hdr.Ttl>>8)&_DO == _DO
}

// SetDo sets the DO (DNSSEC OK) bit.
func (rr *RR_OPT) SetDo() {
	b1 := byte(rr.Hdr.Ttl >> 24)
	b2 := byte(rr.Hdr.Ttl >> 16)
	b3 := byte(rr.Hdr.Ttl >> 8)
	b4 := byte(rr.Hdr.Ttl)
	b3 |= _DO // Set it
	rr.Hdr.Ttl = uint32(b1)<<24 | uint32(b2)<<16 | uint32(b3)<<8 | uint32(b4)
}

// EDNS0 defines an EDNS0 Option. An OPT RR can have multiple option appended to
// it. Basic use pattern for adding an option to and OPT RR:
//
//	// o is the OPT RR, e is the EDNS0 option
//	o.Option = append(o.Option, e)
type EDNS0 interface {
	// Option returns the option code for the option.
	Option() uint16
	// pack returns the bytes of the option data.
	pack() ([]byte, error)
	// unpack sets the data as found in the buffer. Is also sets
	// the length of the slice as the length of the option data.
	unpack([]byte)
	// String returns the string representation of the option.
	String() string
}

// The nsid EDNS0 option is used to retrieve some sort of nameserver
// identifier. When seding a request Nsid must be set to the empty string
// The identifier is an opaque string encoded as hex.
// Basic use pattern for creating an nsid option:
//
//	o := new(dns.RR_OPT)
//	o.Hdr.Name = "."
//	o.Hdr.Rrtype = dns.TypeOPT
//	e := new(dns.EDNS0_NSID)
//	e.Code = dns.EDNS0NSID
//	o.Option = append(o.Option, e)
type EDNS0_NSID struct {
	Code uint16 // Always EDNS0NSID
	Nsid string // This string needs to be hex encoded
}

func (e *EDNS0_NSID) Option() uint16 {
	return EDNS0NSID
}

func (e *EDNS0_NSID) pack() ([]byte, error) {
	h, err := hex.DecodeString(e.Nsid)
	if err != nil {
		return nil, err
	}
	return h, nil
}

func (e *EDNS0_NSID) unpack(b []byte) {
	e.Nsid = hex.EncodeToString(b)
}

func (e *EDNS0_NSID) String() string {
	return string(e.Nsid)
}

// The subnet EDNS0 option is used to give the remote nameserver
// an idea of where the client lives. It can then give back a different
// answer depending on the location or network topology.
// Basic use pattern for creating an subnet option:
//
//	o := new(dns.RR_OPT)
//	o.Hdr.Name = "."
//	o.Hdr.Rrtype = dns.TypeOPT
//	e := new(dns.EDNS0_SUBNET)
//	e.Code = dns.EDNS0SUBNET
//	e.Family = 1	// 1 for IPv4 source address, 2 for IPv6
//	e.NetMask = 32	// 32 for IPV4, 128 for IPv6
//	e.SourceScope = 0
//	e.Address = net.ParseIP("127.0.0.1").To4()	// for IPv4
//	// e.Address = net.ParseIP("2001:7b8:32a::2")	// for IPV6
//	o.Option = append(o.Option, e)
type EDNS0_SUBNET struct {
	Code          uint16 // Always EDNS0SUBNET
	Family        uint16 // 1 for IP, 2 for IP6
	SourceNetmask uint8
	SourceScope   uint8
	Address       net.IP
}

func (e *EDNS0_SUBNET) Option() uint16 {
	return EDNS0SUBNET
}

func (e *EDNS0_SUBNET) pack() ([]byte, error) {
	b := make([]byte, 4)
	b[0], b[1] = packUint16(e.Family)
	b[2] = e.SourceNetmask
	b[3] = e.SourceScope
	switch e.Family {
	case 1:
		if e.SourceNetmask > net.IPv4len*8 {
			return nil, errors.New("bad netmask")
		}
		ip := make([]byte, net.IPv4len)
		a := e.Address.To4().Mask(net.CIDRMask(int(e.SourceNetmask), net.IPv4len*8))
		for i := 0; i < net.IPv4len; i++ {
			if i+1 > len(e.Address) {
				break
			}
			ip[i] = a[i]
		}
		b = append(b, ip...)
	case 2:
		if e.SourceNetmask > net.IPv6len*8 {
			return nil, errors.New("bad netmask")
		}
		ip := make([]byte, net.IPv6len)
		a := e.Address.Mask(net.CIDRMask(int(e.SourceNetmask), net.IPv6len*8))
		for i := 0; i < net.IPv6len; i++ {
			if i+1 > len(e.Address) {
				break
			}
			ip[i] = a[i]
		}
		b = append(b, ip...)
	default:
		return nil, errors.New("bad address family")
	}
	return b, nil
}

func (e *EDNS0_SUBNET) unpack(b []byte) {
	if len(b) < 8 {
		return
	}
	e.Family, _ = unpackUint16(b, 0)
	e.SourceNetmask = b[2]
	e.SourceScope = b[3]
	switch e.Family {
	case 1:
		if len(b) == 8 {
			e.Address = net.IPv4(b[4], b[5], b[6], b[7])
		}
	case 2:
		if len(b) == 20 {
			e.Address = net.IP{b[4], b[4+1], b[4+2], b[4+3], b[4+4],
				b[4+5], b[4+6], b[4+7], b[4+8], b[4+9], b[4+10],
				b[4+11], b[4+12], b[4+13], b[4+14], b[4+15]}
		}
	}
	return
}

func (e *EDNS0_SUBNET) String() (s string) {
	if e.Address == nil {
		s = "<nil>"
	} else if e.Address.To4() != nil {
		s = e.Address.String()
	} else {
		s = "[" + e.Address.String() + "]"
	}
	s += "/" + strconv.Itoa(int(e.SourceNetmask)) + "/" + strconv.Itoa(int(e.SourceScope))
	return
}

// The UPDATE_LEASE EDNS0 (draft RFC) option is used to tell the server to set
// an expiration on an update RR. This is helpful for clients that cannot clean
// up after themselves. This is a draft RFC and more information can be found at
// http://files.dns-sd.org/draft-sekar-dns-ul.txt
//
//	o := new(dns.RR_OPT)
//	o.Hdr.Name = "."
//	o.Hdr.Rrtype = dns.TypeOPT
//	e := new(dns.EDNS0_UPDATE_LEASE)
//	e.Code = dns.EDNS0UPDATELEASE
//	e.Lease = 120 // in seconds
//	o.Option = append(o.Option, e)
const EDNS0UPDATELEASE = 0x02

type EDNS0_UPDATE_LEASE struct {
	Code          uint16 // Always EDNS0UPDATELEASE
	Lease         uint32
}

func (e *EDNS0_UPDATE_LEASE) Option() uint16 {
	return EDNS0UPDATELEASE
}

// Copied: http://golang.org/src/pkg/net/dnsmsg.go
func (e *EDNS0_UPDATE_LEASE) pack() ([]byte, error) {
	lease_byte := make([]byte, 4)
	lease_byte[0] = byte(e.Lease >> 24)
	lease_byte[1] = byte(e.Lease >> 16)
	lease_byte[2] = byte(e.Lease >> 8)
	lease_byte[3] = byte(e.Lease)
	return lease_byte, nil
}

func (e *EDNS0_UPDATE_LEASE) unpack(b []byte) {
	e.Lease = uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3])
}

func (e *EDNS0_UPDATE_LEASE) String() string {
	return strconv.Itoa(int(e.Lease))
}
<commit_msg>Small tweak<commit_after>// EDNS0
//
// EDNS0 is an extension mechanism for the DNS defined in RFC 2671. It defines a
// standard RR type, the OPT RR, which is then completely abused.
// Basic use pattern for creating an (empty) OPT RR:
//
//	o := new(dns.RR_OPT)
//	o.Hdr.Name = "."
//	o.Hdr.Rrtype = dns.TypeOPT
//
// The rdata of an OPT RR consists out of a slice of EDNS0 interfaces. Currently
// only a few have been standardized: EDNS0_NSID (RFC 5001) and EDNS0_SUBNET (draft). Note that
// these options may be combined in an OPT RR.
// Basic use pattern for a server to check if (and which) options are set:
//
//	// o is a dns.RR_OPT
//	for _, s := range o.Options {
//		switch e := s.(type) {
//		case *dns.EDNS0_NSID:
//			// do stuff with e.Nsid
//		case *dns.EDNS0_SUBNET:
//			// access e.Family, e.Address, etc.
//		}
//	}
package dns

import (
	"encoding/hex"
	"errors"
	"net"
	"strconv"
)

// EDNS0 Option codes.
const (
	_           = iota
	EDNS0LLQ             // not used
	EDNS0UL              // not used
	EDNS0NSID            // nsid (RFC5001)
	EDNS0SUBNET = 0x50fa // client-subnet draft
	_DO         = 1 << 7 // dnssec ok
)

type RR_OPT struct {
	Hdr    RR_Header
	Option []EDNS0 `dns:"opt"`
}

func (rr *RR_OPT) Header() *RR_Header {
	return &rr.Hdr
}

func (rr *RR_OPT) String() string {
	s := "\n;; OPT PSEUDOSECTION:\n; EDNS: version " + strconv.Itoa(int(rr.Version())) + "; "
	if rr.Do() {
		s += "flags: do; "
	} else {
		s += "flags: ; "
	}
	s += "udp: " + strconv.Itoa(int(rr.UDPSize()))

	for _, o := range rr.Option {
		switch o.(type) {
		case *EDNS0_NSID:
			s += "\n; NSID: " + o.String()
			h, e := o.pack()
			var r string
			if e == nil {
				for _, c := range h {
					r += "(" + string(c) + ")"
				}
				s += "  " + r
			}
		case *EDNS0_SUBNET:
			s += "\n; SUBNET: " + o.String()
		case *EDNS0_UPDATE_LEASE:
			s += "\n; LEASE: " + o.String()
		}
	}
	return s
}

func (rr *RR_OPT) Len() int {
	l := rr.Hdr.Len()
	for i := 0; i < len(rr.Option); i++ {
		lo, _ := rr.Option[i].pack()
		l += 2 + len(lo)
	}
	return l
}

func (rr *RR_OPT) Copy() RR {
	return &RR_OPT{*rr.Hdr.CopyHeader(), rr.Option}
}

// Version returns the EDNS version used. Only zero is defined.
func (rr *RR_OPT) Version() uint8 {
	return uint8(rr.Hdr.Ttl & 0x00FF00FFFF)
}

// SetVersion sets the version of EDNS. This is usually zero.
func (rr *RR_OPT) SetVersion(v uint8) {
	rr.Hdr.Ttl = rr.Hdr.Ttl&0xFF00FFFF | uint32(v)
}

// UDPSize returns the UDP buffer size.
func (rr *RR_OPT) UDPSize() uint16 {
	return rr.Hdr.Class
}

// SetUDPSize sets the UDP buffer size.
func (rr *RR_OPT) SetUDPSize(size uint16) {
	rr.Hdr.Class = size
}

// Do returns the value of the DO (DNSSEC OK) bit.
func (rr *RR_OPT) Do() bool {
	return byte(rr.Hdr.Ttl>>8)&_DO == _DO
}

// SetDo sets the DO (DNSSEC OK) bit.
func (rr *RR_OPT) SetDo() {
	b1 := byte(rr.Hdr.Ttl >> 24)
	b2 := byte(rr.Hdr.Ttl >> 16)
	b3 := byte(rr.Hdr.Ttl >> 8)
	b4 := byte(rr.Hdr.Ttl)
	b3 |= _DO // Set it
	rr.Hdr.Ttl = uint32(b1)<<24 | uint32(b2)<<16 | uint32(b3)<<8 | uint32(b4)
}

// EDNS0 defines an EDNS0 Option. An OPT RR can have multiple option appended to
// it. Basic use pattern for adding an option to and OPT RR:
//
//	// o is the OPT RR, e is the EDNS0 option
//	o.Option = append(o.Option, e)
type EDNS0 interface {
	// Option returns the option code for the option.
	Option() uint16
	// pack returns the bytes of the option data.
	pack() ([]byte, error)
	// unpack sets the data as found in the buffer. Is also sets
	// the length of the slice as the length of the option data.
	unpack([]byte)
	// String returns the string representation of the option.
	String() string
}

// The nsid EDNS0 option is used to retrieve some sort of nameserver
// identifier. When seding a request Nsid must be set to the empty string
// The identifier is an opaque string encoded as hex.
// Basic use pattern for creating an nsid option:
//
//	o := new(dns.RR_OPT)
//	o.Hdr.Name = "."
//	o.Hdr.Rrtype = dns.TypeOPT
//	e := new(dns.EDNS0_NSID)
//	e.Code = dns.EDNS0NSID
//	o.Option = append(o.Option, e)
type EDNS0_NSID struct {
	Code uint16 // Always EDNS0NSID
	Nsid string // This string needs to be hex encoded
}

func (e *EDNS0_NSID) Option() uint16 {
	return EDNS0NSID
}

func (e *EDNS0_NSID) pack() ([]byte, error) {
	h, err := hex.DecodeString(e.Nsid)
	if err != nil {
		return nil, err
	}
	return h, nil
}

func (e *EDNS0_NSID) unpack(b []byte) {
	e.Nsid = hex.EncodeToString(b)
}

func (e *EDNS0_NSID) String() string {
	return string(e.Nsid)
}

// The subnet EDNS0 option is used to give the remote nameserver
// an idea of where the client lives. It can then give back a different
// answer depending on the location or network topology.
// Basic use pattern for creating an subnet option:
//
//	o := new(dns.RR_OPT)
//	o.Hdr.Name = "."
//	o.Hdr.Rrtype = dns.TypeOPT
//	e := new(dns.EDNS0_SUBNET)
//	e.Code = dns.EDNS0SUBNET
//	e.Family = 1	// 1 for IPv4 source address, 2 for IPv6
//	e.NetMask = 32	// 32 for IPV4, 128 for IPv6
//	e.SourceScope = 0
//	e.Address = net.ParseIP("127.0.0.1").To4()	// for IPv4
//	// e.Address = net.ParseIP("2001:7b8:32a::2")	// for IPV6
//	o.Option = append(o.Option, e)
type EDNS0_SUBNET struct {
	Code          uint16 // Always EDNS0SUBNET
	Family        uint16 // 1 for IP, 2 for IP6
	SourceNetmask uint8
	SourceScope   uint8
	Address       net.IP
}

func (e *EDNS0_SUBNET) Option() uint16 {
	return EDNS0SUBNET
}

func (e *EDNS0_SUBNET) pack() ([]byte, error) {
	b := make([]byte, 4)
	b[0], b[1] = packUint16(e.Family)
	b[2] = e.SourceNetmask
	b[3] = e.SourceScope
	switch e.Family {
	case 1:
		if e.SourceNetmask > net.IPv4len*8 {
			return nil, errors.New("bad netmask")
		}
		ip := make([]byte, net.IPv4len)
		a := e.Address.To4().Mask(net.CIDRMask(int(e.SourceNetmask), net.IPv4len*8))
		for i := 0; i < net.IPv4len; i++ {
			if i+1 > len(e.Address) {
				break
			}
			ip[i] = a[i]
		}
		b = append(b, ip...)
	case 2:
		if e.SourceNetmask > net.IPv6len*8 {
			return nil, errors.New("bad netmask")
		}
		ip := make([]byte, net.IPv6len)
		a := e.Address.Mask(net.CIDRMask(int(e.SourceNetmask), net.IPv6len*8))
		for i := 0; i < net.IPv6len; i++ {
			if i+1 > len(e.Address) {
				break
			}
			ip[i] = a[i]
		}
		b = append(b, ip...)
	default:
		return nil, errors.New("bad address family")
	}
	return b, nil
}

func (e *EDNS0_SUBNET) unpack(b []byte) {
	if len(b) < 8 {
		return
	}
	e.Family, _ = unpackUint16(b, 0)
	e.SourceNetmask = b[2]
	e.SourceScope = b[3]
	switch e.Family {
	case 1:
		if len(b) == 8 {
			e.Address = net.IPv4(b[4], b[5], b[6], b[7])
		}
	case 2:
		if len(b) == 20 {
			e.Address = net.IP{b[4], b[4+1], b[4+2], b[4+3], b[4+4],
				b[4+5], b[4+6], b[4+7], b[4+8], b[4+9], b[4+10],
				b[4+11], b[4+12], b[4+13], b[4+14], b[4+15]}
		}
	}
	return
}

func (e *EDNS0_SUBNET) String() (s string) {
	if e.Address == nil {
		s = "<nil>"
	} else if e.Address.To4() != nil {
		s = e.Address.String()
	} else {
		s = "[" + e.Address.String() + "]"
	}
	s += "/" + strconv.Itoa(int(e.SourceNetmask)) + "/" + strconv.Itoa(int(e.SourceScope))
	return
}

// The UPDATE_LEASE EDNS0 (draft RFC) option is used to tell the server to set
// an expiration on an update RR. This is helpful for clients that cannot clean
// up after themselves. This is a draft RFC and more information can be found at
// http://files.dns-sd.org/draft-sekar-dns-ul.txt
//
//	o := new(dns.RR_OPT)
//	o.Hdr.Name = "."
//	o.Hdr.Rrtype = dns.TypeOPT
//	e := new(dns.EDNS0_UPDATE_LEASE)
//	e.Code = dns.EDNS0UPDATELEASE
//	e.Lease = 120 // in seconds
//	o.Option = append(o.Option, e)
const EDNS0UPDATELEASE = 0x02

type EDNS0_UPDATE_LEASE struct {
	Code          uint16 // Always EDNS0UPDATELEASE
	Lease         uint32
}

func (e *EDNS0_UPDATE_LEASE) Option() uint16 {
	return EDNS0UPDATELEASE
}

// Copied: http://golang.org/src/pkg/net/dnsmsg.go
func (e *EDNS0_UPDATE_LEASE) pack() ([]byte, error) {
	b := make([]byte, 4)
	b[0] = byte(e.Lease >> 24)
	b[1] = byte(e.Lease >> 16)
	b[2] = byte(e.Lease >> 8)
	b[3] = byte(e.Lease)
	return b, nil
}

func (e *EDNS0_UPDATE_LEASE) unpack(b []byte) {
	e.Lease = uint32(b[0])<<24 | uint32(b[1])<<16 | uint32(b[2])<<8 | uint32(b[3])
}

func (e *EDNS0_UPDATE_LEASE) String() string {
	return strconv.Itoa(int(e.Lease))
}
<|endoftext|>


<commit_before>package gst

/*
#include <stdlib.h>
#include <gst/gst.h>
*/
import "C"

import (
	"github.com/lidouf/glib"
)

type Structure C.GstStructure

func (s *Structure) g() *C.GstStructure {
	return (*C.GstStructure)(s)
}

func (s *Structure) GetName() string {
	return C.GoString((*C.char)(C.gst_structure_get_name(s.g())))
}

func (s *Structure) Serialize() glib.Params {
	return serializeGstStructure(s.g())
}

func MakeStructure(name string, fields *glib.Params) *Structure {
	var f glib.Params
	if fields == nil {
		f = glib.Params{}
	} else {
		f = *fields
	}
	return (*Structure)(makeGstStructure(name, f))
}
<commit_msg>add ToString implementation<commit_after>package gst

/*
#include <stdlib.h>
#include <gst/gst.h>
*/
import "C"

import (
	"github.com/lidouf/glib"
)

type Structure C.GstStructure

func (s *Structure) g() *C.GstStructure {
	return (*C.GstStructure)(s)
}

func (s *Structure) GetName() string {
	return C.GoString((*C.char)(C.gst_structure_get_name(s.g())))
}

func (s *Structure) Serialize() glib.Params {
	return serializeGstStructure(s.g())
}

func (s *Structure) ToString() string {
	return C.GoString((*C.char)(C.gst_structure_to_string(s.g())))
}

func MakeStructure(name string, fields *glib.Params) *Structure {
	var f glib.Params
	if fields == nil {
		f = glib.Params{}
	} else {
		f = *fields
	}
	return (*Structure)(makeGstStructure(name, f))
}
<|endoftext|>


<commit_before>package fixity

import (
	"io"
	"os"
)

const (
	// DefaultAverageChunkSize that the chunker will attempt to produce.
	//
	// See WriteRequest.AverageChunkSize documentation for further explanation
	// of chunk sizes.
	//
	// This value is 1024^2*4 bytes, 4MiB, chosen as a compromise for the
	// small and large files.
	//
	// Ideally it should strike a balance between too many chunks written while
	// still mitigating data duplication caused by frequent changes to files
	// and written data.
	//
	// If written data is commonly editing the first X bytes, this value should
	// be overridden in the WriteRequest itself.
	DefaultAverageChunkSize uint64 = 4194304

	// autoChunkCount is the number of chunks SetRollFromBytes/etc will set to.
	//
	// Eg, if SetRollFromBytes is given a byte array that is 1,048,576 bytes,
	// then it will divide 1,048,576 by eg, 10 to set the rollSize to
	// 104,857, effectively rolling the bytes into 10 parts when it's written.
	autoChunkCount = 6

	minAutoChunkSize = 12288
	maxAutoChunkSize = 1073741824
)

// WriteRequest represents a blob to be written alone with metadata.
type WriteRequest struct {
	Id       string
	RollSize uint64
	Fields   Fields
	Blob     io.ReadCloser
}

// NewWrite creates a new WriteRequest to be used with Fixity.WriteRequest.
func NewWrite(id string, rc io.ReadCloser, f ...Field) *WriteRequest {
	return &WriteRequest{
		Id:       id,
		RollSize: DefaultAverageChunkSize,
		Fields:   f,
		Blob:     rc,
	}
}

func (req *WriteRequest) setRollSize(roll uint64) {
	if roll < minAutoChunkSize {
		roll = minAutoChunkSize
	}
	if roll > maxAutoChunkSize {
		roll = maxAutoChunkSize
	}
	req.RollSize = roll
}

func (req *WriteRequest) SetRollFromBytes(b []byte) {
	req.setRollSize(uint64(len(b)) / autoChunkCount)
}

func (req *WriteRequest) SetRollFromFileInfo(fi os.FileInfo) {
	req.setRollSize(uint64(fi.Size()) / autoChunkCount)
}
<commit_msg>feat: tweaked min/max autochunk size<commit_after>package fixity

import (
	"io"
	"os"
)

const (
	// DefaultAverageChunkSize that the chunker will attempt to produce.
	//
	// See WriteRequest.AverageChunkSize documentation for further explanation
	// of chunk sizes.
	//
	// This value is 1024^2*4 bytes, 4MiB, chosen as a compromise for the
	// small and large files.
	//
	// Ideally it should strike a balance between too many chunks written while
	// still mitigating data duplication caused by frequent changes to files
	// and written data.
	//
	// If written data is commonly editing the first X bytes, this value should
	// be overridden in the WriteRequest itself.
	DefaultAverageChunkSize uint64 = 4194304

	// autoChunkCount is the number of chunks SetRollFromBytes/etc will set to.
	//
	// Eg, if SetRollFromBytes is given a byte array that is 1,048,576 bytes,
	// then it will divide 1,048,576 by eg, 10 to set the rollSize to
	// 104,857, effectively rolling the bytes into 10 parts when it's written.
	autoChunkCount = 6

	// 12KiB so that small files/data won't churn with a ton of chunks.
	minAutoChunkSize = 12288
	// 20MiB because chunks too large become hard to store/send/etc. Each
	// chunk should be reasonbly fast to send between nodes.
	maxAutoChunkSize = 20971520
)

// WriteRequest represents a blob to be written alone with metadata.
type WriteRequest struct {
	Id       string
	RollSize uint64
	Fields   Fields
	Blob     io.ReadCloser
}

// NewWrite creates a new WriteRequest to be used with Fixity.WriteRequest.
func NewWrite(id string, rc io.ReadCloser, f ...Field) *WriteRequest {
	return &WriteRequest{
		Id:       id,
		RollSize: DefaultAverageChunkSize,
		Fields:   f,
		Blob:     rc,
	}
}

func (req *WriteRequest) setRollSize(roll uint64) {
	if roll < minAutoChunkSize {
		roll = minAutoChunkSize
	}
	if roll > maxAutoChunkSize {
		roll = maxAutoChunkSize
	}
	req.RollSize = roll
}

func (req *WriteRequest) SetRollFromBytes(b []byte) {
	req.setRollSize(uint64(len(b)) / autoChunkCount)
}

func (req *WriteRequest) SetRollFromFileInfo(fi os.FileInfo) {
	req.setRollSize(uint64(fi.Size()) / autoChunkCount)
}
<|endoftext|>


<commit_before>package fixity

import (
	"io"
	"os"
)

const (
	// DefaultAverageChunkSize that the chunker will attempt to produce.
	//
	// See WriteRequest.AverageChunkSize documentation for further explanation
	// of chunk sizes.
	//
	// This value is 1024^2*4 bytes, 4MiB, chosen as a compromise for the
	// small and large files.
	//
	// Ideally it should strike a balance between too many chunks written while
	// still mitigating data duplication caused by frequent changes to files
	// and written data.
	//
	// If written data is commonly editing the first X bytes, this value should
	// be overridden in the WriteRequest itself.
	DefaultAverageChunkSize uint64 = 4194304

	// autoChunkCount is the number of chunks SetChunkFromBytes/etc will set to.
	//
	// Eg, if SetChunkFromBytes is given a byte array that is 1,048,576 bytes,
	// then it will divide 1,048,576 by eg, 10 to set the rollSize to
	// 104,857, effectively rolling the bytes into 10 parts when it's written.
	autoChunkCount = 6

	// 12KiB so that small files/data won't churn with a ton of chunks.
	minAutoChunkSize = 12288
	// 4MiB because chunks too large become hard to store/send/etc. Each
	// chunk should be reasonbly fast to send between nodes.
	maxAutoChunkSize = DefaultAverageChunkSize
)

// WriteRequest represents a blob to be written alone with metadata.
type WriteRequest struct {
	// Id is the Content Id used to associate this write with previous writes.
	//
	// This association effectively mutates the write from the previous content
	// with the same Id. Important values are inhereted, such as ChunkSize,
	// ensuring consistent writes.
	Id string `json:"id,omitempty"`

	// AverageChunkSize is the average bytes that each chunk is aimed to be.
	//
	// Chunks are separated by Cotent Defined Chunks (CDC) and this value
	// allows mutations of this blob to use the same ChunkSize with each
	// version. This ensures the chunks are chunk'd by the CDC algorithm
	// with the same spacing.
	//
	// Note that the algorithm is decided by the fixity.Store.
	AverageChunkSize uint64 `json:"averageChunkSize,omitempty"`

	// IgnoreDuplicateBlob will ignore this write if the blob exists for the id.
	//
	// An existing blob is determined to be one which *exists* on the chain,
	// not specifically if it is the latest Content for it's id.
	//
	// This flag is used to ensure data exists on the blockchain, without
	// causing any form of versioning contention. Duplicate blobs are allowed
	// for different Ids.
	IgnoreDuplicateBlob bool

	// Fields are the indexable fields that the data will be indexed with.
	//
	// This is often metadata like filename, unix permissions, etc. It can
	// also be used to retrieve this data later, as the Fields is stored
	// on the fixity.Content object within the datastore.
	Fields Fields `json:"fields,omitempty"`

	// Blob is the actual data that is being written, and is required.
	//
	// This will be used to build a fixity.Blob type.
	Blob io.ReadCloser `json:"-"`
}

// NewWrite creates a new WriteRequest to be used with Fixity.WriteRequest.
func NewWrite(id string, rc io.ReadCloser, f ...Field) *WriteRequest {
	return &WriteRequest{
		Id:               id,
		AverageChunkSize: DefaultAverageChunkSize,
		Fields:           f,
		Blob:             rc,
	}
}

func (req *WriteRequest) setChunkSize(averageChunkSize uint64) {
	if averageChunkSize < minAutoChunkSize {
		averageChunkSize = minAutoChunkSize
	}
	if averageChunkSize > maxAutoChunkSize {
		averageChunkSize = maxAutoChunkSize
	}
	req.AverageChunkSize = averageChunkSize
}

func (req *WriteRequest) SetChunkSizeFromBytes(b []byte) {
	req.setChunkSize(uint64(len(b)) / autoChunkCount)
}

func (req *WriteRequest) SetChunkSizeFromFileInfo(fi os.FileInfo) {
	req.setChunkSize(uint64(fi.Size()) / autoChunkCount)
}
<commit_msg>feat: added chunk sizing tiers; 100KiB/1MiB/4MiB<commit_after>package fixity

import (
	"io"
	"os"
)

const (
	// DefaultAverageChunkSize that the chunker will attempt to produce.
	//
	// See WriteRequest.AverageChunkSize documentation for further explanation
	// of chunk sizes.
	//
	// This value is 1024^2*1 bytes, 1MiB, chosen as a compromise for the
	// small and large files.
	//
	// Ideally it should strike a balance between too many chunks written while
	// still mitigating data duplication caused by frequent changes to files
	// and written data.
	//
	// If written data is commonly editing the first X bytes, this value should
	// be overridden in the WriteRequest itself.
	DefaultAverageChunkSize = medChunkSize

	// smallChunkSize is the size SetChunkSizeFromX will use for small data.
	//
	// This value is 1024*100 bytes, 100 KiB, chosen as a balance for very
	// small files being able to be chunked but still being well over the
	// hard limits of the Restic "average bits". At this time, that value is
	// 14bits (12KiB). Reference for this value can be found here:
	//
	//    https://github.com/leeola/fixity/blob/master/chunkers/restic/restic.go#L12
	//
	// Note that that using this smallChunkSize is decided via the smallCutoff,
	// below.
	smallChunkSize uint64 = 102400

	// medChunkSize is the size SetChunkSizeFromX will use for medium data.
	//
	// This value is 1024^2 bytes, 1MiB, chosen as a balance for medium data.
	//
	// Note that using this medChunkSize is decided via the medCutoff, below.
	medChunkSize uint64 = 1048576

	// medChunkSize is the size SetChunkSizeFromX will use for large data.
	//
	// This value is 1024^2*4 bytes, 4MiB, chosen as a balance between total
	// hashes created and size of chunks being sent over the wire for large
	// data. Too big of a chunk would cause data being sent from one node
	// to another to lag. This value should choose a middleground between
	// total hashes and node transfer times.
	//
	// Note that largeChunkSize is chosen only if the total bytes are larger
	// the medCutoff.
	largeChunkSize uint64 = 4194304

	// smallCutoff decides if the smallChunkSize is to be used or not.
	//
	// If the total bytes given to AssertChunkSize() is below this value,
	// the smallChunkSize will be used.
	smallCutoff = smallChunkSize * 10

	// medCutoff decides if the medChunkSize is to be used or not.
	//
	// If the total bytes given to AssertChunkSize() is below this value,
	// the medChunkSize will be used.
	medCutoff = medChunkSize * 200
)

// WriteRequest represents a blob to be written alone with metadata.
type WriteRequest struct {
	// Id is the Content Id used to associate this write with previous writes.
	//
	// This association effectively mutates the write from the previous content
	// with the same Id. Important values are inhereted, such as ChunkSize,
	// ensuring consistent writes.
	Id string `json:"id,omitempty"`

	// AverageChunkSize is the average bytes that each chunk is aimed to be.
	//
	// Chunks are separated by Cotent Defined Chunks (CDC) and this value
	// allows mutations of this blob to use the same ChunkSize with each
	// version. This ensures the chunks are chunk'd by the CDC algorithm
	// with the same spacing.
	//
	// Note that the algorithm is decided by the fixity.Store.
	AverageChunkSize uint64 `json:"averageChunkSize,omitempty"`

	// IgnoreDuplicateBlob will ignore this write if the blob exists for the id.
	//
	// An existing blob is determined to be one which *exists* on the chain,
	// not specifically if it is the latest Content for it's id.
	//
	// This flag is used to ensure data exists on the blockchain, without
	// causing any form of versioning contention. Duplicate blobs are allowed
	// for different Ids.
	IgnoreDuplicateBlob bool

	// Fields are the indexable fields that the data will be indexed with.
	//
	// This is often metadata like filename, unix permissions, etc. It can
	// also be used to retrieve this data later, as the Fields is stored
	// on the fixity.Content object within the datastore.
	Fields Fields `json:"fields,omitempty"`

	// Blob is the actual data that is being written, and is required.
	//
	// This will be used to build a fixity.Blob type.
	Blob io.ReadCloser `json:"-"`
}

// NewWrite creates a new WriteRequest to be used with Fixity.WriteRequest.
func NewWrite(id string, rc io.ReadCloser, f ...Field) *WriteRequest {
	return &WriteRequest{
		Id:               id,
		AverageChunkSize: DefaultAverageChunkSize,
		Fields:           f,
		Blob:             rc,
	}
}

// SetChunkSizeFromBytes sets the chunkSize from the given byte slice.
//
// The chunksize value is decided via AssertChunkSize().
func (req *WriteRequest) SetChunkSizeFromBytes(b []byte) {
	req.AverageChunkSize = AssertChunkSize(uint64(len(b)))
}

// SetChunkSizeFromFileInfo sets the chunkSize from the given FileInfo.
//
// The chunksize value is decided via AssertChunkSize().
func (req *WriteRequest) SetChunkSizeFromFileInfo(fi os.FileInfo) {
	req.AverageChunkSize = AssertChunkSize(uint64(fi.Size()))
}

// AssertChunkSize chooses a chunksize based on the byte size of the input.
//
// It uses a tiered approach, decided by the constant values smallCutoff and
// medCutoff. In short:
//
// - If totalBytes is smaller than 1MiB, a chunkSize of 100KiB is used.
// - If totalBytes is smaller than 200MiB, a chunkSize of 1MiB is used.
// - If totalBytes is larger than 200MiB, a chunkSize of 4MiB is used.
func AssertChunkSize(totalBytes uint64) (chunkSize uint64) {
	switch {
	case totalBytes < smallCutoff:
		return smallChunkSize
	case totalBytes < medCutoff:
		return medChunkSize
	default:
		return largeChunkSize
	}
}
<|endoftext|>


<commit_before>package vshard

import (
	"context"
	"crypto/md5"
	"encoding/hex"
	"errors"
	"log"
	"math/big"
	"strconv"
	"sync"
	"time"

	farm "github.com/dgryski/go-farm"
	jump "github.com/dgryski/go-jump"
	"github.com/youtube/vitess/go/memcache"
	"github.com/youtube/vitess/go/pools"
)

var (
	// ErrKeyNotFound defines the error mensage when key is not found on memcached
	ErrKeyNotFound = errors.New("error: key not found")

	defaultServerStrategy  = FarmhashShardServerStrategy
	defaultHashKeyStrategy = FarmhashKeyStrategy
)

const (
	defaultCapacity          = 5
	defaultMaxCapacity       = 5
	defaultIdleTimeout       = time.Millisecond * 500
	defaultConnectionTimeout = time.Millisecond * 200
)

// VitessResource implements the expected interface for vitess internal pool
type VitessResource struct {
	*memcache.Connection
}

// ServerStrategy defines the signature for the sharding function
type ServerStrategy func(key string, numServers int) int

// HashKeyStrategy defines the signature for the key hashing function
type HashKeyStrategy func(key string) string

// Close closes connections in a pool
func (r VitessResource) Close() {
	r.Connection.Close()
}

// Pool defines the pool
type Pool struct {
	Servers               []string
	pool                  []*pools.ResourcePool
	Capacity, MaxCapacity int
	numServers            int
	ServerStrategy        ServerStrategy
	HashKeyStrategy       HashKeyStrategy
	IdleTimeout           time.Duration
	ConnectionTimeout     time.Duration
	sync.RWMutex
}

// PoolStats defines all stats vitess memcached driver exposes
type PoolStats struct {
	Slot        int
	Server      string
	Capacity    int64
	Available   int64
	MaxCap      int64
	WaitCount   int64
	WaitTime    time.Duration
	IdleTimeout time.Duration
}

// MD5ShardServerStrategy uses md5+jump to pick a server
func MD5ShardServerStrategy(key string, numServers int) int {
	if numServers == 1 {
		return 0
	}

	hash := md5.Sum([]byte(key))
	hashInt := big.NewInt(0)
	hashInt.SetString(hex.EncodeToString(hash[:]), 16)

	return int(jump.Hash(hashInt.Uint64(), numServers))
}

// FarmhashShardServerStrategy uses farmhash+jump to pick a server
func FarmhashShardServerStrategy(key string, numServers int) int {
	if numServers == 1 {
		return 0
	}

	return int(jump.Hash(farm.Fingerprint64([]byte(key)), numServers))
}

// FarmhashKeyStrategy uses farmhash to normalize key names for storage
func FarmhashKeyStrategy(key string) string {
	return strconv.FormatUint(farm.Fingerprint64([]byte(key)), 10)
}

// NoKeyStrategy doesn't hash the key
func NoKeyStrategy(key string) string {
	return key
}

// Start starts the pool
func (v *Pool) Start() {

	v.initialize()

	for i, server := range v.Servers {
		func(_v *Pool, _server string, _i int) {
			_v.pool = append(_v.pool, pools.NewResourcePool(func() (pools.Resource, error) {
				c, err := memcache.Connect(_server, _v.ConnectionTimeout)
				return VitessResource{c}, err
			}, _v.Capacity, _v.MaxCapacity, _v.IdleTimeout))

			conn, err := _v.GetPoolConnection(_i)
			defer _v.ReturnConnection(_i, conn)
			if err != nil {
				log.Fatalf("Can't connect to memcached %d (%s): %s", _i, _server, err)
			}
		}(v, server, i)
	}
}

func (v *Pool) initialize() {
	v.numServers = len(v.Servers)
	v.pool = []*pools.ResourcePool{}

	if v.ServerStrategy == nil {
		v.ServerStrategy = defaultServerStrategy
	}
	if v.HashKeyStrategy == nil {
		v.HashKeyStrategy = defaultHashKeyStrategy
	}
	if v.Capacity == 0 {
		v.Capacity = defaultCapacity
	}
	if v.MaxCapacity == 0 {
		v.MaxCapacity = defaultMaxCapacity
	}
	if v.IdleTimeout == 0 {
		v.IdleTimeout = defaultIdleTimeout
	}
	if v.ConnectionTimeout == 0 {
		v.ConnectionTimeout = defaultConnectionTimeout
	}
}

// GetConnection returns a connection from the sharding pool, based on the key
func (v *Pool) GetConnection(key string) (*VitessResource, int, error) {
	poolNum := v.ServerStrategy(key, v.numServers)

	connection, err := v.GetPoolConnection(poolNum)
	if err != nil {
		return nil, -1, err
	}

	return connection, poolNum, nil
}

// GetPoolConnection returns a connection from a specific pool number
func (v *Pool) GetPoolConnection(poolNum int) (*VitessResource, error) {
	ctx := context.Background()

	v.RLock()
	resource, err := v.pool[poolNum].Get(ctx)
	v.RUnlock()
	if err != nil {
		return nil, err
	}

	connection := resource.(VitessResource)

	return &connection, nil
}

// ReturnConnection returns a connection to the pool
func (v *Pool) ReturnConnection(poolNum int, resource *VitessResource) {
	if poolNum > v.numServers || poolNum < 0 {
		log.Fatalf("error: invalid server %d (of total %d)", poolNum, v.numServers)
	}

	v.RLock()
	v.pool[poolNum].Put(*resource)
	v.RUnlock()
}

// GetKeyMapping returns a mapping of server to a list of keys, useful for Gets()
func (v *Pool) GetKeyMapping(keys ...string) map[int][]string {
	mapping := make(map[int][]string)

	for i := 0; i < v.numServers; i++ {
		mapping[i] = []string{}
	}

	for _, key := range keys {
		poolNum := v.ServerStrategy(key, v.numServers)
		mapping[poolNum] = append(mapping[poolNum], v.HashKeyStrategy(key))
	}

	return mapping
}
<commit_msg>fix locking<commit_after>package vshard

import (
	"context"
	"crypto/md5"
	"encoding/hex"
	"errors"
	"log"
	"math/big"
	"strconv"
	"sync"
	"time"

	farm "github.com/dgryski/go-farm"
	jump "github.com/dgryski/go-jump"
	"github.com/youtube/vitess/go/memcache"
	"github.com/youtube/vitess/go/pools"
)

var (
	// ErrKeyNotFound defines the error mensage when key is not found on memcached
	ErrKeyNotFound = errors.New("error: key not found")

	defaultServerStrategy  = FarmhashShardServerStrategy
	defaultHashKeyStrategy = FarmhashKeyStrategy
)

const (
	defaultCapacity          = 5
	defaultMaxCapacity       = 5
	defaultIdleTimeout       = time.Millisecond * 500
	defaultConnectionTimeout = time.Millisecond * 200
)

// VitessResource implements the expected interface for vitess internal pool
type VitessResource struct {
	*memcache.Connection
}

// ServerStrategy defines the signature for the sharding function
type ServerStrategy func(key string, numServers int) int

// HashKeyStrategy defines the signature for the key hashing function
type HashKeyStrategy func(key string) string

// Close closes connections in a pool
func (r VitessResource) Close() {
	r.Connection.Close()
}

// Pool defines the pool
type Pool struct {
	Servers               []string
	pool                  []*pools.ResourcePool
	Capacity, MaxCapacity int
	numServers            int
	ServerStrategy        ServerStrategy
	HashKeyStrategy       HashKeyStrategy
	IdleTimeout           time.Duration
	ConnectionTimeout     time.Duration
	sync.RWMutex
}

// PoolStats defines all stats vitess memcached driver exposes
type PoolStats struct {
	Slot        int
	Server      string
	Capacity    int64
	Available   int64
	MaxCap      int64
	WaitCount   int64
	WaitTime    time.Duration
	IdleTimeout time.Duration
}

// MD5ShardServerStrategy uses md5+jump to pick a server
func MD5ShardServerStrategy(key string, numServers int) int {
	if numServers == 1 {
		return 0
	}

	hash := md5.Sum([]byte(key))
	hashInt := big.NewInt(0)
	hashInt.SetString(hex.EncodeToString(hash[:]), 16)

	return int(jump.Hash(hashInt.Uint64(), numServers))
}

// FarmhashShardServerStrategy uses farmhash+jump to pick a server
func FarmhashShardServerStrategy(key string, numServers int) int {
	if numServers == 1 {
		return 0
	}

	return int(jump.Hash(farm.Fingerprint64([]byte(key)), numServers))
}

// FarmhashKeyStrategy uses farmhash to normalize key names for storage
func FarmhashKeyStrategy(key string) string {
	return strconv.FormatUint(farm.Fingerprint64([]byte(key)), 10)
}

// NoKeyStrategy doesn't hash the key
func NoKeyStrategy(key string) string {
	return key
}

// Start starts the pool
func (v *Pool) Start() {

	v.initialize()

	for i, server := range v.Servers {
		func(_v *Pool, _server string, _i int) {
			_v.Lock()
			_v.pool = append(_v.pool, pools.NewResourcePool(func() (pools.Resource, error) {
				c, err := memcache.Connect(_server, _v.ConnectionTimeout)
				return VitessResource{c}, err
			}, _v.Capacity, _v.MaxCapacity, _v.IdleTimeout))
			_v.Unlock()

			conn, err := _v.GetPoolConnection(_i)
			if err != nil {
				log.Fatalf("Can't connect to memcached %d (%s): %s", _i, _server, err)
			}
			_v.ReturnConnection(_i, conn)
		}(v, server, i)
	}
}

func (v *Pool) initialize() {
	v.numServers = len(v.Servers)
	v.pool = []*pools.ResourcePool{}

	if v.ServerStrategy == nil {
		v.ServerStrategy = defaultServerStrategy
	}
	if v.HashKeyStrategy == nil {
		v.HashKeyStrategy = defaultHashKeyStrategy
	}
	if v.Capacity == 0 {
		v.Capacity = defaultCapacity
	}
	if v.MaxCapacity == 0 {
		v.MaxCapacity = defaultMaxCapacity
	}
	if v.IdleTimeout == 0 {
		v.IdleTimeout = defaultIdleTimeout
	}
	if v.ConnectionTimeout == 0 {
		v.ConnectionTimeout = defaultConnectionTimeout
	}
}

// GetConnection returns a connection from the sharding pool, based on the key
func (v *Pool) GetConnection(key string) (*VitessResource, int, error) {
	poolNum := v.ServerStrategy(key, v.numServers)

	connection, err := v.GetPoolConnection(poolNum)
	if err != nil {
		return nil, -1, err
	}

	return connection, poolNum, nil
}

// GetPoolConnection returns a connection from a specific pool number
func (v *Pool) GetPoolConnection(poolNum int) (*VitessResource, error) {
	ctx := context.Background()

	v.RLock()
	resource, err := v.pool[poolNum].Get(ctx)
	v.RUnlock()
	if err != nil {
		return nil, err
	}

	connection := resource.(VitessResource)

	return &connection, nil
}

// ReturnConnection returns a connection to the pool
func (v *Pool) ReturnConnection(poolNum int, resource *VitessResource) {
	if poolNum > v.numServers || poolNum < 0 {
		log.Fatalf("error: invalid server %d (of total %d)", poolNum, v.numServers)
	}

	v.RLock()
	v.pool[poolNum].Put(*resource)
	v.RUnlock()
}

// GetKeyMapping returns a mapping of server to a list of keys, useful for Gets()
func (v *Pool) GetKeyMapping(keys ...string) map[int][]string {
	mapping := make(map[int][]string)

	for i := 0; i < v.numServers; i++ {
		mapping[i] = []string{}
	}

	for _, key := range keys {
		poolNum := v.ServerStrategy(key, v.numServers)
		mapping[poolNum] = append(mapping[poolNum], v.HashKeyStrategy(key))
	}

	return mapping
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package provider

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net"
	"os"
	"strings"
	"time"

	etcdcv3 "github.com/coreos/etcd/clientv3"
	log "github.com/sirupsen/logrus"

	"github.com/kubernetes-incubator/external-dns/endpoint"
	"github.com/kubernetes-incubator/external-dns/plan"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

const (
	priority    = 10 // default priority when nothing is set
	etcdTimeout = 5 * time.Second

	coreDNSPrefix = "/skydns/"

	randomPrefixLabel = "prefix"
)

// coreDNSClient is an interface to work with CoreDNS service records in etcd
type coreDNSClient interface {
	GetServices(prefix string) ([]*Service, error)
	SaveService(value *Service) error
	DeleteService(key string) error
}

type coreDNSProvider struct {
	dryRun       bool
	domainFilter DomainFilter
	client       coreDNSClient
}

// Service represents CoreDNS etcd record
type Service struct {
	Host     string `json:"host,omitempty"`
	Port     int    `json:"port,omitempty"`
	Priority int    `json:"priority,omitempty"`
	Weight   int    `json:"weight,omitempty"`
	Text     string `json:"text,omitempty"`
	Mail     bool   `json:"mail,omitempty"` // Be an MX record. Priority becomes Preference.
	TTL      uint32 `json:"ttl,omitempty"`

	// When a SRV record with a "Host: IP-address" is added, we synthesize
	// a srv.Target domain name.  Normally we convert the full Key where
	// the record lives to a DNS name and use this as the srv.Target.  When
	// TargetStrip > 0 we strip the left most TargetStrip labels from the
	// DNS name.
	TargetStrip int `json:"targetstrip,omitempty"`

	// Group is used to group (or *not* to group) different services
	// together. Services with an identical Group are returned in the same
	// answer.
	Group string `json:"group,omitempty"`

	// Etcd key where we found this service and ignored from json un-/marshalling
	Key string `json:"-"`
}

type etcdClient struct {
	client *etcdcv3.Client
	ctx    context.Context
}

var _ coreDNSClient = etcdClient{}

// GetService return all Service records stored in etcd stored anywhere under the given key (recursively)
func (c etcdClient) GetServices(prefix string) ([]*Service, error) {
	ctx, cancel := context.WithTimeout(c.ctx, etcdTimeout)
	defer cancel()

	path := prefix
	r, err := c.client.Get(ctx, path, etcdcv3.WithPrefix())
	if err != nil {
		return nil, err
	}

	var svcs []*Service
	bx := make(map[Service]bool)
	for _, n := range r.Kvs {
		svc := new(Service)
		if err := json.Unmarshal(n.Value, svc); err != nil {
			return nil, fmt.Errorf("%s: %s", n.Key, err.Error())
		}
		b := Service{Host: svc.Host, Port: svc.Port, Priority: svc.Priority, Weight: svc.Weight, Text: svc.Text, Key: string(n.Key)}
		if _, ok := bx[b]; ok {
			// skip the service if already added to service list.
			// the same service might be found in multiple etcd nodes.
			continue
		}
		bx[b] = true

		svc.Key = string(n.Key)
		if svc.Priority == 0 {
			svc.Priority = priority
		}
		svcs = append(svcs, svc)
	}

	return svcs, nil
}

// SaveService persists service data into etcd
func (c etcdClient) SaveService(service *Service) error {
	ctx, cancel := context.WithTimeout(c.ctx, etcdTimeout)
	defer cancel()

	value, err := json.Marshal(&service)
	if err != nil {
		return err
	}
	_, err = c.client.Put(ctx, service.Key, string(value))
	if err != nil {
		return err
	}
	return nil
}

// DeleteService deletes service record from etcd
func (c etcdClient) DeleteService(key string) error {
	ctx, cancel := context.WithTimeout(c.ctx, etcdTimeout)
	defer cancel()

	_, err := c.client.Delete(ctx, key)
	return err
}

// loads TLS artifacts and builds tls.Clonfig object
func newTLSConfig(certPath, keyPath, caPath, serverName string, insecure bool) (*tls.Config, error) {
	if certPath != "" && keyPath == "" || certPath == "" && keyPath != "" {
		return nil, errors.New("either both cert and key or none must be provided")
	}
	var certificates []tls.Certificate
	if certPath != "" {
		cert, err := tls.LoadX509KeyPair(certPath, keyPath)
		if err != nil {
			return nil, fmt.Errorf("could not load TLS cert: %s", err)
		}
		certificates = append(certificates, cert)
	}
	roots, err := loadRoots(caPath)
	if err != nil {
		return nil, err
	}

	return &tls.Config{
		Certificates:       certificates,
		RootCAs:            roots,
		InsecureSkipVerify: insecure,
		ServerName:         serverName,
	}, nil
}

// loads CA cert
func loadRoots(caPath string) (*x509.CertPool, error) {
	if caPath == "" {
		return nil, nil
	}

	roots := x509.NewCertPool()
	pem, err := ioutil.ReadFile(caPath)
	if err != nil {
		return nil, fmt.Errorf("error reading %s: %s", caPath, err)
	}
	ok := roots.AppendCertsFromPEM(pem)
	if !ok {
		return nil, fmt.Errorf("could not read root certs: %s", err)
	}
	return roots, nil
}

// builds etcd client config depending on connection scheme and TLS parameters
func getETCDConfig() (*etcdcv3.Config, error) {
	etcdURLsStr := os.Getenv("ETCD_URLS")
	if etcdURLsStr == "" {
		etcdURLsStr = "http://localhost:2379"
	}
	etcdURLs := strings.Split(etcdURLsStr, ",")
	firstURL := strings.ToLower(etcdURLs[0])
	if strings.HasPrefix(firstURL, "http://") {
		return &etcdcv3.Config{Endpoints: etcdURLs}, nil
	} else if strings.HasPrefix(firstURL, "https://") {
		caFile := os.Getenv("ETCD_CA_FILE")
		certFile := os.Getenv("ETCD_CERT_FILE")
		keyFile := os.Getenv("ETCD_KEY_FILE")
		serverName := os.Getenv("ETCD_TLS_SERVER_NAME")
		isInsecureStr := strings.ToLower(os.Getenv("ETCD_TLS_INSECURE"))
		isInsecure := isInsecureStr == "true" || isInsecureStr == "yes" || isInsecureStr == "1"
		tlsConfig, err := newTLSConfig(certFile, keyFile, caFile, serverName, isInsecure)
		if err != nil {
			return nil, err
		}
		return &etcdcv3.Config{
			Endpoints: etcdURLs,
			TLS:       tlsConfig,
		}, nil
	} else {
		return nil, errors.New("etcd URLs must start with either http:// or https://")
	}
}

//newETCDClient is an etcd client constructor
func newETCDClient() (coreDNSClient, error) {
	cfg, err := getETCDConfig()
	if err != nil {
		return nil, err
	}
	c, err := etcdcv3.New(*cfg)
	if err != nil {
		return nil, err
	}
	return etcdClient{c, context.Background()}, nil
}

// NewCoreDNSProvider is a CoreDNS provider constructor
func NewCoreDNSProvider(domainFilter DomainFilter, dryRun bool) (Provider, error) {
	client, err := newETCDClient()
	if err != nil {
		return nil, err
	}
	return coreDNSProvider{
		client:       client,
		dryRun:       dryRun,
		domainFilter: domainFilter,
	}, nil
}

// Records returns all DNS records found in CoreDNS etcd backend. Depending on the record fields
// it may be mapped to one or two records of type A, CNAME, TXT, A+TXT, CNAME+TXT
func (p coreDNSProvider) Records() ([]*endpoint.Endpoint, error) {
	var result []*endpoint.Endpoint
	services, err := p.client.GetServices(coreDNSPrefix)
	if err != nil {
		return nil, err
	}
	for _, service := range services {
		domains := strings.Split(strings.TrimPrefix(service.Key, coreDNSPrefix), "/")
		reverse(domains)
		dnsName := strings.Join(domains[service.TargetStrip:], ".")
		if !p.domainFilter.Match(dnsName) {
			continue
		}
		prefix := strings.Join(domains[:service.TargetStrip], ".")
		if service.Host != "" {
			ep := endpoint.NewEndpointWithTTL(
				dnsName,
				guessRecordType(service.Host),
				endpoint.TTL(service.TTL),
				service.Host,
			)
			ep.Labels["originalText"] = service.Text
			ep.Labels[randomPrefixLabel] = prefix
			result = append(result, ep)
		}
		if service.Text != "" {
			ep := endpoint.NewEndpoint(
				dnsName,
				endpoint.RecordTypeTXT,
				service.Text,
			)
			ep.Labels[randomPrefixLabel] = prefix
			result = append(result, ep)
		}
	}
	return result, nil
}

// ApplyChanges stores changes back to etcd converting them to CoreDNS format and aggregating A/CNAME and TXT records
func (p coreDNSProvider) ApplyChanges(changes *plan.Changes) error {
	grouped := map[string][]*endpoint.Endpoint{}
	for _, ep := range changes.Create {
		grouped[ep.DNSName] = append(grouped[ep.DNSName], ep)
	}
	for i, ep := range changes.UpdateNew {
		ep.Labels[randomPrefixLabel] = changes.UpdateOld[i].Labels[randomPrefixLabel]
		grouped[ep.DNSName] = append(grouped[ep.DNSName], ep)
	}
	for dnsName, group := range grouped {
		if !p.domainFilter.Match(dnsName) {
			log.Debugf("Skipping record %s because it was filtered out by the specified --domain-filter", dnsName)
			continue
		}
		var services []Service
		for _, ep := range group {
			if ep.RecordType == endpoint.RecordTypeTXT {
				continue
			}
			prefix := ep.Labels[randomPrefixLabel]
			if prefix == "" {
				prefix = fmt.Sprintf("%08x", rand.Int31())
			}
			service := Service{
				Host:        ep.Targets[0],
				Text:        ep.Labels["originalText"],
				Key:         etcdKeyFor(prefix + "." + dnsName),
				TargetStrip: strings.Count(prefix, ".") + 1,
				TTL:         uint32(ep.RecordTTL),
			}
			services = append(services, service)
		}
		index := 0
		for _, ep := range group {
			if ep.RecordType != "TXT" {
				continue
			}
			if index >= len(services) {
				prefix := ep.Labels[randomPrefixLabel]
				if prefix == "" {
					prefix = fmt.Sprintf("%08x", rand.Int31())
				}
				services = append(services, Service{
					Key:         etcdKeyFor(prefix + "." + dnsName),
					TargetStrip: strings.Count(prefix, ".") + 1,
					TTL:         uint32(ep.RecordTTL),
				})
			}
			services[index].Text = ep.Targets[0]
			index++
		}

		for i := index; index > 0 && i < len(services); i++ {
			services[i].Text = ""
		}

		for _, service := range services {
			log.Infof("Add/set key %s to Host=%s, Text=%s, TTL=%d", service.Key, service.Host, service.Text, service.TTL)
			if !p.dryRun {
				err := p.client.SaveService(&service)
				if err != nil {
					return err
				}
			}
		}
	}

	for _, ep := range changes.Delete {
		dnsName := ep.DNSName
		if ep.Labels[randomPrefixLabel] != "" {
			dnsName = ep.Labels[randomPrefixLabel] + "." + dnsName
		}
		key := etcdKeyFor(dnsName)
		log.Infof("Delete key %s", key)
		if !p.dryRun {
			err := p.client.DeleteService(key)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func guessRecordType(target string) string {
	if net.ParseIP(target) != nil {
		return endpoint.RecordTypeA
	}
	return endpoint.RecordTypeCNAME
}

func etcdKeyFor(dnsName string) string {
	domains := strings.Split(dnsName, ".")
	reverse(domains)
	return coreDNSPrefix + strings.Join(domains, "/")
}

func reverse(slice []string) {
	for i := 0; i < len(slice)/2; i++ {
		j := len(slice) - i - 1
		slice[i], slice[j] = slice[j], slice[i]
	}
}
<commit_msg>2 issues: - coredns support more than 1 targets           - delete with prefix to make sure the record is cleaned<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package provider

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net"
	"os"
	"strings"
	"time"

	etcdcv3 "github.com/coreos/etcd/clientv3"
	log "github.com/sirupsen/logrus"

	"github.com/kubernetes-incubator/external-dns/endpoint"
	"github.com/kubernetes-incubator/external-dns/plan"
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

const (
	priority    = 10 // default priority when nothing is set
	etcdTimeout = 5 * time.Second

	coreDNSPrefix = "/skydns/"

	randomPrefixLabel = "prefix"
)

// coreDNSClient is an interface to work with CoreDNS service records in etcd
type coreDNSClient interface {
	GetServices(prefix string) ([]*Service, error)
	SaveService(value *Service) error
	DeleteService(key string) error
}

type coreDNSProvider struct {
	dryRun       bool
	domainFilter DomainFilter
	client       coreDNSClient
}

// Service represents CoreDNS etcd record
type Service struct {
	Host     string `json:"host,omitempty"`
	Port     int    `json:"port,omitempty"`
	Priority int    `json:"priority,omitempty"`
	Weight   int    `json:"weight,omitempty"`
	Text     string `json:"text,omitempty"`
	Mail     bool   `json:"mail,omitempty"` // Be an MX record. Priority becomes Preference.
	TTL      uint32 `json:"ttl,omitempty"`

	// When a SRV record with a "Host: IP-address" is added, we synthesize
	// a srv.Target domain name.  Normally we convert the full Key where
	// the record lives to a DNS name and use this as the srv.Target.  When
	// TargetStrip > 0 we strip the left most TargetStrip labels from the
	// DNS name.
	TargetStrip int `json:"targetstrip,omitempty"`

	// Group is used to group (or *not* to group) different services
	// together. Services with an identical Group are returned in the same
	// answer.
	Group string `json:"group,omitempty"`

	// Etcd key where we found this service and ignored from json un-/marshalling
	Key string `json:"-"`
}

type etcdClient struct {
	client *etcdcv3.Client
	ctx    context.Context
}

var _ coreDNSClient = etcdClient{}

// GetService return all Service records stored in etcd stored anywhere under the given key (recursively)
func (c etcdClient) GetServices(prefix string) ([]*Service, error) {
	ctx, cancel := context.WithTimeout(c.ctx, etcdTimeout)
	defer cancel()

	path := prefix
	r, err := c.client.Get(ctx, path, etcdcv3.WithPrefix())
	if err != nil {
		return nil, err
	}

	var svcs []*Service
	bx := make(map[Service]bool)
	for _, n := range r.Kvs {
		svc := new(Service)
		if err := json.Unmarshal(n.Value, svc); err != nil {
			return nil, fmt.Errorf("%s: %s", n.Key, err.Error())
		}
		b := Service{Host: svc.Host, Port: svc.Port, Priority: svc.Priority, Weight: svc.Weight, Text: svc.Text, Key: string(n.Key)}
		if _, ok := bx[b]; ok {
			// skip the service if already added to service list.
			// the same service might be found in multiple etcd nodes.
			continue
		}
		bx[b] = true

		svc.Key = string(n.Key)
		if svc.Priority == 0 {
			svc.Priority = priority
		}
		svcs = append(svcs, svc)
	}

	return svcs, nil
}

// SaveService persists service data into etcd
func (c etcdClient) SaveService(service *Service) error {
	ctx, cancel := context.WithTimeout(c.ctx, etcdTimeout)
	defer cancel()

	value, err := json.Marshal(&service)
	if err != nil {
		return err
	}
	_, err = c.client.Put(ctx, service.Key, string(value))
	if err != nil {
		return err
	}
	return nil
}

// DeleteService deletes service record from etcd
func (c etcdClient) DeleteService(key string) error {
	ctx, cancel := context.WithTimeout(c.ctx, etcdTimeout)
	defer cancel()

	_, err := c.client.Delete(ctx, key, etcdcv3.WithPrefix())
	return err
}

// loads TLS artifacts and builds tls.Clonfig object
func newTLSConfig(certPath, keyPath, caPath, serverName string, insecure bool) (*tls.Config, error) {
	if certPath != "" && keyPath == "" || certPath == "" && keyPath != "" {
		return nil, errors.New("either both cert and key or none must be provided")
	}
	var certificates []tls.Certificate
	if certPath != "" {
		cert, err := tls.LoadX509KeyPair(certPath, keyPath)
		if err != nil {
			return nil, fmt.Errorf("could not load TLS cert: %s", err)
		}
		certificates = append(certificates, cert)
	}
	roots, err := loadRoots(caPath)
	if err != nil {
		return nil, err
	}

	return &tls.Config{
		Certificates:       certificates,
		RootCAs:            roots,
		InsecureSkipVerify: insecure,
		ServerName:         serverName,
	}, nil
}

// loads CA cert
func loadRoots(caPath string) (*x509.CertPool, error) {
	if caPath == "" {
		return nil, nil
	}

	roots := x509.NewCertPool()
	pem, err := ioutil.ReadFile(caPath)
	if err != nil {
		return nil, fmt.Errorf("error reading %s: %s", caPath, err)
	}
	ok := roots.AppendCertsFromPEM(pem)
	if !ok {
		return nil, fmt.Errorf("could not read root certs: %s", err)
	}
	return roots, nil
}

// builds etcd client config depending on connection scheme and TLS parameters
func getETCDConfig() (*etcdcv3.Config, error) {
	etcdURLsStr := os.Getenv("ETCD_URLS")
	if etcdURLsStr == "" {
		etcdURLsStr = "http://localhost:2379"
	}
	etcdURLs := strings.Split(etcdURLsStr, ",")
	firstURL := strings.ToLower(etcdURLs[0])
	if strings.HasPrefix(firstURL, "http://") {
		return &etcdcv3.Config{Endpoints: etcdURLs}, nil
	} else if strings.HasPrefix(firstURL, "https://") {
		caFile := os.Getenv("ETCD_CA_FILE")
		certFile := os.Getenv("ETCD_CERT_FILE")
		keyFile := os.Getenv("ETCD_KEY_FILE")
		serverName := os.Getenv("ETCD_TLS_SERVER_NAME")
		isInsecureStr := strings.ToLower(os.Getenv("ETCD_TLS_INSECURE"))
		isInsecure := isInsecureStr == "true" || isInsecureStr == "yes" || isInsecureStr == "1"
		tlsConfig, err := newTLSConfig(certFile, keyFile, caFile, serverName, isInsecure)
		if err != nil {
			return nil, err
		}
		return &etcdcv3.Config{
			Endpoints: etcdURLs,
			TLS:       tlsConfig,
		}, nil
	} else {
		return nil, errors.New("etcd URLs must start with either http:// or https://")
	}
}

//newETCDClient is an etcd client constructor
func newETCDClient() (coreDNSClient, error) {
	cfg, err := getETCDConfig()
	if err != nil {
		return nil, err
	}
	c, err := etcdcv3.New(*cfg)
	if err != nil {
		return nil, err
	}
	return etcdClient{c, context.Background()}, nil
}

// NewCoreDNSProvider is a CoreDNS provider constructor
func NewCoreDNSProvider(domainFilter DomainFilter, dryRun bool) (Provider, error) {
	client, err := newETCDClient()
	if err != nil {
		return nil, err
	}
	return coreDNSProvider{
		client:       client,
		dryRun:       dryRun,
		domainFilter: domainFilter,
	}, nil
}

// Records returns all DNS records found in CoreDNS etcd backend. Depending on the record fields
// it may be mapped to one or two records of type A, CNAME, TXT, A+TXT, CNAME+TXT
func (p coreDNSProvider) Records() ([]*endpoint.Endpoint, error) {
	var result []*endpoint.Endpoint
	services, err := p.client.GetServices(coreDNSPrefix)
	if err != nil {
		return nil, err
	}
	for _, service := range services {
		domains := strings.Split(strings.TrimPrefix(service.Key, coreDNSPrefix), "/")
		reverse(domains)
		dnsName := strings.Join(domains[service.TargetStrip:], ".")
		if !p.domainFilter.Match(dnsName) {
			continue
		}
		prefix := strings.Join(domains[:service.TargetStrip], ".")
		if service.Host != "" {
			ep := endpoint.NewEndpointWithTTL(
				dnsName,
				guessRecordType(service.Host),
				endpoint.TTL(service.TTL),
				service.Host,
			)
			ep.Labels["originalText"] = service.Text
			ep.Labels[randomPrefixLabel] = prefix
			result = append(result, ep)
		}
		if service.Text != "" {
			ep := endpoint.NewEndpoint(
				dnsName,
				endpoint.RecordTypeTXT,
				service.Text,
			)
			ep.Labels[randomPrefixLabel] = prefix
			result = append(result, ep)
		}
	}
	return result, nil
}

// ApplyChanges stores changes back to etcd converting them to CoreDNS format and aggregating A/CNAME and TXT records
func (p coreDNSProvider) ApplyChanges(changes *plan.Changes) error {
	grouped := map[string][]*endpoint.Endpoint{}
	for _, ep := range changes.Create {
		grouped[ep.DNSName] = append(grouped[ep.DNSName], ep)
	}
	for i, ep := range changes.UpdateNew {
		ep.Labels[randomPrefixLabel] = changes.UpdateOld[i].Labels[randomPrefixLabel]
		grouped[ep.DNSName] = append(grouped[ep.DNSName], ep)
	}
	for dnsName, group := range grouped {
		if !p.domainFilter.Match(dnsName) {
			log.Debugf("Skipping record %s because it was filtered out by the specified --domain-filter", dnsName)
			continue
		}
		var services []Service
		for _, ep := range group {
			if ep.RecordType == endpoint.RecordTypeTXT {
				continue
			}

			for _, target := range ep.Targets {
				prefix := ep.Labels[randomPrefixLabel]
				if prefix == "" {
					prefix = fmt.Sprintf("%08x", rand.Int31())
				}

				service := Service{
					Host:        target,
					Text:        ep.Labels["originalText"],
					Key:         etcdKeyFor(prefix + "." + dnsName),
					TargetStrip: strings.Count(prefix, ".") + 1,
					TTL:         uint32(ep.RecordTTL),
				}
				services = append(services, service)
			}
		}
		index := 0
		for _, ep := range group {
			if ep.RecordType != endpoint.RecordTypeTXT {
				continue
			}
			if index >= len(services) {
				prefix := ep.Labels[randomPrefixLabel]
				if prefix == "" {
					prefix = fmt.Sprintf("%08x", rand.Int31())
				}
				services = append(services, Service{
					Key:         etcdKeyFor(prefix + "." + dnsName),
					TargetStrip: strings.Count(prefix, ".") + 1,
					TTL:         uint32(ep.RecordTTL),
				})
			}
			services[index].Text = ep.Targets[0]
			index++
		}

		for i := index; index > 0 && i < len(services); i++ {
			services[i].Text = ""
		}

		for _, service := range services {
			log.Infof("Add/set key %s to Host=%s, Text=%s, TTL=%d", service.Key, service.Host, service.Text, service.TTL)
			if !p.dryRun {
				err := p.client.SaveService(&service)
				if err != nil {
					return err
				}
			}
		}
	}

	for _, ep := range changes.Delete {
		dnsName := ep.DNSName
		if ep.Labels[randomPrefixLabel] != "" {
			dnsName = ep.Labels[randomPrefixLabel] + "." + dnsName
		}
		key := etcdKeyFor(dnsName)
		log.Infof("Delete key %s", key)
		if !p.dryRun {
			err := p.client.DeleteService(key)
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func guessRecordType(target string) string {
	if net.ParseIP(target) != nil {
		return endpoint.RecordTypeA
	}
	return endpoint.RecordTypeCNAME
}

func etcdKeyFor(dnsName string) string {
	domains := strings.Split(dnsName, ".")
	reverse(domains)
	return coreDNSPrefix + strings.Join(domains, "/")
}

func reverse(slice []string) {
	for i := 0; i < len(slice)/2; i++ {
		j := len(slice) - i - 1
		slice[i], slice[j] = slice[j], slice[i]
	}
}
<|endoftext|>


<commit_before>package glue

import (
	"errors"
	"fmt"
	"sync"

	"github.com/segmentio/glue/log"
	"github.com/segmentio/glue/provider"
	"github.com/segmentio/glue/writer"
	"golang.org/x/tools/go/loader"
)

// A Walker walks along supplied directions, visits server RPC code, and
// generates RPC client code along the way with the help of others.
type Walker struct {
	// Provider answers RPC-implementation-specific (e.g. stl, gorilla, etc.) questions.
	Provider provider.Provider
	Writer   writer.Writer
}

// Directions tell the Walker where to walk and what to pay attention to along the way.
type Directions struct {
	// Path determines the source code path to walk along. It's required
	Path string
	// Name is the name of the RPC declaration (e.g. `type MathService struct{}`).
	Name string
	// Service is the name of the RPC service. (e.g. `Math` in `Math.Sum`)
	Service string
}

// Walk is the logical entrypoint for Glue. It walks the source code and asks
func (w *Walker) Walk(directions Directions) error {
	var conf loader.Config
	conf.Import(directions.Path)

	prgm, err := conf.Load()
	if err != nil {
		log.Printf("failed to parse Go code: %s", err.Error())
		return err
	}

	var wg sync.WaitGroup
	for _, pkg := range prgm.InitialPackages() {
		wg.Add(1)
		go func(p *loader.PackageInfo) {
			defer wg.Done()
			w.walkPackage(p, directions.Name, directions.Service)
		}(pkg)
	}

	wg.Wait()
	return nil
}

func (w *Walker) walkPackage(pkg *loader.PackageInfo, decl, service string) error {
	visitor := NewVisitor(VisitorConfig{
		Pkg:         pkg,
		Provider:    w.Provider,
		Declaration: decl,
	})
	funcsByRecv := visitor.Go()

	if len(funcsByRecv) == 0 {
		log.Print("could not find RPC declaration")
		return errors.New("not found")
	}

	for _, funcs := range funcsByRecv {
		generator := Generator{
			Provider: w.Provider,
		}

		ident := fmt.Sprintf("%sClient", service)
		src, err := generator.Generate(GenerateInput{
			PackageName: "client",
			Service:     service,
			Funcs:       funcs,
		})
		if err != nil {
			return err
		}

		fname := fmt.Sprintf("generated_%s.go", ident)
		if err := w.Writer.Write(fname, src); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>log: print when we generate something to make glue more visible (#18)<commit_after>package glue

import (
	"errors"
	"fmt"
	"sync"

	"github.com/segmentio/glue/log"
	"github.com/segmentio/glue/provider"
	"github.com/segmentio/glue/writer"
	"golang.org/x/tools/go/loader"
)

// A Walker walks along supplied directions, visits server RPC code, and
// generates RPC client code along the way with the help of others.
type Walker struct {
	// Provider answers RPC-implementation-specific (e.g. stl, gorilla, etc.) questions.
	Provider provider.Provider
	Writer   writer.Writer
}

// Directions tell the Walker where to walk and what to pay attention to along the way.
type Directions struct {
	// Path determines the source code path to walk along. It's required
	Path string
	// Name is the name of the RPC declaration (e.g. `type MathService struct{}`).
	Name string
	// Service is the name of the RPC service. (e.g. `Math` in `Math.Sum`)
	Service string
}

// Walk is the logical entrypoint for Glue. It walks the source code and asks
func (w *Walker) Walk(directions Directions) error {
	var conf loader.Config
	conf.Import(directions.Path)

	prgm, err := conf.Load()
	if err != nil {
		log.Printf("failed to parse Go code: %s", err.Error())
		return err
	}

	var wg sync.WaitGroup
	for _, pkg := range prgm.InitialPackages() {
		wg.Add(1)
		go func(p *loader.PackageInfo) {
			defer wg.Done()
			w.walkPackage(p, directions.Name, directions.Service)
		}(pkg)
	}

	wg.Wait()
	return nil
}

func (w *Walker) walkPackage(pkg *loader.PackageInfo, decl, service string) error {
	visitor := NewVisitor(VisitorConfig{
		Pkg:         pkg,
		Provider:    w.Provider,
		Declaration: decl,
	})
	funcsByRecv := visitor.Go()

	if len(funcsByRecv) == 0 {
		log.Print("could not find RPC declaration")
		return errors.New("not found")
	}

	for _, funcs := range funcsByRecv {
		generator := Generator{
			Provider: w.Provider,
		}

		ident := fmt.Sprintf("%sClient", service)
		src, err := generator.Generate(GenerateInput{
			PackageName: "client",
			Service:     service,
			Funcs:       funcs,
		})
		if err != nil {
			return err
		}

		fname := fmt.Sprintf("generated_%s.go", ident)
		if err := w.Writer.Write(fname, src); err != nil {
			return err
		}

		log.Printf("glue: generated %s", ident)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"go/build"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

func getVendorSubmodules() (map[string]string, error) {
	output, err := execute(
		exec.Command("git", "submodule", "status"),
	)
	if err != nil {
		return nil, err
	}

	vendors := map[string]string{}

	lines := strings.Split(output, "\n")
	for _, line := range lines {
		if line == "" {
			continue
		}

		parts := strings.Split(strings.TrimLeft(line, " -"), " ")
		if len(parts) >= 2 {
			path := parts[1]
			commit := parts[0]
			if strings.HasPrefix(path, "vendor/") {
				path = strings.TrimPrefix(path, "vendor/")
				vendors[path] = commit
			}
		}
	}

	return vendors, nil
}

func addVendorSubmodule(importpath string) error {
	var (
		target   = "vendor/" + importpath
		prefixes = []string{
			"https://",
			"git+ssh://",
			"git://",
		}

		errs []string
	)

	for _, prefix := range prefixes {
		var url string
		if prefix == "https://" {
			var err error
			url, err = getHttpsURLForImportPath(importpath)
			if err != nil {
				errs = append(errs, err.Error())
				continue
			}
		} else {
			url = prefix + importpath
		}

		_, err := execute(
			exec.Command("git", "submodule", "add", "-f", url, target),
		)
		if err == nil {
			return nil
		}

		errs = append(errs, err.Error())
	}

	return errors.New(strings.Join(errs, "\n"))
}

// NOTE: This list is copied from
// https://github.com/golang/go/blob/10538a8f9e2e718a47633ac5a6e90415a2c3f5f1/src/cmd/go/vcs.go#L821-L861
var wellKnownSites = []string{
	"github.com/",
	"bitbucket.org/",
	"hub.jazz.net/git/",
	"git.apache.org/",
	"git.openstack.org/",
}

func getHttpsURLForImportPath(importpath string) (url string, err error) {
	url = "https://" + importpath
	for _, site := range wellKnownSites {
		if strings.HasPrefix(importpath, site) {
			return
		}
	}

	// NOTE: Parse <meta name="go-import" content="import-prefix vcs repo-root">
	// For detail, see the output of "go help importpath"
	var doc *goquery.Document
	doc, err = goquery.NewDocument(url)
	if err != nil {
		return
	}
	doc.Find("meta[name=go-import]").Each(func(i int, s *goquery.Selection) {
		if err != nil {
			return
		}
		content, exists := s.Attr("content")
		if !exists {
			err = fmt.Errorf(`"content" attribute not found in meta name="go-import" at %s`, url)
			return
		}
		terms := strings.Fields(content)
		if len(terms) != 3 {
			err = fmt.Errorf(`invalid formatted "content" attribute in meta name="go-import" at %s`, url)
			return
		}
		prefix := terms[0]
		vcs := terms[1]
		repoRoot := terms[2]
		if strings.HasPrefix(importpath, prefix) && vcs == "git" {
			url = repoRoot
		}
	})
	return
}

func removeVendorSubmodule(importpath string) error {
	vendor := "vendor/" + importpath

	_, err := execute(
		exec.Command("git", "submodule", "deinit", "-f", vendor),
	)
	if err != nil {
		return fmt.Errorf(
			"can't deinit submodule: %s", err,
		)
	}

	_, err = execute(
		exec.Command("git", "rm", "--force", vendor),
	)
	if err != nil {
		return fmt.Errorf(
			"can't remove submodule directory: %s", err,
		)
	}

	_, err = execute(
		exec.Command("rm", "-r", filepath.Join(".git", "modules", vendor)),
	)
	if err != nil {
		return fmt.Errorf(
			"can't remove submodule directory in .git/modules: %s", err,
		)
	}

	return nil
}

func updateVendorSubmodule(importpath string) error {
	cwd, err := os.Getwd()
	if err != nil {
		return err
	}

	cmd := exec.Command("git", "pull", "origin", "master")
	cmd.Dir = filepath.Join(cwd, "vendor", importpath)

	_, err = execute(cmd)

	return err
}

func getRootImportpath(importpath string) (string, error) {
	pkg, err := build.Import(importpath, "", build.IgnoreVendor)
	if err != nil {
		return "", err
	}

	cmd := exec.Command("git", "rev-parse", "--show-toplevel")
	cmd.Dir = filepath.Join(pkg.SrcRoot, importpath)

	rootdir, err := execute(cmd)
	if err != nil {
		return "", err
	}

	return strings.Trim(
		strings.TrimSpace(strings.TrimPrefix(rootdir, pkg.SrcRoot)),
		"/",
	), nil
}
<commit_msg>Do not use naked return<commit_after>package main

import (
	"errors"
	"fmt"
	"go/build"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/PuerkitoBio/goquery"
)

func getVendorSubmodules() (map[string]string, error) {
	output, err := execute(
		exec.Command("git", "submodule", "status"),
	)
	if err != nil {
		return nil, err
	}

	vendors := map[string]string{}

	lines := strings.Split(output, "\n")
	for _, line := range lines {
		if line == "" {
			continue
		}

		parts := strings.Split(strings.TrimLeft(line, " -"), " ")
		if len(parts) >= 2 {
			path := parts[1]
			commit := parts[0]
			if strings.HasPrefix(path, "vendor/") {
				path = strings.TrimPrefix(path, "vendor/")
				vendors[path] = commit
			}
		}
	}

	return vendors, nil
}

func addVendorSubmodule(importpath string) error {
	var (
		target   = "vendor/" + importpath
		prefixes = []string{
			"https://",
			"git+ssh://",
			"git://",
		}

		errs []string
	)

	for _, prefix := range prefixes {
		var url string
		if prefix == "https://" {
			var err error
			url, err = getHttpsURLForImportPath(importpath)
			if err != nil {
				errs = append(errs, err.Error())
				continue
			}
		} else {
			url = prefix + importpath
		}

		_, err := execute(
			exec.Command("git", "submodule", "add", "-f", url, target),
		)
		if err == nil {
			return nil
		}

		errs = append(errs, err.Error())
	}

	return errors.New(strings.Join(errs, "\n"))
}

// NOTE: This list is copied from
// https://github.com/golang/go/blob/10538a8f9e2e718a47633ac5a6e90415a2c3f5f1/src/cmd/go/vcs.go#L821-L861
var wellKnownSites = []string{
	"github.com/",
	"bitbucket.org/",
	"hub.jazz.net/git/",
	"git.apache.org/",
	"git.openstack.org/",
}

func getHttpsURLForImportPath(importpath string) (url string, err error) {
	url = "https://" + importpath
	for _, site := range wellKnownSites {
		if strings.HasPrefix(importpath, site) {
			return url, nil
		}
	}

	// NOTE: Parse <meta name="go-import" content="import-prefix vcs repo-root">
	// For detail, see the output of "go help importpath"
	var doc *goquery.Document
	doc, err = goquery.NewDocument(url)
	if err != nil {
		return
	}
	doc.Find("meta[name=go-import]").Each(func(i int, s *goquery.Selection) {
		if err != nil {
			return
		}
		content, exists := s.Attr("content")
		if !exists {
			err = fmt.Errorf(`"content" attribute not found in meta name="go-import" at %s`, url)
			return
		}
		terms := strings.Fields(content)
		if len(terms) != 3 {
			err = fmt.Errorf(`invalid formatted "content" attribute in meta name="go-import" at %s`, url)
			return
		}
		prefix := terms[0]
		vcs := terms[1]
		repoRoot := terms[2]
		if strings.HasPrefix(importpath, prefix) && vcs == "git" {
			url = repoRoot
		}
	})
	if err != nil {
		return "", err
	}
	return url, nil
}

func removeVendorSubmodule(importpath string) error {
	vendor := "vendor/" + importpath

	_, err := execute(
		exec.Command("git", "submodule", "deinit", "-f", vendor),
	)
	if err != nil {
		return fmt.Errorf(
			"can't deinit submodule: %s", err,
		)
	}

	_, err = execute(
		exec.Command("git", "rm", "--force", vendor),
	)
	if err != nil {
		return fmt.Errorf(
			"can't remove submodule directory: %s", err,
		)
	}

	_, err = execute(
		exec.Command("rm", "-r", filepath.Join(".git", "modules", vendor)),
	)
	if err != nil {
		return fmt.Errorf(
			"can't remove submodule directory in .git/modules: %s", err,
		)
	}

	return nil
}

func updateVendorSubmodule(importpath string) error {
	cwd, err := os.Getwd()
	if err != nil {
		return err
	}

	cmd := exec.Command("git", "pull", "origin", "master")
	cmd.Dir = filepath.Join(cwd, "vendor", importpath)

	_, err = execute(cmd)

	return err
}

func getRootImportpath(importpath string) (string, error) {
	pkg, err := build.Import(importpath, "", build.IgnoreVendor)
	if err != nil {
		return "", err
	}

	cmd := exec.Command("git", "rev-parse", "--show-toplevel")
	cmd.Dir = filepath.Join(pkg.SrcRoot, importpath)

	rootdir, err := execute(cmd)
	if err != nil {
		return "", err
	}

	return strings.Trim(
		strings.TrimSpace(strings.TrimPrefix(rootdir, pkg.SrcRoot)),
		"/",
	), nil
}
<|endoftext|>


<commit_before>package task

import (
	"errors"
	"fmt"
)

var (
	// ErrTaskfileAlreadyExists is returned on creating a Taskfile if one already exists
	ErrTaskfileAlreadyExists = errors.New("task: A Taskfile already exists")
)

type taskFileNotFound struct {
	taskFile string
}

func (err taskFileNotFound) Error() string {
	return fmt.Sprintf(`task: No task file found (is it named "%s"?). Use "task --init" to create a new one`, err.taskFile)
}

type taskNotFoundError struct {
	taskName string
}

func (err *taskNotFoundError) Error() string {
	return fmt.Sprintf(`task: Task "%s" not found`, err.taskName)
}

type taskRunError struct {
	taskName string
	err      error
}

func (err *taskRunError) Error() string {
	return fmt.Sprintf(`task: Failed to run task "%s": %v`, err.taskName, err.err)
}

type cyclicDepError struct {
	taskName string
}

func (err *cyclicDepError) Error() string {
	return fmt.Sprintf(`task: Cyclic dependency of task "%s" detected`, err.taskName)
}

type cantWatchNoSourcesError struct {
	taskName string
}

func (err *cantWatchNoSourcesError) Error() string {
	return fmt.Sprintf(`task: Can't watch task "%s" because it has no specified sources`, err.taskName)
}

// MaximumTaskCallExceededError is returned when a task is called too
// many times. In this case you probably have a cyclic dependendy or
// infinite loop
type MaximumTaskCallExceededError struct {
	task string
}

func (e *MaximumTaskCallExceededError) Error() string {
	return fmt.Sprintf(
		`task: maximum task call exceeded (%d) for task "%s": probably an cyclic dep or infinite loop`,
		MaximumTaskCall,
		e.task,
	)
}
<commit_msg>Delete unused errors<commit_after>package task

import (
	"errors"
	"fmt"
)

var (
	// ErrTaskfileAlreadyExists is returned on creating a Taskfile if one already exists
	ErrTaskfileAlreadyExists = errors.New("task: A Taskfile already exists")
)

type taskFileNotFound struct {
	taskFile string
}

func (err taskFileNotFound) Error() string {
	return fmt.Sprintf(`task: No task file found (is it named "%s"?). Use "task --init" to create a new one`, err.taskFile)
}

type taskNotFoundError struct {
	taskName string
}

func (err *taskNotFoundError) Error() string {
	return fmt.Sprintf(`task: Task "%s" not found`, err.taskName)
}

type taskRunError struct {
	taskName string
	err      error
}

func (err *taskRunError) Error() string {
	return fmt.Sprintf(`task: Failed to run task "%s": %v`, err.taskName, err.err)
}

// MaximumTaskCallExceededError is returned when a task is called too
// many times. In this case you probably have a cyclic dependendy or
// infinite loop
type MaximumTaskCallExceededError struct {
	task string
}

func (e *MaximumTaskCallExceededError) Error() string {
	return fmt.Sprintf(
		`task: maximum task call exceeded (%d) for task "%s": probably an cyclic dep or infinite loop`,
		MaximumTaskCall,
		e.task,
	)
}
<|endoftext|>