<commit_before>package main

import (
	"bufio"
	"bytes"
	"code.google.com/p/go.net/html"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
)

type (
	ResultType  interface{}
	TextFunc    func(n *html.Node) ResultType
	ElementFunc func(n *html.Node, results []ResultType) ResultType
)

func htmlwalk(n *html.Node, textf TextFunc, elementf ElementFunc) ResultType {
	if n.Type == html.TextNode {
		return textf(n)
	}
	results := make([]ResultType, 0)
	child := n.FirstChild
	for {
		if child == nil {
			break
		}
		result := htmlwalk(child, textf, elementf)
		if multiresults, ok := result.([]ResultType); ok {
			results = append(results, multiresults...)
		} else {
			results = append(results, result)
		}
		child = child.NextSibling
	}
	return elementf(n, results)
}

func stringValue(v interface{}) string {
	if s, ok := v.(string); ok {
		return s
	}
	return ""
}

func untagText(n *html.Node) ResultType {
	return n.Data
}

func untagElement(n *html.Node, results []ResultType) ResultType {
	var b bytes.Buffer
	for _, v := range results {
		s := stringValue(v)
		_, err := b.WriteString(s)
		if err != nil {
			panic(err)
		}
	}
	return b.String()
}

type TagType int16

const (
	Maybe TagType = iota
	Keep
)

type tag struct {
	Type  TagType
	Value string
}

func tagValue(v interface{}) (*tag, []ResultType) {
	if t, ok := v.(*tag); ok {
		return t, nil
	} else if t, ok := v.([]ResultType); ok {
		return nil, t
	}
	panic(fmt.Sprintf("Unkown value %v", v))
}

func promoteText(n *html.Node) ResultType {
	return &tag{Maybe, n.Data}
}

func promoteElementIf(tagname string) func(n *html.Node, results []ResultType) ResultType {
	return func(n *html.Node, results []ResultType) ResultType {
		if n.Data == tagname {
			var b bytes.Buffer
			for _, v := range results {
				t, _ := tagValue(v)
				_, err := b.WriteString(t.Value)
				if err != nil {
					panic(err)
				}
			}
			return &tag{Keep, b.String()}
		}
		return results
	}
}

func extractPromoted(n *html.Node) string {
	_, results := tagValue(htmlwalk(n, promoteText, promoteElementIf("h1")))

	if results != nil {
		var b bytes.Buffer
		for _, r := range results {
			tag, _ := tagValue(r)
			if tag.Type == Keep {
				_, err := b.WriteString(tag.Value + " ")
				if err != nil {
					panic(err)
				}
			}
		}
		return b.String()
	}
	return ""
}

var url = flag.Bool("u", false, "Use -u for url")

func main() {
	flag.Parse()

	if len(os.Args) < 2 {
		fmt.Printf("Usage %s [-u] NAME\n", os.Args[0])
		os.Exit(0)
	}

	var in io.Reader
	if *url {
		resp, err := http.Get(os.Args[2])
		if err != nil {
			panic(err)
		}
		defer resp.Body.Close()
		in = resp.Body
	} else {
		fi, err := os.Open(os.Args[1])
		if err != nil {
			panic(err)
		}
		defer fi.Close()
		in = bufio.NewReader(fi)
	}

	n, err := html.Parse(in)
	if err != nil {
		panic(err)
	}

	fmt.Println(stringValue(htmlwalk(n, untagText, untagElement)))
	fmt.Println(extractPromoted(n))
}
<commit_msg>1.7: Html: interface{} to ResultType<commit_after>package main

import (
	"bufio"
	"bytes"
	"code.google.com/p/go.net/html"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
)

type (
	ResultType  interface{}
	TextFunc    func(n *html.Node) ResultType
	ElementFunc func(n *html.Node, results []ResultType) ResultType
)

func htmlwalk(n *html.Node, textf TextFunc, elementf ElementFunc) ResultType {
	if n.Type == html.TextNode {
		return textf(n)
	}
	results := make([]ResultType, 0)
	child := n.FirstChild
	for {
		if child == nil {
			break
		}
		result := htmlwalk(child, textf, elementf)
		if multiresults, ok := result.([]ResultType); ok {
			results = append(results, multiresults...)
		} else {
			results = append(results, result)
		}
		child = child.NextSibling
	}
	return elementf(n, results)
}

func stringValue(v interface{}) string {
	if s, ok := v.(string); ok {
		return s
	}
	return ""
}

func untagText(n *html.Node) ResultType {
	return n.Data
}

func untagElement(n *html.Node, results []ResultType) ResultType {
	var b bytes.Buffer
	for _, v := range results {
		s := stringValue(v)
		_, err := b.WriteString(s)
		if err != nil {
			panic(err)
		}
	}
	return b.String()
}

type TagType int16

const (
	Maybe TagType = iota
	Keep
)

type tag struct {
	Type  TagType
	Value string
}

func tagValue(v ResultType) (*tag, []ResultType) {
	if t, ok := v.(*tag); ok {
		return t, nil
	} else if t, ok := v.([]ResultType); ok {
		return nil, t
	}
	panic(fmt.Sprintf("Unkown value %v", v))
}

func promoteText(n *html.Node) ResultType {
	return &tag{Maybe, n.Data}
}

func promoteElementIf(tagname string) func(n *html.Node, results []ResultType) ResultType {
	return func(n *html.Node, results []ResultType) ResultType {
		if n.Data == tagname {
			var b bytes.Buffer
			for _, v := range results {
				t, _ := tagValue(v)
				_, err := b.WriteString(t.Value)
				if err != nil {
					panic(err)
				}
			}
			return &tag{Keep, b.String()}
		}
		return results
	}
}

func extractPromoted(n *html.Node) string {
	_, results := tagValue(htmlwalk(n, promoteText, promoteElementIf("h1")))

	if results != nil {
		var b bytes.Buffer
		for _, r := range results {
			tag, _ := tagValue(r)
			if tag.Type == Keep {
				_, err := b.WriteString(tag.Value + " ")
				if err != nil {
					panic(err)
				}
			}
		}
		return b.String()
	}
	return ""
}

var url = flag.Bool("u", false, "Use -u for url")

func main() {
	flag.Parse()

	if len(os.Args) < 2 {
		fmt.Printf("Usage %s [-u] NAME\n", os.Args[0])
		os.Exit(0)
	}

	var in io.Reader
	if *url {
		resp, err := http.Get(os.Args[2])
		if err != nil {
			panic(err)
		}
		defer resp.Body.Close()
		in = resp.Body
	} else {
		fi, err := os.Open(os.Args[1])
		if err != nil {
			panic(err)
		}
		defer fi.Close()
		in = bufio.NewReader(fi)
	}

	n, err := html.Parse(in)
	if err != nil {
		panic(err)
	}

	fmt.Println(stringValue(htmlwalk(n, untagText, untagElement)))
	fmt.Println(extractPromoted(n))
}
<|endoftext|>


<commit_before>package cache

import (
	"fmt"
	"log"
	"sync"

	humanize "github.com/dustin/go-humanize"
)

type Key = string

type Cache struct {
	mu     sync.Mutex
	filled int64
	Policy Policy
	Items  map[Key]ItemMeta
}

type ItemMeta struct {
	Size     int64
	CanEvict bool
	Usage
}

type Item struct {
	Key
	ItemMeta
}

func (me *Cache) Remove(k Key) {
	me.mu.Lock()
	i := me.Items[k]
	me.filled -= i.Size
	delete(me.Items, k)
	me.Policy.Forget(k)
	me.mu.Unlock()
}

func (me *Cache) Update(i Item) {
	me.mu.Lock()
	m := me.Items[i.Key]
	me.filled -= m.Size
	me.filled += i.Size
	if me.Items == nil {
		me.Items = make(map[Key]ItemMeta)
	}
	me.Items[i.Key] = i.ItemMeta
	if i.CanEvict {
		me.Policy.Update(i.Key, i.Usage)
	} else {
		me.Policy.Forget(i.Key)
	}
	me.mu.Unlock()
}

func (me *Cache) logState() {
	log.Print(me)
}

func (me *Cache) String() string {
	me.mu.Lock()
	defer me.mu.Unlock()
	return fmt.Sprintf(
		"%p: %d items, %v bytes used, lru: %s",
		me, len(me.Items), humanize.Bytes(uint64(me.filled)),
		func() string {
			k, ok := me.Policy.Candidate()
			if ok {
				i := me.Items[k]
				return fmt.Sprintf(
					"%q (%v: %v)",
					k, humanize.Bytes(uint64(i.Size)), i.Usage)
			}
			return "none"
		}(),
	)
}

func (me *Cache) Used() int64 {
	return me.filled
}

func (me *Cache) NumItems() int {
	return len(me.Items)
}

func (me *Cache) Clear() {
	for k := range me.Items {
		delete(me.Items, k)
		me.Policy.Forget(k)
	}
	me.Items = nil
	me.filled = 0
}

func (me *Cache) Filled() int64 {
	me.mu.Lock()
	defer me.mu.Unlock()
	return me.filled
}
<commit_msg>Add cache.Cache.Candidate<commit_after>package cache

import (
	"fmt"
	"log"
	"sync"

	humanize "github.com/dustin/go-humanize"
)

type Key = string

type Cache struct {
	mu     sync.Mutex
	filled int64
	Policy Policy
	Items  map[Key]ItemMeta
}

type ItemMeta struct {
	Size     int64
	CanEvict bool
	Usage
}

type Item struct {
	Key
	ItemMeta
}

func (me *Cache) Remove(k Key) {
	me.mu.Lock()
	i := me.Items[k]
	me.filled -= i.Size
	delete(me.Items, k)
	me.Policy.Forget(k)
	me.mu.Unlock()
}

func (me *Cache) Update(i Item) {
	me.mu.Lock()
	m := me.Items[i.Key]
	me.filled -= m.Size
	me.filled += i.Size
	if me.Items == nil {
		me.Items = make(map[Key]ItemMeta)
	}
	me.Items[i.Key] = i.ItemMeta
	if i.CanEvict {
		me.Policy.Update(i.Key, i.Usage)
	} else {
		me.Policy.Forget(i.Key)
	}
	me.mu.Unlock()
}

func (me *Cache) logState() {
	log.Print(me)
}

func (me *Cache) String() string {
	me.mu.Lock()
	defer me.mu.Unlock()
	return fmt.Sprintf(
		"%p: %d items, %v bytes used, lru: %s",
		me, len(me.Items), humanize.Bytes(uint64(me.filled)),
		func() string {
			k, ok := me.Policy.Candidate()
			if ok {
				i := me.Items[k]
				return fmt.Sprintf(
					"%q (%v: %v)",
					k, humanize.Bytes(uint64(i.Size)), i.Usage)
			}
			return "none"
		}(),
	)
}

func (me *Cache) Used() int64 {
	return me.filled
}

func (me *Cache) NumItems() int {
	return len(me.Items)
}

func (me *Cache) Clear() {
	for k := range me.Items {
		delete(me.Items, k)
		me.Policy.Forget(k)
	}
	me.Items = nil
	me.filled = 0
}

func (me *Cache) Filled() int64 {
	me.mu.Lock()
	defer me.mu.Unlock()
	return me.filled
}

func (me *Cache) Candidate() (Item, bool) {
	me.mu.Lock()
	defer me.mu.Unlock()
	k, ok := me.Policy.Candidate()
	return Item{
		Key:      k,
		ItemMeta: me.Items[k],
	}, ok
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package common

import (
	"fmt"
	"io"
	"os"
	"path"
	"strings"
	"time"

	"github.com/juju/loggo"
	"github.com/juju/utils"
	"github.com/juju/utils/parallel"
	"github.com/juju/utils/shell"

	coreCloudinit "github.com/juju/juju/cloudinit"
	"github.com/juju/juju/cloudinit/sshinit"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/bootstrap"
	"github.com/juju/juju/environs/cloudinit"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/network"
	coretools "github.com/juju/juju/tools"
	"github.com/juju/juju/utils/ssh"
)

var logger = loggo.GetLogger("juju.provider.common")

// Bootstrap is a common implementation of the Bootstrap method defined on
// environs.Environ; we strongly recommend that this implementation be used
// when writing a new provider.
func Bootstrap(ctx environs.BootstrapContext, env environs.Environ, args environs.BootstrapParams) (err error) {
	// TODO make safe in the case of racing Bootstraps
	// If two Bootstraps are called concurrently, there's
	// no way to make sure that only one succeeds.

	var inst instance.Instance
	defer func() { handleBootstrapError(err, ctx, inst, args.KeepBroken, env) }()

	// First thing, ensure we have tools otherwise there's no point.
	selectedTools, err := EnsureBootstrapTools(ctx, env, config.PreferredSeries(env.Config()), args.Constraints.Arch)
	if err != nil {
		return err
	}

	// Get the bootstrap SSH client. Do this early, so we know
	// not to bother with any of the below if we can't finish the job.
	client := ssh.DefaultClient
	if client == nil {
		// This should never happen: if we don't have OpenSSH, then
		// go.crypto/ssh should be used with an auto-generated key.
		return fmt.Errorf("no SSH client available")
	}

	privateKey, err := GenerateSystemSSHKey(env)
	if err != nil {
		return err
	}
	machineConfig := environs.NewBootstrapMachineConfig(privateKey)

	fmt.Fprintln(ctx.GetStderr(), "Launching instance")
	inst, hw, _, err := env.StartInstance(environs.StartInstanceParams{
		Constraints:   args.Constraints,
		Tools:         selectedTools,
		MachineConfig: machineConfig,
		Placement:     args.Placement,
	})
	if err != nil {
		return fmt.Errorf("cannot start bootstrap instance: %v", err)
	}
	fmt.Fprintf(ctx.GetStderr(), " - %s\n", inst.Id())
	machineConfig.InstanceId = inst.Id()
	machineConfig.HardwareCharacteristics = hw

	err = bootstrap.SaveState(
		env.Storage(),
		&bootstrap.BootstrapState{
			StateInstances: []instance.Id{inst.Id()},
		})
	if err != nil {
		return fmt.Errorf("cannot save state: %v", err)
	}
	return FinishBootstrap(ctx, client, inst, machineConfig)
}

// GenerateSystemSSHKey creates a new key for the system identity. The
// authorized_keys in the environment config is updated to include the public
// key for the generated key.
func GenerateSystemSSHKey(env environs.Environ) (privateKey string, err error) {
	logger.Debugf("generate a system ssh key")
	// Create a new system ssh key and add that to the authorized keys.
	privateKey, publicKey, err := ssh.GenerateKey(config.JujuSystemKey)
	if err != nil {
		return "", fmt.Errorf("failed to create system key: %v", err)
	}
	authorized_keys := config.ConcatAuthKeys(env.Config().AuthorizedKeys(), publicKey)
	newConfig, err := env.Config().Apply(map[string]interface{}{
		config.AuthKeysConfig: authorized_keys,
	})
	if err != nil {
		return "", fmt.Errorf("failed to create new config: %v", err)
	}
	if err = env.SetConfig(newConfig); err != nil {
		return "", fmt.Errorf("failed to set new config: %v", err)
	}
	return privateKey, nil
}

// handleBootstrapError cleans up after a failed bootstrap.
func handleBootstrapError(err error, ctx environs.BootstrapContext, inst instance.Instance, keepInst bool, env environs.Environ) {
	if err == nil {
		return
	}

	logger.Errorf("bootstrap failed: %v", err)
	ch := make(chan os.Signal, 1)
	ctx.InterruptNotify(ch)
	defer ctx.StopInterruptNotify(ch)
	defer close(ch)
	go func() {
		for _ = range ch {
			fmt.Fprintln(ctx.GetStderr(), "Cleaning up failed bootstrap")
		}
	}()

	if inst != nil && keepInst {
		fmt.Fprintln(ctx.GetStderr(), "Not stopping instance", inst.Id())
	} else if inst != nil && !keepInst {
		fmt.Fprintln(ctx.GetStderr(), "Stopping instance...")
		if stoperr := env.StopInstances(inst.Id()); stoperr != nil {
			logger.Errorf("cannot stop failed bootstrap instance %q: %v", inst.Id(), stoperr)
		} else {
			// set to nil so we know we can safely delete the state file
			inst = nil
		}
	}
	// We only delete the bootstrap state file if either we didn't
	// start an instance, or we managed to cleanly stop it.
	if inst == nil {
		if rmerr := bootstrap.DeleteStateFile(env.Storage()); rmerr != nil {
			logger.Errorf("cannot delete bootstrap state file: %v", rmerr)
		}
	}
}

// FinishBootstrap completes the bootstrap process by connecting
// to the instance via SSH and carrying out the cloud-config.
//
// Note: FinishBootstrap is exposed so it can be replaced for testing.
var FinishBootstrap = func(ctx environs.BootstrapContext, client ssh.Client, inst instance.Instance, machineConfig *cloudinit.MachineConfig) error {
	interrupted := make(chan os.Signal, 1)
	ctx.InterruptNotify(interrupted)
	defer ctx.StopInterruptNotify(interrupted)
	// Each attempt to connect to an address must verify the machine is the
	// bootstrap machine by checking its nonce file exists and contains the
	// nonce in the MachineConfig. This also blocks sshinit from proceeding
	// until cloud-init has completed, which is necessary to ensure apt
	// invocations don't trample each other.
	nonceFile := utils.ShQuote(path.Join(machineConfig.DataDir, cloudinit.NonceFile))
	checkNonceCommand := fmt.Sprintf(`
	noncefile=%s
	if [ ! -e "$noncefile" ]; then
		echo "$noncefile does not exist" >&2
		exit 1
	fi
	content=$(cat $noncefile)
	if [ "$content" != %s ]; then
		echo "$noncefile contents do not match machine nonce" >&2
		exit 1
	fi
	`, nonceFile, utils.ShQuote(machineConfig.MachineNonce))
	addr, err := waitSSH(
		ctx,
		interrupted,
		client,
		checkNonceCommand,
		inst,
		machineConfig.Config.BootstrapSSHOpts(),
	)
	if err != nil {
		return err
	}
	// Bootstrap is synchronous, and will spawn a subprocess
	// to complete the procedure. If the user hits Ctrl-C,
	// SIGINT is sent to the foreground process attached to
	// the terminal, which will be the ssh subprocess at this
	// point. For that reason, we do not call StopInterruptNotify
	// until this function completes.
	cloudcfg := coreCloudinit.New()
	if err := cloudinit.ConfigureJuju(machineConfig, cloudcfg); err != nil {
		return err
	}
	configScript, err := sshinit.ConfigureScript(cloudcfg)
	if err != nil {
		return err
	}
	script := shell.DumpFileOnErrorScript(machineConfig.CloudInitOutputLog) + configScript
	return sshinit.RunConfigureScript(script, sshinit.ConfigureParams{
		Host:           "ubuntu@" + addr,
		Client:         client,
		Config:         cloudcfg,
		ProgressWriter: ctx.GetStderr(),
	})
}

type addresser interface {
	// Refresh refreshes the addresses for the instance.
	Refresh() error

	// Addresses returns the addresses for the instance.
	// To ensure that the results are up to date, call
	// Refresh first.
	Addresses() ([]network.Address, error)
}

type hostChecker struct {
	addr   network.Address
	client ssh.Client

	// checkDelay is the amount of time to wait between retries.
	checkDelay time.Duration

	// checkHostScript is executed on the host via SSH.
	// hostChecker.loop will return once the script
	// runs without error.
	checkHostScript string

	// closed is closed to indicate that the host checker should
	// return, without waiting for the result of any ongoing
	// attempts.
	closed <-chan struct{}
}

// Close implements io.Closer, as required by parallel.Try.
func (*hostChecker) Close() error {
	return nil
}

func (hc *hostChecker) loop(dying <-chan struct{}) (io.Closer, error) {
	// The value of connectSSH is taken outside the goroutine that may outlive
	// hostChecker.loop, or we evoke the wrath of the race detector.
	connectSSH := connectSSH
	done := make(chan error, 1)
	var lastErr error
	for {
		go func() {
			done <- connectSSH(hc.client, hc.addr.Value, hc.checkHostScript)
		}()
		select {
		case <-hc.closed:
			return hc, lastErr
		case <-dying:
			return hc, lastErr
		case lastErr = <-done:
			if lastErr == nil {
				return hc, nil
			}
		}
		select {
		case <-hc.closed:
		case <-dying:
		case <-time.After(hc.checkDelay):
		}
	}
}

type parallelHostChecker struct {
	*parallel.Try
	client ssh.Client
	stderr io.Writer

	// active is a map of adresses to channels for addresses actively
	// being tested. The goroutine testing the address will continue
	// to attempt connecting to the address until it succeeds, the Try
	// is killed, or the corresponding channel in this map is closed.
	active map[network.Address]chan struct{}

	// checkDelay is how long each hostChecker waits between attempts.
	checkDelay time.Duration

	// checkHostScript is the script to run on each host to check that
	// it is the host we expect.
	checkHostScript string
}

func (p *parallelHostChecker) UpdateAddresses(addrs []network.Address) {
	for _, addr := range addrs {
		if _, ok := p.active[addr]; ok {
			continue
		}
		fmt.Fprintf(p.stderr, "Attempting to connect to %s:22\n", addr.Value)
		closed := make(chan struct{})
		hc := &hostChecker{
			addr:            addr,
			client:          p.client,
			checkDelay:      p.checkDelay,
			checkHostScript: p.checkHostScript,
			closed:          closed,
		}
		p.active[addr] = closed
		p.Start(hc.loop)
	}
}

// Close prevents additional functions from being added to
// the Try, and tells each active hostChecker to exit.
func (p *parallelHostChecker) Close() error {
	// We signal each checker to stop and wait for them
	// each to complete; this allows us to get the error,
	// as opposed to when using try.Kill which does not
	// wait for the functions to complete.
	p.Try.Close()
	for _, ch := range p.active {
		close(ch)
	}
	return nil
}

// connectSSH is called to connect to the specified host and
// execute the "checkHostScript" bash script on it.
var connectSSH = func(client ssh.Client, host, checkHostScript string) error {
	cmd := client.Command("ubuntu@"+host, []string{"/bin/bash"}, nil)
	cmd.Stdin = strings.NewReader(checkHostScript)
	output, err := cmd.CombinedOutput()
	if err != nil && len(output) > 0 {
		err = fmt.Errorf("%s", strings.TrimSpace(string(output)))
	}
	return err
}

// waitSSH waits for the instance to be assigned a routable
// address, then waits until we can connect to it via SSH.
//
// waitSSH attempts on all addresses returned by the instance
// in parallel; the first succeeding one wins. We ensure that
// private addresses are for the correct machine by checking
// the presence of a file on the machine that contains the
// machine's nonce. The "checkHostScript" is a bash script
// that performs this file check.
func waitSSH(ctx environs.BootstrapContext, interrupted <-chan os.Signal, client ssh.Client, checkHostScript string, inst addresser, timeout config.SSHTimeoutOpts) (addr string, err error) {
	globalTimeout := time.After(timeout.Timeout)
	pollAddresses := time.NewTimer(0)

	// checker checks each address in a loop, in parallel,
	// until one succeeds, the global timeout is reached,
	// or the tomb is killed.
	checker := parallelHostChecker{
		Try:             parallel.NewTry(0, nil),
		client:          client,
		stderr:          ctx.GetStderr(),
		active:          make(map[network.Address]chan struct{}),
		checkDelay:      timeout.RetryDelay,
		checkHostScript: checkHostScript,
	}
	defer checker.Kill()

	fmt.Fprintln(ctx.GetStderr(), "Waiting for address")
	for {
		select {
		case <-pollAddresses.C:
			pollAddresses.Reset(timeout.AddressesDelay)
			if err := inst.Refresh(); err != nil {
				return "", fmt.Errorf("refreshing addresses: %v", err)
			}
			addresses, err := inst.Addresses()
			if err != nil {
				return "", fmt.Errorf("getting addresses: %v", err)
			}
			checker.UpdateAddresses(addresses)
		case <-globalTimeout:
			checker.Close()
			lastErr := checker.Wait()
			format := "waited for %v "
			args := []interface{}{timeout.Timeout}
			if len(checker.active) == 0 {
				format += "without getting any addresses"
			} else {
				format += "without being able to connect"
			}
			if lastErr != nil && lastErr != parallel.ErrStopped {
				format += ": %v"
				args = append(args, lastErr)
			}
			return "", fmt.Errorf(format, args...)
		case <-interrupted:
			return "", fmt.Errorf("interrupted")
		case <-checker.Dead():
			result, err := checker.Result()
			if err != nil {
				return "", err
			}
			return result.(*hostChecker).addr.Value, nil
		}
	}
}

// EnsureBootstrapTools finds tools, syncing with an external tools source as
// necessary; it then selects the newest tools to bootstrap with, and sets
// agent-version.
func EnsureBootstrapTools(ctx environs.BootstrapContext, env environs.Environ, series string, arch *string) (coretools.List, error) {
	possibleTools, err := bootstrap.EnsureToolsAvailability(ctx, env, series, arch)
	if err != nil {
		return nil, err
	}
	return bootstrap.SetBootstrapTools(env, possibleTools)
}
<commit_msg>Do not call handleBoostrapError at all if keep-broken is true<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package common

import (
	"fmt"
	"io"
	"os"
	"path"
	"strings"
	"time"

	"github.com/juju/loggo"
	"github.com/juju/utils"
	"github.com/juju/utils/parallel"
	"github.com/juju/utils/shell"

	coreCloudinit "github.com/juju/juju/cloudinit"
	"github.com/juju/juju/cloudinit/sshinit"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/bootstrap"
	"github.com/juju/juju/environs/cloudinit"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/network"
	coretools "github.com/juju/juju/tools"
	"github.com/juju/juju/utils/ssh"
)

var logger = loggo.GetLogger("juju.provider.common")

// Bootstrap is a common implementation of the Bootstrap method defined on
// environs.Environ; we strongly recommend that this implementation be used
// when writing a new provider.
func Bootstrap(ctx environs.BootstrapContext, env environs.Environ, args environs.BootstrapParams) (err error) {
	// TODO make safe in the case of racing Bootstraps
	// If two Bootstraps are called concurrently, there's
	// no way to make sure that only one succeeds.

	var inst instance.Instance
	// If we have not been asked to keep a broken bootstrap, make sure we clean up.
	if !args.KeepBroken {
		defer func() { handleBootstrapError(err, ctx, inst, env) }()
	}

	// First thing, ensure we have tools otherwise there's no point.
	selectedTools, err := EnsureBootstrapTools(ctx, env, config.PreferredSeries(env.Config()), args.Constraints.Arch)
	if err != nil {
		return err
	}

	// Get the bootstrap SSH client. Do this early, so we know
	// not to bother with any of the below if we can't finish the job.
	client := ssh.DefaultClient
	if client == nil {
		// This should never happen: if we don't have OpenSSH, then
		// go.crypto/ssh should be used with an auto-generated key.
		return fmt.Errorf("no SSH client available")
	}

	privateKey, err := GenerateSystemSSHKey(env)
	if err != nil {
		return err
	}
	machineConfig := environs.NewBootstrapMachineConfig(privateKey)

	fmt.Fprintln(ctx.GetStderr(), "Launching instance")
	inst, hw, _, err := env.StartInstance(environs.StartInstanceParams{
		Constraints:   args.Constraints,
		Tools:         selectedTools,
		MachineConfig: machineConfig,
		Placement:     args.Placement,
	})
	if err != nil {
		return fmt.Errorf("cannot start bootstrap instance: %v", err)
	}
	fmt.Fprintf(ctx.GetStderr(), " - %s\n", inst.Id())
	machineConfig.InstanceId = inst.Id()
	machineConfig.HardwareCharacteristics = hw

	err = bootstrap.SaveState(
		env.Storage(),
		&bootstrap.BootstrapState{
			StateInstances: []instance.Id{inst.Id()},
		})
	if err != nil {
		return fmt.Errorf("cannot save state: %v", err)
	}
	return FinishBootstrap(ctx, client, inst, machineConfig)
}

// GenerateSystemSSHKey creates a new key for the system identity. The
// authorized_keys in the environment config is updated to include the public
// key for the generated key.
func GenerateSystemSSHKey(env environs.Environ) (privateKey string, err error) {
	logger.Debugf("generate a system ssh key")
	// Create a new system ssh key and add that to the authorized keys.
	privateKey, publicKey, err := ssh.GenerateKey(config.JujuSystemKey)
	if err != nil {
		return "", fmt.Errorf("failed to create system key: %v", err)
	}
	authorized_keys := config.ConcatAuthKeys(env.Config().AuthorizedKeys(), publicKey)
	newConfig, err := env.Config().Apply(map[string]interface{}{
		config.AuthKeysConfig: authorized_keys,
	})
	if err != nil {
		return "", fmt.Errorf("failed to create new config: %v", err)
	}
	if err = env.SetConfig(newConfig); err != nil {
		return "", fmt.Errorf("failed to set new config: %v", err)
	}
	return privateKey, nil
}

// handleBootstrapError cleans up after a failed bootstrap.
func handleBootstrapError(err error, ctx environs.BootstrapContext, inst instance.Instance, env environs.Environ) {
	if err == nil {
		return
	}

	logger.Errorf("bootstrap failed: %v", err)
	ch := make(chan os.Signal, 1)
	ctx.InterruptNotify(ch)
	defer ctx.StopInterruptNotify(ch)
	defer close(ch)
	go func() {
		for _ = range ch {
			fmt.Fprintln(ctx.GetStderr(), "Cleaning up failed bootstrap")
		}
	}()

	if inst != nil {
		fmt.Fprintln(ctx.GetStderr(), "Stopping instance...")
		if stoperr := env.StopInstances(inst.Id()); stoperr != nil {
			logger.Errorf("cannot stop failed bootstrap instance %q: %v", inst.Id(), stoperr)
		} else {
			// set to nil so we know we can safely delete the state file
			inst = nil
		}
	}
	// We only delete the bootstrap state file if either we didn't
	// start an instance, or we managed to cleanly stop it.
	if inst == nil {
		if rmerr := bootstrap.DeleteStateFile(env.Storage()); rmerr != nil {
			logger.Errorf("cannot delete bootstrap state file: %v", rmerr)
		}
	}
}

// FinishBootstrap completes the bootstrap process by connecting
// to the instance via SSH and carrying out the cloud-config.
//
// Note: FinishBootstrap is exposed so it can be replaced for testing.
var FinishBootstrap = func(ctx environs.BootstrapContext, client ssh.Client, inst instance.Instance, machineConfig *cloudinit.MachineConfig) error {
	interrupted := make(chan os.Signal, 1)
	ctx.InterruptNotify(interrupted)
	defer ctx.StopInterruptNotify(interrupted)
	// Each attempt to connect to an address must verify the machine is the
	// bootstrap machine by checking its nonce file exists and contains the
	// nonce in the MachineConfig. This also blocks sshinit from proceeding
	// until cloud-init has completed, which is necessary to ensure apt
	// invocations don't trample each other.
	nonceFile := utils.ShQuote(path.Join(machineConfig.DataDir, cloudinit.NonceFile))
	checkNonceCommand := fmt.Sprintf(`
	noncefile=%s
	if [ ! -e "$noncefile" ]; then
		echo "$noncefile does not exist" >&2
		exit 1
	fi
	content=$(cat $noncefile)
	if [ "$content" != %s ]; then
		echo "$noncefile contents do not match machine nonce" >&2
		exit 1
	fi
	`, nonceFile, utils.ShQuote(machineConfig.MachineNonce))
	addr, err := waitSSH(
		ctx,
		interrupted,
		client,
		checkNonceCommand,
		inst,
		machineConfig.Config.BootstrapSSHOpts(),
	)
	if err != nil {
		return err
	}
	// Bootstrap is synchronous, and will spawn a subprocess
	// to complete the procedure. If the user hits Ctrl-C,
	// SIGINT is sent to the foreground process attached to
	// the terminal, which will be the ssh subprocess at this
	// point. For that reason, we do not call StopInterruptNotify
	// until this function completes.
	cloudcfg := coreCloudinit.New()
	if err := cloudinit.ConfigureJuju(machineConfig, cloudcfg); err != nil {
		return err
	}
	configScript, err := sshinit.ConfigureScript(cloudcfg)
	if err != nil {
		return err
	}
	script := shell.DumpFileOnErrorScript(machineConfig.CloudInitOutputLog) + configScript
	return sshinit.RunConfigureScript(script, sshinit.ConfigureParams{
		Host:           "ubuntu@" + addr,
		Client:         client,
		Config:         cloudcfg,
		ProgressWriter: ctx.GetStderr(),
	})
}

type addresser interface {
	// Refresh refreshes the addresses for the instance.
	Refresh() error

	// Addresses returns the addresses for the instance.
	// To ensure that the results are up to date, call
	// Refresh first.
	Addresses() ([]network.Address, error)
}

type hostChecker struct {
	addr   network.Address
	client ssh.Client

	// checkDelay is the amount of time to wait between retries.
	checkDelay time.Duration

	// checkHostScript is executed on the host via SSH.
	// hostChecker.loop will return once the script
	// runs without error.
	checkHostScript string

	// closed is closed to indicate that the host checker should
	// return, without waiting for the result of any ongoing
	// attempts.
	closed <-chan struct{}
}

// Close implements io.Closer, as required by parallel.Try.
func (*hostChecker) Close() error {
	return nil
}

func (hc *hostChecker) loop(dying <-chan struct{}) (io.Closer, error) {
	// The value of connectSSH is taken outside the goroutine that may outlive
	// hostChecker.loop, or we evoke the wrath of the race detector.
	connectSSH := connectSSH
	done := make(chan error, 1)
	var lastErr error
	for {
		go func() {
			done <- connectSSH(hc.client, hc.addr.Value, hc.checkHostScript)
		}()
		select {
		case <-hc.closed:
			return hc, lastErr
		case <-dying:
			return hc, lastErr
		case lastErr = <-done:
			if lastErr == nil {
				return hc, nil
			}
		}
		select {
		case <-hc.closed:
		case <-dying:
		case <-time.After(hc.checkDelay):
		}
	}
}

type parallelHostChecker struct {
	*parallel.Try
	client ssh.Client
	stderr io.Writer

	// active is a map of adresses to channels for addresses actively
	// being tested. The goroutine testing the address will continue
	// to attempt connecting to the address until it succeeds, the Try
	// is killed, or the corresponding channel in this map is closed.
	active map[network.Address]chan struct{}

	// checkDelay is how long each hostChecker waits between attempts.
	checkDelay time.Duration

	// checkHostScript is the script to run on each host to check that
	// it is the host we expect.
	checkHostScript string
}

func (p *parallelHostChecker) UpdateAddresses(addrs []network.Address) {
	for _, addr := range addrs {
		if _, ok := p.active[addr]; ok {
			continue
		}
		fmt.Fprintf(p.stderr, "Attempting to connect to %s:22\n", addr.Value)
		closed := make(chan struct{})
		hc := &hostChecker{
			addr:            addr,
			client:          p.client,
			checkDelay:      p.checkDelay,
			checkHostScript: p.checkHostScript,
			closed:          closed,
		}
		p.active[addr] = closed
		p.Start(hc.loop)
	}
}

// Close prevents additional functions from being added to
// the Try, and tells each active hostChecker to exit.
func (p *parallelHostChecker) Close() error {
	// We signal each checker to stop and wait for them
	// each to complete; this allows us to get the error,
	// as opposed to when using try.Kill which does not
	// wait for the functions to complete.
	p.Try.Close()
	for _, ch := range p.active {
		close(ch)
	}
	return nil
}

// connectSSH is called to connect to the specified host and
// execute the "checkHostScript" bash script on it.
var connectSSH = func(client ssh.Client, host, checkHostScript string) error {
	cmd := client.Command("ubuntu@"+host, []string{"/bin/bash"}, nil)
	cmd.Stdin = strings.NewReader(checkHostScript)
	output, err := cmd.CombinedOutput()
	if err != nil && len(output) > 0 {
		err = fmt.Errorf("%s", strings.TrimSpace(string(output)))
	}
	return err
}

// waitSSH waits for the instance to be assigned a routable
// address, then waits until we can connect to it via SSH.
//
// waitSSH attempts on all addresses returned by the instance
// in parallel; the first succeeding one wins. We ensure that
// private addresses are for the correct machine by checking
// the presence of a file on the machine that contains the
// machine's nonce. The "checkHostScript" is a bash script
// that performs this file check.
func waitSSH(ctx environs.BootstrapContext, interrupted <-chan os.Signal, client ssh.Client, checkHostScript string, inst addresser, timeout config.SSHTimeoutOpts) (addr string, err error) {
	globalTimeout := time.After(timeout.Timeout)
	pollAddresses := time.NewTimer(0)

	// checker checks each address in a loop, in parallel,
	// until one succeeds, the global timeout is reached,
	// or the tomb is killed.
	checker := parallelHostChecker{
		Try:             parallel.NewTry(0, nil),
		client:          client,
		stderr:          ctx.GetStderr(),
		active:          make(map[network.Address]chan struct{}),
		checkDelay:      timeout.RetryDelay,
		checkHostScript: checkHostScript,
	}
	defer checker.Kill()

	fmt.Fprintln(ctx.GetStderr(), "Waiting for address")
	for {
		select {
		case <-pollAddresses.C:
			pollAddresses.Reset(timeout.AddressesDelay)
			if err := inst.Refresh(); err != nil {
				return "", fmt.Errorf("refreshing addresses: %v", err)
			}
			addresses, err := inst.Addresses()
			if err != nil {
				return "", fmt.Errorf("getting addresses: %v", err)
			}
			checker.UpdateAddresses(addresses)
		case <-globalTimeout:
			checker.Close()
			lastErr := checker.Wait()
			format := "waited for %v "
			args := []interface{}{timeout.Timeout}
			if len(checker.active) == 0 {
				format += "without getting any addresses"
			} else {
				format += "without being able to connect"
			}
			if lastErr != nil && lastErr != parallel.ErrStopped {
				format += ": %v"
				args = append(args, lastErr)
			}
			return "", fmt.Errorf(format, args...)
		case <-interrupted:
			return "", fmt.Errorf("interrupted")
		case <-checker.Dead():
			result, err := checker.Result()
			if err != nil {
				return "", err
			}
			return result.(*hostChecker).addr.Value, nil
		}
	}
}

// EnsureBootstrapTools finds tools, syncing with an external tools source as
// necessary; it then selects the newest tools to bootstrap with, and sets
// agent-version.
func EnsureBootstrapTools(ctx environs.BootstrapContext, env environs.Environ, series string, arch *string) (coretools.List, error) {
	possibleTools, err := bootstrap.EnsureToolsAvailability(ctx, env, series, arch)
	if err != nil {
		return nil, err
	}
	return bootstrap.SetBootstrapTools(env, possibleTools)
}
<|endoftext|>


<commit_before>/**********************************************************************************
* Copyright (c) 2009-2018 Misakai Ltd.
* This program is free software: you can redistribute it and/or modify it under the
* terms of the GNU Affero General Public License as published by the  Free Software
* Foundation, either version 3 of the License, or(at your option) any later version.
*
* This program is distributed  in the hope that it  will be useful, but WITHOUT ANY
* WARRANTY;  without even  the implied warranty of MERCHANTABILITY or FITNESS FOR A
* PARTICULAR PURPOSE.  See the GNU Affero General Public License  for  more details.
*
* You should have  received a copy  of the  GNU Affero General Public License along
* with this program. If not, see<http://www.gnu.org/licenses/>.
************************************************************************************/
package storage

import (
	"bytes"
	"context"
	"crypto/rand"
	"fmt"
	"io/ioutil"
	"os"
	"runtime"
	"strconv"
	"sync"
	"testing"
	"time"

	"github.com/emitter-io/emitter/message"
	"github.com/emitter-io/stats"
	"github.com/kelindar/binary"
	"github.com/stretchr/testify/assert"
)

func getNTestMessages(count int) (frame message.Frame) {
	for i := 0; i < count; i++ {
		id := message.NewID(message.Ssid{0, uint32(i / 2), 2, uint32(i)})
		frame = append(frame, message.Message{
			ID:      id,
			Channel: []byte("test/channel/"),
			Payload: []byte(fmt.Sprintf("%v,%v,%v", 1, 2, i)),
			TTL:     100000,
		})
	}
	return
}

// Opens an NewSSD and runs a a test on it.
func runSSDTest(test func(store *SSD)) {

	// Prepare a store
	dir, err := ioutil.TempDir("", "emitter")
	if err != nil {
		println(err.Error())
	}

	store := NewSSD(nil)
	store.Configure(map[string]interface{}{
		"dir": dir,
	})

	// Close once we're done and delete data
	defer os.RemoveAll(dir)
	defer store.Close()

	test(store)
}

func TestSSD_Store(t *testing.T) {
	runSSDTest(func(store *SSD) {
		err := store.storeFrame(getNTestMessages(10))
		assert.NoError(t, err)
	})
}

func TestSSD_Query(t *testing.T) {
	runSSDTest(func(store *SSD) {
		err := store.storeFrame(getNTestMessages(10))
		assert.NoError(t, err)

		zero := time.Unix(0, 0)
		f, err := store.Query([]uint32{0, 3, 2, 6}, zero, zero, 5)
		assert.NoError(t, err)
		assert.Len(t, f, 1)
	})
}

func TestSSD_QueryOrdered(t *testing.T) {
	runSSDTest(func(store *SSD) {
		for i := int64(0); i < 100; i++ {
			msg := message.New(message.Ssid{0, 1, uint32(i / 10), 3}, []byte("a/b/c/"), []byte(strconv.FormatInt(i, 10)))
			msg.TTL = 100000
			//binary.BigEndian.PutUint32(msg.ID[12:16], newUnique())
			err := store.Store(msg)
			assert.NoError(t, err)
		}

		zero := time.Unix(0, 0)
		f, err := store.Query([]uint32{0, 1, 2, 3}, zero, zero, 5)
		assert.NoError(t, err)

		assert.Len(t, f, 5)
		assert.Equal(t, "25", string(f[0].Payload))
		assert.Equal(t, "26", string(f[1].Payload))
		assert.Equal(t, "27", string(f[2].Payload))
		assert.Equal(t, "28", string(f[3].Payload))
		assert.Equal(t, "29", string(f[4].Payload))
	})
}

func TestSSD_QueryOrdered2(t *testing.T) {
	s0 := newUnique()
	s1 := newUnique()
	runSSDTest(func(store *SSD) {
		for i := int64(0); i < 100; i++ {
			msg := message.New(message.Ssid{s0, s1, newUnique(), uint32(i / 10), uint32(i / 10)}, []byte("a/b/c/"), []byte(strconv.FormatInt(i, 10)))
			msg.ID.SetTime(msg.Time() + i)
			msg.TTL = 100000
			//binary.BigEndian.PutUint32(msg.ID[12:16], newUnique())
			err := store.Store(msg)
			assert.NoError(t, err)
		}

		zero := time.Unix(0, 0)
		f, err := store.Query([]uint32{s0, s1, uint32(1815237614), 2, 2}, zero, zero, 5)
		assert.NoError(t, err)

		assert.Len(t, f, 5)
		assert.Equal(t, "25", string(f[0].Payload))
		assert.Equal(t, "26", string(f[1].Payload))
		assert.Equal(t, "27", string(f[2].Payload))
		assert.Equal(t, "28", string(f[3].Payload))
		assert.Equal(t, "29", string(f[4].Payload))
	})
}

func TestSSD_QuerySurveyed(t *testing.T) {
	runSSDTest(func(s *SSD) {
		const wildcard = uint32(1815237614)
		msgs := getNTestMessages(10)
		s.storeFrame(msgs)
		zero := time.Unix(0, 0)
		tests := []struct {
			query    []uint32
			limit    int
			count    int
			gathered []byte
		}{
			{query: []uint32{0, 3, 2, 7}, limit: 10, count: 1},
			{query: []uint32{0, 1}, limit: 5, count: 2},
			{query: []uint32{0, 1}, limit: 5, count: 5, gathered: msgs.Encode()},
		}

		for _, tc := range tests {
			if tc.gathered == nil {
				s.cluster = nil
			} else {
				s.cluster = survey(func(string, []byte) (message.Awaiter, error) {
					return &mockAwaiter{f: func(_ time.Duration) [][]byte { return [][]byte{tc.gathered} }}, nil
				})
			}

			out, err := s.Query(tc.query, zero, zero, tc.limit)
			assert.NoError(t, err)
			count := 0
			for range out {
				count++
			}
			assert.Equal(t, tc.count, count)
		}
	})
}

func TestSSD_OnSurvey(t *testing.T) {
	runSSDTest(func(s *SSD) {
		s.storeFrame(getNTestMessages(10))
		zero := time.Unix(0, 0)
		tests := []struct {
			name        string
			query       lookupQuery
			expectOk    bool
			expectCount int
		}{
			{name: "dummy"},
			{name: "ssdstore"},
			{
				name:        "ssdstore",
				query:       newLookupQuery(message.Ssid{0, 1}, zero, zero, 1),
				expectOk:    true,
				expectCount: 1,
			},
			{
				name:        "ssdstore",
				query:       newLookupQuery(message.Ssid{0, 1}, zero, zero, 10),
				expectOk:    true,
				expectCount: 2,
			},
		}

		for _, tc := range tests {
			q, _ := binary.Marshal(tc.query)
			resp, ok := s.OnSurvey(tc.name, q)
			assert.Equal(t, tc.expectOk, ok)
			if tc.expectOk && ok {
				msgs, err := message.DecodeFrame(resp)
				assert.NoError(t, err)
				assert.Equal(t, tc.expectCount, len(msgs))
			}
		}

		// Special, wrong payload case
		_, ok := s.OnSurvey("ssdstore", []byte{})
		assert.Equal(t, false, ok)
	})

}

// batch=1         batch/s=43444   msg/s=43444
// batch=10        batch/s=18445   msg/s=184452
// batch=100       batch/s=2620    msg/s=262004
// batch=1000      batch/s=260     msg/s=260115
// batch=10000     batch/s=23      msg/s=228987
func BenchmarkStore(b *testing.B) {
	runSSDTest(func(store *SSD) {
		benchmarkStoreSingle(b, store, 1)
		benchmarkStoreSingle(b, store, 10)
		benchmarkStoreSingle(b, store, 100)
		benchmarkStoreSingle(b, store, 1000)
		benchmarkStoreSingle(b, store, 10000)
	})
}

// batch=1         batch/s=173956  msg/s=173956
// batch=10        batch/s=39658   msg/s=396576
// batch=100       batch/s=4400    msg/s=439988
// batch=1000      batch/s=448     msg/s=448376
// batch=10000     batch/s=31      msg/s=314984
func BenchmarkStoreParallel(b *testing.B) {
	runSSDTest(func(store *SSD) {
		benchmarkStoreParallel(b, store, 1, runtime.NumCPU())
		benchmarkStoreParallel(b, store, 10, runtime.NumCPU())
		benchmarkStoreParallel(b, store, 100, runtime.NumCPU())
		benchmarkStoreParallel(b, store, 1000, runtime.NumCPU())
		benchmarkStoreParallel(b, store, 10000, runtime.NumCPU())
	})
}

func benchmarkStoreParallel(b *testing.B, store *SSD, batchSize int, proceses int) {
	var wg sync.WaitGroup
	wg.Add(proceses)
	m := stats.NewMetric("elapsed")
	for i := 0; i < proceses; i++ {
		go func() {
			benchmarkStore(b, store, batchSize, m)
			wg.Done()
		}()
	}
	wg.Wait()
	fmt.Printf("batch=%v  \tbatch/s=%.0f\tmsg/s=%.0f \n", batchSize, m.Rate(), m.Rate()*float64(batchSize))
}

func benchmarkStoreSingle(b *testing.B, store *SSD, batchSize int) {
	m := stats.NewMetric("elapsed")
	benchmarkStore(b, store, batchSize, m)
	fmt.Printf("batch=%v  \tbatch/s=%.0f\tmsg/s=%.0f \n", batchSize, m.Rate(), m.Rate()*float64(batchSize))
}

func benchmarkStore(b *testing.B, store *SSD, batchSize int, m *stats.Metric) {
	batch := getNTestMessages(batchSize)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func() {
		for {
			select {
			case <-ctx.Done():
				return

			default:
				store.storeFrame(batch)
				m.Update(int32(batchSize))
			}
		}
	}()

	time.Sleep(2 * time.Second)
}

// last=1          query/s=45053   msg/s=45053
// last=10         query/s=8154    msg/s=81539
// last=100        query/s=872     msg/s=87200
// last=1000       query/s=81      msg/s=80517
func BenchmarkQuery(b *testing.B) {
	runSSDTest(func(store *SSD) {
		batchSize := 10
		for i := int64(0); i < 100000; i++ {
			store.storeFrame(getNTestMessages(batchSize))
		}

		benchmarkQuerySingle(b, store, 1)
		benchmarkQuerySingle(b, store, 10)
		benchmarkQuerySingle(b, store, 100)
		benchmarkQuerySingle(b, store, 1000)
	})
}

// last=1          query/s=152868  msg/s=152868
// last=10         query/s=26351   msg/s=263507
// last=100        query/s=3023    msg/s=302346
// last=1000       query/s=287     msg/s=287481
func BenchmarkQueryParallel(b *testing.B) {
	runSSDTest(func(store *SSD) {
		batchSize := 10
		for i := int64(0); i < 100000; i++ {
			store.storeFrame(getNTestMessages(batchSize))
		}

		benchmarkQueryParallel(b, store, 1, runtime.NumCPU())
		benchmarkQueryParallel(b, store, 10, runtime.NumCPU())
		benchmarkQueryParallel(b, store, 100, runtime.NumCPU())
		benchmarkQueryParallel(b, store, 1000, runtime.NumCPU())
	})
}

func benchmarkQueryParallel(b *testing.B, store *SSD, last int, proceses int) {
	var wg sync.WaitGroup
	wg.Add(proceses)
	m := stats.NewMetric("elapsed")
	for i := 0; i < proceses; i++ {
		go func() {
			benchmarkQuery(b, store, last, m)
			wg.Done()
		}()
	}
	wg.Wait()
	fmt.Printf("last=%v  \tquery/s=%.0f \n", last, m.Rate())
}

func benchmarkQuerySingle(b *testing.B, store *SSD, last int) {
	m := stats.NewMetric("elapsed")
	benchmarkQuery(b, store, last, m)
	fmt.Printf("last=%v  \tquery/s=%.0f \n", last, m.Rate())
}

func benchmarkQuery(b *testing.B, store *SSD, last int, m *stats.Metric) {
	t0 := time.Unix(0, 0)
	t1 := time.Unix(time.Now().Unix(), 0)

	ssid := []uint32{0, 3, 2, 6}
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func() {
		for {
			select {
			case <-ctx.Done():
				return

			default:
				store.Query(ssid, t0, t1, last)
				m.Update(int32(last))
			}
		}
	}()

	time.Sleep(5 * time.Second)
}

func TestBackup(t *testing.T) {
	runSSDTest(func(store *SSD) {
		err := store.storeFrame(getNTestMessages(10))
		assert.NoError(t, err)

		// Do the backup
		buffer := bytes.NewBuffer(nil)
		err = store.Backup(buffer)
		assert.NoError(t, err)

		// Restore the backup
		reader := bytes.NewReader(buffer.Bytes())
		err = store.Restore(reader)
		assert.NoError(t, err)
	})
}

func newUnique() uint32 {
	b := make([]byte, 4)
	rand.Read(b)
	return binary.BigEndian.Uint32(b)
}
<commit_msg>test reset<commit_after>/**********************************************************************************
* Copyright (c) 2009-2018 Misakai Ltd.
* This program is free software: you can redistribute it and/or modify it under the
* terms of the GNU Affero General Public License as published by the  Free Software
* Foundation, either version 3 of the License, or(at your option) any later version.
*
* This program is distributed  in the hope that it  will be useful, but WITHOUT ANY
* WARRANTY;  without even  the implied warranty of MERCHANTABILITY or FITNESS FOR A
* PARTICULAR PURPOSE.  See the GNU Affero General Public License  for  more details.
*
* You should have  received a copy  of the  GNU Affero General Public License along
* with this program. If not, see<http://www.gnu.org/licenses/>.
************************************************************************************/
package storage

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"runtime"
	"strconv"
	"sync"
	"testing"
	"time"

	"github.com/emitter-io/emitter/message"
	"github.com/emitter-io/stats"
	"github.com/kelindar/binary"
	"github.com/stretchr/testify/assert"
)

func getNTestMessages(count int) (frame message.Frame) {
	for i := 0; i < count; i++ {
		id := message.NewID(message.Ssid{0, uint32(i / 2), 2, uint32(i)})
		frame = append(frame, message.Message{
			ID:      id,
			Channel: []byte("test/channel/"),
			Payload: []byte(fmt.Sprintf("%v,%v,%v", 1, 2, i)),
			TTL:     100000,
		})
	}
	return
}

// Opens an NewSSD and runs a a test on it.
func runSSDTest(test func(store *SSD)) {

	// Prepare a store
	dir, _ := ioutil.TempDir("", "emitter")
	store := NewSSD(nil)
	store.Configure(map[string]interface{}{
		"dir": dir,
	})

	// Close once we're done and delete data
	defer os.RemoveAll(dir)
	defer store.Close()

	test(store)
}

func TestSSD_Store(t *testing.T) {
	runSSDTest(func(store *SSD) {
		err := store.storeFrame(getNTestMessages(10))
		assert.NoError(t, err)
	})
}

func TestSSD_Query(t *testing.T) {
	runSSDTest(func(store *SSD) {
		err := store.storeFrame(getNTestMessages(10))
		assert.NoError(t, err)

		zero := time.Unix(0, 0)
		f, err := store.Query([]uint32{0, 3, 2, 6}, zero, zero, 5)
		assert.NoError(t, err)
		assert.Len(t, f, 1)
	})
}

func TestSSD_QueryOrdered(t *testing.T) {
	runSSDTest(func(store *SSD) {
		for i := int64(0); i < 100; i++ {
			msg := message.New(message.Ssid{0, 1, 2}, []byte("a/b/c/"), []byte(strconv.FormatInt(i, 10)))
			msg.ID.SetTime(msg.ID.Time() + (i * 10))
			err := store.Store(msg)
			assert.NoError(t, err)
		}

		zero := time.Unix(0, 0)
		f, err := store.Query([]uint32{0, 1, 2}, zero, zero, 5)
		assert.NoError(t, err)

		assert.Len(t, f, 5)
		assert.Equal(t, message.Ssid{0, 1, 2}, f[0].Ssid())
		assert.Equal(t, "95", string(f[0].Payload))
		assert.Equal(t, "96", string(f[1].Payload))
		assert.Equal(t, "97", string(f[2].Payload))
		assert.Equal(t, "98", string(f[3].Payload))
		assert.Equal(t, "99", string(f[4].Payload))

	})
}

func TestSSD_QuerySurveyed(t *testing.T) {
	runSSDTest(func(s *SSD) {
		const wildcard = uint32(1815237614)
		msgs := getNTestMessages(10)
		s.storeFrame(msgs)
		zero := time.Unix(0, 0)
		tests := []struct {
			query    []uint32
			limit    int
			count    int
			gathered []byte
		}{
			{query: []uint32{0, 3, 2, 7}, limit: 10, count: 1},
			{query: []uint32{0, 1}, limit: 5, count: 2},
			{query: []uint32{0, 1}, limit: 5, count: 5, gathered: msgs.Encode()},
		}

		for _, tc := range tests {
			if tc.gathered == nil {
				s.cluster = nil
			} else {
				s.cluster = survey(func(string, []byte) (message.Awaiter, error) {
					return &mockAwaiter{f: func(_ time.Duration) [][]byte { return [][]byte{tc.gathered} }}, nil
				})
			}

			out, err := s.Query(tc.query, zero, zero, tc.limit)
			assert.NoError(t, err)
			count := 0
			for range out {
				count++
			}
			assert.Equal(t, tc.count, count)
		}
	})
}

func TestSSD_OnSurvey(t *testing.T) {
	runSSDTest(func(s *SSD) {
		s.storeFrame(getNTestMessages(10))
		zero := time.Unix(0, 0)
		tests := []struct {
			name        string
			query       lookupQuery
			expectOk    bool
			expectCount int
		}{
			{name: "dummy"},
			{name: "ssdstore"},
			{
				name:        "ssdstore",
				query:       newLookupQuery(message.Ssid{0, 1}, zero, zero, 1),
				expectOk:    true,
				expectCount: 1,
			},
			{
				name:        "ssdstore",
				query:       newLookupQuery(message.Ssid{0, 1}, zero, zero, 10),
				expectOk:    true,
				expectCount: 2,
			},
		}

		for _, tc := range tests {
			q, _ := binary.Marshal(tc.query)
			resp, ok := s.OnSurvey(tc.name, q)
			assert.Equal(t, tc.expectOk, ok)
			if tc.expectOk && ok {
				msgs, err := message.DecodeFrame(resp)
				assert.NoError(t, err)
				assert.Equal(t, tc.expectCount, len(msgs))
			}
		}

		// Special, wrong payload case
		_, ok := s.OnSurvey("ssdstore", []byte{})
		assert.Equal(t, false, ok)
	})

}

// batch=1         batch/s=43444   msg/s=43444
// batch=10        batch/s=18445   msg/s=184452
// batch=100       batch/s=2620    msg/s=262004
// batch=1000      batch/s=260     msg/s=260115
// batch=10000     batch/s=23      msg/s=228987
func BenchmarkStore(b *testing.B) {
	runSSDTest(func(store *SSD) {
		benchmarkStoreSingle(b, store, 1)
		benchmarkStoreSingle(b, store, 10)
		benchmarkStoreSingle(b, store, 100)
		benchmarkStoreSingle(b, store, 1000)
		benchmarkStoreSingle(b, store, 10000)
	})
}

// batch=1         batch/s=173956  msg/s=173956
// batch=10        batch/s=39658   msg/s=396576
// batch=100       batch/s=4400    msg/s=439988
// batch=1000      batch/s=448     msg/s=448376
// batch=10000     batch/s=31      msg/s=314984
func BenchmarkStoreParallel(b *testing.B) {
	runSSDTest(func(store *SSD) {
		benchmarkStoreParallel(b, store, 1, runtime.NumCPU())
		benchmarkStoreParallel(b, store, 10, runtime.NumCPU())
		benchmarkStoreParallel(b, store, 100, runtime.NumCPU())
		benchmarkStoreParallel(b, store, 1000, runtime.NumCPU())
		benchmarkStoreParallel(b, store, 10000, runtime.NumCPU())
	})
}

func benchmarkStoreParallel(b *testing.B, store *SSD, batchSize int, proceses int) {
	var wg sync.WaitGroup
	wg.Add(proceses)
	m := stats.NewMetric("elapsed")
	for i := 0; i < proceses; i++ {
		go func() {
			benchmarkStore(b, store, batchSize, m)
			wg.Done()
		}()
	}
	wg.Wait()
	fmt.Printf("batch=%v  \tbatch/s=%.0f\tmsg/s=%.0f \n", batchSize, m.Rate(), m.Rate()*float64(batchSize))
}

func benchmarkStoreSingle(b *testing.B, store *SSD, batchSize int) {
	m := stats.NewMetric("elapsed")
	benchmarkStore(b, store, batchSize, m)
	fmt.Printf("batch=%v  \tbatch/s=%.0f\tmsg/s=%.0f \n", batchSize, m.Rate(), m.Rate()*float64(batchSize))
}

func benchmarkStore(b *testing.B, store *SSD, batchSize int, m *stats.Metric) {
	batch := getNTestMessages(batchSize)
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func() {
		for {
			select {
			case <-ctx.Done():
				return

			default:
				store.storeFrame(batch)
				m.Update(int32(batchSize))
			}
		}
	}()

	time.Sleep(2 * time.Second)
}

// last=1          query/s=45053   msg/s=45053
// last=10         query/s=8154    msg/s=81539
// last=100        query/s=872     msg/s=87200
// last=1000       query/s=81      msg/s=80517
func BenchmarkQuery(b *testing.B) {
	runSSDTest(func(store *SSD) {
		batchSize := 10
		for i := int64(0); i < 100000; i++ {
			store.storeFrame(getNTestMessages(batchSize))
		}

		benchmarkQuerySingle(b, store, 1)
		benchmarkQuerySingle(b, store, 10)
		benchmarkQuerySingle(b, store, 100)
		benchmarkQuerySingle(b, store, 1000)
	})
}

// last=1          query/s=152868  msg/s=152868
// last=10         query/s=26351   msg/s=263507
// last=100        query/s=3023    msg/s=302346
// last=1000       query/s=287     msg/s=287481
func BenchmarkQueryParallel(b *testing.B) {
	runSSDTest(func(store *SSD) {
		batchSize := 10
		for i := int64(0); i < 100000; i++ {
			store.storeFrame(getNTestMessages(batchSize))
		}

		benchmarkQueryParallel(b, store, 1, runtime.NumCPU())
		benchmarkQueryParallel(b, store, 10, runtime.NumCPU())
		benchmarkQueryParallel(b, store, 100, runtime.NumCPU())
		benchmarkQueryParallel(b, store, 1000, runtime.NumCPU())
	})
}

func benchmarkQueryParallel(b *testing.B, store *SSD, last int, proceses int) {
	var wg sync.WaitGroup
	wg.Add(proceses)
	m := stats.NewMetric("elapsed")
	for i := 0; i < proceses; i++ {
		go func() {
			benchmarkQuery(b, store, last, m)
			wg.Done()
		}()
	}
	wg.Wait()
	fmt.Printf("last=%v  \tquery/s=%.0f \n", last, m.Rate())
}

func benchmarkQuerySingle(b *testing.B, store *SSD, last int) {
	m := stats.NewMetric("elapsed")
	benchmarkQuery(b, store, last, m)
	fmt.Printf("last=%v  \tquery/s=%.0f \n", last, m.Rate())
}

func benchmarkQuery(b *testing.B, store *SSD, last int, m *stats.Metric) {
	t0 := time.Unix(0, 0)
	t1 := time.Unix(time.Now().Unix(), 0)

	ssid := []uint32{0, 3, 2, 6}
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func() {
		for {
			select {
			case <-ctx.Done():
				return

			default:
				store.Query(ssid, t0, t1, last)
				m.Update(int32(last))
			}
		}
	}()

	time.Sleep(5 * time.Second)
}

func TestBackup(t *testing.T) {
	runSSDTest(func(store *SSD) {
		err := store.storeFrame(getNTestMessages(10))
		assert.NoError(t, err)

		// Do the backup
		buffer := bytes.NewBuffer(nil)
		err = store.Backup(buffer)
		assert.NoError(t, err)

		// Restore the backup
		reader := bytes.NewReader(buffer.Bytes())
		err = store.Restore(reader)
		assert.NoError(t, err)
	})
}
<|endoftext|>


<commit_before>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package lb

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/r3labs/terraform/helper/schema"

	aes "github.com/ernestio/crypto/aes"
	"github.com/ernestio/ernestprovider/event"
	"github.com/ernestio/ernestprovider/providers/azure"
	"github.com/fatih/structs"
)

// Event : This is the Ernest representation of an azure lb
type Event struct {
	event.Base
	ID                       string                    `json:"id"`
	Name                     string                    `json:"name" validate:"required"`
	ResourceGroupName        string                    `json:"resource_group_name" validate:"required"`
	Location                 string                    `json:"location" validate:"required"`
	FrontendIPConfigurations []FrontendIPConfiguration `json:"frontend_ip_configurations" validate:"required"`
	Tags                     map[string]string         `json:"tags"`
	ClientID                 string                    `json:"azure_client_id"`
	ClientSecret             string                    `json:"azure_client_secret"`
	TenantID                 string                    `json:"azure_tenant_id"`
	SubscriptionID           string                    `json:"azure_subscription_id"`
	Environment              string                    `json:"environment"`
	ErrorMessage             string                    `json:"error,omitempty"`
	Components               []json.RawMessage         `json:"components"`
	CryptoKey                string                    `json:"-"`
	Validator                *event.Validator          `json:"-"`
}

// FrontendIPConfiguration ...
type FrontendIPConfiguration struct {
	Name                       string `json:"name" validate:"required" structs:"name"`
	Subnet                     string `json:"subnet" structs:"-"`
	SubnetID                   string `json:"subnet_id" structs:"subnet_id"`
	PrivateIPAddress           string `json:"private_ip_address" structs:"private_ip_address"`
	PrivateIPAddressAllocation string `json:"private_ip_address_allocation" structs:"private_ip_address_allocation"`
	PublicIPAddress            string `json:"public_ip_address" structs:"-"`
	PublicIPAddressID          string `json:"public_ip_address_id" structs:"public_ip_address_id"`
}

// New : Constructor
func New(subject, cryptoKey string, body []byte, val *event.Validator) (event.Event, error) {
	var ev event.Resource
	ev = &Event{CryptoKey: cryptoKey, Validator: val}
	body = []byte(strings.Replace(string(body), `"_component":"lbs"`, `"_component":"lb"`, 1))
	if err := json.Unmarshal(body, &ev); err != nil {
		err := fmt.Errorf("Error on input message : %s", err)
		return nil, err
	}

	return azure.New(subject, "azurerm_lb", body, val, ev)
}

// SetComponents : ....
func (ev *Event) SetComponents(components []event.Event) {
	for _, v := range components {
		ev.Components = append(ev.Components, v.GetBody())
	}
}

// ValidateID : determines if the given id is valid for this resource type
func (ev *Event) ValidateID(id string) bool {
	parts := strings.Split(strings.ToLower(id), "/")
	if len(parts) != 9 {
		return false
	}
	if parts[6] != "microsoft.network" {
		return false
	}
	if parts[7] != "loadbalancers" {
		return false
	}
	return true
}

// SetID : id setter
func (ev *Event) SetID(id string) {
	ev.ID = id
}

// GetID : id getter
func (ev *Event) GetID() string {
	return ev.ID
}

// SetState : state setter
func (ev *Event) SetState(state string) {
	ev.State = state
}

// ResourceDataToEvent : Translates a ResourceData on a valid Ernest Event
func (ev *Event) ResourceDataToEvent(d *schema.ResourceData) error {
	ev.ID = d.Id()
	ev.Name = d.Get("name").(string)
	ev.ComponentID = "lb::" + ev.Name
	ev.ResourceGroupName = d.Get("resource_group_name").(string)
	ev.Location = d.Get("location").(string)
	configs := d.Get("frontend_ip_configurations").([]interface{})
	for _, c := range configs {
		cfg := c.(map[string]interface{})
		ev.FrontendIPConfigurations = append(ev.FrontendIPConfigurations, FrontendIPConfiguration{
			Name:                       cfg["name"].(string),
			SubnetID:                   cfg["subnet_id"].(string),
			PrivateIPAddress:           cfg["private_ip_address"].(string),
			PrivateIPAddressAllocation: cfg["public_ip_address_allocation"].(string),
			PublicIPAddressID:          cfg["public_ip_address_id"].(string),
		})
	}

	tags := make(map[string]string, 0)
	for k, v := range d.Get("tags").(map[string]interface{}) {
		tags[k] = v.(string)
	}
	ev.Tags = tags

	return nil
}

// EventToResourceData : Translates the current event on a valid ResourceData
func (ev *Event) EventToResourceData(d *schema.ResourceData) error {
	crypto := aes.New()

	encFields := make(map[string]string)
	encFields["subscription_id"] = ev.SubscriptionID
	encFields["client_id"] = ev.ClientID
	encFields["client_secret"] = ev.ClientSecret
	encFields["tenant_id"] = ev.TenantID
	encFields["environment"] = ev.Environment
	for k, v := range encFields {
		dec, err := crypto.Decrypt(v, ev.CryptoKey)
		if err != nil {
			err := fmt.Errorf("Field '%s' not valid : %s", k, err)
			ev.Log("error", err.Error())
			return err
		}
		if err := d.Set(k, dec); err != nil {
			err := fmt.Errorf("Field '%s' not valid : %s", k, err)
			ev.Log("error", err.Error())
			return err
		}
	}

	fields := make(map[string]interface{})
	fields["name"] = ev.Name
	fields["location"] = ev.Location
	fields["resource_group_name"] = ev.ResourceGroupName
	var configs []interface{}
	for _, c := range ev.FrontendIPConfigurations {
		configs = append(configs, structs.Map(c))
	}
	fields["frontend_ip_configuration"] = configs
	fields["tags"] = ev.Tags
	for k, v := range fields {
		if err := d.Set(k, v); err != nil {
			err := fmt.Errorf("Field '%s' not valid : %s", k, err)
			ev.Log("error", err.Error())
			return err
		}
	}

	return nil
}

// Clone : will mark the event as errored
func (ev *Event) Clone() (event.Event, error) {
	body, _ := json.Marshal(ev)
	return New(ev.Subject, ev.CryptoKey, body, ev.Validator)
}

// Error : will mark the event as errored
func (ev *Event) Error(err error) {
	ev.ErrorMessage = err.Error()
	ev.Body, err = json.Marshal(ev)
}
<commit_msg>load balancer validateID fix<commit_after>/* This Source Code Form is subject to the terms of the Mozilla Public
 * License, v. 2.0. If a copy of the MPL was not distributed with this
 * file, You can obtain one at http://mozilla.org/MPL/2.0/. */

package lb

import (
	"encoding/json"
	"fmt"
	"strings"

	"github.com/r3labs/terraform/helper/schema"

	aes "github.com/ernestio/crypto/aes"
	"github.com/ernestio/ernestprovider/event"
	"github.com/ernestio/ernestprovider/providers/azure"
	"github.com/fatih/structs"
)

// Event : This is the Ernest representation of an azure lb
type Event struct {
	event.Base
	ID                       string                    `json:"id"`
	Name                     string                    `json:"name" validate:"required"`
	ResourceGroupName        string                    `json:"resource_group_name" validate:"required"`
	Location                 string                    `json:"location" validate:"required"`
	FrontendIPConfigurations []FrontendIPConfiguration `json:"frontend_ip_configurations" validate:"required"`
	Tags                     map[string]string         `json:"tags"`
	ClientID                 string                    `json:"azure_client_id"`
	ClientSecret             string                    `json:"azure_client_secret"`
	TenantID                 string                    `json:"azure_tenant_id"`
	SubscriptionID           string                    `json:"azure_subscription_id"`
	Environment              string                    `json:"environment"`
	ErrorMessage             string                    `json:"error,omitempty"`
	Components               []json.RawMessage         `json:"components"`
	CryptoKey                string                    `json:"-"`
	Validator                *event.Validator          `json:"-"`
}

// FrontendIPConfiguration ...
type FrontendIPConfiguration struct {
	Name                       string `json:"name" validate:"required" structs:"name"`
	Subnet                     string `json:"subnet" structs:"-"`
	SubnetID                   string `json:"subnet_id" structs:"subnet_id"`
	PrivateIPAddress           string `json:"private_ip_address" structs:"private_ip_address"`
	PrivateIPAddressAllocation string `json:"private_ip_address_allocation" structs:"private_ip_address_allocation"`
	PublicIPAddress            string `json:"public_ip_address" structs:"-"`
	PublicIPAddressID          string `json:"public_ip_address_id" structs:"public_ip_address_id"`
}

// New : Constructor
func New(subject, cryptoKey string, body []byte, val *event.Validator) (event.Event, error) {
	var ev event.Resource
	ev = &Event{CryptoKey: cryptoKey, Validator: val}
	body = []byte(strings.Replace(string(body), `"_component":"lbs"`, `"_component":"lb"`, 1))
	if err := json.Unmarshal(body, &ev); err != nil {
		err := fmt.Errorf("Error on input message : %s", err)
		return nil, err
	}

	return azure.New(subject, "azurerm_lb", body, val, ev)
}

// SetComponents : ....
func (ev *Event) SetComponents(components []event.Event) {
	for _, v := range components {
		ev.Components = append(ev.Components, v.GetBody())
	}
}

// ValidateID : determines if the given id is valid for this resource type
func (ev *Event) ValidateID(id string) bool {
	parts := strings.Split(strings.ToLower(id), "/")
	if len(parts) != 9 {
		return false
	}
	if parts[6] != "microsoft.network" {
		return false
	}
	if parts[7] != "loadbalancers" {
		return false
	}
	return true
}

// SetID : id setter
func (ev *Event) SetID(id string) {
	ev.ID = id
}

// GetID : id getter
func (ev *Event) GetID() string {
	return ev.ID
}

// SetState : state setter
func (ev *Event) SetState(state string) {
	ev.State = state
}

// ResourceDataToEvent : Translates a ResourceData on a valid Ernest Event
func (ev *Event) ResourceDataToEvent(d *schema.ResourceData) error {
	ev.ID = d.Id()
	ev.Name = d.Get("name").(string)
	ev.ComponentID = "lb::" + ev.Name
	ev.ResourceGroupName = d.Get("resource_group_name").(string)
	ev.Location = d.Get("location").(string)
	configs := d.Get("frontend_ip_configuration").([]interface{})
	for _, c := range configs {
		cfg := c.(map[string]interface{})
		ev.FrontendIPConfigurations = append(ev.FrontendIPConfigurations, FrontendIPConfiguration{
			Name:                       cfg["name"].(string),
			SubnetID:                   cfg["subnet_id"].(string),
			PrivateIPAddress:           cfg["private_ip_address"].(string),
			PrivateIPAddressAllocation: cfg["private_ip_address_allocation"].(string),
			PublicIPAddressID:          cfg["public_ip_address_id"].(string),
		})
	}

	tags := make(map[string]string, 0)
	for k, v := range d.Get("tags").(map[string]interface{}) {
		tags[k] = v.(string)
	}
	ev.Tags = tags

	return nil
}

// EventToResourceData : Translates the current event on a valid ResourceData
func (ev *Event) EventToResourceData(d *schema.ResourceData) error {
	crypto := aes.New()

	encFields := make(map[string]string)
	encFields["subscription_id"] = ev.SubscriptionID
	encFields["client_id"] = ev.ClientID
	encFields["client_secret"] = ev.ClientSecret
	encFields["tenant_id"] = ev.TenantID
	encFields["environment"] = ev.Environment
	for k, v := range encFields {
		dec, err := crypto.Decrypt(v, ev.CryptoKey)
		if err != nil {
			err := fmt.Errorf("Field '%s' not valid : %s", k, err)
			ev.Log("error", err.Error())
			return err
		}
		if err := d.Set(k, dec); err != nil {
			err := fmt.Errorf("Field '%s' not valid : %s", k, err)
			ev.Log("error", err.Error())
			return err
		}
	}

	fields := make(map[string]interface{})
	fields["name"] = ev.Name
	fields["location"] = ev.Location
	fields["resource_group_name"] = ev.ResourceGroupName
	var configs []interface{}
	for _, c := range ev.FrontendIPConfigurations {
		configs = append(configs, structs.Map(c))
	}
	fields["frontend_ip_configuration"] = configs
	fields["tags"] = ev.Tags
	for k, v := range fields {
		if err := d.Set(k, v); err != nil {
			err := fmt.Errorf("Field '%s' not valid : %s", k, err)
			ev.Log("error", err.Error())
			return err
		}
	}

	return nil
}

// Clone : will mark the event as errored
func (ev *Event) Clone() (event.Event, error) {
	body, _ := json.Marshal(ev)
	return New(ev.Subject, ev.CryptoKey, body, ev.Validator)
}

// Error : will mark the event as errored
func (ev *Event) Error(err error) {
	ev.ErrorMessage = err.Error()
	ev.Body, err = json.Marshal(ev)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package feature

import (
	"fmt"
	"sort"
	"strconv"
	"strings"

	"github.com/golang/glog"
	"github.com/spf13/pflag"
)

type Feature string

const (
	flagName = "feature-gates"

	// allAlphaGate is a global toggle for alpha features. Per-feature key
	// values override the default set by allAlphaGate. Examples:
	//   AllAlpha=false,NewFeature=true  will result in newFeature=true
	//   AllAlpha=true,NewFeature=false  will result in newFeature=false
	allAlphaGate Feature = "AllAlpha"
)

var (
	// The generic features.
	defaultFeatures = map[Feature]FeatureSpec{
		allAlphaGate: {Default: false, PreRelease: Alpha},
	}

	// Special handling for a few gates.
	specialFeatures = map[Feature]func(f *featureGate, val bool){
		allAlphaGate: setUnsetAlphaGates,
	}

	// DefaultFeatureGate is a shared global FeatureGate.
	DefaultFeatureGate FeatureGate = NewFeatureGate()
)

type FeatureSpec struct {
	Default    bool
	PreRelease prerelease
}

type prerelease string

const (
	// Values for PreRelease.
	Alpha = prerelease("ALPHA")
	Beta  = prerelease("BETA")
	GA    = prerelease("")
)

// FeatureGate parses and stores flag gates for known features from
// a string like feature1=true,feature2=false,...
type FeatureGate interface {
	AddFlag(fs *pflag.FlagSet)
	Set(value string) error
	Enabled(key Feature) bool
	Add(features map[Feature]FeatureSpec) error
	KnownFeatures() []string
}

// featureGate implements FeatureGate as well as pflag.Value for flag parsing.
type featureGate struct {
	known   map[Feature]FeatureSpec
	special map[Feature]func(*featureGate, bool)
	enabled map[Feature]bool

	// is set to true when AddFlag is called. Note: initialization is not go-routine safe, lookup is
	closed bool
}

func setUnsetAlphaGates(f *featureGate, val bool) {
	for k, v := range f.known {
		if v.PreRelease == Alpha {
			if _, found := f.enabled[k]; !found {
				f.enabled[k] = val
			}
		}
	}
}

// Set, String, and Type implement pflag.Value
var _ pflag.Value = &featureGate{}

func NewFeatureGate() *featureGate {
	f := &featureGate{
		known:   map[Feature]FeatureSpec{},
		special: specialFeatures,
		enabled: map[Feature]bool{},
	}
	for k, v := range defaultFeatures {
		f.known[k] = v
	}
	return f
}

// Set Parses a string of the form // "key1=value1,key2=value2,..." into a
// map[string]bool of known keys or returns an error.
func (f *featureGate) Set(value string) error {
	for _, s := range strings.Split(value, ",") {
		if len(s) == 0 {
			continue
		}
		arr := strings.SplitN(s, "=", 2)
		k := Feature(strings.TrimSpace(arr[0]))
		_, ok := f.known[Feature(k)]
		if !ok {
			return fmt.Errorf("unrecognized key: %s", k)
		}
		if len(arr) != 2 {
			return fmt.Errorf("missing bool value for %s", k)
		}
		v := strings.TrimSpace(arr[1])
		boolValue, err := strconv.ParseBool(v)
		if err != nil {
			return fmt.Errorf("invalid value of %s: %s, err: %v", k, v, err)
		}
		f.enabled[k] = boolValue

		// Handle "special" features like "all alpha gates"
		if fn, found := f.special[k]; found {
			fn(f, boolValue)
		}
	}

	glog.Infof("feature gates: %v", f.enabled)
	return nil
}

func (f *featureGate) String() string {
	pairs := []string{}
	for k, v := range f.enabled {
		pairs = append(pairs, fmt.Sprintf("%s=%t", k, v))
	}
	sort.Strings(pairs)
	return strings.Join(pairs, ",")
}

func (f *featureGate) Type() string {
	return "mapStringBool"
}

func (f *featureGate) Add(features map[Feature]FeatureSpec) error {
	if f.closed {
		return fmt.Errorf("cannot add a feature gate after adding it to the flag set")
	}

	for name, spec := range features {
		if existingSpec, found := f.known[name]; found {
			if existingSpec == spec {
				continue
			}
			return fmt.Errorf("feature gate %q with different spec already exists: %v", name, existingSpec)
		}

		f.known[name] = spec
	}
	return nil
}

func (f *featureGate) Enabled(key Feature) bool {
	defaultValue := f.known[key].Default
	if f.enabled != nil {
		if v, ok := f.enabled[key]; ok {
			return v
		}
	}
	return defaultValue
}

// AddFlag adds a flag for setting global feature gates to the specified FlagSet.
func (f *featureGate) AddFlag(fs *pflag.FlagSet) {
	f.closed = true

	known := f.KnownFeatures()
	fs.Var(f, flagName, ""+
		"A set of key=value pairs that describe feature gates for alpha/experimental features. "+
		"Options are:\n"+strings.Join(known, "\n"))
}

// Returns a string describing the FeatureGate's known features.
func (f *featureGate) KnownFeatures() []string {
	var known []string
	for k, v := range f.known {
		pre := ""
		if v.PreRelease != GA {
			pre = fmt.Sprintf("%s - ", v.PreRelease)
		}
		known = append(known, fmt.Sprintf("%s=true|false (%sdefault=%t)", k, pre, v.Default))
	}
	sort.Strings(known)
	return known
}
<commit_msg>FeatureGate: update comments<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package feature

import (
	"fmt"
	"sort"
	"strconv"
	"strings"

	"github.com/golang/glog"
	"github.com/spf13/pflag"
)

type Feature string

const (
	flagName = "feature-gates"

	// allAlphaGate is a global toggle for alpha features. Per-feature key
	// values override the default set by allAlphaGate. Examples:
	//   AllAlpha=false,NewFeature=true  will result in newFeature=true
	//   AllAlpha=true,NewFeature=false  will result in newFeature=false
	allAlphaGate Feature = "AllAlpha"
)

var (
	// The generic features.
	defaultFeatures = map[Feature]FeatureSpec{
		allAlphaGate: {Default: false, PreRelease: Alpha},
	}

	// Special handling for a few gates.
	specialFeatures = map[Feature]func(f *featureGate, val bool){
		allAlphaGate: setUnsetAlphaGates,
	}

	// DefaultFeatureGate is a shared global FeatureGate.
	DefaultFeatureGate FeatureGate = NewFeatureGate()
)

type FeatureSpec struct {
	Default    bool
	PreRelease prerelease
}

type prerelease string

const (
	// Values for PreRelease.
	Alpha = prerelease("ALPHA")
	Beta  = prerelease("BETA")
	GA    = prerelease("")
)

// FeatureGate parses and stores flag gates for known features from
// a string like feature1=true,feature2=false,...
type FeatureGate interface {
	// AddFlag adds a flag for setting global feature gates to the specified FlagSet.
	AddFlag(fs *pflag.FlagSet)
	// Set parses and stores flag gates for known features
	// from a string like feature1=true,feature2=false,...
	Set(value string) error
	// Enabled returns true if the key is enabled.
	Enabled(key Feature) bool
	// Add adds features to the featureGate.
	Add(features map[Feature]FeatureSpec) error
	// KnownFeatures returns a slice of strings describing the FeatureGate's known features.
	KnownFeatures() []string
}

// featureGate implements FeatureGate as well as pflag.Value for flag parsing.
type featureGate struct {
	known   map[Feature]FeatureSpec
	special map[Feature]func(*featureGate, bool)
	enabled map[Feature]bool

	// is set to true when AddFlag is called. Note: initialization is not go-routine safe, lookup is
	closed bool
}

func setUnsetAlphaGates(f *featureGate, val bool) {
	for k, v := range f.known {
		if v.PreRelease == Alpha {
			if _, found := f.enabled[k]; !found {
				f.enabled[k] = val
			}
		}
	}
}

// Set, String, and Type implement pflag.Value
var _ pflag.Value = &featureGate{}

func NewFeatureGate() *featureGate {
	f := &featureGate{
		known:   map[Feature]FeatureSpec{},
		special: specialFeatures,
		enabled: map[Feature]bool{},
	}
	for k, v := range defaultFeatures {
		f.known[k] = v
	}
	return f
}

// Set Parses a string of the form "key1=value1,key2=value2,..." into a
// map[string]bool of known keys or returns an error.
func (f *featureGate) Set(value string) error {
	for _, s := range strings.Split(value, ",") {
		if len(s) == 0 {
			continue
		}
		arr := strings.SplitN(s, "=", 2)
		k := Feature(strings.TrimSpace(arr[0]))
		_, ok := f.known[Feature(k)]
		if !ok {
			return fmt.Errorf("unrecognized key: %s", k)
		}
		if len(arr) != 2 {
			return fmt.Errorf("missing bool value for %s", k)
		}
		v := strings.TrimSpace(arr[1])
		boolValue, err := strconv.ParseBool(v)
		if err != nil {
			return fmt.Errorf("invalid value of %s: %s, err: %v", k, v, err)
		}
		f.enabled[k] = boolValue

		// Handle "special" features like "all alpha gates"
		if fn, found := f.special[k]; found {
			fn(f, boolValue)
		}
	}

	glog.Infof("feature gates: %v", f.enabled)
	return nil
}

// String returns a string containing all enabled feature gates, formatted as "key1=value1,key2=value2,...".
func (f *featureGate) String() string {
	pairs := []string{}
	for k, v := range f.enabled {
		pairs = append(pairs, fmt.Sprintf("%s=%t", k, v))
	}
	sort.Strings(pairs)
	return strings.Join(pairs, ",")
}

func (f *featureGate) Type() string {
	return "mapStringBool"
}

// Add adds features to the featureGate.
func (f *featureGate) Add(features map[Feature]FeatureSpec) error {
	if f.closed {
		return fmt.Errorf("cannot add a feature gate after adding it to the flag set")
	}

	for name, spec := range features {
		if existingSpec, found := f.known[name]; found {
			if existingSpec == spec {
				continue
			}
			return fmt.Errorf("feature gate %q with different spec already exists: %v", name, existingSpec)
		}

		f.known[name] = spec
	}
	return nil
}

// Enabled returns true if the key is enabled.
func (f *featureGate) Enabled(key Feature) bool {
	defaultValue := f.known[key].Default
	if f.enabled != nil {
		if v, ok := f.enabled[key]; ok {
			return v
		}
	}
	return defaultValue
}

// AddFlag adds a flag for setting global feature gates to the specified FlagSet.
func (f *featureGate) AddFlag(fs *pflag.FlagSet) {
	f.closed = true

	known := f.KnownFeatures()
	fs.Var(f, flagName, ""+
		"A set of key=value pairs that describe feature gates for alpha/experimental features. "+
		"Options are:\n"+strings.Join(known, "\n"))
}

// KnownFeatures returns a slice of strings describing the FeatureGate's known features.
func (f *featureGate) KnownFeatures() []string {
	var known []string
	for k, v := range f.known {
		pre := ""
		if v.PreRelease != GA {
			pre = fmt.Sprintf("%s - ", v.PreRelease)
		}
		known = append(known, fmt.Sprintf("%s=true|false (%sdefault=%t)", k, pre, v.Default))
	}
	sort.Strings(known)
	return known
}
<|endoftext|>


<commit_before>// Package patreon implements the OAuth protocol for authenticating users through Patreon.
package patreon

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/markbates/goth"
	"golang.org/x/oauth2"
)

const (
	// AuthorizationURL specifies Patreon's OAuth2 authorization endpoint (see https://tools.ietf.org/html/rfc6749#section-3.1).
	// See Example_refreshToken for examples.
	authorizationURL = "https://www.patreon.com/oauth2/authorize"

	// AccessTokenURL specifies Patreon's OAuth2 token endpoint (see https://tools.ietf.org/html/rfc6749#section-3.2).
	// See Example_refreshToken for examples.
	accessTokenURL = "https://www.patreon.com/api/oauth2/token"

	profileURL = "https://www.patreon.com/api/oauth2/v2/identity"
)

//goland:noinspection GoUnusedConst
const (
	// ScopeIdentity provides read access to data about the user. See the /identity endpoint documentation for details about what data is available.
	ScopeIdentity = "identity"

	// ScopeIdentityEmail provides read access to the user’s email.
	ScopeIdentityEmail = "identity[email]"

	// ScopeIdentityMemberships provides read access to the user’s memberships.
	ScopeIdentityMemberships = "identity.memberships"

	// ScopeCampaigns provides read access to basic campaign data. See the /campaign endpoint documentation for details about what data is available.
	ScopeCampaigns = "campaigns"

	// ScopeCampaignsWebhook provides read, write, update, and delete access to the campaign’s webhooks created by the client.
	ScopeCampaignsWebhook = "w:campaigns.webhook"

	// ScopeCampaignsMembers provides read access to data about a campaign’s members. See the /members endpoint documentation for details about what data is available. Also allows the same information to be sent via webhooks created by your client.
	ScopeCampaignsMembers = "campaigns.members"

	// ScopeCampaignsMembersEmail provides read access to the member’s email. Also allows the same information to be sent via webhooks created by your client.
	ScopeCampaignsMembersEmail = "campaigns.members[email]"

	// ScopeCampaignsMembersAddress provides read access to the member’s address, if an address was collected in the pledge flow. Also allows the same information to be sent via webhooks created by your client.
	ScopeCampaignsMembersAddress = "campaigns.members.address"

	// ScopeCampaignsPosts provides read access to the posts on a campaign.
	ScopeCampaignsPosts = "campaigns.posts"
)

// New creates a new Patreon provider and sets up important connection details.
// You should always call `Patreon.New` to get a new Provider.  Never try to
// create one manually.
func New(clientKey, secret, callbackURL string, scopes ...string) *Provider {
	p := &Provider{
		ClientKey:    clientKey,
		Secret:       secret,
		CallbackURL:  callbackURL,
		providerName: "patreon",
	}
	p.config = newConfig(p, scopes)
	return p
}

// Provider is the implementation of `goth.Provider` for accessing Patreon.
type Provider struct {
	ClientKey    string
	Secret       string
	CallbackURL  string
	HTTPClient   *http.Client
	config       *oauth2.Config
	providerName string
}

// Name gets the name used to retrieve this provider.
func (p *Provider) Name() string {
	return p.providerName
}

// SetName is to update the name of the provider (needed in case of multiple providers of 1 type)
func (p *Provider) SetName(name string) {
	p.providerName = name
}

func (p *Provider) Client() *http.Client {
	return goth.HTTPClientWithFallBack(p.HTTPClient)
}

// Debug is a no-op for the Patreon package.
func (p *Provider) Debug(debug bool) {}

// BeginAuth asks Patreon for an authentication end-point.
func (p *Provider) BeginAuth(state string) (goth.Session, error) {
	url := p.config.AuthCodeURL(state)
	session := &Session{
		AuthURL: url,
	}
	return session, nil
}

// FetchUser will go to Patreon and access basic information about the user.
func (p *Provider) FetchUser(session goth.Session) (goth.User, error) {
	s := session.(*Session)
	user := goth.User{
		AccessToken:  s.AccessToken,
		Provider:     p.Name(),
		RefreshToken: s.RefreshToken,
		ExpiresAt:    s.ExpiresAt,
	}

	if user.AccessToken == "" {
		// data is not yet retrieved since accessToken is still empty
		return user, fmt.Errorf("%s cannot get user information without accessToken", p.providerName)
	}

	req, err := http.NewRequest("GET", profileURL, nil)
	if err != nil {
		return user, err
	}
	req.Header.Set("Authorization", "Bearer "+s.AccessToken)
	resp, err := p.Client().Do(req)
	if err != nil {
		return user, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return user, fmt.Errorf("%s responded with a %d trying to fetch user information", p.providerName, resp.StatusCode)
	}

	u := User{}
	err = json.NewDecoder(resp.Body).Decode(&u)
	if err != nil {
		return user, err
	}

	user.Name = u.Data.Attributes.FullName
	user.Email = u.Data.Attributes.Email
	user.UserID = u.Data.ID

	return user, err
}

func newConfig(p *Provider, scopes []string) *oauth2.Config {
	c := &oauth2.Config{
		ClientID:     p.ClientKey,
		ClientSecret: p.Secret,
		RedirectURL:  p.CallbackURL,
		Endpoint: oauth2.Endpoint{
			AuthURL:  authorizationURL,
			TokenURL: accessTokenURL,
		},
		Scopes: []string{ScopeIdentity, ScopeIdentityEmail},
	}

	defaultScopes := map[string]struct{}{
		ScopeIdentity:      {},
		ScopeIdentityEmail: {},
	}

	for _, scope := range scopes {
		if _, exists := defaultScopes[scope]; !exists {
			c.Scopes = append(c.Scopes, scope)
		}
	}

	return c
}

// RefreshTokenAvailable refresh token is provided by auth provider or not
func (p *Provider) RefreshTokenAvailable() bool {
	return true
}

// RefreshToken get new access token based on the refresh token
func (p *Provider) RefreshToken(refreshToken string) (*oauth2.Token, error) {
	token := &oauth2.Token{RefreshToken: refreshToken}
	ts := p.config.TokenSource(goth.ContextForClient(p.Client()), token)
	newToken, err := ts.Token()
	if err != nil {
		return nil, err
	}
	return newToken, err
}

type User struct {
	Data struct {
		Attributes struct {
			Email    string `json:"email"`
			FullName string `json:"full_name"`
		} `json:"attributes"`
		ID string `json:"id"`
	} `json:"data"`
}
<commit_msg>Scopes len check<commit_after>// Package patreon implements the OAuth protocol for authenticating users through Patreon.
package patreon

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/markbates/goth"
	"golang.org/x/oauth2"
)

const (
	// AuthorizationURL specifies Patreon's OAuth2 authorization endpoint (see https://tools.ietf.org/html/rfc6749#section-3.1).
	// See Example_refreshToken for examples.
	authorizationURL = "https://www.patreon.com/oauth2/authorize"

	// AccessTokenURL specifies Patreon's OAuth2 token endpoint (see https://tools.ietf.org/html/rfc6749#section-3.2).
	// See Example_refreshToken for examples.
	accessTokenURL = "https://www.patreon.com/api/oauth2/token"

	profileURL = "https://www.patreon.com/api/oauth2/v2/identity"
)

//goland:noinspection GoUnusedConst
const (
	// ScopeIdentity provides read access to data about the user. See the /identity endpoint documentation for details about what data is available.
	ScopeIdentity = "identity"

	// ScopeIdentityEmail provides read access to the user’s email.
	ScopeIdentityEmail = "identity[email]"

	// ScopeIdentityMemberships provides read access to the user’s memberships.
	ScopeIdentityMemberships = "identity.memberships"

	// ScopeCampaigns provides read access to basic campaign data. See the /campaign endpoint documentation for details about what data is available.
	ScopeCampaigns = "campaigns"

	// ScopeCampaignsWebhook provides read, write, update, and delete access to the campaign’s webhooks created by the client.
	ScopeCampaignsWebhook = "w:campaigns.webhook"

	// ScopeCampaignsMembers provides read access to data about a campaign’s members. See the /members endpoint documentation for details about what data is available. Also allows the same information to be sent via webhooks created by your client.
	ScopeCampaignsMembers = "campaigns.members"

	// ScopeCampaignsMembersEmail provides read access to the member’s email. Also allows the same information to be sent via webhooks created by your client.
	ScopeCampaignsMembersEmail = "campaigns.members[email]"

	// ScopeCampaignsMembersAddress provides read access to the member’s address, if an address was collected in the pledge flow. Also allows the same information to be sent via webhooks created by your client.
	ScopeCampaignsMembersAddress = "campaigns.members.address"

	// ScopeCampaignsPosts provides read access to the posts on a campaign.
	ScopeCampaignsPosts = "campaigns.posts"
)

// New creates a new Patreon provider and sets up important connection details.
// You should always call `Patreon.New` to get a new Provider.  Never try to
// create one manually.
func New(clientKey, secret, callbackURL string, scopes ...string) *Provider {
	p := &Provider{
		ClientKey:    clientKey,
		Secret:       secret,
		CallbackURL:  callbackURL,
		providerName: "patreon",
	}
	p.config = newConfig(p, scopes)
	return p
}

// Provider is the implementation of `goth.Provider` for accessing Patreon.
type Provider struct {
	ClientKey    string
	Secret       string
	CallbackURL  string
	HTTPClient   *http.Client
	config       *oauth2.Config
	providerName string
}

// Name gets the name used to retrieve this provider.
func (p *Provider) Name() string {
	return p.providerName
}

// SetName is to update the name of the provider (needed in case of multiple providers of 1 type)
func (p *Provider) SetName(name string) {
	p.providerName = name
}

func (p *Provider) Client() *http.Client {
	return goth.HTTPClientWithFallBack(p.HTTPClient)
}

// Debug is a no-op for the Patreon package.
func (p *Provider) Debug(debug bool) {}

// BeginAuth asks Patreon for an authentication end-point.
func (p *Provider) BeginAuth(state string) (goth.Session, error) {
	url := p.config.AuthCodeURL(state)
	session := &Session{
		AuthURL: url,
	}
	return session, nil
}

// FetchUser will go to Patreon and access basic information about the user.
func (p *Provider) FetchUser(session goth.Session) (goth.User, error) {
	s := session.(*Session)
	user := goth.User{
		AccessToken:  s.AccessToken,
		Provider:     p.Name(),
		RefreshToken: s.RefreshToken,
		ExpiresAt:    s.ExpiresAt,
	}

	if user.AccessToken == "" {
		// data is not yet retrieved since accessToken is still empty
		return user, fmt.Errorf("%s cannot get user information without accessToken", p.providerName)
	}

	req, err := http.NewRequest("GET", profileURL, nil)
	if err != nil {
		return user, err
	}
	req.Header.Set("Authorization", "Bearer "+s.AccessToken)
	resp, err := p.Client().Do(req)
	if err != nil {
		return user, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return user, fmt.Errorf("%s responded with a %d trying to fetch user information", p.providerName, resp.StatusCode)
	}

	u := User{}
	err = json.NewDecoder(resp.Body).Decode(&u)
	if err != nil {
		return user, err
	}

	user.Name = u.Data.Attributes.FullName
	user.Email = u.Data.Attributes.Email
	user.UserID = u.Data.ID

	return user, err
}

func newConfig(p *Provider, scopes []string) *oauth2.Config {
	c := &oauth2.Config{
		ClientID:     p.ClientKey,
		ClientSecret: p.Secret,
		RedirectURL:  p.CallbackURL,
		Endpoint: oauth2.Endpoint{
			AuthURL:  authorizationURL,
			TokenURL: accessTokenURL,
		},
		Scopes: []string{ScopeIdentity, ScopeIdentityEmail},
	}

	defaultScopes := map[string]struct{}{
		ScopeIdentity:      {},
		ScopeIdentityEmail: {},
	}

	if len(scopes) > 0 {
		for _, scope := range scopes {
			c.Scopes = append(c.Scopes, scope)
		}
	}
	return c
}

// RefreshTokenAvailable refresh token is provided by auth provider or not
func (p *Provider) RefreshTokenAvailable() bool {
	return true
}

// RefreshToken get new access token based on the refresh token
func (p *Provider) RefreshToken(refreshToken string) (*oauth2.Token, error) {
	token := &oauth2.Token{RefreshToken: refreshToken}
	ts := p.config.TokenSource(goth.ContextForClient(p.Client()), token)
	newToken, err := ts.Token()
	if err != nil {
		return nil, err
	}
	return newToken, err
}

type User struct {
	Data struct {
		Attributes struct {
			Email    string `json:"email"`
			FullName string `json:"full_name"`
		} `json:"attributes"`
		ID string `json:"id"`
	} `json:"data"`
}
<|endoftext|>


<commit_before>package models

import (
	"github.com/nsqio/go-nsq"
)

type IngestManifest struct {
	NSQMessage         *nsq.Message  `json:"-"`
	WorkItemId         int
	S3Bucket           string
	S3Key              string
	ETag               string
	Fetch              *WorkSummary
	Untar              *WorkSummary
	Validate           *WorkSummary
	Store              *WorkSummary
	Record             *WorkSummary
	Replicate          *WorkSummary
	Cleanup            *WorkSummary
	Object             *IntellectualObject
}

func NewIngestManifest() (*IngestManifest) {
	return &IngestManifest{
		Fetch: NewWorkSummary(),
		Untar: NewWorkSummary(),
		Validate: NewWorkSummary(),
		Store: NewWorkSummary(),
		Record: NewWorkSummary(),
		Replicate: NewWorkSummary(),
		Cleanup: NewWorkSummary(),
		Object: NewIntellectualObject(),
	}
}

// TODO: Write method to get first error, all errors.
<commit_msg>Moved NSQMessage from IngestManifest to FetchData<commit_after>package models

type IngestManifest struct {
	WorkItemId         int
	S3Bucket           string
	S3Key              string
	ETag               string
	Fetch              *WorkSummary
	Untar              *WorkSummary
	Validate           *WorkSummary
	Store              *WorkSummary
	Record             *WorkSummary
	Replicate          *WorkSummary
	Cleanup            *WorkSummary
	Object             *IntellectualObject
}

func NewIngestManifest() (*IngestManifest) {
	return &IngestManifest{
		Fetch: NewWorkSummary(),
		Untar: NewWorkSummary(),
		Validate: NewWorkSummary(),
		Store: NewWorkSummary(),
		Record: NewWorkSummary(),
		Replicate: NewWorkSummary(),
		Cleanup: NewWorkSummary(),
		Object: NewIntellectualObject(),
	}
}

// TODO: Write method to get first error, all errors.
<|endoftext|>