<commit_before>package redis

import (
	"errors"
	"fmt"
	"net/url"
	"strconv"
	"strings"

	"github.com/go-redis/redis/v7"
	"github.com/koron/nvgd/internal/httperror"
	"github.com/koron/nvgd/resource"
)

type getHandler func(*redis.Client, string, []string) (*resource.Resource, error)

var getHandlers = map[string]getHandler{
	"string": getString,
	"list":   getList,
	"set":    getSet,
	"zset":   getZset,
	"hash":   getHash,
	"none":   getNone,
}

func get(c *redis.Client, args []string) (*resource.Resource, error) {
	if len(args) < 1 {
		return nil, errors.New("require a key at least")
	}
	key, err := url.PathUnescape(args[0])
	if err != nil {
		return nil, fmt.Errorf("key contains invalid sequence: %s", err)
	}

	typ, err := c.Type(key).Result()
	if err != nil {
		return nil, fmt.Errorf("failed to TYPE: %s", err)
	}
	h, ok := getHandlers[strings.ToLower(typ)]
	if !ok {
		return nil, fmt.Errorf("unsupported redis value type: %s", typ)
	}
	return h(c, key, args[1:])
}

func getString(c *redis.Client, k string, args []string) (*resource.Resource, error) {
	// GET
	if len(args) == 0 {
		s, err := c.Get(k).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(s), nil
	}

	// GETBIT
	if len(args) == 1 {
		off, err := strconv.ParseInt(args[0], 10, 64)
		if err != nil {
			return nil, err
		}
		n, err := c.GetBit(k, off).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(strconv.FormatInt(n, 10)), nil
	}

	// GETRANGE
	if len(args) == 2 {
		start, err := strconv.ParseInt(args[0], 10, 64)
		if err != nil {
			return nil, err
		}
		end, err := strconv.ParseInt(args[1], 10, 64)
		if err != nil {
			return nil, err
		}
		s, err := c.GetRange(k, start, end).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(s), nil
	}

	return nil, errors.New("too many arguments")
}

func getList(c *redis.Client, k string, args []string) (*resource.Resource, error) {
	// LLEN
	if len(args) == 0 {
		n, err := c.LLen(k).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(strconv.FormatInt(n, 10)), nil
	}

	// LINDEX
	if len(args) == 1 {
		index, err := strconv.ParseInt(args[0], 10, 64)
		if err != nil {
			return nil, err
		}
		s, err := c.LIndex(k, index).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(s), nil
	}

	// LRANGE
	if len(args) == 2 {
		start, err := strconv.ParseInt(args[0], 10, 64)
		if err != nil {
			return nil, err
		}
		stop, err := strconv.ParseInt(args[1], 10, 64)
		if err != nil {
			return nil, err
		}
		ss, err := c.LRange(k, start, stop).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(strings.Join(ss, "\n")), nil
	}

	return nil, errors.New("too many arguments")
}

func getSet(c *redis.Client, k string, args []string) (*resource.Resource, error) {
	// SCARD
	if len(args) == 0 {
		n, err := c.SCard(k).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(strconv.FormatInt(n, 10)), nil
	}

	// SISMEMBER
	if len(args) == 1 {
		member := args[0]
		b, err := c.SIsMember(k, member).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(strconv.FormatBool(b)), nil
	}

	return nil, errors.New("too many arguments")
}

func getZset(c *redis.Client, k string, args []string) (*resource.Resource, error) {
	// ZCARD
	if len(args) == 0 {
		n, err := c.ZCard(k).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(strconv.FormatInt(n, 10)), nil
	}

	// ZRANK
	if len(args) == 1 {
		member := args[0]
		n, err := c.ZRank(k, member).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(strconv.FormatInt(n, 10)), nil
	}

	// ZRANGE
	if len(args) == 2 {
		start, err := strconv.ParseInt(args[0], 10, 64)
		if err != nil {
			return nil, err
		}
		stop, err := strconv.ParseInt(args[1], 10, 64)
		if err != nil {
			return nil, err
		}
		ss, err := c.ZRange(k, start, stop).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(strings.Join(ss, "\n")), nil
	}

	return nil, errors.New("too many arguments")
}

func getHash(c *redis.Client, k string, args []string) (*resource.Resource, error) {
	// HLEN
	if len(args) == 0 {
		n, err := c.HLen(k).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(strconv.FormatInt(n, 10)), nil
	}

	// HGET
	if len(args) == 0 {
		member := args[0]
		s, err := c.HGet(k, member).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(s), nil
	}

	return nil, errors.New("too many arguments")
}

func getNone(c *redis.Client, k string, args []string) (*resource.Resource, error) {
	return nil, httperror.Newf(404, "not found a key: %s", k)
}
<commit_msg>Fix a condition of redis HGet<commit_after>package redis

import (
	"errors"
	"fmt"
	"net/url"
	"strconv"
	"strings"

	"github.com/go-redis/redis/v7"
	"github.com/koron/nvgd/internal/httperror"
	"github.com/koron/nvgd/resource"
)

type getHandler func(*redis.Client, string, []string) (*resource.Resource, error)

var getHandlers = map[string]getHandler{
	"string": getString,
	"list":   getList,
	"set":    getSet,
	"zset":   getZset,
	"hash":   getHash,
	"none":   getNone,
}

func get(c *redis.Client, args []string) (*resource.Resource, error) {
	if len(args) < 1 {
		return nil, errors.New("require a key at least")
	}
	key, err := url.PathUnescape(args[0])
	if err != nil {
		return nil, fmt.Errorf("key contains invalid sequence: %s", err)
	}

	typ, err := c.Type(key).Result()
	if err != nil {
		return nil, fmt.Errorf("failed to TYPE: %s", err)
	}
	h, ok := getHandlers[strings.ToLower(typ)]
	if !ok {
		return nil, fmt.Errorf("unsupported redis value type: %s", typ)
	}
	return h(c, key, args[1:])
}

func getString(c *redis.Client, k string, args []string) (*resource.Resource, error) {
	// GET
	if len(args) == 0 {
		s, err := c.Get(k).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(s), nil
	}

	// GETBIT
	if len(args) == 1 {
		off, err := strconv.ParseInt(args[0], 10, 64)
		if err != nil {
			return nil, err
		}
		n, err := c.GetBit(k, off).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(strconv.FormatInt(n, 10)), nil
	}

	// GETRANGE
	if len(args) == 2 {
		start, err := strconv.ParseInt(args[0], 10, 64)
		if err != nil {
			return nil, err
		}
		end, err := strconv.ParseInt(args[1], 10, 64)
		if err != nil {
			return nil, err
		}
		s, err := c.GetRange(k, start, end).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(s), nil
	}

	return nil, errors.New("too many arguments")
}

func getList(c *redis.Client, k string, args []string) (*resource.Resource, error) {
	// LLEN
	if len(args) == 0 {
		n, err := c.LLen(k).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(strconv.FormatInt(n, 10)), nil
	}

	// LINDEX
	if len(args) == 1 {
		index, err := strconv.ParseInt(args[0], 10, 64)
		if err != nil {
			return nil, err
		}
		s, err := c.LIndex(k, index).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(s), nil
	}

	// LRANGE
	if len(args) == 2 {
		start, err := strconv.ParseInt(args[0], 10, 64)
		if err != nil {
			return nil, err
		}
		stop, err := strconv.ParseInt(args[1], 10, 64)
		if err != nil {
			return nil, err
		}
		ss, err := c.LRange(k, start, stop).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(strings.Join(ss, "\n")), nil
	}

	return nil, errors.New("too many arguments")
}

func getSet(c *redis.Client, k string, args []string) (*resource.Resource, error) {
	// SCARD
	if len(args) == 0 {
		n, err := c.SCard(k).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(strconv.FormatInt(n, 10)), nil
	}

	// SISMEMBER
	if len(args) == 1 {
		member := args[0]
		b, err := c.SIsMember(k, member).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(strconv.FormatBool(b)), nil
	}

	return nil, errors.New("too many arguments")
}

func getZset(c *redis.Client, k string, args []string) (*resource.Resource, error) {
	// ZCARD
	if len(args) == 0 {
		n, err := c.ZCard(k).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(strconv.FormatInt(n, 10)), nil
	}

	// ZRANK
	if len(args) == 1 {
		member := args[0]
		n, err := c.ZRank(k, member).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(strconv.FormatInt(n, 10)), nil
	}

	// ZRANGE
	if len(args) == 2 {
		start, err := strconv.ParseInt(args[0], 10, 64)
		if err != nil {
			return nil, err
		}
		stop, err := strconv.ParseInt(args[1], 10, 64)
		if err != nil {
			return nil, err
		}
		ss, err := c.ZRange(k, start, stop).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(strings.Join(ss, "\n")), nil
	}

	return nil, errors.New("too many arguments")
}

func getHash(c *redis.Client, k string, args []string) (*resource.Resource, error) {
	// HLEN
	if len(args) == 0 {
		n, err := c.HLen(k).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(strconv.FormatInt(n, 10)), nil
	}

	// HGET
	if len(args) == 1 {
		member := args[0]
		s, err := c.HGet(k, member).Result()
		if err != nil {
			return nil, err
		}
		return resource.NewString(s), nil
	}

	return nil, errors.New("too many arguments")
}

func getNone(c *redis.Client, k string, args []string) (*resource.Resource, error) {
	return nil, httperror.Newf(404, "not found a key: %s", k)
}
<|endoftext|>


<commit_before>package html2data

import (
	"reflect"
	"strings"
	"testing"
)

func Test_GetDataSingle(t *testing.T) {
	test_data := []struct {
		html string
		css  string
		out  string
	}{
		{
			"one
head
two",
			"h1",
			"head",
		}, {
			"one
head
two
head2
",
			"h1",
			"head",
		},
	}

	for _, item := range test_data {
		reader := strings.NewReader(item.html)
		out, err := FromReader(reader).GetDataSingle(item.css)

		if err != nil {
			t.Errorf("Got error:", err)
		}

		if out != item.out {
			t.Errorf("expected: %#v, real: %#v", item.out, out)
		}
	}
}

func Test_GetData(t *testing.T) {
	test_data := []struct {
		html string
		css  map[string]string
		out  map[string][]string
	}{
		{
			"one
head
two",
			map[string]string{"h1": "h1"},
			map[string][]string{"h1": {"head"}},
		}, {
			"<title>Title</title>one
head
two
Head 2
",
			map[string]string{"title": "title", "h1": "h1"},
			map[string][]string{"title": {"Title"}, "h1": {"head", "Head 2"}},
		},
	}

	for _, item := range test_data {
		reader := strings.NewReader(item.html)
		out, err := FromReader(reader).GetData(item.css)

		if err != nil {
			t.Errorf("Got error:", err)
		}

		if !reflect.DeepEqual(item.out, out) {
			t.Errorf("expected: %#v, real: %#v", item.out, out)
		}
	}
}
<commit_msg>Changes after golint/go vet<commit_after>package html2data

import (
	"reflect"
	"strings"
	"testing"
)

func Test_GetDataSingle(t *testing.T) {
	testData := []struct {
		html string
		css  string
		out  string
	}{
		{
			"one
head
two",
			"h1",
			"head",
		}, {
			"one
head
two
head2
",
			"h1",
			"head",
		},
	}

	for _, item := range testData {
		reader := strings.NewReader(item.html)
		out, err := FromReader(reader).GetDataSingle(item.css)

		if err != nil {
			t.Errorf("Got error: %s", err)
		}

		if out != item.out {
			t.Errorf("expected: %#v, real: %#v", item.out, out)
		}
	}
}

func Test_GetData(t *testing.T) {
	testData := []struct {
		html string
		css  map[string]string
		out  map[string][]string
	}{
		{
			"one
head
two",
			map[string]string{"h1": "h1"},
			map[string][]string{"h1": {"head"}},
		}, {
			"<title>Title</title>one
head
two
Head 2
",
			map[string]string{"title": "title", "h1": "h1"},
			map[string][]string{"title": {"Title"}, "h1": {"head", "Head 2"}},
		},
	}

	for _, item := range testData {
		reader := strings.NewReader(item.html)
		out, err := FromReader(reader).GetData(item.css)

		if err != nil {
			t.Errorf("Got error: %s", err)
		}

		if !reflect.DeepEqual(item.out, out) {
			t.Errorf("expected: %#v, real: %#v", item.out, out)
		}
	}
}
<|endoftext|>


<commit_before>package memberlist

import (
	"context"
	"fmt"
	"net"
	"strconv"
	"strings"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/prometheus/alertmanager/cluster"
	"github.com/prometheus/client_golang/prometheus"
	"k8s.io/klog/glog"
	"k8s.io/kops/protokube/pkg/gossip"
)

type MemberlistGossiper struct {
	peer       *cluster.Peer
	seeds      gossip.SeedProvider
	listenPort int

	state *state
	bcast func([]byte)
}

func NewMemberlistGossiper(listen string, channelName string, nodeName string, password []byte, seeds gossip.SeedProvider) (*MemberlistGossiper, error) {
	_, portString, err := net.SplitHostPort(listen)
	if err != nil {
		return nil, fmt.Errorf("cannot parse -listen flag: %v", listen)
	}
	port, err := strconv.Atoi(portString)
	if err != nil {
		return nil, fmt.Errorf("cannot parse -listen flag: %v", listen)
	}

	initialPeers, err := seeds.GetSeeds()
	if err != nil {
		return nil, err
	}
	// TODO: get port from other config?
	for i, initialPeer := range initialPeers {
		if !strings.Contains(initialPeer, ":") {
			initialPeers[i] = initialPeer + ":" + strconv.Itoa(port)
		}
	}

	peer, err := cluster.Create(
		log.NewNopLogger(),
		prometheus.DefaultRegisterer,
		listen,
		"", //*clusterAdvertiseAddr,
		initialPeers,
		true,
		cluster.DefaultPushPullInterval,
		cluster.DefaultGossipInterval,
		cluster.DefaultTcpTimeout,
		cluster.DefaultProbeTimeout,
		cluster.DefaultProbeInterval,
	)
	if err != nil {
		return nil, err
	}

	s := &state{}

	return &MemberlistGossiper{
		peer:       peer,
		seeds:      seeds,
		listenPort: port,
		state:      s,
		bcast:      peer.AddState(channelName, s, prometheus.DefaultRegisterer).Broadcast,
	}, nil
}

func (g *MemberlistGossiper) Start() error {
	if err := g.peer.Join(cluster.DefaultReconnectInterval, cluster.DefaultReconnectTimeout); err != nil {
		return err
	}
	ctx, cancel := context.WithTimeout(context.Background(), cluster.DefaultPushPullInterval)
	defer func() {
		cancel()
		if err := g.peer.Leave(10 * time.Second); err != nil {
			glog.V(2).Infof("unable to leave gossip mesh: %v", err)
		}
	}()

	g.peer.Settle(ctx, cluster.DefaultGossipInterval*10)
	g.runSeeding()

	return nil
}

func (g *MemberlistGossiper) runSeeding() {
SEED_LOOP:
	for {
		glog.V(2).Infof("Querying for seeds")

		seeds, err := g.seeds.GetSeeds()
		if err != nil {
			glog.Warningf("error getting seeds: %v", err)
			time.Sleep(1 * time.Minute)
			continue
		}
		glog.Infof("Got seeds: %s", seeds)

		for _, seed := range seeds {
			if !strings.Contains(seed, ":") {
				seed = seed + ":" + strconv.Itoa(g.listenPort)
			}
			if err := g.peer.AddPeer(seed); err != nil {
				glog.Infof("error connecting to seeds: %v", err)
				time.Sleep(1 * time.Minute)
				continue SEED_LOOP
			}
		}

		glog.V(2).Infof("Seeding successful")

		// Reseed periodically, just in case of partitions
		// TODO: Make it so that only one node polls, or at least statistically get close
		time.Sleep(60 * time.Minute)
	}
}

func (g *MemberlistGossiper) Snapshot() *gossip.GossipStateSnapshot {
	return g.state.snapshot()
}

func (g *MemberlistGossiper) UpdateValues(removeKeys []string, putKeys map[string]string) error {
	glog.V(2).Infof("UpdateValues: remove=%s, put=%s", removeKeys, putKeys)
	g.state.updateValues(removeKeys, putKeys)
	b, err := g.state.MarshalBinary()
	if err != nil {
		return err
	}
	g.bcast(b)
	return nil
}
<commit_msg>Move to fork<commit_after>package memberlist

import (
	"context"
	"fmt"
	"net"
	"strconv"
	"strings"
	"time"

	"github.com/go-kit/kit/log"
	cluster "github.com/jacksontj/memberlistmesh"
	"github.com/prometheus/client_golang/prometheus"
	"k8s.io/klog/glog"
	"k8s.io/kops/protokube/pkg/gossip"
)

type MemberlistGossiper struct {
	peer       *cluster.Peer
	seeds      gossip.SeedProvider
	listenPort int

	state *state
	bcast func([]byte)
}

func NewMemberlistGossiper(listen string, channelName string, nodeName string, password []byte, seeds gossip.SeedProvider) (*MemberlistGossiper, error) {
	_, portString, err := net.SplitHostPort(listen)
	if err != nil {
		return nil, fmt.Errorf("cannot parse -listen flag: %v", listen)
	}
	port, err := strconv.Atoi(portString)
	if err != nil {
		return nil, fmt.Errorf("cannot parse -listen flag: %v", listen)
	}

	initialPeers, err := seeds.GetSeeds()
	if err != nil {
		return nil, err
	}
	// TODO: get port from other config?
	for i, initialPeer := range initialPeers {
		if !strings.Contains(initialPeer, ":") {
			initialPeers[i] = initialPeer + ":" + strconv.Itoa(port)
		}
	}

	peer, err := cluster.Create(
		log.NewNopLogger(),
		prometheus.DefaultRegisterer,
		listen,
		"", //*clusterAdvertiseAddr,
		initialPeers,
		true,
		cluster.DefaultPushPullInterval,
		cluster.DefaultGossipInterval,
		cluster.DefaultTcpTimeout,
		cluster.DefaultProbeTimeout,
		cluster.DefaultProbeInterval,
	)
	if err != nil {
		return nil, err
	}

	s := &state{}

	return &MemberlistGossiper{
		peer:       peer,
		seeds:      seeds,
		listenPort: port,
		state:      s,
		bcast:      peer.AddState(channelName, s, prometheus.DefaultRegisterer).Broadcast,
	}, nil
}

func (g *MemberlistGossiper) Start() error {
	if err := g.peer.Join(cluster.DefaultReconnectInterval, cluster.DefaultReconnectTimeout); err != nil {
		return err
	}
	ctx, cancel := context.WithTimeout(context.Background(), cluster.DefaultPushPullInterval)
	defer func() {
		cancel()
		if err := g.peer.Leave(10 * time.Second); err != nil {
			glog.V(2).Infof("unable to leave gossip mesh: %v", err)
		}
	}()

	g.peer.Settle(ctx, cluster.DefaultGossipInterval*10)
	g.runSeeding()

	return nil
}

func (g *MemberlistGossiper) runSeeding() {
SEED_LOOP:
	for {
		glog.V(2).Infof("Querying for seeds")

		seeds, err := g.seeds.GetSeeds()
		if err != nil {
			glog.Warningf("error getting seeds: %v", err)
			time.Sleep(1 * time.Minute)
			continue
		}
		glog.Infof("Got seeds: %s", seeds)

		for _, seed := range seeds {
			if !strings.Contains(seed, ":") {
				seed = seed + ":" + strconv.Itoa(g.listenPort)
			}
			if err := g.peer.AddPeer(seed); err != nil {
				glog.Infof("error connecting to seeds: %v", err)
				time.Sleep(1 * time.Minute)
				continue SEED_LOOP
			}
		}

		glog.V(2).Infof("Seeding successful")

		// Reseed periodically, just in case of partitions
		// TODO: Make it so that only one node polls, or at least statistically get close
		time.Sleep(60 * time.Minute)
	}
}

func (g *MemberlistGossiper) Snapshot() *gossip.GossipStateSnapshot {
	return g.state.snapshot()
}

func (g *MemberlistGossiper) UpdateValues(removeKeys []string, putKeys map[string]string) error {
	glog.V(2).Infof("UpdateValues: remove=%s, put=%s", removeKeys, putKeys)
	g.state.updateValues(removeKeys, putKeys)
	b, err := g.state.MarshalBinary()
	if err != nil {
		return err
	}
	g.bcast(b)
	return nil
}
<|endoftext|>


<commit_before>/*
Package bugsnag provides Bugsnag integration.

Bugsnag recommends to create a global scope error handler.
Although that works, makes writing software harder,
so this package recommends creating a separate instance:

	APIKey := "key"

	handler := bugsnag.NewHandler(APIKey)

If you need access to the underlying Notifier instance (or need more advanced construction), you can access it from the handler:

	handler.Notifier.Config.AppVersion = "1.0.0"

Bugsnag provides an extensive set of configuration options, so it might make sense to manually construct it:

	// Note: there is a conflict in package names which we resolved with the "_" (underline) prefix.
	handler := &_bugsnag.Handler {
		Notifier: bugsnag.New(bugsnag.Configuration{
			APIKey:      APIKey,
			Synchronous: true,
		})
	}
*/
package bugsnag

import bugsnag "github.com/bugsnag/bugsnag-go"

// Handler is responsible for sending errors to Bugsnag.
type Handler struct {
	Notifier *bugsnag.Notifier
}

// NewHandler creates a new Bugsnag handler.
func NewHandler(APIKey string) *Handler {
	return &Handler{
		Notifier: bugsnag.New(bugsnag.Configuration{
			APIKey:      APIKey,
			Synchronous: true,
		}),
	}
}

// Handle calls the underlying Airbrake notifier.
func (h *Handler) Handle(err error) {
	h.Notifier.Notify(err)
}
<commit_msg>Fix typo<commit_after>/*
Package bugsnag provides Bugsnag integration.

Bugsnag recommends to create a global scope error handler.
Although that works, makes writing software harder,
so this package recommends creating a separate instance:

	APIKey := "key"

	handler := bugsnag.NewHandler(APIKey)

If you need access to the underlying Notifier instance (or need more advanced construction), you can access it from the handler:

	handler.Notifier.Config.AppVersion = "1.0.0"

Bugsnag provides an extensive set of configuration options, so it might make sense to manually construct it:

	// Note: there is a conflict in package names which we resolved with the "_" (underline) prefix.
	handler := &_bugsnag.Handler {
		Notifier: bugsnag.New(bugsnag.Configuration{
			APIKey:      APIKey,
			Synchronous: true,
		})
	}
*/
package bugsnag

import bugsnag "github.com/bugsnag/bugsnag-go"

// Handler is responsible for sending errors to Bugsnag.
type Handler struct {
	Notifier *bugsnag.Notifier
}

// NewHandler creates a new Bugsnag handler.
func NewHandler(APIKey string) *Handler {
	return &Handler{
		Notifier: bugsnag.New(bugsnag.Configuration{
			APIKey:      APIKey,
			Synchronous: true,
		}),
	}
}

// Handle calls the underlying Bugsnag notifier.
func (h *Handler) Handle(err error) {
	h.Notifier.Notify(err)
}
<|endoftext|>


<commit_before>package http

import (
	"encoding/base64"
	"encoding/hex"
	"errors"
	"fmt"
	"net/http"

	"github.com/hashicorp/vault/helper/pgpkeys"
	"github.com/hashicorp/vault/vault"
)

func handleSysRekeyInit(core *vault.Core, recovery bool) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		switch {
		case recovery && !core.SealAccess().RecoveryKeySupported():
			respondError(w, http.StatusBadRequest, fmt.Errorf("recovery rekeying not supported"))
		case r.Method == "GET":
			handleSysRekeyInitGet(core, recovery, w, r)
		case r.Method == "POST" || r.Method == "PUT":
			handleSysRekeyInitPut(core, recovery, w, r)
		case r.Method == "DELETE":
			handleSysRekeyInitDelete(core, recovery, w, r)
		default:
			respondError(w, http.StatusMethodNotAllowed, nil)
		}
	})
}

func handleSysRekeyInitGet(core *vault.Core, recovery bool, w http.ResponseWriter, r *http.Request) {
	barrierConfig, err := core.SealAccess().BarrierConfig()
	if err != nil {
		respondError(w, http.StatusInternalServerError, err)
		return
	}
	if barrierConfig == nil {
		respondError(w, http.StatusBadRequest, fmt.Errorf(
			"server is not yet initialized"))
		return
	}

	// Get the rekey configuration
	rekeyConf, err := core.RekeyConfig(recovery)
	if err != nil {
		respondError(w, http.StatusInternalServerError, err)
		return
	}

	// Get the progress
	progress, err := core.RekeyProgress(recovery)
	if err != nil {
		respondError(w, http.StatusInternalServerError, err)
		return
	}

	sealThreshold, err := core.RekeyThreshold(recovery)
	if err != nil {
		respondError(w, http.StatusInternalServerError, err)
		return
	}

	// Format the status
	status := &RekeyStatusResponse{
		Started:  false,
		T:        0,
		N:        0,
		Progress: progress,
		Required: sealThreshold,
	}
	if rekeyConf != nil {
		status.Nonce = rekeyConf.Nonce
		status.Started = true
		status.T = rekeyConf.SecretThreshold
		status.N = rekeyConf.SecretShares
		if rekeyConf.PGPKeys != nil && len(rekeyConf.PGPKeys) != 0 {
			pgpFingerprints, err := pgpkeys.GetFingerprints(rekeyConf.PGPKeys, nil)
			if err != nil {
				respondError(w, http.StatusInternalServerError, err)
				return
			}
			status.PGPFingerprints = pgpFingerprints
			status.Backup = rekeyConf.Backup
		}
	}
	respondOk(w, status)
}

func handleSysRekeyInitPut(core *vault.Core, recovery bool, w http.ResponseWriter, r *http.Request) {
	// Parse the request
	var req RekeyRequest
	if err := parseRequest(r, &req); err != nil {
		respondError(w, http.StatusBadRequest, err)
		return
	}

	if req.Backup && len(req.PGPKeys) == 0 {
		respondError(w, http.StatusBadRequest, fmt.Errorf("cannot request a backup of the new keys without providing PGP keys for encryption"))
		return
	}

	// Right now we don't support this, but the rest of the code is ready for
	// when we do, hence the check below for this to be false if
	// StoredShares is greater than zero
	if core.SealAccess().StoredKeysSupported() {
		respondError(w, http.StatusBadRequest, fmt.Errorf("rekeying of barrier not supported when stored key support is available"))
		return
	}

	// Initialize the rekey
	err := core.RekeyInit(&vault.SealConfig{
		SecretShares:    req.SecretShares,
		SecretThreshold: req.SecretThreshold,
		StoredShares:    req.StoredShares,
		PGPKeys:         req.PGPKeys,
		Backup:          req.Backup,
	}, recovery)
	if err != nil {
		respondError(w, http.StatusBadRequest, err)
		return
	}

	handleSysRekeyInitGet(core, recovery, w, r)
}

func handleSysRekeyInitDelete(core *vault.Core, recovery bool, w http.ResponseWriter, r *http.Request) {
	err := core.RekeyCancel(recovery)
	if err != nil {
		respondError(w, http.StatusInternalServerError, err)
		return
	}
	respondOk(w, nil)
}

func handleSysRekeyUpdate(core *vault.Core, recovery bool) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Parse the request
		var req RekeyUpdateRequest
		if err := parseRequest(r, &req); err != nil {
			respondError(w, http.StatusBadRequest, err)
			return
		}
		if req.Key == "" {
			respondError(
				w, http.StatusBadRequest,
				errors.New("'key' must specified in request body as JSON"))
			return
		}

		// Decode the key, which is base64 or hex encoded
		min, max := core.BarrierKeyLength()
		key, err := hex.DecodeString(req.Key)
		// We check min and max here to ensure that a string that is base64
		// encoded but also valid hex will not be valid and we instead base64
		// decode it
		if err != nil || len(key) < min || len(key) > max {
			key, err = base64.StdEncoding.DecodeString(req.Key)
			if err != nil {
				respondError(
					w, http.StatusBadRequest,
					errors.New("'key' must be a valid hex or base64 string"))
				return
			}
		}

		// Use the key to make progress on rekey
		result, err := core.RekeyUpdate(key, req.Nonce, recovery)
		if err != nil {
			respondError(w, http.StatusBadRequest, err)
			return
		}

		// Format the response
		resp := &RekeyUpdateResponse{}
		if result != nil {
			resp.Complete = true
			resp.Nonce = req.Nonce
			resp.Backup = result.Backup
			resp.PGPFingerprints = result.PGPFingerprints

			// Encode the keys
			keys := make([]string, 0, len(result.SecretShares))
			keysB64 := make([]string, 0, len(result.SecretShares))
			for _, k := range result.SecretShares {
				keys = append(keys, hex.EncodeToString(k))
				keysB64 = append(keysB64, base64.StdEncoding.EncodeToString(k))
			}
			resp.Keys = keys
			resp.KeysB64 = keysB64
		}
		respondOk(w, resp)
	})
}

type RekeyRequest struct {
	SecretShares    int      `json:"secret_shares"`
	SecretThreshold int      `json:"secret_threshold"`
	StoredShares    int      `json:"stored_shares"`
	PGPKeys         []string `json:"pgp_keys"`
	Backup          bool     `json:"backup"`
}

type RekeyStatusResponse struct {
	Nonce           string   `json:"nonce"`
	Started         bool     `json:"started"`
	T               int      `json:"t"`
	N               int      `json:"n"`
	Progress        int      `json:"progress"`
	Required        int      `json:"required"`
	PGPFingerprints []string `json:"pgp_fingerprints"`
	Backup          bool     `json:"backup"`
}

type RekeyUpdateRequest struct {
	Nonce string
	Key   string
}

type RekeyUpdateResponse struct {
	Nonce           string   `json:"nonce"`
	Complete        bool     `json:"complete"`
	Keys            []string `json:"keys"`
	KeysB64         []string `json:"keys_base64"`
	PGPFingerprints []string `json:"pgp_fingerprints"`
	Backup          bool     `json:"backup"`
}
<commit_msg>Redirect rekey operation from standby to master (#1868)<commit_after>package http

import (
	"encoding/base64"
	"encoding/hex"
	"errors"
	"fmt"
	"net/http"

	"github.com/hashicorp/vault/helper/pgpkeys"
	"github.com/hashicorp/vault/vault"
)

func handleSysRekeyInit(core *vault.Core, recovery bool) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		standby, _ := core.Standby()
		if standby {
			respondStandby(core, w, r.URL)
			return
		}

		switch {
		case recovery && !core.SealAccess().RecoveryKeySupported():
			respondError(w, http.StatusBadRequest, fmt.Errorf("recovery rekeying not supported"))
		case r.Method == "GET":
			handleSysRekeyInitGet(core, recovery, w, r)
		case r.Method == "POST" || r.Method == "PUT":
			handleSysRekeyInitPut(core, recovery, w, r)
		case r.Method == "DELETE":
			handleSysRekeyInitDelete(core, recovery, w, r)
		default:
			respondError(w, http.StatusMethodNotAllowed, nil)
		}
	})
}

func handleSysRekeyInitGet(core *vault.Core, recovery bool, w http.ResponseWriter, r *http.Request) {
	barrierConfig, err := core.SealAccess().BarrierConfig()
	if err != nil {
		respondError(w, http.StatusInternalServerError, err)
		return
	}
	if barrierConfig == nil {
		respondError(w, http.StatusBadRequest, fmt.Errorf(
			"server is not yet initialized"))
		return
	}

	// Get the rekey configuration
	rekeyConf, err := core.RekeyConfig(recovery)
	if err != nil {
		respondError(w, http.StatusInternalServerError, err)
		return
	}

	// Get the progress
	progress, err := core.RekeyProgress(recovery)
	if err != nil {
		respondError(w, http.StatusInternalServerError, err)
		return
	}

	sealThreshold, err := core.RekeyThreshold(recovery)
	if err != nil {
		respondError(w, http.StatusInternalServerError, err)
		return
	}

	// Format the status
	status := &RekeyStatusResponse{
		Started:  false,
		T:        0,
		N:        0,
		Progress: progress,
		Required: sealThreshold,
	}
	if rekeyConf != nil {
		status.Nonce = rekeyConf.Nonce
		status.Started = true
		status.T = rekeyConf.SecretThreshold
		status.N = rekeyConf.SecretShares
		if rekeyConf.PGPKeys != nil && len(rekeyConf.PGPKeys) != 0 {
			pgpFingerprints, err := pgpkeys.GetFingerprints(rekeyConf.PGPKeys, nil)
			if err != nil {
				respondError(w, http.StatusInternalServerError, err)
				return
			}
			status.PGPFingerprints = pgpFingerprints
			status.Backup = rekeyConf.Backup
		}
	}
	respondOk(w, status)
}

func handleSysRekeyInitPut(core *vault.Core, recovery bool, w http.ResponseWriter, r *http.Request) {
	// Parse the request
	var req RekeyRequest
	if err := parseRequest(r, &req); err != nil {
		respondError(w, http.StatusBadRequest, err)
		return
	}

	if req.Backup && len(req.PGPKeys) == 0 {
		respondError(w, http.StatusBadRequest, fmt.Errorf("cannot request a backup of the new keys without providing PGP keys for encryption"))
		return
	}

	// Right now we don't support this, but the rest of the code is ready for
	// when we do, hence the check below for this to be false if
	// StoredShares is greater than zero
	if core.SealAccess().StoredKeysSupported() {
		respondError(w, http.StatusBadRequest, fmt.Errorf("rekeying of barrier not supported when stored key support is available"))
		return
	}

	// Initialize the rekey
	err := core.RekeyInit(&vault.SealConfig{
		SecretShares:    req.SecretShares,
		SecretThreshold: req.SecretThreshold,
		StoredShares:    req.StoredShares,
		PGPKeys:         req.PGPKeys,
		Backup:          req.Backup,
	}, recovery)
	if err != nil {
		respondError(w, http.StatusBadRequest, err)
		return
	}

	handleSysRekeyInitGet(core, recovery, w, r)
}

func handleSysRekeyInitDelete(core *vault.Core, recovery bool, w http.ResponseWriter, r *http.Request) {
	err := core.RekeyCancel(recovery)
	if err != nil {
		respondError(w, http.StatusInternalServerError, err)
		return
	}
	respondOk(w, nil)
}

func handleSysRekeyUpdate(core *vault.Core, recovery bool) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		standby, _ := core.Standby()
		if standby {
			respondStandby(core, w, r.URL)
			return
		}

		// Parse the request
		var req RekeyUpdateRequest
		if err := parseRequest(r, &req); err != nil {
			respondError(w, http.StatusBadRequest, err)
			return
		}
		if req.Key == "" {
			respondError(
				w, http.StatusBadRequest,
				errors.New("'key' must specified in request body as JSON"))
			return
		}

		// Decode the key, which is base64 or hex encoded
		min, max := core.BarrierKeyLength()
		key, err := hex.DecodeString(req.Key)
		// We check min and max here to ensure that a string that is base64
		// encoded but also valid hex will not be valid and we instead base64
		// decode it
		if err != nil || len(key) < min || len(key) > max {
			key, err = base64.StdEncoding.DecodeString(req.Key)
			if err != nil {
				respondError(
					w, http.StatusBadRequest,
					errors.New("'key' must be a valid hex or base64 string"))
				return
			}
		}

		// Use the key to make progress on rekey
		result, err := core.RekeyUpdate(key, req.Nonce, recovery)
		if err != nil {
			respondError(w, http.StatusBadRequest, err)
			return
		}

		// Format the response
		resp := &RekeyUpdateResponse{}
		if result != nil {
			resp.Complete = true
			resp.Nonce = req.Nonce
			resp.Backup = result.Backup
			resp.PGPFingerprints = result.PGPFingerprints

			// Encode the keys
			keys := make([]string, 0, len(result.SecretShares))
			keysB64 := make([]string, 0, len(result.SecretShares))
			for _, k := range result.SecretShares {
				keys = append(keys, hex.EncodeToString(k))
				keysB64 = append(keysB64, base64.StdEncoding.EncodeToString(k))
			}
			resp.Keys = keys
			resp.KeysB64 = keysB64
		}
		respondOk(w, resp)
	})
}

type RekeyRequest struct {
	SecretShares    int      `json:"secret_shares"`
	SecretThreshold int      `json:"secret_threshold"`
	StoredShares    int      `json:"stored_shares"`
	PGPKeys         []string `json:"pgp_keys"`
	Backup          bool     `json:"backup"`
}

type RekeyStatusResponse struct {
	Nonce           string   `json:"nonce"`
	Started         bool     `json:"started"`
	T               int      `json:"t"`
	N               int      `json:"n"`
	Progress        int      `json:"progress"`
	Required        int      `json:"required"`
	PGPFingerprints []string `json:"pgp_fingerprints"`
	Backup          bool     `json:"backup"`
}

type RekeyUpdateRequest struct {
	Nonce string
	Key   string
}

type RekeyUpdateResponse struct {
	Nonce           string   `json:"nonce"`
	Complete        bool     `json:"complete"`
	Keys            []string `json:"keys"`
	KeysB64         []string `json:"keys_base64"`
	PGPFingerprints []string `json:"pgp_fingerprints"`
	Backup          bool     `json:"backup"`
}
<|endoftext|>


<commit_before>package serf

import (
	"encoding/json"
	"fmt"
	"github.com/hashicorp/memberlist"
	"io"
	"io/ioutil"
	"os"
	"time"
)

// ProtocolVersionMap is the mapping of Serf delegate protocol versions
// to memberlist protocol versions. We mask the memberlist protocols using
// our own protocol version.
var ProtocolVersionMap map[uint8]uint8

func init() {
	ProtocolVersionMap = map[uint8]uint8{
		4: 2,
		3: 2,
		2: 2,
	}
}

// Config is the configuration for creating a Serf instance.
type Config struct {
	// The name of this node. This must be unique in the cluster. If this
	// is not set, Serf will set it to the hostname of the running machine.
	NodeName string

	// The tags for this role, if any. This is used to provide arbitrary
	// key/value metadata per-node. For example, a "role" tag may be used to
	// differentiate "load-balancer" from a "web" role as parts of the same cluster.
	// Tags are deprecating 'Role', and instead it acts as a special key in this
	// map.
	Tags map[string]string

	// TagsFile is the path to a file where Serf can store its tags. Tag
	// persistence is desirable since tags may be set or deleted while the
	// agent is running. Tags can be reloaded from this file on later starts.
	TagsFile string

	// EventCh is a channel that receives all the Serf events. The events
	// are sent on this channel in proper ordering. Care must be taken that
	// this channel doesn't block, either by processing the events quick
	// enough or buffering the channel, otherwise it can block state updates
	// within Serf itself. If no EventCh is specified, no events will be fired,
	// but point-in-time snapshots of members can still be retrieved by
	// calling Members on Serf.
	EventCh chan<- Event

	// ProtocolVersion is the protocol version to speak. This must be between
	// ProtocolVersionMin and ProtocolVersionMax.
	ProtocolVersion uint8

	// BroadcastTimeout is the amount of time to wait for a broadcast
	// message to be sent to the cluster. Broadcast messages are used for
	// things like leave messages and force remove messages. If this is not
	// set, a timeout of 5 seconds will be set.
	BroadcastTimeout time.Duration

	// The settings below relate to Serf's event coalescence feature. Serf
	// is able to coalesce multiple events into single events in order to
	// reduce the amount of noise that is sent along the EventCh. For example
	// if five nodes quickly join, the EventCh will be sent one EventMemberJoin
	// containing the five nodes rather than five individual EventMemberJoin
	// events. Coalescence can mitigate potential flapping behavior.
	//
	// Coalescence is disabled by default and can be enabled by setting
	// CoalescePeriod.
	//
	// CoalescePeriod specifies the time duration to coalesce events.
	// For example, if this is set to 5 seconds, then all events received
	// within 5 seconds that can be coalesced will be.
	//
	// QuiescentPeriod specifies the duration of time where if no events
	// are received, coalescence immediately happens. For example, if
	// CoalscePeriod is set to 10 seconds but QuiscentPeriod is set to 2
	// seconds, then the events will be coalesced and dispatched if no
	// new events are received within 2 seconds of the last event. Otherwise,
	// every event will always be delayed by at least 10 seconds.
	CoalescePeriod  time.Duration
	QuiescentPeriod time.Duration

	// The settings below relate to Serf's user event coalescing feature.
	// The settings operate like above but only affect user messages and
	// not the Member* messages that Serf generates.
	UserCoalescePeriod  time.Duration
	UserQuiescentPeriod time.Duration

	// The settings below relate to Serf keeping track of recently
	// failed/left nodes and attempting reconnects.
	//
	// ReapInterval is the interval when the reaper runs. If this is not
	// set (it is zero), it will be set to a reasonable default.
	//
	// ReconnectInterval is the interval when we attempt to reconnect
	// to failed nodes. If this is not set (it is zero), it will be set
	// to a reasonable default.
	//
	// ReconnectTimeout is the amount of time to attempt to reconnect to
	// a failed node before giving up and considering it completely gone.
	//
	// TombstoneTimeout is the amount of time to keep around nodes
	// that gracefully left as tombstones for syncing state with other
	// Serf nodes.
	ReapInterval      time.Duration
	ReconnectInterval time.Duration
	ReconnectTimeout  time.Duration
	TombstoneTimeout  time.Duration

	// QueueDepthWarning is used to generate warning message if the
	// number of queued messages to broadcast exceeds this number. This
	// is to provide the user feedback if events are being triggered
	// faster than they can be disseminated
	QueueDepthWarning int

	// MaxQueueDepth is used to start dropping messages if the number
	// of queued messages to broadcast exceeds this number. This is to
	// prevent an unbounded growth of memory utilization
	MaxQueueDepth int

	// RecentIntentBuffer is used to set the size of recent join and leave intent
	// messages that will be buffered. This is used to guard against
	// the case where Serf broadcasts an intent that arrives before the
	// Memberlist event. It is important that this not be too small to avoid
	// continuous rebroadcasting of dead events.
	RecentIntentBuffer int

	// EventBuffer is used to control how many events are buffered.
	// This is used to prevent re-delivery of events to a client. The buffer
	// must be large enough to handle all "recent" events, since Serf will
	// not deliver messages that are older than the oldest entry in the buffer.
	// Thus if a client is generating too many events, it's possible that the
	// buffer gets overrun and messages are not delivered.
	EventBuffer int

	// QueryBuffer is used to control how many queries are buffered.
	// This is used to prevent re-delivery of queries to a client. The buffer
	// must be large enough to handle all "recent" events, since Serf will not
	// deliver queries older than the oldest entry in the buffer.
	// Thus if a client is generating too many queries, it's possible that the
	// buffer gets overrun and messages are not delivered.
	QueryBuffer int

	// QueryTimeoutMult configures the default timeout multipler for a query to run if no
	// specific value is provided. Queries are real-time by nature, where the
	// reply is time sensitive. As a result, results are collected in an async
	// fashion, however the query must have a bounded duration. We want the timeout
	// to be long enough that all nodes have time to receive the message, run a handler,
	// and generate a reply. Once the timeout is exceeded, any further replies are ignored.
	// The default value is
	//
	// Timeout = GossipInterval * QueryTimeoutMult * log(N+1)
	//
	QueryTimeoutMult int

	// MemberlistConfig is the memberlist configuration that Serf will
	// use to do the underlying membership management and gossip. Some
	// fields in the MemberlistConfig will be overwritten by Serf no
	// matter what:
	//
	//   * Name - This will always be set to the same as the NodeName
	//     in this configuration.
	//
	//   * Events - Serf uses a custom event delegate.
	//
	//   * Delegate - Serf uses a custom delegate.
	//
	MemberlistConfig *memberlist.Config

	// LogOutput is the location to write logs to. If this is not set,
	// logs will go to stderr.
	LogOutput io.Writer

	// SnapshotPath if provided is used to snapshot live nodes as well
	// as lamport clock values. When Serf is started with a snapshot,
	// it will attempt to join all the previously known nodes until one
	// succeeds and will also avoid replaying old user events.
	SnapshotPath string

	// EnableNameConflictResolution controls if Serf will actively attempt
	// to resolve a name conflict. Since each Serf member must have a unique
	// name, a cluster can run into issues if multiple nodes claim the same name.
	// Without automatic resolution, Serf merely logs some warnings, but otherwise
	// does not take any action. Automatic resolution detects the conflict and issues
	// a special query which asks the cluster for the Name -> IP:Port mapping. If there
	// is a simple majority of votes, that node stays while the other node will leave
	// the cluster and exit.
	EnableNameConflictResolution bool
}

// Init allocates the subdata structures
func (c *Config) Init() {
	if c.Tags == nil {
		c.Tags = make(map[string]string)
	}
}

func (c *Config) PersistTags() error {
	encoded, err := json.MarshalIndent(c.Tags, "", "  ")
	if err != nil {
		return fmt.Errorf("Failed to encode tags: %s", err)
	}
	// Use 0600 for permissions, in case tag data is sensitive
	if err = ioutil.WriteFile(c.TagsFile, encoded, 0600); err != nil {
		return fmt.Errorf("Failed to write tags file: %s", err)
	}

	// Success!
	return nil
}

// DefaultConfig returns a Config struct that contains reasonable defaults
// for most of the configurations.
func DefaultConfig() *Config {
	hostname, err := os.Hostname()
	if err != nil {
		panic(err)
	}

	return &Config{
		NodeName:                     hostname,
		BroadcastTimeout:             5 * time.Second,
		EventBuffer:                  512,
		QueryBuffer:                  512,
		LogOutput:                    os.Stderr,
		ProtocolVersion:              ProtocolVersionMax,
		ReapInterval:                 15 * time.Second,
		RecentIntentBuffer:           128,
		ReconnectInterval:            30 * time.Second,
		ReconnectTimeout:             24 * time.Hour,
		QueueDepthWarning:            128,
		MaxQueueDepth:                4096,
		TombstoneTimeout:             24 * time.Hour,
		MemberlistConfig:             memberlist.DefaultLANConfig(),
		QueryTimeoutMult:             16,
		EnableNameConflictResolution: true,
	}
}
<commit_msg>serf: only try to persist tags if a tags file is configured<commit_after>package serf

import (
	"encoding/json"
	"fmt"
	"github.com/hashicorp/memberlist"
	"io"
	"io/ioutil"
	"os"
	"time"
)

// ProtocolVersionMap is the mapping of Serf delegate protocol versions
// to memberlist protocol versions. We mask the memberlist protocols using
// our own protocol version.
var ProtocolVersionMap map[uint8]uint8

func init() {
	ProtocolVersionMap = map[uint8]uint8{
		4: 2,
		3: 2,
		2: 2,
	}
}

// Config is the configuration for creating a Serf instance.
type Config struct {
	// The name of this node. This must be unique in the cluster. If this
	// is not set, Serf will set it to the hostname of the running machine.
	NodeName string

	// The tags for this role, if any. This is used to provide arbitrary
	// key/value metadata per-node. For example, a "role" tag may be used to
	// differentiate "load-balancer" from a "web" role as parts of the same cluster.
	// Tags are deprecating 'Role', and instead it acts as a special key in this
	// map.
	Tags map[string]string

	// TagsFile is the path to a file where Serf can store its tags. Tag
	// persistence is desirable since tags may be set or deleted while the
	// agent is running. Tags can be reloaded from this file on later starts.
	TagsFile string

	// EventCh is a channel that receives all the Serf events. The events
	// are sent on this channel in proper ordering. Care must be taken that
	// this channel doesn't block, either by processing the events quick
	// enough or buffering the channel, otherwise it can block state updates
	// within Serf itself. If no EventCh is specified, no events will be fired,
	// but point-in-time snapshots of members can still be retrieved by
	// calling Members on Serf.
	EventCh chan<- Event

	// ProtocolVersion is the protocol version to speak. This must be between
	// ProtocolVersionMin and ProtocolVersionMax.
	ProtocolVersion uint8

	// BroadcastTimeout is the amount of time to wait for a broadcast
	// message to be sent to the cluster. Broadcast messages are used for
	// things like leave messages and force remove messages. If this is not
	// set, a timeout of 5 seconds will be set.
	BroadcastTimeout time.Duration

	// The settings below relate to Serf's event coalescence feature. Serf
	// is able to coalesce multiple events into single events in order to
	// reduce the amount of noise that is sent along the EventCh. For example
	// if five nodes quickly join, the EventCh will be sent one EventMemberJoin
	// containing the five nodes rather than five individual EventMemberJoin
	// events. Coalescence can mitigate potential flapping behavior.
	//
	// Coalescence is disabled by default and can be enabled by setting
	// CoalescePeriod.
	//
	// CoalescePeriod specifies the time duration to coalesce events.
	// For example, if this is set to 5 seconds, then all events received
	// within 5 seconds that can be coalesced will be.
	//
	// QuiescentPeriod specifies the duration of time where if no events
	// are received, coalescence immediately happens. For example, if
	// CoalscePeriod is set to 10 seconds but QuiscentPeriod is set to 2
	// seconds, then the events will be coalesced and dispatched if no
	// new events are received within 2 seconds of the last event. Otherwise,
	// every event will always be delayed by at least 10 seconds.
	CoalescePeriod  time.Duration
	QuiescentPeriod time.Duration

	// The settings below relate to Serf's user event coalescing feature.
	// The settings operate like above but only affect user messages and
	// not the Member* messages that Serf generates.
	UserCoalescePeriod  time.Duration
	UserQuiescentPeriod time.Duration

	// The settings below relate to Serf keeping track of recently
	// failed/left nodes and attempting reconnects.
	//
	// ReapInterval is the interval when the reaper runs. If this is not
	// set (it is zero), it will be set to a reasonable default.
	//
	// ReconnectInterval is the interval when we attempt to reconnect
	// to failed nodes. If this is not set (it is zero), it will be set
	// to a reasonable default.
	//
	// ReconnectTimeout is the amount of time to attempt to reconnect to
	// a failed node before giving up and considering it completely gone.
	//
	// TombstoneTimeout is the amount of time to keep around nodes
	// that gracefully left as tombstones for syncing state with other
	// Serf nodes.
	ReapInterval      time.Duration
	ReconnectInterval time.Duration
	ReconnectTimeout  time.Duration
	TombstoneTimeout  time.Duration

	// QueueDepthWarning is used to generate warning message if the
	// number of queued messages to broadcast exceeds this number. This
	// is to provide the user feedback if events are being triggered
	// faster than they can be disseminated
	QueueDepthWarning int

	// MaxQueueDepth is used to start dropping messages if the number
	// of queued messages to broadcast exceeds this number. This is to
	// prevent an unbounded growth of memory utilization
	MaxQueueDepth int

	// RecentIntentBuffer is used to set the size of recent join and leave intent
	// messages that will be buffered. This is used to guard against
	// the case where Serf broadcasts an intent that arrives before the
	// Memberlist event. It is important that this not be too small to avoid
	// continuous rebroadcasting of dead events.
	RecentIntentBuffer int

	// EventBuffer is used to control how many events are buffered.
	// This is used to prevent re-delivery of events to a client. The buffer
	// must be large enough to handle all "recent" events, since Serf will
	// not deliver messages that are older than the oldest entry in the buffer.
	// Thus if a client is generating too many events, it's possible that the
	// buffer gets overrun and messages are not delivered.
	EventBuffer int

	// QueryBuffer is used to control how many queries are buffered.
	// This is used to prevent re-delivery of queries to a client. The buffer
	// must be large enough to handle all "recent" events, since Serf will not
	// deliver queries older than the oldest entry in the buffer.
	// Thus if a client is generating too many queries, it's possible that the
	// buffer gets overrun and messages are not delivered.
	QueryBuffer int

	// QueryTimeoutMult configures the default timeout multipler for a query to run if no
	// specific value is provided. Queries are real-time by nature, where the
	// reply is time sensitive. As a result, results are collected in an async
	// fashion, however the query must have a bounded duration. We want the timeout
	// to be long enough that all nodes have time to receive the message, run a handler,
	// and generate a reply. Once the timeout is exceeded, any further replies are ignored.
	// The default value is
	//
	// Timeout = GossipInterval * QueryTimeoutMult * log(N+1)
	//
	QueryTimeoutMult int

	// MemberlistConfig is the memberlist configuration that Serf will
	// use to do the underlying membership management and gossip. Some
	// fields in the MemberlistConfig will be overwritten by Serf no
	// matter what:
	//
	//   * Name - This will always be set to the same as the NodeName
	//     in this configuration.
	//
	//   * Events - Serf uses a custom event delegate.
	//
	//   * Delegate - Serf uses a custom delegate.
	//
	MemberlistConfig *memberlist.Config

	// LogOutput is the location to write logs to. If this is not set,
	// logs will go to stderr.
	LogOutput io.Writer

	// SnapshotPath if provided is used to snapshot live nodes as well
	// as lamport clock values. When Serf is started with a snapshot,
	// it will attempt to join all the previously known nodes until one
	// succeeds and will also avoid replaying old user events.
	SnapshotPath string

	// EnableNameConflictResolution controls if Serf will actively attempt
	// to resolve a name conflict. Since each Serf member must have a unique
	// name, a cluster can run into issues if multiple nodes claim the same name.
	// Without automatic resolution, Serf merely logs some warnings, but otherwise
	// does not take any action. Automatic resolution detects the conflict and issues
	// a special query which asks the cluster for the Name -> IP:Port mapping. If there
	// is a simple majority of votes, that node stays while the other node will leave
	// the cluster and exit.
	EnableNameConflictResolution bool
}

// Init allocates the subdata structures
func (c *Config) Init() {
	if c.Tags == nil {
		c.Tags = make(map[string]string)
	}
}

func (c *Config) PersistTags() error {
	if c.TagsFile != "" {
		encoded, err := json.MarshalIndent(c.Tags, "", "  ")
		if err != nil {
			return fmt.Errorf("Failed to encode tags: %s", err)
		}

		// Use 0600 for permissions, in case tag data is sensitive
		if err = ioutil.WriteFile(c.TagsFile, encoded, 0600); err != nil {
			return fmt.Errorf("Failed to write tags file: %s", err)
		}
	}

	// Success!
	return nil
}

// DefaultConfig returns a Config struct that contains reasonable defaults
// for most of the configurations.
func DefaultConfig() *Config {
	hostname, err := os.Hostname()
	if err != nil {
		panic(err)
	}

	return &Config{
		NodeName:                     hostname,
		BroadcastTimeout:             5 * time.Second,
		EventBuffer:                  512,
		QueryBuffer:                  512,
		LogOutput:                    os.Stderr,
		ProtocolVersion:              ProtocolVersionMax,
		ReapInterval:                 15 * time.Second,
		RecentIntentBuffer:           128,
		ReconnectInterval:            30 * time.Second,
		ReconnectTimeout:             24 * time.Hour,
		QueueDepthWarning:            128,
		MaxQueueDepth:                4096,
		TombstoneTimeout:             24 * time.Hour,
		MemberlistConfig:             memberlist.DefaultLANConfig(),
		QueryTimeoutMult:             16,
		EnableNameConflictResolution: true,
	}
}
<|endoftext|>


<commit_before>package worker

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"

	"github.com/bitly/go-simplejson"
	"github.com/travis-ci/worker/backend"

	gocontext "golang.org/x/net/context"
)

// HTTPJobQueue is a JobQueue that uses http
type HTTPJobQueue struct {
	processorPool *ProcessorPool
	jobBoardURL   *url.URL
	site          string
	queue         string
	workerID      string

	DefaultLanguage, DefaultDist, DefaultGroup, DefaultOS string
}

type httpFetchJobsRequest struct {
	Jobs []string `json:"jobs"`
}

type httpFetchJobsResponse struct {
	Jobs []string `json:"jobs"`
}

// NewHTTPJobQueue creates a new job-board job queue
func NewHTTPJobQueue(pool *ProcessorPool, jobBoardURL *url.URL, site, queue, workerID string) (*HTTPJobQueue, error) {
	return &HTTPJobQueue{
		processorPool: pool,
		jobBoardURL:   jobBoardURL,
		site:          site,
		queue:         queue,
		workerID:      workerID,
	}, nil
}

// Jobs consumes new jobs from job-board
func (q *HTTPJobQueue) Jobs(ctx gocontext.Context) (outChan <-chan Job, err error) {
	buildJobChan := make(chan Job)
	outChan = buildJobChan

	go func() {
		for {
			jobIds, err := q.fetchJobs()
			if err != nil {
				fmt.Fprintf(os.Stderr, "TODO: handle error from httpJobQueue.fetchJobs: %#v", err)
				panic("whoops!")
			}
			for _, id := range jobIds {
				go func(id uint64) {
					buildJob, err := q.fetchJob(id)
					if err != nil {
						fmt.Fprintf(os.Stderr, "TODO: handle error from httpJobQueue.fetchJob: %#v", err)
					}
					buildJobChan <- buildJob
				}(id)
			}

			time.Sleep(time.Second)
		}
	}()

	return outChan, nil
}

func (q *HTTPJobQueue) fetchJobs() ([]uint64, error) {
	// POST /jobs?count=17&queue=flah
	// Content-Type: application/json
	// Travis-Site: ${SITE}
	// Authorization: Basic ${BASE64_BASIC_AUTH}
	// From: ${UNIQUE_ID}

	fetchRequestPayload := &httpFetchJobsRequest{Jobs: []string{}}
	numWaiting := 0
	q.processorPool.Each(func(i int, p *Processor) {
		// CurrentStatus is one of "new", "waiting", "processing" or "done"
		switch p.CurrentStatus {
		case "processing":
			fetchRequestPayload.Jobs = append(fetchRequestPayload.Jobs, fmt.Sprintf("%d", p.LastJobID))
		case "waiting", "new":
			numWaiting++
		}
	})

	jobIdsJSON, err := json.Marshal(fetchRequestPayload)
	if err != nil {
		return nil, err
	}

	// copy jobBoardURL
	url := *q.jobBoardURL

	query := url.Query()
	query.Add("count", fmt.Sprintf("%d", numWaiting))
	query.Add("queue", q.queue)

	url.Path = "/jobs"
	url.RawQuery = query.Encode()

	client := &http.Client{}

	req, err := http.NewRequest("POST", url.String(), bytes.NewReader(jobIdsJSON))

	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Travis-Site", q.site)
	req.Header.Add("From", q.workerID)

	resp, err := client.Do(req)

	fetchResponsePayload := &httpFetchJobsResponse{}
	err = json.NewDecoder(resp.Body).Decode(&fetchResponsePayload)
	if err != nil {
		return nil, err
	}

	var jobIds []uint64
	for _, strID := range fetchResponsePayload.Jobs {
		alreadyRunning := false
		for _, prevStrID := range fetchRequestPayload.Jobs {
			if strID == prevStrID {
				alreadyRunning = true
			}
		}
		if alreadyRunning {
			continue
		}

		id, err := strconv.ParseUint(strID, 10, 64)
		if err != nil {
			return nil, err
		}
		jobIds = append(jobIds, id)
	}

	return jobIds, nil
}

func (q *HTTPJobQueue) fetchJob(id uint64) (Job, error) {
	// GET /jobs/:id
	// Authorization: Basic ${BASE64_BASIC_AUTH}
	// Travis-Site: ${SITE}
	// From: ${UNIQUE_ID}

	buildJob := &httpJob{
		payload: &httpJobPayload{
			Data: &JobPayload{},
		},
		startAttributes: &backend.StartAttributes{},
	}
	startAttrs := &httpJobPayloadStartAttrs{
		Data: &jobPayloadStartAttrs{
			Config: &backend.StartAttributes{},
		},
	}

	// copy jobBoardURL
	url := *q.jobBoardURL
	url.Path = fmt.Sprintf("/jobs/%d", id)

	client := &http.Client{}

	req, err := http.NewRequest("GET", url.String(), nil)

	req.Header.Add("Travis-Site", q.site)
	req.Header.Add("From", q.workerID)

	resp, err := client.Do(req)

	body, err := ioutil.ReadAll(resp.Body)

	err = json.Unmarshal(body, buildJob.payload)

	err = json.Unmarshal(body, &startAttrs)

	rawPayload, err := simplejson.NewJson(body)
	buildJob.rawPayload = rawPayload.Get("data")

	buildJob.startAttributes = startAttrs.Data.Config
	buildJob.startAttributes.VMType = buildJob.payload.Data.VMType
	buildJob.startAttributes.SetDefaults(q.DefaultLanguage, q.DefaultDist, q.DefaultGroup, q.DefaultOS, VMTypeDefault)

	return buildJob, err
}

// Cleanup does not do anything!
func (q *HTTPJobQueue) Cleanup() error {
	return nil
}
<commit_msg>http-job-queue: handle all errors<commit_after>package worker

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"time"

	"github.com/bitly/go-simplejson"
	"github.com/pkg/errors"
	"github.com/travis-ci/worker/backend"

	gocontext "golang.org/x/net/context"
)

// HTTPJobQueue is a JobQueue that uses http
type HTTPJobQueue struct {
	processorPool *ProcessorPool
	jobBoardURL   *url.URL
	site          string
	queue         string
	workerID      string

	DefaultLanguage, DefaultDist, DefaultGroup, DefaultOS string
}

type httpFetchJobsRequest struct {
	Jobs []string `json:"jobs"`
}

type httpFetchJobsResponse struct {
	Jobs []string `json:"jobs"`
}

// NewHTTPJobQueue creates a new job-board job queue
func NewHTTPJobQueue(pool *ProcessorPool, jobBoardURL *url.URL, site, queue, workerID string) (*HTTPJobQueue, error) {
	return &HTTPJobQueue{
		processorPool: pool,
		jobBoardURL:   jobBoardURL,
		site:          site,
		queue:         queue,
		workerID:      workerID,
	}, nil
}

// Jobs consumes new jobs from job-board
func (q *HTTPJobQueue) Jobs(ctx gocontext.Context) (outChan <-chan Job, err error) {
	buildJobChan := make(chan Job)
	outChan = buildJobChan

	go func() {
		for {
			jobIds, err := q.fetchJobs()
			if err != nil {
				fmt.Fprintf(os.Stderr, "TODO: handle error from httpJobQueue.fetchJobs: %#v", err)
				panic("whoops!")
			}
			for _, id := range jobIds {
				go func(id uint64) {
					buildJob, err := q.fetchJob(id)
					if err != nil {
						fmt.Fprintf(os.Stderr, "TODO: handle error from httpJobQueue.fetchJob: %#v", err)
					}
					buildJobChan <- buildJob
				}(id)
			}

			time.Sleep(time.Second)
		}
	}()

	return outChan, nil
}

func (q *HTTPJobQueue) fetchJobs() ([]uint64, error) {
	// POST /jobs?count=17&queue=flah
	// Content-Type: application/json
	// Travis-Site: ${SITE}
	// Authorization: Basic ${BASE64_BASIC_AUTH}
	// From: ${UNIQUE_ID}

	fetchRequestPayload := &httpFetchJobsRequest{Jobs: []string{}}
	numWaiting := 0
	q.processorPool.Each(func(i int, p *Processor) {
		// CurrentStatus is one of "new", "waiting", "processing" or "done"
		switch p.CurrentStatus {
		case "processing":
			fetchRequestPayload.Jobs = append(fetchRequestPayload.Jobs, fmt.Sprintf("%d", p.LastJobID))
		case "waiting", "new":
			numWaiting++
		}
	})

	jobIdsJSON, err := json.Marshal(fetchRequestPayload)
	if err != nil {
		return nil, errors.Wrap(err, "failed to marshal job board jobs request payload")
	}

	// copy jobBoardURL
	url := *q.jobBoardURL

	query := url.Query()
	query.Add("count", fmt.Sprintf("%d", numWaiting))
	query.Add("queue", q.queue)

	url.Path = "/jobs"
	url.RawQuery = query.Encode()

	client := &http.Client{}

	req, err := http.NewRequest("POST", url.String(), bytes.NewReader(jobIdsJSON))
	if err != nil {
		return nil, errors.Wrap(err, "failed to create job board jobs request")
	}

	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Travis-Site", q.site)
	req.Header.Add("From", q.workerID)

	resp, err := client.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, "failed to make job board jobs request")
	}

	fetchResponsePayload := &httpFetchJobsResponse{}
	err = json.NewDecoder(resp.Body).Decode(&fetchResponsePayload)
	if err != nil {
		return nil, errors.Wrap(err, "failed to decode job board jobs response")
	}

	var jobIds []uint64
	for _, strID := range fetchResponsePayload.Jobs {
		alreadyRunning := false
		for _, prevStrID := range fetchRequestPayload.Jobs {
			if strID == prevStrID {
				alreadyRunning = true
			}
		}
		if alreadyRunning {
			continue
		}

		id, err := strconv.ParseUint(strID, 10, 64)
		if err != nil {
			return nil, errors.Wrap(err, "failed to parse job ID")
		}
		jobIds = append(jobIds, id)
	}

	return jobIds, nil
}

func (q *HTTPJobQueue) fetchJob(id uint64) (Job, error) {
	// GET /jobs/:id
	// Authorization: Basic ${BASE64_BASIC_AUTH}
	// Travis-Site: ${SITE}
	// From: ${UNIQUE_ID}

	buildJob := &httpJob{
		payload: &httpJobPayload{
			Data: &JobPayload{},
		},
		startAttributes: &backend.StartAttributes{},
	}
	startAttrs := &httpJobPayloadStartAttrs{
		Data: &jobPayloadStartAttrs{
			Config: &backend.StartAttributes{},
		},
	}

	// copy jobBoardURL
	url := *q.jobBoardURL
	url.Path = fmt.Sprintf("/jobs/%d", id)

	client := &http.Client{}

	req, err := http.NewRequest("GET", url.String(), nil)
	if err != nil {
		return nil, errors.Wrap(err, "couldn't make job board job request")
	}

	req.Header.Add("Travis-Site", q.site)
	req.Header.Add("From", q.workerID)

	resp, err := client.Do(req)
	if err != nil {
		return nil, errors.Wrap(err, "error making job board job request")
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, errors.Wrap(err, "error reading body from job board job request")
	}

	err = json.Unmarshal(body, buildJob.payload)
	if err != nil {
		return nil, errors.Wrap(err, "failed to unmarshal job board payload")
	}

	err = json.Unmarshal(body, &startAttrs)
	if err != nil {
		return nil, errors.Wrap(err, "failed to unmarshal start attributes from job board")
	}

	rawPayload, err := simplejson.NewJson(body)
	if err != nil {
		return nil, errors.Wrap(err, "failed to parse raw payload with simplejson")
	}
	buildJob.rawPayload = rawPayload.Get("data")

	buildJob.startAttributes = startAttrs.Data.Config
	buildJob.startAttributes.VMType = buildJob.payload.Data.VMType
	buildJob.startAttributes.SetDefaults(q.DefaultLanguage, q.DefaultDist, q.DefaultGroup, q.DefaultOS, VMTypeDefault)

	return buildJob, nil
}

// Cleanup does not do anything!
func (q *HTTPJobQueue) Cleanup() error {
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1beta3

import (
	"fmt"

	newer "github.com/GoogleCloudPlatform/kubernetes/pkg/api"
)

func init() {
	// Add field conversion funcs.
	err := newer.Scheme.AddFieldLabelConversionFunc("v1beta3", "Pod",
		func(label, value string) (string, string, error) {
			switch label {
			case "metadata.name",
				"status.phase",
				"spec.host":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = newer.Scheme.AddFieldLabelConversionFunc("v1beta3", "Node",
		func(label, value string) (string, string, error) {
			switch label {
			case "metadata.name":
				return label, value, nil
			case "spec.unschedulable":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = newer.Scheme.AddFieldLabelConversionFunc("v1beta3", "ReplicationController",
		func(label, value string) (string, string, error) {
			switch label {
			case "metadata.name",
				"status.replicas":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = newer.Scheme.AddFieldLabelConversionFunc("v1beta3", "Event",
		func(label, value string) (string, string, error) {
			switch label {
			case "involvedObject.kind",
				"involvedObject.namespace",
				"involvedObject.name",
				"involvedObject.uid",
				"involvedObject.apiVersion",
				"involvedObject.resourceVersion",
				"involvedObject.fieldPath",
				"reason",
				"source":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = newer.Scheme.AddFieldLabelConversionFunc("v1beta1", "Namespace",
		func(label, value string) (string, string, error) {
			switch label {
			case "status.phase":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
}
<commit_msg>Generating conversion methods.<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1beta3

import (
	"fmt"

	newer "github.com/GoogleCloudPlatform/kubernetes/pkg/api"
)

func init() {
	err := newer.Scheme.AddConversionFuncs()

	// Add field conversion funcs.
	err = newer.Scheme.AddFieldLabelConversionFunc("v1beta3", "Pod",
		func(label, value string) (string, string, error) {
			switch label {
			case "metadata.name",
				"status.phase",
				"spec.host":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = newer.Scheme.AddFieldLabelConversionFunc("v1beta3", "Node",
		func(label, value string) (string, string, error) {
			switch label {
			case "metadata.name":
				return label, value, nil
			case "spec.unschedulable":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = newer.Scheme.AddFieldLabelConversionFunc("v1beta3", "ReplicationController",
		func(label, value string) (string, string, error) {
			switch label {
			case "metadata.name",
				"status.replicas":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = newer.Scheme.AddFieldLabelConversionFunc("v1beta3", "Event",
		func(label, value string) (string, string, error) {
			switch label {
			case "involvedObject.kind",
				"involvedObject.namespace",
				"involvedObject.name",
				"involvedObject.uid",
				"involvedObject.apiVersion",
				"involvedObject.resourceVersion",
				"involvedObject.fieldPath",
				"reason",
				"source":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
	err = newer.Scheme.AddFieldLabelConversionFunc("v1beta1", "Namespace",
		func(label, value string) (string, string, error) {
			switch label {
			case "status.phase":
				return label, value, nil
			default:
				return "", "", fmt.Errorf("field label not supported: %s", label)
			}
		})
	if err != nil {
		// If one of the conversion functions is malformed, detect it immediately.
		panic(err)
	}
}
<|endoftext|>


<commit_before>package types

import (
	"github.com/atlassian/smith/pkg/cleanup"

	sc_v1a1 "github.com/kubernetes-incubator/service-catalog/pkg/apis/servicecatalog/v1alpha1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	unstructured_conversion "k8s.io/apimachinery/pkg/conversion/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	api_v1 "k8s.io/client-go/pkg/api/v1"
	apps_v1b1 "k8s.io/client-go/pkg/apis/apps/v1beta1"
)

var MainKnownTypes = map[schema.GroupKind]cleanup.SpecCleanup{
	{Group: apps_v1b1.GroupName, Kind: "Deployment"}: deploymentCleanup,
	{Group: api_v1.GroupName, Kind: "Service"}:       serviceCleanup,
}

var ServiceCatalogKnownTypes = map[schema.GroupKind]cleanup.SpecCleanup{
	{Group: sc_v1a1.GroupName, Kind: "Binding"}:  scBindingCleanup,
	{Group: sc_v1a1.GroupName, Kind: "Instance"}: scInstanceCleanup,
}

func deploymentCleanup(spec, actual *unstructured.Unstructured) (*unstructured.Unstructured, error) {
	var deployment apps_v1b1.Deployment
	if err := unstructured_conversion.DefaultConverter.FromUnstructured(spec.Object, &deployment); err != nil {
		return nil, err
	}

	// TODO: implement cleanup

	return spec, nil
}

func serviceCleanup(spec, actual *unstructured.Unstructured) (*unstructured.Unstructured, error) {
	var serviceSpec api_v1.Service
	if err := unstructured_conversion.DefaultConverter.FromUnstructured(spec.Object, &serviceSpec); err != nil {
		return nil, err
	}
	var serviceActual api_v1.Service
	if err := unstructured_conversion.DefaultConverter.FromUnstructured(actual.Object, &serviceActual); err != nil {
		return nil, err
	}

	serviceSpec.Spec.ClusterIP = serviceActual.Spec.ClusterIP
	serviceSpec.Status = serviceActual.Status

	updatedObj := &unstructured.Unstructured{
		Object: make(map[string]interface{}),
	}
	if err := unstructured_conversion.DefaultConverter.ToUnstructured(&serviceSpec, &updatedObj.Object); err != nil {
		return nil, err
	}
	return updatedObj, nil
}

func scBindingCleanup(spec, actual *unstructured.Unstructured) (*unstructured.Unstructured, error) {
	var bindingSpec sc_v1a1.Binding
	if err := unstructured_conversion.DefaultConverter.FromUnstructured(spec.Object, &bindingSpec); err != nil {
		return nil, err
	}
	var bindingActual sc_v1a1.Binding
	if err := unstructured_conversion.DefaultConverter.FromUnstructured(actual.Object, &bindingActual); err != nil {
		return nil, err
	}

	bindingSpec.Spec.ExternalID = bindingActual.Spec.ExternalID
	bindingSpec.Status = bindingActual.Status

	updatedObj := &unstructured.Unstructured{
		Object: make(map[string]interface{}),
	}
	if err := unstructured_conversion.DefaultConverter.ToUnstructured(&bindingSpec, &updatedObj.Object); err != nil {
		return nil, err
	}
	return updatedObj, nil
}

func scInstanceCleanup(spec, actual *unstructured.Unstructured) (*unstructured.Unstructured, error) {
	var instanceSpec sc_v1a1.Instance
	if err := unstructured_conversion.DefaultConverter.FromUnstructured(spec.Object, &instanceSpec); err != nil {
		return nil, err
	}
	var instanceActual sc_v1a1.Instance
	if err := unstructured_conversion.DefaultConverter.FromUnstructured(actual.Object, &instanceActual); err != nil {
		return nil, err
	}

	instanceSpec.Spec.ExternalID = instanceActual.Spec.ExternalID

	updatedObj := &unstructured.Unstructured{
		Object: make(map[string]interface{}),
	}
	if err := unstructured_conversion.DefaultConverter.ToUnstructured(&instanceSpec, &updatedObj.Object); err != nil {
		return nil, err
	}
	return updatedObj, nil
}
<commit_msg>NodePort cleanup<commit_after>package types

import (
	"github.com/atlassian/smith/pkg/cleanup"

	sc_v1a1 "github.com/kubernetes-incubator/service-catalog/pkg/apis/servicecatalog/v1alpha1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	unstructured_conversion "k8s.io/apimachinery/pkg/conversion/unstructured"
	"k8s.io/apimachinery/pkg/runtime/schema"
	api_v1 "k8s.io/client-go/pkg/api/v1"
	apps_v1b1 "k8s.io/client-go/pkg/apis/apps/v1beta1"
)

var MainKnownTypes = map[schema.GroupKind]cleanup.SpecCleanup{
	{Group: apps_v1b1.GroupName, Kind: "Deployment"}: deploymentCleanup,
	{Group: api_v1.GroupName, Kind: "Service"}:       serviceCleanup,
}

var ServiceCatalogKnownTypes = map[schema.GroupKind]cleanup.SpecCleanup{
	{Group: sc_v1a1.GroupName, Kind: "Binding"}:  scBindingCleanup,
	{Group: sc_v1a1.GroupName, Kind: "Instance"}: scInstanceCleanup,
}

func deploymentCleanup(spec, actual *unstructured.Unstructured) (*unstructured.Unstructured, error) {
	var deployment apps_v1b1.Deployment
	if err := unstructured_conversion.DefaultConverter.FromUnstructured(spec.Object, &deployment); err != nil {
		return nil, err
	}

	// TODO: implement cleanup

	return spec, nil
}

func serviceCleanup(spec, actual *unstructured.Unstructured) (*unstructured.Unstructured, error) {
	var serviceSpec api_v1.Service
	if err := unstructured_conversion.DefaultConverter.FromUnstructured(spec.Object, &serviceSpec); err != nil {
		return nil, err
	}
	var serviceActual api_v1.Service
	if err := unstructured_conversion.DefaultConverter.FromUnstructured(actual.Object, &serviceActual); err != nil {
		return nil, err
	}

	serviceSpec.Spec.ClusterIP = serviceActual.Spec.ClusterIP
	serviceSpec.Status = serviceActual.Status

	if len(serviceActual.Spec.Ports) == len(serviceSpec.Spec.Ports) {
		for i, port := range serviceSpec.Spec.Ports {
			if port.NodePort == 0 {
				actualPort := serviceActual.Spec.Ports[i]
				port.NodePort = actualPort.NodePort
				if port == actualPort { // NodePort field is the only difference, other fields are the same
					// Copy port from actual if port is not specified in spec
					serviceSpec.Spec.Ports[i].NodePort = actualPort.NodePort
				}
			}
		}
	}

	updatedObj := &unstructured.Unstructured{
		Object: make(map[string]interface{}),
	}
	if err := unstructured_conversion.DefaultConverter.ToUnstructured(&serviceSpec, &updatedObj.Object); err != nil {
		return nil, err
	}
	return updatedObj, nil
}

func scBindingCleanup(spec, actual *unstructured.Unstructured) (*unstructured.Unstructured, error) {
	var bindingSpec sc_v1a1.Binding
	if err := unstructured_conversion.DefaultConverter.FromUnstructured(spec.Object, &bindingSpec); err != nil {
		return nil, err
	}
	var bindingActual sc_v1a1.Binding
	if err := unstructured_conversion.DefaultConverter.FromUnstructured(actual.Object, &bindingActual); err != nil {
		return nil, err
	}

	bindingSpec.Spec.ExternalID = bindingActual.Spec.ExternalID
	bindingSpec.Status = bindingActual.Status

	updatedObj := &unstructured.Unstructured{
		Object: make(map[string]interface{}),
	}
	if err := unstructured_conversion.DefaultConverter.ToUnstructured(&bindingSpec, &updatedObj.Object); err != nil {
		return nil, err
	}
	return updatedObj, nil
}

func scInstanceCleanup(spec, actual *unstructured.Unstructured) (*unstructured.Unstructured, error) {
	var instanceSpec sc_v1a1.Instance
	if err := unstructured_conversion.DefaultConverter.FromUnstructured(spec.Object, &instanceSpec); err != nil {
		return nil, err
	}
	var instanceActual sc_v1a1.Instance
	if err := unstructured_conversion.DefaultConverter.FromUnstructured(actual.Object, &instanceActual); err != nil {
		return nil, err
	}

	instanceSpec.Spec.ExternalID = instanceActual.Spec.ExternalID

	updatedObj := &unstructured.Unstructured{
		Object: make(map[string]interface{}),
	}
	if err := unstructured_conversion.DefaultConverter.ToUnstructured(&instanceSpec, &updatedObj.Object); err != nil {
		return nil, err
	}
	return updatedObj, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package notify

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/blang/semver"
	"github.com/golang/glog"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/version"
)

const updateLinkPrefix = "https://github.com/kubernetes/minikube/releases/tag/v"

var (
	timeLayout              = time.RFC1123
	lastUpdateCheckFilePath = constants.MakeMiniPath("last_update_check")
)

func MaybePrintUpdateTextFromGithub(output io.Writer) {
	MaybePrintUpdateText(output, constants.GithubMinikubeReleasesURL, lastUpdateCheckFilePath)
}

func MaybePrintUpdateText(output io.Writer, url string, lastUpdatePath string) {
	if !shouldCheckURLVersion(lastUpdatePath) {
		return
	}
	latestVersion, err := getLatestVersionFromURL(url)
	if err != nil {
		glog.Errorln(err)
		return
	}
	localVersion, err := version.GetSemverVersion()
	if err != nil {
		glog.Errorln(err)
		return
	}
	if localVersion.Compare(latestVersion) < 0 {
		writeTimeToFile(lastUpdateCheckFilePath, time.Now().UTC())
		fmt.Fprintf(output, `There is a newer version of minikube available (%s%s).  Download it here:
%s%s

To disable this notification, run the following:
minikube config set WantUpdateNotification false
`,
			version.VersionPrefix, latestVersion, updateLinkPrefix, latestVersion)
	}
}

func shouldCheckURLVersion(filePath string) bool {
	if !viper.GetBool(config.WantUpdateNotification) {
		return false
	}
	lastUpdateTime := getTimeFromFileIfExists(filePath)
	return time.Since(lastUpdateTime).Hours() >= viper.GetFloat64(config.ReminderWaitPeriodInHours)
}

type Release struct {
	Name      string
	Checksums map[string]string
}

type Releases []Release

func getJson(url string, target *Releases) error {
	r, err := http.Get(url)
	if err != nil {
		return errors.Wrap(err, "Error getting minikube version url via http")
	}
	defer r.Body.Close()

	return json.NewDecoder(r.Body).Decode(target)
}

func getLatestVersionFromURL(url string) (semver.Version, error) {
	r, err := GetAllVersionsFromURL(url)
	if err != nil {
		return semver.Version{}, err
	}
	return semver.Make(strings.TrimPrefix(r[0].Name, version.VersionPrefix))
}

func GetAllVersionsFromURL(url string) (Releases, error) {
	var releases Releases
	glog.Infof("Checking for updates...")
	if err := getJson(url, &releases); err != nil {
		return releases, errors.Wrap(err, "Error getting json from minikube version url")
	}
	if len(releases) == 0 {
		return releases, errors.Errorf("There were no json releases at the url specified: %s", url)
	}
	return releases, nil
}

func writeTimeToFile(path string, inputTime time.Time) error {
	err := ioutil.WriteFile(path, []byte(inputTime.Format(timeLayout)), 0644)
	if err != nil {
		return errors.Wrap(err, "Error writing current update time to file: ")
	}
	return nil
}

func getTimeFromFileIfExists(path string) time.Time {
	lastUpdateCheckTime, err := ioutil.ReadFile(path)
	if err != nil {
		return time.Time{}
	}
	timeInFile, err := time.Parse(timeLayout, string(lastUpdateCheckTime))
	if err != nil {
		return time.Time{}
	}
	return timeInFile
}
<commit_msg>Only warn when can't get all localkube versions<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package notify

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/blang/semver"
	"github.com/golang/glog"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/version"
)

const updateLinkPrefix = "https://github.com/kubernetes/minikube/releases/tag/v"

var (
	timeLayout              = time.RFC1123
	lastUpdateCheckFilePath = constants.MakeMiniPath("last_update_check")
)

func MaybePrintUpdateTextFromGithub(output io.Writer) {
	MaybePrintUpdateText(output, constants.GithubMinikubeReleasesURL, lastUpdateCheckFilePath)
}

func MaybePrintUpdateText(output io.Writer, url string, lastUpdatePath string) {
	if !shouldCheckURLVersion(lastUpdatePath) {
		return
	}
	latestVersion, err := getLatestVersionFromURL(url)
	if err != nil {
		glog.Warning(err)
		return
	}
	localVersion, err := version.GetSemverVersion()
	if err != nil {
		glog.Warning(err)
		return
	}
	if localVersion.Compare(latestVersion) < 0 {
		writeTimeToFile(lastUpdateCheckFilePath, time.Now().UTC())
		fmt.Fprintf(output, `There is a newer version of minikube available (%s%s).  Download it here:
%s%s

To disable this notification, run the following:
minikube config set WantUpdateNotification false
`,
			version.VersionPrefix, latestVersion, updateLinkPrefix, latestVersion)
	}
}

func shouldCheckURLVersion(filePath string) bool {
	if !viper.GetBool(config.WantUpdateNotification) {
		return false
	}
	lastUpdateTime := getTimeFromFileIfExists(filePath)
	return time.Since(lastUpdateTime).Hours() >= viper.GetFloat64(config.ReminderWaitPeriodInHours)
}

type Release struct {
	Name      string
	Checksums map[string]string
}

type Releases []Release

func getJson(url string, target *Releases) error {
	r, err := http.Get(url)
	if err != nil {
		return errors.Wrap(err, "Error getting minikube version url via http")
	}
	defer r.Body.Close()

	return json.NewDecoder(r.Body).Decode(target)
}

func getLatestVersionFromURL(url string) (semver.Version, error) {
	r, err := GetAllVersionsFromURL(url)
	if err != nil {
		return semver.Version{}, err
	}
	return semver.Make(strings.TrimPrefix(r[0].Name, version.VersionPrefix))
}

func GetAllVersionsFromURL(url string) (Releases, error) {
	var releases Releases
	glog.Infof("Checking for updates...")
	if err := getJson(url, &releases); err != nil {
		return releases, errors.Wrap(err, "Error getting json from minikube version url")
	}
	if len(releases) == 0 {
		return releases, errors.Errorf("There were no json releases at the url specified: %s", url)
	}
	return releases, nil
}

func writeTimeToFile(path string, inputTime time.Time) error {
	err := ioutil.WriteFile(path, []byte(inputTime.Format(timeLayout)), 0644)
	if err != nil {
		return errors.Wrap(err, "Error writing current update time to file: ")
	}
	return nil
}

func getTimeFromFileIfExists(path string) time.Time {
	lastUpdateCheckTime, err := ioutil.ReadFile(path)
	if err != nil {
		return time.Time{}
	}
	timeInFile, err := time.Parse(timeLayout, string(lastUpdateCheckTime))
	if err != nil {
		return time.Time{}
	}
	return timeInFile
}
<|endoftext|>