<commit_before>/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package etcdserver

import (
	"crypto/sha1"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"log"
	"net/url"
	"path"
	"reflect"
	"sort"
	"strings"

	etcdErr "github.com/coreos/etcd/error"
	"github.com/coreos/etcd/pkg/flags"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/store"
)

const (
	raftAttributesSuffix = "raftAttributes"
	attributesSuffix     = "attributes"
)

type ClusterInfo interface {
	ID() types.ID
	ClientURLs() []string
	// Members returns a slice of members sorted by their ID
	Members() []*Member
	Member(id types.ID) *Member
}

// Cluster is a list of Members that belong to the same raft cluster
type Cluster struct {
	id      types.ID
	token   string
	members map[types.ID]*Member
	// removed contains the ids of removed members in the cluster.
	// removed id cannot be reused.
	removed map[types.ID]bool
	store   store.Store
}

// NewClusterFromString returns Cluster through given cluster token and parsing
// members from a sets of names to IPs discovery formatted like:
// mach0=http://1.1.1.1,mach0=http://2.2.2.2,mach1=http://3.3.3.3,mach2=http://4.4.4.4
func NewClusterFromString(token string, cluster string) (*Cluster, error) {
	c := newCluster(token)

	v, err := url.ParseQuery(strings.Replace(cluster, ",", "&", -1))
	if err != nil {
		return nil, err
	}
	for name, urls := range v {
		if len(urls) == 0 || urls[0] == "" {
			return nil, fmt.Errorf("Empty URL given for %q", name)
		}
		purls := &flags.URLsValue{}
		if err := purls.Set(strings.Join(urls, ",")); err != nil {
			return nil, err
		}
		m := NewMember(name, types.URLs(*purls), c.token, nil)
		if _, ok := c.members[m.ID]; ok {
			return nil, fmt.Errorf("Member exists with identical ID %v", m)
		}
		c.members[m.ID] = m
	}
	c.genID()
	return c, nil
}

func NewClusterFromStore(token string, st store.Store) *Cluster {
	c := newCluster(token)
	c.store = st

	e, err := c.store.Get(storeMembersPrefix, true, true)
	if err != nil {
		if isKeyNotFound(err) {
			return c
		}
		log.Panicf("get storeMembers should never fail: %v", err)
	}
	for _, n := range e.Node.Nodes {
		m, err := nodeToMember(n)
		if err != nil {
			log.Panicf("nodeToMember should never fail: %v", err)
		}
		c.members[m.ID] = m
	}

	e, err = c.store.Get(storeRemovedMembersPrefix, true, true)
	if err != nil {
		if isKeyNotFound(err) {
			return c
		}
		log.Panicf("get storeRemovedMembers should never fail: %v", err)
	}
	for _, n := range e.Node.Nodes {
		c.removed[mustParseMemberIDFromKey(n.Key)] = true
	}

	return c
}

func NewClusterFromMembers(token string, id types.ID, membs []*Member) *Cluster {
	c := newCluster(token)
	c.id = id
	for _, m := range membs {
		c.members[m.ID] = m
	}
	return c
}

func newCluster(token string) *Cluster {
	return &Cluster{
		token:   token,
		members: make(map[types.ID]*Member),
		removed: make(map[types.ID]bool),
	}
}

func (c Cluster) ID() types.ID { return c.id }

func (c Cluster) Members() []*Member {
	var sms SortableMemberSlice
	for _, m := range c.members {
		sms = append(sms, m)
	}
	sort.Sort(sms)
	return []*Member(sms)
}

type SortableMemberSlice []*Member

func (s SortableMemberSlice) Len() int           { return len(s) }
func (s SortableMemberSlice) Less(i, j int) bool { return s[i].ID < s[j].ID }
func (s SortableMemberSlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

func (c *Cluster) Member(id types.ID) *Member {
	return c.members[id]
}

// MemberByName returns a Member with the given name if exists.
// If more than one member has the given name, it will panic.
func (c *Cluster) MemberByName(name string) *Member {
	var memb *Member
	for _, m := range c.members {
		if m.Name == name {
			if memb != nil {
				log.Panicf("two members with the given name %s exist", name)
			}
			memb = m
		}
	}
	return memb
}

func (c Cluster) MemberIDs() []types.ID {
	var ids []types.ID
	for _, m := range c.members {
		ids = append(ids, m.ID)
	}
	sort.Sort(types.IDSlice(ids))
	return ids
}

func (c *Cluster) IsIDRemoved(id types.ID) bool {
	return c.removed[id]
}

// PeerURLs returns a list of all peer addresses. Each address is prefixed
// with the scheme (currently "http://"). The returned list is sorted in
// ascending lexicographical order.
func (c Cluster) PeerURLs() []string {
	endpoints := make([]string, 0)
	for _, p := range c.members {
		for _, addr := range p.PeerURLs {
			endpoints = append(endpoints, addr)
		}
	}
	sort.Strings(endpoints)
	return endpoints
}

// ClientURLs returns a list of all client addresses. Each address is prefixed
// with the scheme (currently "http://"). The returned list is sorted in
// ascending lexicographical order.
func (c Cluster) ClientURLs() []string {
	urls := make([]string, 0)
	for _, p := range c.members {
		for _, url := range p.ClientURLs {
			urls = append(urls, url)
		}
	}
	sort.Strings(urls)
	return urls
}

func (c Cluster) String() string {
	sl := []string{}
	for _, m := range c.members {
		for _, u := range m.PeerURLs {
			sl = append(sl, fmt.Sprintf("%s=%s", m.Name, u))
		}
	}
	sort.Strings(sl)
	return strings.Join(sl, ",")
}

// ValidateAndAssignIDs validates the given members by matching their PeerURLs
// with the existing members in the cluster. If the validation succeeds, it
// assigns the IDs from the given members to the existing members in the
// cluster. If the validation fails, an error will be returned.
func (c *Cluster) ValidateAndAssignIDs(membs []*Member) error {
	if len(c.members) != len(membs) {
		return fmt.Errorf("member count is unequal")
	}
	omembs := make([]*Member, 0)
	for _, m := range c.members {
		omembs = append(omembs, m)
	}
	sort.Sort(SortableMemberSliceByPeerURLs(omembs))
	sort.Sort(SortableMemberSliceByPeerURLs(membs))
	for i := range omembs {
		if !reflect.DeepEqual(omembs[i].PeerURLs, membs[i].PeerURLs) {
			return fmt.Errorf("unmatched member while checking PeerURLs")
		}
		omembs[i].ID = membs[i].ID
	}
	c.members = make(map[types.ID]*Member)
	for _, m := range omembs {
		c.members[m.ID] = m
	}
	return nil
}

func (c *Cluster) genID() {
	mIDs := c.MemberIDs()
	b := make([]byte, 8*len(mIDs))
	for i, id := range mIDs {
		binary.BigEndian.PutUint64(b[8*i:], uint64(id))
	}
	hash := sha1.Sum(b)
	c.id = types.ID(binary.BigEndian.Uint64(hash[:8]))
}

func (c *Cluster) SetID(id types.ID) { c.id = id }

func (c *Cluster) SetStore(st store.Store) { c.store = st }

// AddMember puts a new Member into the store.
// A Member with a matching id must not exist.
func (c *Cluster) AddMember(m *Member) {
	b, err := json.Marshal(m.RaftAttributes)
	if err != nil {
		log.Panicf("marshal raftAttributes should never fail: %v", err)
	}
	p := path.Join(memberStoreKey(m.ID), raftAttributesSuffix)
	if _, err := c.store.Create(p, false, string(b), false, store.Permanent); err != nil {
		log.Panicf("create raftAttributes should never fail: %v", err)
	}
	b, err = json.Marshal(m.Attributes)
	if err != nil {
		log.Panicf("marshal attributes should never fail: %v", err)
	}
	p = path.Join(memberStoreKey(m.ID), attributesSuffix)
	if _, err := c.store.Create(p, false, string(b), false, store.Permanent); err != nil {
		log.Panicf("create attributes should never fail: %v", err)
	}
	c.members[m.ID] = m
}

// RemoveMember removes a member from the store.
// The given id MUST exist, or the function panics.
func (c *Cluster) RemoveMember(id types.ID) {
	if _, err := c.store.Delete(memberStoreKey(id), true, true); err != nil {
		log.Panicf("delete member should never fail: %v", err)
	}
	delete(c.members, id)
	if _, err := c.store.Create(removedMemberStoreKey(id), false, "", false, store.Permanent); err != nil {
		log.Panicf("create removedMember should never fail: %v", err)
	}
	c.removed[id] = true
}

// nodeToMember builds member through a store node.
// the child nodes of the given node should be sorted by key.
func nodeToMember(n *store.NodeExtern) (*Member, error) {
	m := &Member{ID: mustParseMemberIDFromKey(n.Key)}
	if len(n.Nodes) != 2 {
		return m, fmt.Errorf("len(nodes) = %d, want 2", len(n.Nodes))
	}
	if w := path.Join(n.Key, attributesSuffix); n.Nodes[0].Key != w {
		return m, fmt.Errorf("key = %v, want %v", n.Nodes[0].Key, w)
	}
	if err := json.Unmarshal([]byte(*n.Nodes[0].Value), &m.Attributes); err != nil {
		return m, fmt.Errorf("unmarshal attributes error: %v", err)
	}
	if w := path.Join(n.Key, raftAttributesSuffix); n.Nodes[1].Key != w {
		return m, fmt.Errorf("key = %v, want %v", n.Nodes[1].Key, w)
	}
	if err := json.Unmarshal([]byte(*n.Nodes[1].Value), &m.RaftAttributes); err != nil {
		return m, fmt.Errorf("unmarshal raftAttributes error: %v", err)
	}
	return m, nil
}

func isKeyNotFound(err error) bool {
	e, ok := err.(*etcdErr.Error)
	return ok && e.ErrorCode == etcdErr.EcodeKeyNotFound
}
<commit_msg>etcdserver: improve panic message in Cluster<commit_after>/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package etcdserver

import (
	"crypto/sha1"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"log"
	"net/url"
	"path"
	"reflect"
	"sort"
	"strings"

	etcdErr "github.com/coreos/etcd/error"
	"github.com/coreos/etcd/pkg/flags"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/store"
)

const (
	raftAttributesSuffix = "raftAttributes"
	attributesSuffix     = "attributes"
)

type ClusterInfo interface {
	ID() types.ID
	ClientURLs() []string
	// Members returns a slice of members sorted by their ID
	Members() []*Member
	Member(id types.ID) *Member
}

// Cluster is a list of Members that belong to the same raft cluster
type Cluster struct {
	id      types.ID
	token   string
	members map[types.ID]*Member
	// removed contains the ids of removed members in the cluster.
	// removed id cannot be reused.
	removed map[types.ID]bool
	store   store.Store
}

// NewClusterFromString returns Cluster through given cluster token and parsing
// members from a sets of names to IPs discovery formatted like:
// mach0=http://1.1.1.1,mach0=http://2.2.2.2,mach1=http://3.3.3.3,mach2=http://4.4.4.4
func NewClusterFromString(token string, cluster string) (*Cluster, error) {
	c := newCluster(token)

	v, err := url.ParseQuery(strings.Replace(cluster, ",", "&", -1))
	if err != nil {
		return nil, err
	}
	for name, urls := range v {
		if len(urls) == 0 || urls[0] == "" {
			return nil, fmt.Errorf("Empty URL given for %q", name)
		}
		purls := &flags.URLsValue{}
		if err := purls.Set(strings.Join(urls, ",")); err != nil {
			return nil, err
		}
		m := NewMember(name, types.URLs(*purls), c.token, nil)
		if _, ok := c.members[m.ID]; ok {
			return nil, fmt.Errorf("Member exists with identical ID %v", m)
		}
		c.members[m.ID] = m
	}
	c.genID()
	return c, nil
}

func NewClusterFromStore(token string, st store.Store) *Cluster {
	c := newCluster(token)
	c.store = st

	e, err := c.store.Get(storeMembersPrefix, true, true)
	if err != nil {
		if isKeyNotFound(err) {
			return c
		}
		log.Panicf("get storeMembers should never fail: %v", err)
	}
	for _, n := range e.Node.Nodes {
		m, err := nodeToMember(n)
		if err != nil {
			log.Panicf("nodeToMember should never fail: %v", err)
		}
		c.members[m.ID] = m
	}

	e, err = c.store.Get(storeRemovedMembersPrefix, true, true)
	if err != nil {
		if isKeyNotFound(err) {
			return c
		}
		log.Panicf("get storeRemovedMembers should never fail: %v", err)
	}
	for _, n := range e.Node.Nodes {
		c.removed[mustParseMemberIDFromKey(n.Key)] = true
	}

	return c
}

func NewClusterFromMembers(token string, id types.ID, membs []*Member) *Cluster {
	c := newCluster(token)
	c.id = id
	for _, m := range membs {
		c.members[m.ID] = m
	}
	return c
}

func newCluster(token string) *Cluster {
	return &Cluster{
		token:   token,
		members: make(map[types.ID]*Member),
		removed: make(map[types.ID]bool),
	}
}

func (c Cluster) ID() types.ID { return c.id }

func (c Cluster) Members() []*Member {
	var sms SortableMemberSlice
	for _, m := range c.members {
		sms = append(sms, m)
	}
	sort.Sort(sms)
	return []*Member(sms)
}

type SortableMemberSlice []*Member

func (s SortableMemberSlice) Len() int           { return len(s) }
func (s SortableMemberSlice) Less(i, j int) bool { return s[i].ID < s[j].ID }
func (s SortableMemberSlice) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }

func (c *Cluster) Member(id types.ID) *Member {
	return c.members[id]
}

// MemberByName returns a Member with the given name if exists.
// If more than one member has the given name, it will panic.
func (c *Cluster) MemberByName(name string) *Member {
	var memb *Member
	for _, m := range c.members {
		if m.Name == name {
			if memb != nil {
				log.Panicf("two members with the given name %q exist", name)
			}
			memb = m
		}
	}
	return memb
}

func (c Cluster) MemberIDs() []types.ID {
	var ids []types.ID
	for _, m := range c.members {
		ids = append(ids, m.ID)
	}
	sort.Sort(types.IDSlice(ids))
	return ids
}

func (c *Cluster) IsIDRemoved(id types.ID) bool {
	return c.removed[id]
}

// PeerURLs returns a list of all peer addresses. Each address is prefixed
// with the scheme (currently "http://"). The returned list is sorted in
// ascending lexicographical order.
func (c Cluster) PeerURLs() []string {
	endpoints := make([]string, 0)
	for _, p := range c.members {
		for _, addr := range p.PeerURLs {
			endpoints = append(endpoints, addr)
		}
	}
	sort.Strings(endpoints)
	return endpoints
}

// ClientURLs returns a list of all client addresses. Each address is prefixed
// with the scheme (currently "http://"). The returned list is sorted in
// ascending lexicographical order.
func (c Cluster) ClientURLs() []string {
	urls := make([]string, 0)
	for _, p := range c.members {
		for _, url := range p.ClientURLs {
			urls = append(urls, url)
		}
	}
	sort.Strings(urls)
	return urls
}

func (c Cluster) String() string {
	sl := []string{}
	for _, m := range c.members {
		for _, u := range m.PeerURLs {
			sl = append(sl, fmt.Sprintf("%s=%s", m.Name, u))
		}
	}
	sort.Strings(sl)
	return strings.Join(sl, ",")
}

// ValidateAndAssignIDs validates the given members by matching their PeerURLs
// with the existing members in the cluster. If the validation succeeds, it
// assigns the IDs from the given members to the existing members in the
// cluster. If the validation fails, an error will be returned.
func (c *Cluster) ValidateAndAssignIDs(membs []*Member) error {
	if len(c.members) != len(membs) {
		return fmt.Errorf("member count is unequal")
	}
	omembs := make([]*Member, 0)
	for _, m := range c.members {
		omembs = append(omembs, m)
	}
	sort.Sort(SortableMemberSliceByPeerURLs(omembs))
	sort.Sort(SortableMemberSliceByPeerURLs(membs))
	for i := range omembs {
		if !reflect.DeepEqual(omembs[i].PeerURLs, membs[i].PeerURLs) {
			return fmt.Errorf("unmatched member while checking PeerURLs")
		}
		omembs[i].ID = membs[i].ID
	}
	c.members = make(map[types.ID]*Member)
	for _, m := range omembs {
		c.members[m.ID] = m
	}
	return nil
}

func (c *Cluster) genID() {
	mIDs := c.MemberIDs()
	b := make([]byte, 8*len(mIDs))
	for i, id := range mIDs {
		binary.BigEndian.PutUint64(b[8*i:], uint64(id))
	}
	hash := sha1.Sum(b)
	c.id = types.ID(binary.BigEndian.Uint64(hash[:8]))
}

func (c *Cluster) SetID(id types.ID) { c.id = id }

func (c *Cluster) SetStore(st store.Store) { c.store = st }

// AddMember puts a new Member into the store.
// A Member with a matching id must not exist.
func (c *Cluster) AddMember(m *Member) {
	b, err := json.Marshal(m.RaftAttributes)
	if err != nil {
		log.Panicf("marshal raftAttributes should never fail: %v", err)
	}
	p := path.Join(memberStoreKey(m.ID), raftAttributesSuffix)
	if _, err := c.store.Create(p, false, string(b), false, store.Permanent); err != nil {
		log.Panicf("create raftAttributes should never fail: %v", err)
	}
	b, err = json.Marshal(m.Attributes)
	if err != nil {
		log.Panicf("marshal attributes should never fail: %v", err)
	}
	p = path.Join(memberStoreKey(m.ID), attributesSuffix)
	if _, err := c.store.Create(p, false, string(b), false, store.Permanent); err != nil {
		log.Panicf("create attributes should never fail: %v", err)
	}
	c.members[m.ID] = m
}

// RemoveMember removes a member from the store.
// The given id MUST exist, or the function panics.
func (c *Cluster) RemoveMember(id types.ID) {
	if _, err := c.store.Delete(memberStoreKey(id), true, true); err != nil {
		log.Panicf("delete member should never fail: %v", err)
	}
	delete(c.members, id)
	if _, err := c.store.Create(removedMemberStoreKey(id), false, "", false, store.Permanent); err != nil {
		log.Panicf("create removedMember should never fail: %v", err)
	}
	c.removed[id] = true
}

// nodeToMember builds member through a store node.
// the child nodes of the given node should be sorted by key.
func nodeToMember(n *store.NodeExtern) (*Member, error) {
	m := &Member{ID: mustParseMemberIDFromKey(n.Key)}
	if len(n.Nodes) != 2 {
		return m, fmt.Errorf("len(nodes) = %d, want 2", len(n.Nodes))
	}
	if w := path.Join(n.Key, attributesSuffix); n.Nodes[0].Key != w {
		return m, fmt.Errorf("key = %v, want %v", n.Nodes[0].Key, w)
	}
	if err := json.Unmarshal([]byte(*n.Nodes[0].Value), &m.Attributes); err != nil {
		return m, fmt.Errorf("unmarshal attributes error: %v", err)
	}
	if w := path.Join(n.Key, raftAttributesSuffix); n.Nodes[1].Key != w {
		return m, fmt.Errorf("key = %v, want %v", n.Nodes[1].Key, w)
	}
	if err := json.Unmarshal([]byte(*n.Nodes[1].Value), &m.RaftAttributes); err != nil {
		return m, fmt.Errorf("unmarshal raftAttributes error: %v", err)
	}
	return m, nil
}

func isKeyNotFound(err error) bool {
	e, ok := err.(*etcdErr.Error)
	return ok && e.ErrorCode == etcdErr.EcodeKeyNotFound
}
<|endoftext|>


<commit_before>package xmmsclient

import (
	"bytes"
	"encoding/hex"
	"reflect"
	"testing"
)

func checkBuffer(t *testing.T, expected []byte, actual []byte) {
	if !reflect.DeepEqual(expected, actual) {
		t.Fatalf("\nwant:\n%s\nhave:\n%s", hex.Dump(expected), hex.Dump(actual))
	}
}

func TestSerializeInt(t *testing.T) {
	var expected = []byte{
		0x00, 0x00, 0x00, 0x02,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a,
	}
	var buffer bytes.Buffer

	var err = SerializeXmmsValue(XmmsInt(42), &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}

func TestSerializeString(t *testing.T) {
	var expected = []byte{
		0x00, 0x00, 0x00, 0x03,
		0x00, 0x00, 0x00, 0x05,
		0x74, 0x65, 0x73, 0x74, 0x00,
	}
	var buffer bytes.Buffer

	var err = SerializeXmmsValue(XmmsString("test"), &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}

func TestSerializeList(t *testing.T) {
	var expected = []byte{
		0x00, 0x00, 0x00, 0x06,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x02,
		0x00, 0x00, 0x00, 0x02,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a,
		0x00, 0x00, 0x00, 0x03,
		0x00, 0x00, 0x00, 0x05,
		0x74, 0x65, 0x73, 0x74, 0x00,
	}
	var buffer bytes.Buffer

	var err = SerializeXmmsValue(NewXmmsList(XmmsInt(42), XmmsString("test")), &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}

/* No sorted map in Go, need some other testing strategy
func TestSerializeDict(t *testing.T) {
	var expected = []byte{
		0x00, 0x00, 0x00, 0x07,
		0x00, 0x00, 0x00, 0x02,
		0x00, 0x00, 0x00, 0x04,
		0x69, 0x6e, 0x74, 0x00,
		0x00, 0x00, 0x00, 0x02,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a,
		0x00, 0x00, 0x00, 0x04,
		0x73, 0x74, 0x72, 0x00,
		0x00, 0x00, 0x00, 0x03,
		0x00, 0x00, 0x00, 0x05,
		0x74, 0x65, 0x73, 0x74, 0x00,
	}
	var buffer bytes.Buffer

	var err = SerializeXmmsValue(XmmsDict{"int": XmmsInt(42), "str": XmmsString("test")}, &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}
*/
<commit_msg>Enable XmmsDict test with only one entry.<commit_after>package xmmsclient

import (
	"bytes"
	"encoding/hex"
	"reflect"
	"testing"
)

func checkBuffer(t *testing.T, expected []byte, actual []byte) {
	if !reflect.DeepEqual(expected, actual) {
		t.Fatalf("\nwant:\n%s\nhave:\n%s", hex.Dump(expected), hex.Dump(actual))
	}
}

func TestSerializeInt(t *testing.T) {
	var expected = []byte{
		0x00, 0x00, 0x00, 0x02,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a,
	}
	var buffer bytes.Buffer

	var err = SerializeXmmsValue(XmmsInt(42), &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}

func TestSerializeString(t *testing.T) {
	var expected = []byte{
		0x00, 0x00, 0x00, 0x03,
		0x00, 0x00, 0x00, 0x05,
		0x74, 0x65, 0x73, 0x74, 0x00,
	}
	var buffer bytes.Buffer

	var err = SerializeXmmsValue(XmmsString("test"), &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}

func TestSerializeList(t *testing.T) {
	var expected = []byte{
		0x00, 0x00, 0x00, 0x06,
		0x00, 0x00, 0x00, 0x00,
		0x00, 0x00, 0x00, 0x02,
		0x00, 0x00, 0x00, 0x02,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a,
		0x00, 0x00, 0x00, 0x03,
		0x00, 0x00, 0x00, 0x05,
		0x74, 0x65, 0x73, 0x74, 0x00,
	}
	var buffer bytes.Buffer

	var err = SerializeXmmsValue(NewXmmsList(XmmsInt(42), XmmsString("test")), &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}

func TestSerializeDict(t *testing.T) {
	var expected = []byte{
		0x00, 0x00, 0x00, 0x07,
		0x00, 0x00, 0x00, 0x01,
		0x00, 0x00, 0x00, 0x04,
		0x69, 0x6e, 0x74, 0x00,
		0x00, 0x00, 0x00, 0x02,
		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2a,
	}
	var buffer bytes.Buffer

	var err = SerializeXmmsValue(XmmsDict{"int": XmmsInt(42)}, &buffer)
	if err != nil {
		t.Fatal(err)
	}

	checkBuffer(t, expected, buffer.Bytes())
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
)

var (
	packageName string
	outputFile  string
	inputDir    string
	relativeTo  string
)

func init() {
	flag.StringVar(&packageName, "p", "", "Package name of generated source file")
	flag.StringVar(&outputFile, "o", "", "Filename for generated file")
	flag.StringVar(&inputDir, "i", "", "Input directory to process")
}

type module struct {
	jsonType string
	keys     []*moduleKeys
	name     string
}

type moduleKeys struct {
	name     string
	jsonType string
	suffix   string
}

func main() {
	flag.Parse()

	if packageName == "" {
		fmt.Println("Package name required")
		os.Exit(1)
	}
	if outputFile == "" {
		fmt.Println("Output file name required")
		os.Exit(1)
	}
	if inputDir == "" {
		fmt.Println("Input dir name required")
		os.Exit(1)
	}

	file, err := os.Create(outputFile)
	if err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}
	defer file.Close()

	if err := writeHeader(file); err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}

	modules, err := getModules(file)
	if err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}

	if err := writeModules(file, modules); err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}

	if err := writeControl(file, modules); err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}
}

func writeHeader(w io.Writer) error {
	_, err := w.Write([]byte(fmt.Sprintf(`package %s

// DO NOT EDIT. This file is generated by ./cmd/generateModuleTypes/main.go.

import "fmt"

`, packageName)))
	return err
}

func getModules(w io.Writer) ([]*module, error) {
	files, err := ioutil.ReadDir(inputDir)
	if err != nil {
		return nil, err
	}

	var modules []*module

	for _, file := range files {
		filename := path.Join(inputDir, file.Name())
		f, err := os.Open(filename)
		if err != nil {
			return nil, err
		}
		reader := bufio.NewReader(f)
		_, _, err = reader.ReadLine()
		if err != nil {
			f.Close()
			return nil, err
		}

		genLine, _, err := reader.ReadLine()
		if err != nil {
			f.Close()
			return nil, err
		}
		f.Close()

		if !bytes.Equal(genLine[:11], []byte("#gen:module")) {
			log.Printf("Skipping file %s", file.Name())
			continue
		}

		genLineParts := bytes.SplitN(genLine, []byte(" "), 3)
		if len(genLineParts) != 3 {
			log.Printf("Invalid gen line in file %s", filename)
		}

		mod := &module{
			name:     file.Name()[:len(file.Name())-len(filepath.Ext(file.Name()))],
			jsonType: string(genLineParts[1]),
		}
		keys := bytes.Split(genLineParts[2], []byte(","))
		mod.keys = make([]*moduleKeys, 0, len(keys))

		for _, key := range keys {
			keyParts := strings.Split(string(key), ":")
			newKey := &moduleKeys{
				name:     keyParts[0],
				jsonType: keyParts[1],
			}

			if len(keyParts) == 3 {
				newKey.suffix = keyParts[2]
			}

			mod.keys = append(mod.keys, newKey)
		}

		modules = append(modules, mod)
	}
	return modules, nil
}

func writeModules(w io.Writer, modules []*module) error {
	for _, mod := range modules {
		fmt.Fprintf(w, "type %s struct {\n", goName(mod.name))
		for _, key := range mod.keys {
			fmt.Fprintf(w, "	%s %s `json:\"%s\"`\n", goName(key.name), key.jsonType, key.name)
		}
		fmt.Fprint(w, "}\n\n")

		if mod.jsonType == "a" {
			writeArrayPrint(w, mod)
		} else if mod.jsonType == "o" {
			writeObjPrint(w, mod)
		} else {
			return fmt.Errorf("Json type %s not supported", mod.jsonType)
		}
	}
	return nil
}

func writeArrayPrint(w io.Writer, mod *module) {
	srcName := goName(mod.name)
	fancyName := displayName(mod.name)

	fmt.Fprintf(w, `func printLong%s(a []*%s) {
	fmt.Println("    %s:")
	for _, o := range a {
`, srcName, srcName, fancyName)

	for _, key := range mod.keys {
		if key.suffix == "%" {
			key.suffix = "%%"
		}

		fmt.Fprintf(w, "		fmt.Printf(\"        %s: %s%s\\n\", o.%s)\n",
			displayName(key.name),
			fmtType(key.jsonType),
			key.suffix,
			goName(key.name),
		)
	}

	fmt.Fprint(w, "		fmt.Println(\"\")\n	}\n}\n\n")
}

func writeObjPrint(w io.Writer, mod *module) {
	srcName := goName(mod.name)
	fancyName := displayName(mod.name)

	fmt.Fprintf(w, `func printLong%s(a *%s) {
	fmt.Println("    %s:")
`, srcName, srcName, fancyName)

	for _, key := range mod.keys {
		if key.suffix == "%" {
			key.suffix = "%%"
		}

		fmt.Fprintf(w, "	fmt.Printf(\"        %s: %s%s\\n\", a.%s)\n",
			displayName(key.name),
			fmtType(key.jsonType),
			key.suffix,
			goName(key.name),
		)
	}

	fmt.Fprint(w, "}\n\n")
}

func writeControl(w io.Writer, modules []*module) error {
	fmt.Fprint(w, "type HostResponse struct {\n	Host *ConfigHost `json:\"host\"`\n")
	for _, mod := range modules {
		srcName := goName(mod.name)
		fmt.Fprintf(w, "	%s %s*%s `json:\"%s,omitempty\"`\n",
			srcName, srcType(mod.jsonType), srcName, mod.name,
		)
	}
	fmt.Fprint(w, `}

func (r *HostResponse) Print(short bool) {
	if r == nil {
		return
	}

	if short {
		r.printShort()
		return
	}
	r.printLong()
}

func (r *HostResponse) printShort() {
	// TODO
}

`)

	fmt.Fprint(w, "func (r *HostResponse) printLong() {\n")

	for _, mod := range modules {
		srcName := goName(mod.name)
		if mod.jsonType == "a" {
			fmt.Fprintf(w, `	if len(r.%s) > 0 {
		printLong%s(r.%s)
		fmt.Println("")
	}
`, srcName, srcName, srcName)
		} else if mod.jsonType == "o" {
			fmt.Fprintf(w, `	if r.%s != nil {
		printLong%s(r.%s)
		fmt.Println("")
	}
`, srcName, srcName, srcName)
		}
	}

	fmt.Fprint(w, "}\n\n")
	return nil
}

func srcType(t string) string {
	switch t {
	case "a":
		return "[]"
	}
	return ""
}

func fmtType(t string) string {
	switch t {
	case "string":
		return "%s"
	case "float64":
		return "%.2f"
	case "bool":
		return "%t"
	case "int":
		return "%d"
	}
	return ""
}

func displayName(s string) string {
	s = strings.Replace(s, "_", " ", -1)
	s = strings.Title(s)
	return s
}

func goName(s string) string {
	s = displayName(s)
	s = strings.Replace(s, " ", "", -1)
	s = strings.Replace(s, "(", "", -1)
	s = strings.Replace(s, ")", "", -1)
	s = strings.Replace(s, "-", "", -1)
	return s
}
<commit_msg>Implemented more flexible module type definitions<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path"
	"path/filepath"
	"strings"
)

var (
	packageName string
	outputFile  string
	inputDir    string
	relativeTo  string
)

func init() {
	flag.StringVar(&packageName, "p", "", "Package name of generated source file")
	flag.StringVar(&outputFile, "o", "", "Filename for generated file")
	flag.StringVar(&inputDir, "i", "", "Input directory to process")
}

type module struct {
	jsonType string
	keys     []*moduleKey
	types    map[string][]*moduleKey
	name     string
}

type moduleKey struct {
	name    string
	keyType string
	suffix  string
}

func main() {
	flag.Parse()

	if packageName == "" {
		fmt.Println("Package name required")
		os.Exit(1)
	}
	if outputFile == "" {
		fmt.Println("Output file name required")
		os.Exit(1)
	}
	if inputDir == "" {
		fmt.Println("Input dir name required")
		os.Exit(1)
	}

	modules, err := getModules()
	if err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}

	file, err := os.Create(outputFile)
	if err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}
	defer file.Close()

	if err := writeHeader(file); err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}

	if err := writeModules(file, modules); err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}

	if err := writeControl(file, modules); err != nil {
		fmt.Println(err.Error())
		os.Exit(1)
	}
}

func getModules() ([]*module, error) {
	files, err := ioutil.ReadDir(inputDir)
	if err != nil {
		return nil, err
	}

	var modules []*module

	for _, file := range files {
		filename := path.Join(inputDir, file.Name())
		f, err := os.Open(filename)
		if err != nil {
			return nil, err
		}
		defer f.Close()
		reader := bufio.NewReader(f)
		_, _, err = reader.ReadLine()
		if err != nil {
			f.Close()
			return nil, err
		}

		genLine, _, err := reader.ReadLine()
		if err != nil {
			return nil, err
		}

		var mod *module

		if bytes.Equal(genLine[:12], []byte("#gen:module2")) {
			mod, err = parseNewFormat(genLine, reader, filename, file)
		} else if bytes.Equal(genLine[:11], []byte("#gen:module")) {
			mod, err = parseOldFormat(genLine, filename, file)
		} else {
			log.Printf("Skipping file %s", file.Name())
			continue
		}

		if err != nil {
			return nil, err
		}

		modules = append(modules, mod)
		f.Close()
	}
	return modules, nil
}

func parseNewFormat(firstline []byte, reader *bufio.Reader, filename string, file os.FileInfo) (*module, error) {
	genHeaderLine := bytes.SplitN(firstline, []byte(" "), 2)
	if len(genHeaderLine) != 2 {
		return nil, fmt.Errorf("Invalid gen line in file %s", filename)
	}

	mod := &module{
		name:     file.Name()[:len(file.Name())-len(filepath.Ext(file.Name()))],
		jsonType: string(genHeaderLine[1]),
		keys:     make([]*moduleKey, 0),
		types:    make(map[string][]*moduleKey),
	}

	mode := "key"
	t := ""
	for {
		line, _, err := reader.ReadLine()
		if err != nil {
			return nil, err
		}
		line = bytes.TrimSpace(line)
		line = bytes.TrimLeft(line, "#")
		line = bytes.TrimSpace(line)

		if len(line) == 0 {
			continue
		}

		if bytes.Equal(line[:12], []byte("!gen:module2")) {
			if mode != "key" {
				return nil, fmt.Errorf("Missing 'endtype' in module %s", mod.name)
			}
			break
		}

		lineParts := bytes.Split(line, []byte{' '})
		if mode == "key" { // root key mode
			if string(lineParts[0]) == "type" {
				if len(lineParts) != 2 {
					return nil, fmt.Errorf("Missing type name in module %s", mod.name)
				}
				t = string(lineParts[1])
				if _, exists := mod.types[t]; exists {
					return nil, fmt.Errorf("Duplicate type definition in module %s", mod.name)
				}
				mod.types[t] = make([]*moduleKey, 0, 1)
				mode = "type"
				continue
			} else if string(lineParts[0]) == "key" {
				if len(lineParts) < 3 {
					return nil, fmt.Errorf("Invalid key in module %s", mod.name)
				}
				key := &moduleKey{
					name:    string(lineParts[1]),
					keyType: string(lineParts[2]),
				}
				if len(lineParts) == 4 {
					key.suffix = string(lineParts[3])
				}
				mod.keys = append(mod.keys, key)
				continue
			} else {
				return nil, fmt.Errorf("Unknown symbol %s in module %s", lineParts[0], mod.name)
			}
		} else { // type mode
			if bytes.Equal(lineParts[0], []byte("endtype")) {
				t = ""
				mode = "key"
				continue
			}

			if len(lineParts) < 2 {
				return nil, fmt.Errorf("Invalid key in type %s in module %s", t, mod.name)
			}
			key := &moduleKey{
				name:    string(lineParts[0]),
				keyType: string(lineParts[1]),
			}
			if len(lineParts) == 3 {
				key.suffix = string(lineParts[2])
			}
			mod.types[t] = append(mod.types[t], key)
			continue
		}
	}
	return mod, nil
}

func parseOldFormat(line []byte, filename string, file os.FileInfo) (*module, error) {
	genLineParts := bytes.SplitN(line, []byte(" "), 3)
	if len(genLineParts) != 3 {
		return nil, fmt.Errorf("Invalid gen line in file %s", filename)
	}

	mod := &module{
		name:     file.Name()[:len(file.Name())-len(filepath.Ext(file.Name()))],
		jsonType: string(genLineParts[1]),
	}

	keys := bytes.Split(genLineParts[2], []byte(","))
	mod.keys = make([]*moduleKey, 0, len(keys))

	for _, key := range keys {
		keyParts := strings.Split(string(key), ":")
		newKey := &moduleKey{
			name:    keyParts[0],
			keyType: keyParts[1],
		}

		if len(keyParts) == 3 {
			newKey.suffix = keyParts[2]
		}

		mod.keys = append(mod.keys, newKey)
	}
	return mod, nil
}

func writeHeader(w io.Writer) error {
	_, err := w.Write([]byte(fmt.Sprintf(`package %s

// DO NOT EDIT. This file is generated by cmd/generateModuleTypes/main.go.

import (
	"fmt"
	"strings"
)

`, packageName)))
	return err
}

func writeModules(w io.Writer, modules []*module) error {
	for _, mod := range modules {
		modName := goName(mod.name)
		fmt.Fprintf(w, "type %s struct {\n", modName)
		for _, key := range mod.keys {
			ktype := key.keyType
			if ktype[:2] == "[]" {
				ktype = "[]*" + modName + goName(ktype[2:])
			}
			fmt.Fprintf(w, "	%s %s `json:\"%s\"`\n", goName(key.name), ktype, key.name)
		}
		fmt.Fprint(w, "}\n\n")

		for t, keys := range mod.types {
			modName := goName(mod.name)
			fmt.Fprintf(w, "type %s struct {\n", modName+goName(t))
			for _, key := range keys {
				ktype := key.keyType
				if ktype[:2] == "[]" {
					ktype = "[]*" + modName + goName(ktype[2:])
				}
				fmt.Fprintf(w, "	%s %s `json:\"%s\"`\n", goName(key.name), ktype, key.name)
			}
			fmt.Fprint(w, "}\n\n")
		}

		if mod.jsonType != "a" && mod.jsonType != "o" {
			return fmt.Errorf("Json type %s not supported", mod.jsonType)
		}

		writePrint(w, mod)
	}
	return nil
}

func writePrint(w io.Writer, mod *module) {
	srcName := goName(mod.name)
	fancyName := displayName(mod.name)

	headerType := "a []"
	if mod.jsonType == "o" {
		headerType = "o "
	}

	fmt.Fprintf(w, `func printLong%s(depth int, %s*%s) {
	indent := strings.Repeat(" ", depth*2)
	fmt.Printf("%%s%s:\n", indent)
`, srcName, headerType, srcName, fancyName)

	if mod.jsonType == "a" {
		fmt.Fprintf(w, "	for _, o := range a {\n")
	}

	for _, key := range mod.keys {
		if key.suffix == "%" {
			key.suffix = "%%"
		}

		ktype := key.keyType
		isSlice := false
		if ktype[:2] == "[]" {
			ktype = ktype[2:]
			isSlice = true
		}
		if _, exists := mod.types[ktype]; exists {
			fmt.Fprintf(w, "	fmt.Printf(\"%%s%s: \\n\", indent)\n", displayName(key.name))
			if isSlice {
				fmt.Fprintf(w, "	for _, p := range o.%s {\n", goName(key.name))
				fmt.Fprintf(w, "		printLong%s(depth+1, p)\n", srcName+goName(ktype))
				fmt.Fprintf(w, "	}\n")
			} else {
				fmt.Fprintf(w, "	printLong%s(depth+1, o.%s)\n", srcName+goName(ktype), goName(key.name))
			}
		} else {
			fmt.Fprintf(w, "	fmt.Printf(\"%%s%s: %s%s\\n\", indent, o.%s)\n",
				displayName(key.name),
				fmtType(ktype),
				key.suffix,
				goName(key.name),
			)
		}
	}
	if mod.jsonType == "a" {
		fmt.Fprint(w, "	fmt.Println(\"\")\n	}\n}\n\n")
	} else {
		fmt.Fprint(w, "	fmt.Println(\"\")\n}\n\n")
	}

	for t, keys := range mod.types {
		fullType := srcName + goName(t)
		fmt.Fprintf(w, `func printLong%s(depth int, o *%s) {
	indent := strings.Repeat(" ", depth*2)
`, fullType, fullType)

		for _, key := range keys {
			if key.suffix == "%" {
				key.suffix = "%%"
			}

			ktype := key.keyType
			if _, exists := mod.types[ktype]; exists {
				fmt.Fprintf(w, "	fmt.Printf(\"%%s%s: \\n\", indent)\n", displayName(key.name))
				fmt.Fprintf(w, "	printLong%s(depth+1, o.%s)\n", srcName+goName(ktype), goName(key.name))
			} else {
				fmt.Fprintf(w, "	fmt.Printf(\"%%s%s: %s%s\\n\", indent, o.%s)\n",
					displayName(key.name),
					fmtType(ktype),
					key.suffix,
					goName(key.name),
				)
			}
		}
	}

	if len(mod.types) > 0 {
		fmt.Fprint(w, "	fmt.Println(\"\")\n}\n\n")
	}
}

func writeControl(w io.Writer, modules []*module) error {
	fmt.Fprint(w, "type HostResponse struct {\n	Host *ConfigHost `json:\"host\"`\n")
	for _, mod := range modules {
		srcName := goName(mod.name)
		fmt.Fprintf(w, "	%s %s*%s `json:\"%s,omitempty\"`\n",
			srcName, srcType(mod.jsonType), srcName, mod.name,
		)
	}
	fmt.Fprint(w, `}

func (r *HostResponse) Print(short bool) {
	if r == nil {
		return
	}

	if short {
		r.printShort()
		return
	}
	r.printLong()
}

func (r *HostResponse) printShort() {
	// TODO
}

`)

	fmt.Fprint(w, "func (r *HostResponse) printLong() {\n")

	for _, mod := range modules {
		srcName := goName(mod.name)
		if mod.jsonType == "a" {
			fmt.Fprintf(w, `	if len(r.%s) > 0 {
		printLong%s(1, r.%s)
		fmt.Println("")
	}
`, srcName, srcName, srcName)
		} else if mod.jsonType == "o" {
			fmt.Fprintf(w, `	if r.%s != nil {
		printLong%s(1, r.%s)
		fmt.Println("")
	}
`, srcName, srcName, srcName)
		}
	}

	fmt.Fprint(w, "}\n")
	return nil
}

func srcType(t string) string {
	switch t {
	case "a":
		return "[]"
	}
	return ""
}

func fmtType(t string) string {
	switch t {
	case "string":
		return "%s"
	case "float64":
		return "%.2f"
	case "bool":
		return "%t"
	case "int":
		return "%d"
	}
	return ""
}

func displayName(s string) string {
	s = strings.Replace(s, "_", " ", -1)
	s = strings.Title(s)
	return s
}

func goName(s string) string {
	s = displayName(s)
	s = strings.Replace(s, " ", "", -1)
	s = strings.Replace(s, "(", "", -1)
	s = strings.Replace(s, ")", "", -1)
	s = strings.Replace(s, "-", "", -1)
	return s
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

type stageTest struct {
	// name of the stage
	name string

	// commands of the stage
	commands []string

	// fail is true if the stage must fail, else false
	fail bool

	// stderr is the text that the stderr file must contain
	stderr string

	// stdout is the text that the stdout file must contain
	stdout string
}

func TestCanBeTested(t *testing.T) {
	assert := assert.New(t)

	pr := &PullRequest{}
	assert.Error(pr.canBeTested())

	pr.Commits = append(pr.Commits, PullRequestCommit{})
	assert.Error(pr.canBeTested())

	pr.CommentTrigger = &PullRequestComment{}
	assert.Error(pr.canBeTested())

	pr.Mergeable = true
	assert.NoError(pr.canBeTested())

	pr.Commits[0].Time = time.Now()
	assert.Error(pr.canBeTested())
}

func TestEqual(t *testing.T) {
	assert := assert.New(t)

	pr1 := &PullRequest{}
	pr2 := PullRequest{}
	assert.True(pr1.Equal(pr2))

	pr2.Commits = append(pr2.Commits, PullRequestCommit{Sha: "abc"})
	assert.False(pr1.Equal(pr2))

	pr1.Commits = pr2.Commits
	assert.True(pr1.Equal(pr2))

	pr1.Commits = []PullRequestCommit{{Sha: "xyz"}}
	assert.False(pr1.Equal(pr2))
}

func TestRunStage(t *testing.T) {
	var err error
	var stdout, stderr []byte
	assert := assert.New(t)

	pr := &PullRequest{}

	pr.LogDir, err = ioutil.TempDir("/tmp", ".logs")
	assert.NoError(err)
	defer os.RemoveAll(pr.LogDir)

	tests := []stageTest{
		{
			name:     "1",
			commands: []string{"echo -n 1"},
			fail:     false,
			stderr:   "",
			stdout:   "1",
		},
		{
			name:     "2",
			commands: []string{"(echo -n 2 >&2)"},
			fail:     false,
			stderr:   "2",
			stdout:   "",
		},
		{
			name:     "3",
			commands: []string{"(echo -n 3 >&2 && exit 1)"},
			fail:     true,
			stderr:   "3",
			stdout:   "",
		},
		{
			name:     "4",
			commands: []string{"(echo -n 4 && exit 1)"},
			fail:     true,
			stderr:   "",
			stdout:   "4",
		},
	}

	for _, t := range tests {
		err = pr.runStage(t.name, t.commands)
		if t.fail {
			assert.Error(err, "stage: %+v", t)
		} else {
			assert.NoError(err, "stage: %+v", t)
		}

		stderr, err = ioutil.ReadFile(fmt.Sprintf("%s/%s.stderr", pr.LogDir, t.name))
		assert.NoError(err)
		assert.Equal(t.stderr, string(stderr))

		stdout, err = ioutil.ReadFile(fmt.Sprintf("%s/%s.stdout", pr.LogDir, t.name))
		assert.NoError(err)
		assert.Equal(t.stdout, string(stdout))
	}
}
<commit_msg>localCI: fix unit tests<commit_after>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

type stageTest struct {
	// name of the stage
	name string

	// commands of the stage
	commands []string

	// fail is true if the stage must fail, else false
	fail bool

	// output is the text that the output file must contain
	output string
}

func TestCanBeTested(t *testing.T) {
	assert := assert.New(t)

	pr := &PullRequest{}
	assert.Error(pr.canBeTested())

	pr.Commits = append(pr.Commits, PullRequestCommit{})
	assert.Error(pr.canBeTested())

	pr.CommentTrigger = &PullRequestComment{}
	assert.Error(pr.canBeTested())

	pr.Mergeable = true
	assert.NoError(pr.canBeTested())

	pr.Commits[0].Time = time.Now()
	assert.Error(pr.canBeTested())
}

func TestEqual(t *testing.T) {
	assert := assert.New(t)

	pr1 := &PullRequest{}
	pr2 := PullRequest{}
	assert.True(pr1.Equal(pr2))

	pr2.Commits = append(pr2.Commits, PullRequestCommit{Sha: "abc"})
	assert.False(pr1.Equal(pr2))

	pr1.Commits = pr2.Commits
	assert.True(pr1.Equal(pr2))

	pr1.Commits = []PullRequestCommit{{Sha: "xyz"}}
	assert.False(pr1.Equal(pr2))
}

func TestRunStage(t *testing.T) {
	var err error
	var output []byte
	assert := assert.New(t)

	pr := &PullRequest{}

	pr.LogDir, err = ioutil.TempDir("/tmp", ".logs")
	assert.NoError(err)
	defer os.RemoveAll(pr.LogDir)

	tests := []stageTest{
		{
			name:     "1",
			commands: []string{"echo -n 1"},
			fail:     false,
			output:   "1",
		},
		{
			name:     "2",
			commands: []string{"(echo -n 2 >&2)"},
			fail:     false,
			output:   "2",
		},
		{
			name:     "3",
			commands: []string{"(echo -n 3 >&2 && exit 1)"},
			fail:     true,
			output:   "3",
		},
		{
			name:     "4",
			commands: []string{"(echo -n 4 && exit 1)"},
			fail:     true,
			output:   "4",
		},
	}

	for _, t := range tests {
		err = pr.runStage(t.name, t.commands)
		if t.fail {
			assert.Error(err, "stage: %+v", t)
		} else {
			assert.NoError(err, "stage: %+v", t)
		}

		output, err = ioutil.ReadFile(filepath.Join(pr.LogDir, t.name))
		assert.NoError(err)
		assert.Equal(t.output, string(output))
	}
}
<|endoftext|>


<commit_before>package stun

import (
  "bytes"
  "encoding/binary"
  "errors"
  "net"
)

type XorMappedAddressAttribute struct {
  Family  uint16
  Port    uint16
  Address net.IP
}

func (h *XorMappedAddressAttribute) Type() (StunAttributeType) {
  return XorMappedAddress
}

func (h *XorMappedAddressAttribute) Encode(msg *StunMessage) ([]byte, error) {
  buf := new(bytes.Buffer)
  err := binary.Write(buf, binary.BigEndian, attributeHeader(StunAttribute(h)))
  err = binary.Write(buf, binary.BigEndian, h.Family)
  err = binary.Write(buf, binary.BigEndian, h.Port)
  err = binary.Write(buf, binary.BigEndian, h.Address)

  if err != nil {
    return nil, err
  }
  return buf.Bytes(), nil
}

func (h *XorMappedAddressAttribute) Decode(data []byte, _ uint16, header *Header) (error) {
  if data[0] != 0 && data[1] != 1 && data[0] != 2 {
    return errors.New("Incorrect Mapped Address Family.")
  }
  h.Family = uint16(data[1])
  if (h.Family == 1 && len(data) < 8) || (h.Family == 2 && len(data) < 20) {
    return errors.New("Mapped Address Attribute unexpectedly Truncated.")
  }
  h.Port = uint16(data[2]) << 8 + uint16(data[3])
  if h.Family == 1 {
    h.Address = data[4:8]
  } else {
    h.Address = data[4:20]
  }
  return nil
}

func (h *XorMappedAddressAttribute) Length() (uint16) {
  if h.Family == 1 {
    return 8
  } else {
    return 20
  }
}
<commit_msg>xor mapping does xor on decode<commit_after>package stun

import (
  "bytes"
  "encoding/binary"
  "errors"
  "net"
)

type XorMappedAddressAttribute struct {
  Family  uint16
  Port    uint16
  Address net.IP
}

func (h *XorMappedAddressAttribute) Type() (StunAttributeType) {
  return XorMappedAddress
}

func (h *XorMappedAddressAttribute) Encode(msg *StunMessage) ([]byte, error) {
  buf := new(bytes.Buffer)
  err := binary.Write(buf, binary.BigEndian, attributeHeader(StunAttribute(h)))
  err = binary.Write(buf, binary.BigEndian, h.Family)
  xport := h.Port ^ uint16(magicCookie >> 16)
  err = binary.Write(buf, binary.BigEndian, xport)
  err = binary.Write(buf, binary.BigEndian, h.Address)

  if err != nil {
    return nil, err
  }
  return buf.Bytes(), nil
}

func (h *XorMappedAddressAttribute) Decode(data []byte, _ uint16, header *Header) (error) {
  if data[0] != 0 && data[1] != 1 && data[0] != 2 {
    return errors.New("Incorrect Mapped Address Family.")
  }
  h.Family = uint16(data[1])
  if (h.Family == 1 && len(data) < 8) || (h.Family == 2 && len(data) < 20) {
    return errors.New("Mapped Address Attribute unexpectedly Truncated.")
  }
  h.Port = uint16(data[2]) << 8 + uint16(data[3])
  // X-port is XOR'ed with the 16 most significant bits of the magic Cookie
  h.Port ^= uint16(magicCookie >> 16)

  var xoraddress []byte
  if h.Family == 1 {
    xoraddress = make([]byte, 4)
    binary.BigEndian.PutUint32(xoraddress, magicCookie)
    h.Address = data[4:8]
  } else {
    xoraddress = make([]byte, 16)
    binary.BigEndian.PutUint32(xoraddress, magicCookie)
    copy(xoraddress[4:16], header.Id[:])
    h.Address = data[4:20]
  }
  for i, _ := range xoraddress {
    h.Address[i] ^= xoraddress[i]
  }
  return nil
}

func (h *XorMappedAddressAttribute) Length() (uint16) {
  if h.Family == 1 {
    return 8
  } else {
    return 20
  }
}
<|endoftext|>


<commit_before>// Copyright 2017 flannel authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package extension

import (
	"encoding/json"
	"sync"

	log "github.com/golang/glog"
	"golang.org/x/net/context"

	"fmt"

	"github.com/coreos/flannel/backend"
	"github.com/coreos/flannel/subnet"
)

type network struct {
	name                string
	extIface            *backend.ExternalInterface
	lease               *subnet.Lease
	sm                  subnet.Manager
	preStartupCommand   string
	postStartupCommand  string
	subnetAddCommand    string
	subnetRemoveCommand string
}

func (n *network) Lease() *subnet.Lease {
	return n.lease
}

func (n *network) MTU() int {
	return n.extIface.Iface.MTU
}

func (n *network) Run(ctx context.Context) {
	wg := sync.WaitGroup{}

	log.Info("Watching for new subnet leases")
	evts := make(chan []subnet.Event)
	wg.Add(1)
	go func() {
		subnet.WatchLeases(ctx, n.sm, n.lease, evts)
		wg.Done()
	}()

	defer wg.Wait()

	for {
		select {
		case evtBatch := <-evts:
			n.handleSubnetEvents(evtBatch)

		case <-ctx.Done():
			return
		}
	}
}

func (n *network) handleSubnetEvents(batch []subnet.Event) {
	for _, evt := range batch {
		switch evt.Type {
		case subnet.EventAdded:
			log.Infof("Subnet added: %v via %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP)

			if evt.Lease.Attrs.BackendType != "extension" {
				log.Warningf("Ignoring non-extension subnet: type=%v", evt.Lease.Attrs.BackendType)
				continue
			}

			if len(n.subnetAddCommand) > 0 {
				var dat interface{}
				if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &dat); err != nil {
					log.Errorf("failed to unmarshal BackendData: %v", err)
				} else {
					backendData := dat.(string)
					cmd_output, err := runCmd([]string{
						fmt.Sprintf("SUBNET=%s", evt.Lease.Subnet),
						fmt.Sprintf("PUBLIC_IP=%s", evt.Lease.Attrs.PublicIP)},
						backendData,
						"sh", "-c", n.subnetAddCommand)

					if err != nil {
						log.Errorf("failed to run command: %s Err: %v Output: %s", n.subnetAddCommand, err, cmd_output)
					} else {
						log.Infof("Ran command: %s\n Output: %s", n.subnetAddCommand, cmd_output)
					}
				}
			}

		case subnet.EventRemoved:
			log.Info("Subnet removed: ", evt.Lease.Subnet)

			if evt.Lease.Attrs.BackendType != "extension" {
				log.Warningf("Ignoring non-extension subnet: type=%v", evt.Lease.Attrs.BackendType)
				continue
			}

			if len(n.subnetRemoveCommand) > 0 {
				var dat interface{}
				if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &dat); err != nil {
					log.Errorf("failed to unmarshal BackendData: %v", err)
				} else {
					backendData := dat.(string)
					cmd_output, err := runCmd([]string{
						fmt.Sprintf("SUBNET=%s", evt.Lease.Subnet),
						fmt.Sprintf("PUBLIC_IP=%s", evt.Lease.Attrs.PublicIP)},
						backendData,
						"sh", "-c", n.subnetRemoveCommand)

					if err != nil {
						log.Errorf("failed to run command: %s Err: %v Output: %s", n.subnetRemoveCommand, err, cmd_output)
					} else {
						log.Infof("Ran command: %s\n Output: %s", n.subnetRemoveCommand, cmd_output)
					}
				}
			}

		default:
			log.Error("Internal error: unknown event type: ", int(evt.Type))
		}
	}
}
<commit_msg>backend/extension: Allow the backend data to be empty<commit_after>// Copyright 2017 flannel authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package extension

import (
	"encoding/json"
	"sync"

	log "github.com/golang/glog"
	"golang.org/x/net/context"

	"fmt"

	"github.com/coreos/flannel/backend"
	"github.com/coreos/flannel/subnet"
)

type network struct {
	name                string
	extIface            *backend.ExternalInterface
	lease               *subnet.Lease
	sm                  subnet.Manager
	preStartupCommand   string
	postStartupCommand  string
	subnetAddCommand    string
	subnetRemoveCommand string
}

func (n *network) Lease() *subnet.Lease {
	return n.lease
}

func (n *network) MTU() int {
	return n.extIface.Iface.MTU
}

func (n *network) Run(ctx context.Context) {
	wg := sync.WaitGroup{}

	log.Info("Watching for new subnet leases")
	evts := make(chan []subnet.Event)
	wg.Add(1)
	go func() {
		subnet.WatchLeases(ctx, n.sm, n.lease, evts)
		wg.Done()
	}()

	defer wg.Wait()

	for {
		select {
		case evtBatch := <-evts:
			n.handleSubnetEvents(evtBatch)

		case <-ctx.Done():
			return
		}
	}
}

func (n *network) handleSubnetEvents(batch []subnet.Event) {
	for _, evt := range batch {
		switch evt.Type {
		case subnet.EventAdded:
			log.Infof("Subnet added: %v via %v", evt.Lease.Subnet, evt.Lease.Attrs.PublicIP)

			if evt.Lease.Attrs.BackendType != "extension" {
				log.Warningf("Ignoring non-extension subnet: type=%v", evt.Lease.Attrs.BackendType)
				continue
			}

			if len(n.subnetAddCommand) > 0 {
				backendData := ""

				if len(evt.Lease.Attrs.BackendData) > 0 {
					if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &backendData); err != nil {
						log.Errorf("failed to unmarshal BackendData: %v", err)
						continue
					}
				}

				cmd_output, err := runCmd([]string{
					fmt.Sprintf("SUBNET=%s", evt.Lease.Subnet),
					fmt.Sprintf("PUBLIC_IP=%s", evt.Lease.Attrs.PublicIP)},
					backendData,
					"sh", "-c", n.subnetAddCommand)

				if err != nil {
					log.Errorf("failed to run command: %s Err: %v Output: %s", n.subnetAddCommand, err, cmd_output)
				} else {
					log.Infof("Ran command: %s\n Output: %s", n.subnetAddCommand, cmd_output)
				}
			}

		case subnet.EventRemoved:
			log.Info("Subnet removed: ", evt.Lease.Subnet)

			if evt.Lease.Attrs.BackendType != "extension" {
				log.Warningf("Ignoring non-extension subnet: type=%v", evt.Lease.Attrs.BackendType)
				continue
			}

			if len(n.subnetRemoveCommand) > 0 {
				backendData := ""

				if len(evt.Lease.Attrs.BackendData) > 0 {
					if err := json.Unmarshal(evt.Lease.Attrs.BackendData, &backendData); err != nil {
						log.Errorf("failed to unmarshal BackendData: %v", err)
						continue
					}
				}
				cmd_output, err := runCmd([]string{
					fmt.Sprintf("SUBNET=%s", evt.Lease.Subnet),
					fmt.Sprintf("PUBLIC_IP=%s", evt.Lease.Attrs.PublicIP)},
					backendData,
					"sh", "-c", n.subnetRemoveCommand)

				if err != nil {
					log.Errorf("failed to run command: %s Err: %v Output: %s", n.subnetRemoveCommand, err, cmd_output)
				} else {
					log.Infof("Ran command: %s\n Output: %s", n.subnetRemoveCommand, cmd_output)
				}
			}

		default:
			log.Error("Internal error: unknown event type: ", int(evt.Type))
		}
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"os"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	tmflags "github.com/tendermint/tendermint/cmd/tendermint/commands/flags"
	cfg "github.com/tendermint/tendermint/config"
	"github.com/tendermint/tmlibs/log"
)

var (
	config = cfg.DefaultConfig()
	logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout)).With("module", "main")
)

func init() {
	RootCmd.PersistentFlags().String("log_level", config.LogLevel, "Log level")
}

var RootCmd = &cobra.Command{
	Use:   "tendermint",
	Short: "Tendermint Core (BFT Consensus) in Go",
	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
		err := viper.Unmarshal(config)
		if err != nil {
			return err
		}
		config.SetRoot(config.RootDir)
		cfg.EnsureRoot(config.RootDir)
		logger, err = tmflags.ParseLogLevel(config.LogLevel, logger)
		if err != nil {
			return err
		}
		return nil
	},
}
<commit_msg>tracing logger (Refs #506)<commit_after>package commands

import (
	"os"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"

	tmflags "github.com/tendermint/tendermint/cmd/tendermint/commands/flags"
	cfg "github.com/tendermint/tendermint/config"
	"github.com/tendermint/tmlibs/cli"
	"github.com/tendermint/tmlibs/log"
)

var (
	config = cfg.DefaultConfig()
	logger = log.NewTMLogger(log.NewSyncWriter(os.Stdout)).With("module", "main")
)

func init() {
	RootCmd.PersistentFlags().String("log_level", config.LogLevel, "Log level")
}

var RootCmd = &cobra.Command{
	Use:   "tendermint",
	Short: "Tendermint Core (BFT Consensus) in Go",
	PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
		err := viper.Unmarshal(config)
		if err != nil {
			return err
		}
		config.SetRoot(config.RootDir)
		cfg.EnsureRoot(config.RootDir)
		logger, err = tmflags.ParseLogLevel(config.LogLevel, logger)
		if err != nil {
			return err
		}
		if viper.GetBool(cli.TraceFlag) {
			logger = log.NewTracingLogger(logger)
		}
		return nil
	},
}
<|endoftext|>


<commit_before>package egoscale

// APIKeyType holds the type of the API key
type APIKeyType string

const (
	// APIKeyTypeUnrestricted is unrestricted
	APIKeyTypeUnrestricted APIKeyType = "unrestricted"
	// APIKeyTypeRestricted is restricted
	APIKeyTypeRestricted APIKeyType = "restricted"
)

// APIKey represents an API key
type APIKey struct {
	Name       string     `json:"name"`
	Key        string     `json:"key"`
	Secret     string     `json:"secret,omitempty"`
	Operations []string   `json:"operations,omitempty"`
	Type       APIKeyType `json:"type"`
}

// CreateAPIKey represents an API key creation
type CreateAPIKey struct {
	Name       string `json:"name"`
	Operations string `json:"operations,omitempty"`
	_          bool   `name:"createApiKey" description:"Create an API key."`
}

// Response returns the struct to unmarshal
func (CreateAPIKey) Response() interface{} {
	return new(APIKey)
}

// ListAPIKeys represents a search for API keys
type ListAPIKeys struct {
	_ bool `name:"listApiKeys" description:"List API keys."`
}

// ListAPIKeysResponse represents a list of API keys
type ListAPIKeysResponse struct {
	Count   int      `json:"count"`
	APIKeys []APIKey `json:"apikeys"`
}

// Response returns the struct to unmarshal
func (ListAPIKeys) Response() interface{} {
	return new(ListAPIKeysResponse)
}

// ListAPIKeyOperations represents a search for operations for the current API key
type ListAPIKeyOperations struct {
	_ bool `name:"listApiKeyOperations" description:"List operations allowed for the current API key."`
}

// ListAPIKeyOperationsResponse represents a list of operations for the current API key
type ListAPIKeyOperationsResponse struct {
	Operations []string `json:"operations"`
}

// Response returns the struct to unmarshal
func (ListAPIKeyOperations) Response() interface{} {
	return new(ListAPIKeyOperationsResponse)
}

// GetAPIKey get an API key
type GetAPIKey struct {
	Key string `json:"key"`
	_   bool   `name:"getApiKey" description:"Get an API key."`
}

// Response returns the struct to unmarshal
func (GetAPIKey) Response() interface{} {
	return new(APIKey)
}

// RevokeAPIKey represents a revocation of an API key
type RevokeAPIKey struct {
	Key string `json:"key"`
	_   bool   `name:"revokeApiKey" description:"Revoke an API key."`
}

// RevokeAPIKeyResponse represents the response to an API key revocation
type RevokeAPIKeyResponse struct {
	Success bool `json:"success"`
}

// Response returns the struct to unmarshal
func (RevokeAPIKey) Response() interface{} {
	return new(RevokeAPIKeyResponse)
}
<commit_msg>Fix IAM ListAPIKeysResponse field (#415)<commit_after>package egoscale

// APIKeyType holds the type of the API key
type APIKeyType string

const (
	// APIKeyTypeUnrestricted is unrestricted
	APIKeyTypeUnrestricted APIKeyType = "unrestricted"
	// APIKeyTypeRestricted is restricted
	APIKeyTypeRestricted APIKeyType = "restricted"
)

// APIKey represents an API key
type APIKey struct {
	Name       string     `json:"name"`
	Key        string     `json:"key"`
	Secret     string     `json:"secret,omitempty"`
	Operations []string   `json:"operations,omitempty"`
	Type       APIKeyType `json:"type"`
}

// CreateAPIKey represents an API key creation
type CreateAPIKey struct {
	Name       string `json:"name"`
	Operations string `json:"operations,omitempty"`
	_          bool   `name:"createApiKey" description:"Create an API key."`
}

// Response returns the struct to unmarshal
func (CreateAPIKey) Response() interface{} {
	return new(APIKey)
}

// ListAPIKeys represents a search for API keys
type ListAPIKeys struct {
	_ bool `name:"listApiKeys" description:"List API keys."`
}

// ListAPIKeysResponse represents a list of API keys
type ListAPIKeysResponse struct {
	Count   int      `json:"count"`
	APIKeys []APIKey `json:"apikey"`
}

// Response returns the struct to unmarshal
func (ListAPIKeys) Response() interface{} {
	return new(ListAPIKeysResponse)
}

// ListAPIKeyOperations represents a search for operations for the current API key
type ListAPIKeyOperations struct {
	_ bool `name:"listApiKeyOperations" description:"List operations allowed for the current API key."`
}

// ListAPIKeyOperationsResponse represents a list of operations for the current API key
type ListAPIKeyOperationsResponse struct {
	Operations []string `json:"operations"`
}

// Response returns the struct to unmarshal
func (ListAPIKeyOperations) Response() interface{} {
	return new(ListAPIKeyOperationsResponse)
}

// GetAPIKey get an API key
type GetAPIKey struct {
	Key string `json:"key"`
	_   bool   `name:"getApiKey" description:"Get an API key."`
}

// Response returns the struct to unmarshal
func (GetAPIKey) Response() interface{} {
	return new(APIKey)
}

// RevokeAPIKey represents a revocation of an API key
type RevokeAPIKey struct {
	Key string `json:"key"`
	_   bool   `name:"revokeApiKey" description:"Revoke an API key."`
}

// RevokeAPIKeyResponse represents the response to an API key revocation
type RevokeAPIKeyResponse struct {
	Success bool `json:"success"`
}

// Response returns the struct to unmarshal
func (RevokeAPIKey) Response() interface{} {
	return new(RevokeAPIKeyResponse)
}
<|endoftext|>


<commit_before>// 15 april 2015
package main

import (
	"fmt"
	"os"
	"github.com/andlabs/pgidl"
)

var pkgtypes = map[string]string{}

func typedecl(t *pgidl.Type, name string) string {
	if t == nil {
		return "void " + name
	}
	if t.IsFuncPtr {
		return cfuncptrdecl(t.FuncType, name)
	}
	s := t.Name + " "
	if pkgtypes[t.Name] != "" {
		s = pkgtypes[t.Name] + " "
	}
	for i := uint(0); i < t.NumPtrs; i++ {
		s += "*"
	}
	return s + name
}

func arglist(a []*pgidl.Arg) string {
	if len(a) == 0 {
		return "void"
	}
	s := typedecl(a[0].Type, a[0].Name)
	for i := 1; i < len(a); i++ {
		s += ", " + typedecl(a[i].Type, a[i].Name)
	}
	return s
}

func cfuncdecl(f *pgidl.Func, name string) string {
	fd := name + "(" + arglist(f.Args) + ")"
	return "extern " + typedecl(f.Ret, fd) + ";"
}

func cfuncptrdecl(f *pgidl.Func, name string) string {
	name = "(*" + name + ")"
	fd := name + "(" + arglist(f.Args) + ")"
	return typedecl(f.Ret, fd)
}

func cmethodmacro(f *pgidl.Func, typename string) string {
	s := "#define " + typename + f.Name + "(this"
	for _, a := range f.Args {
		s += ", " + a.Name
	}
	s += ") ("
	s += "(*((this)->" + f.Name + "))"
	s += "((this)"
	for _, a := range f.Args {
		s += ", (" + a.Name + ")"
	}
	s += ")"
	s += ")"
	return s
}

func genpkgfunc(f *pgidl.Func, prefix string) {
	fmt.Printf("%s\n", cfuncdecl(f, prefix + f.Name))
}

func genstruct(s *pgidl.Struct, prefix string) {
	fmt.Printf("struct %s%s {\n", prefix, s.Name)
	for _, f := range s.Fields {
		fmt.Printf("\t%s;\n", typedecl(f.Type, f.Name))
	}
	fmt.Printf("};\n")
}

func geniface(i *pgidl.Interface, prefix string) {
	fmt.Printf("struct %s%s {\n", prefix, i.Name)
	if i.From != "" {
		fmt.Printf("\t%s%s base;\n", prefix, i.From)
	}
	for _, f := range i.Fields {
		fmt.Printf("\t%s;\n", typedecl(f.Type, f.Name))
	}
	for _, m := range i.Methods {
		fmt.Printf("\t%s;\n", cfuncptrdecl(m, m.Name))
		fmt.Printf("%s\n", cmethodmacro(m, prefix + i.Name))
	}
	fmt.Printf("};\n")
	fmt.Printf("#define %s%s(this) ((%s%s *) (this))\n",
		prefix, i.Name,
		prefix, i.Name)
}

func genpkg(p *pgidl.Package) {
	for _, s := range p.Structs {
		fmt.Printf("typedef struct %s%s %s%s;\n",
			p.Name, s.Name,
			p.Name, s.Name)
		pkgtypes[s.Name] = p.Name + s.Name
	}
	for _, i := range p.Interfaces {
		fmt.Printf("typedef struct %s%s %s%s;\n",
			p.Name, i.Name,
			p.Name, i.Name)
		pkgtypes[i.Name] = p.Name + i.Name
	}
	for _, o := range p.Order {
		switch o.Which {
		case pgidl.Funcs:
			genpkgfunc(p.Funcs[o.Index], p.Name)
		case pgidl.Structs:
			genstruct(p.Structs[o.Index], p.Name)
		case pgidl.Interfaces:
			geniface(p.Interfaces[o.Index], p.Name)
		case pgidl.Raws:
			fmt.Printf("%s\n", p.Raws[o.Index])
		}
	}
}

func main() {
	idl, errs := pgidl.Parse(os.Stdin, "<stdin>")
	if len(errs) != 0 {
		for _, e := range errs {
			fmt.Fprintf(os.Stderr, "%s\n", e)
		}
		os.Exit(1)
	}
	fmt.Printf("// generated by idl2h; do not edit\n")
	for _, p := range idl {
		genpkg(p)
	}
}
<commit_msg>Properly inserted this pointer into methods.<commit_after>// 15 april 2015
package main

import (
	"fmt"
	"os"
	"github.com/andlabs/pgidl"
)

var pkgtypes = map[string]string{}

func typedecl(t *pgidl.Type, name string) string {
	if t == nil {
		return "void " + name
	}
	if t.IsFuncPtr {
		return cfuncptrdecl(t.FuncType, name)
	}
	s := t.Name + " "
	if pkgtypes[t.Name] != "" {
		s = pkgtypes[t.Name] + " "
	}
	for i := uint(0); i < t.NumPtrs; i++ {
		s += "*"
	}
	return s + name
}

func arglist(a []*pgidl.Arg) string {
	if len(a) == 0 {
		return "void"
	}
	s := typedecl(a[0].Type, a[0].Name)
	for i := 1; i < len(a); i++ {
		s += ", " + typedecl(a[i].Type, a[i].Name)
	}
	return s
}

func cfuncdecl(f *pgidl.Func, name string) string {
	fd := name + "(" + arglist(f.Args) + ")"
	return "extern " + typedecl(f.Ret, fd) + ";"
}

func cfuncptrdecl(f *pgidl.Func, name string) string {
	name = "(*" + name + ")"
	fd := name + "(" + arglist(f.Args) + ")"
	return typedecl(f.Ret, fd)
}

func cmethodmacro(f *pgidl.Func, typename string) string {
	s := "#define " + typename + f.Name + "("
	first := true
	for _, a := range f.Args {
		if !first {
			s += ", "
		}
		s += a.Name
		first = false
	}
	s += ") ("
	s += "(*((this)->" + f.Name + "))"
	s += "("
	first = true
	for _, a := range f.Args {
		if !first {
			s += ", "
		}
		s += "(" + a.Name + ")"
		first = false
	}
	s += ")"
	s += ")"
	return s
}

func genpkgfunc(f *pgidl.Func, prefix string) {
	fmt.Printf("%s\n", cfuncdecl(f, prefix + f.Name))
}

func genstruct(s *pgidl.Struct, prefix string) {
	fmt.Printf("struct %s%s {\n", prefix, s.Name)
	for _, f := range s.Fields {
		fmt.Printf("\t%s;\n", typedecl(f.Type, f.Name))
	}
	fmt.Printf("};\n")
}

func geniface(i *pgidl.Interface, prefix string) {
	fmt.Printf("struct %s%s {\n", prefix, i.Name)
	if i.From != "" {
		fmt.Printf("\t%s%s base;\n", prefix, i.From)
	}
	for _, f := range i.Fields {
		fmt.Printf("\t%s;\n", typedecl(f.Type, f.Name))
	}
	for _, m := range i.Methods {
		// hack our this pointer in
		m.Args = append([]*pgidl.Arg{
			&pgidl.Arg{
				Name:	"this",
				Type:	&pgidl.Type{
					Name:	prefix + i.Name,
					NumPtrs:	1,
				},
			},
		}, m.Args...)
		fmt.Printf("\t%s;\n", cfuncptrdecl(m, m.Name))
		fmt.Printf("%s\n", cmethodmacro(m, prefix + i.Name))
	}
	fmt.Printf("};\n")
	fmt.Printf("#define %s%s(this) ((%s%s *) (this))\n",
		prefix, i.Name,
		prefix, i.Name)
}

func genpkg(p *pgidl.Package) {
	for _, s := range p.Structs {
		fmt.Printf("typedef struct %s%s %s%s;\n",
			p.Name, s.Name,
			p.Name, s.Name)
		pkgtypes[s.Name] = p.Name + s.Name
	}
	for _, i := range p.Interfaces {
		fmt.Printf("typedef struct %s%s %s%s;\n",
			p.Name, i.Name,
			p.Name, i.Name)
		pkgtypes[i.Name] = p.Name + i.Name
	}
	for _, o := range p.Order {
		switch o.Which {
		case pgidl.Funcs:
			genpkgfunc(p.Funcs[o.Index], p.Name)
		case pgidl.Structs:
			genstruct(p.Structs[o.Index], p.Name)
		case pgidl.Interfaces:
			geniface(p.Interfaces[o.Index], p.Name)
		case pgidl.Raws:
			fmt.Printf("%s\n", p.Raws[o.Index])
		}
	}
}

func main() {
	idl, errs := pgidl.Parse(os.Stdin, "<stdin>")
	if len(errs) != 0 {
		for _, e := range errs {
			fmt.Fprintf(os.Stderr, "%s\n", e)
		}
		os.Exit(1)
	}
	fmt.Printf("// generated by idl2h; do not edit\n")
	for _, p := range idl {
		genpkg(p)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package collector

import (
	"regexp"
	"testing"

	"github.com/coreos/go-systemd/dbus"
	"github.com/prometheus/client_golang/prometheus"
)

// Creates mock UnitLists
func getUnitListFixtures() [][]dbus.UnitStatus {
	fixture1 := []dbus.UnitStatus{
		{
			Name:        "foo",
			Description: "foo desc",
			LoadState:   "loaded",
			ActiveState: "active",
			SubState:    "running",
			Followed:    "",
			Path:        "/org/freedesktop/systemd1/unit/foo",
			JobId:       0,
			JobType:     "",
			JobPath:     "/",
		},
		{
			Name:        "bar",
			Description: "bar desc",
			LoadState:   "not-found",
			ActiveState: "inactive",
			SubState:    "dead",
			Followed:    "",
			Path:        "/org/freedesktop/systemd1/unit/bar",
			JobId:       0,
			JobType:     "",
			JobPath:     "/",
		},
		{
			Name:        "foobar",
			Description: "bar desc",
			LoadState:   "not-found",
			ActiveState: "inactive",
			SubState:    "dead",
			Followed:    "",
			Path:        "/org/freedesktop/systemd1/unit/bar",
			JobId:       0,
			JobType:     "",
			JobPath:     "/",
		},
		{
			Name:        "baz",
			Description: "bar desc",
			LoadState:   "not-found",
			ActiveState: "inactive",
			SubState:    "dead",
			Followed:    "",
			Path:        "/org/freedesktop/systemd1/unit/bar",
			JobId:       0,
			JobType:     "",
			JobPath:     "/",
		},
	}

	fixture2 := []dbus.UnitStatus{}

	return [][]dbus.UnitStatus{fixture1, fixture2}
}

func TestSystemdCollectorDoesntCrash(t *testing.T) {
	c, err := NewSystemdCollector()
	if err != nil {
		t.Fatal(err)
	}
	sink := make(chan prometheus.Metric)
	go func() {
		for {
			<-sink
		}
	}()

	fixtures := getUnitListFixtures()
	collector := (c).(*systemdCollector)
	for _, units := range fixtures {
		collector.collectUnitStatusMetrics(sink, units)
	}
}

func TestSystemdIgnoreFilter(t *testing.T) {
	fixtures := getUnitListFixtures()
	whitelistPattern := regexp.MustCompile("foo")
	blacklistPattern := regexp.MustCompile("bar")
	filtered := filterUnits(fixtures[0], whitelistPattern, blacklistPattern)
	for _, unit := range filtered {
		if blacklistPattern.MatchString(unit.Name) || !whitelistPattern.MatchString(unit.Name) {
			t.Error(unit.Name, "should not be in the filtered list")
		}
	}
}
func TestSystemdIgnoreFilterDefaultKeepsAll(t *testing.T) {
	c, err := NewSystemdCollector()
	if err != nil {
		t.Fatal(err)
	}
	fixtures := getUnitListFixtures()
	collector := c.(*systemdCollector)
	filtered := filterUnits(fixtures[0], collector.unitWhitelistPattern, collector.unitBlacklistPattern)
	if len(filtered) != len(fixtures[0]) {
		t.Error("Default filters removed units")
	}
}

func TestSystemdSummary(t *testing.T) {
	fixtures := getUnitListFixtures()
	summary := summarizeUnits(fixtures[0])

	for _, state := range unitStatesName {
		if state == "inactive" {
			testSummaryHelper(t, state, summary[state], 3.0)
		} else if state == "active" {
			testSummaryHelper(t, state, summary[state], 1.0)
		} else {
			testSummaryHelper(t, state, summary[state], 0.0)
		}
	}
}

func testSummaryHelper(t *testing.T, state string, actual float64, expected float64) {
	if actual != expected {
		t.Errorf("Summary mode didn't count %s jobs correctly. Actual: %f, expected: %f", state, actual, expected)
	}
}
<commit_msg>Fix tests.<commit_after>// Copyright 2015 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package collector

import (
	"regexp"
	"testing"

	"github.com/coreos/go-systemd/dbus"
	"github.com/prometheus/client_golang/prometheus"
)

// Creates mock UnitLists
func getUnitListFixtures() [][]unit {
	fixture1 := []unit{
		{
			UnitStatus: dbus.UnitStatus{
				Name:        "foo",
				Description: "foo desc",
				LoadState:   "loaded",
				ActiveState: "active",
				SubState:    "running",
				Followed:    "",
				Path:        "/org/freedesktop/systemd1/unit/foo",
				JobId:       0,
				JobType:     "",
				JobPath:     "/",
			},
		},
		{
			UnitStatus: dbus.UnitStatus{
				Name:        "bar",
				Description: "bar desc",
				LoadState:   "not-found",
				ActiveState: "inactive",
				SubState:    "dead",
				Followed:    "",
				Path:        "/org/freedesktop/systemd1/unit/bar",
				JobId:       0,
				JobType:     "",
				JobPath:     "/",
			},
		},
		{
			UnitStatus: dbus.UnitStatus{
				Name:        "foobar",
				Description: "bar desc",
				LoadState:   "not-found",
				ActiveState: "inactive",
				SubState:    "dead",
				Followed:    "",
				Path:        "/org/freedesktop/systemd1/unit/bar",
				JobId:       0,
				JobType:     "",
				JobPath:     "/",
			},
		},
		{
			UnitStatus: dbus.UnitStatus{
				Name:        "baz",
				Description: "bar desc",
				LoadState:   "not-found",
				ActiveState: "inactive",
				SubState:    "dead",
				Followed:    "",
				Path:        "/org/freedesktop/systemd1/unit/bar",
				JobId:       0,
				JobType:     "",
				JobPath:     "/",
			},
		},
	}

	fixture2 := []unit{}

	return [][]unit{fixture1, fixture2}
}

func TestSystemdCollectorDoesntCrash(t *testing.T) {
	c, err := NewSystemdCollector()
	if err != nil {
		t.Fatal(err)
	}
	sink := make(chan prometheus.Metric)
	go func() {
		for {
			<-sink
		}
	}()

	fixtures := getUnitListFixtures()
	collector := (c).(*systemdCollector)
	for _, units := range fixtures {
		collector.collectUnitStatusMetrics(sink, units)
	}
}

func TestSystemdIgnoreFilter(t *testing.T) {
	fixtures := getUnitListFixtures()
	whitelistPattern := regexp.MustCompile("foo")
	blacklistPattern := regexp.MustCompile("bar")
	filtered := filterUnits(fixtures[0], whitelistPattern, blacklistPattern)
	for _, unit := range filtered {
		if blacklistPattern.MatchString(unit.Name) || !whitelistPattern.MatchString(unit.Name) {
			t.Error(unit.Name, "should not be in the filtered list")
		}
	}
}
func TestSystemdIgnoreFilterDefaultKeepsAll(t *testing.T) {
	c, err := NewSystemdCollector()
	if err != nil {
		t.Fatal(err)
	}
	fixtures := getUnitListFixtures()
	collector := c.(*systemdCollector)
	filtered := filterUnits(fixtures[0], collector.unitWhitelistPattern, collector.unitBlacklistPattern)
	if len(filtered) != len(fixtures[0]) {
		t.Error("Default filters removed units")
	}
}

func TestSystemdSummary(t *testing.T) {
	fixtures := getUnitListFixtures()
	summary := summarizeUnits(fixtures[0])

	for _, state := range unitStatesName {
		if state == "inactive" {
			testSummaryHelper(t, state, summary[state], 3.0)
		} else if state == "active" {
			testSummaryHelper(t, state, summary[state], 1.0)
		} else {
			testSummaryHelper(t, state, summary[state], 0.0)
		}
	}
}

func testSummaryHelper(t *testing.T, state string, actual float64, expected float64) {
	if actual != expected {
		t.Errorf("Summary mode didn't count %s jobs correctly. Actual: %f, expected: %f", state, actual, expected)
	}
}
<|endoftext|>


<commit_before>package producer

import (
	"github.com/artyom/scribe"
	"github.com/artyom/thrift"
	"github.com/trivago/gollum/shared"
	"strconv"
	"sync"
	"time"
)

// Scribe producer plugin
// Configuration example
//
// - "producer.Scribe":
//   Enable: true
//   Host: "192.168.222.30"
//   Port: 1463
//   BatchSize: 4096
//   BatchSizeThreshold: 16777216
//   BatchTimeoutSec: 2
//   Stream:
//     - "console"
//     - "_GOLLUM_"
//   Category:
//     "console" : "default"
//     "_GOLLUM_"  : "default"
//
// Host and Port should be clear
//
// Category maps a stream to a specific scribe category. You can define the
// wildcard stream (*) here, too. All streams that do not have a specific
// mapping will go to this stream (including _GOLLUM_).
// If no category mappings are set all messages will be send to "default".
//
// BatchSize defines the number of bytes to be buffered before they are written
// to scribe. By default this is set to 8KB.
//
// BatchSizeThreshold defines the maximum number of bytes to buffer before
// messages get dropped. If a message crosses the threshold it is still buffered
// but additional messages will be dropped. By default this is set to 8MB.
//
// BatchTimeoutSec defines the maximum number of seconds to wait after the last
// message arrived before a batch is flushed automatically. By default this is
// set to 5.
type Scribe struct {
	standardProducer
	scribe          *scribe.ScribeClient
	transport       *thrift.TFramedTransport
	socket          *thrift.TSocket
	batch           *scribeMessageBuffer
	category        map[shared.MessageStreamID]string
	batchSize       int
	batchTimeoutSec int
	bufferSizeKB    int
	defaultCategory string
	sendLock        *sync.Mutex
}

func init() {
	shared.Plugin.Register(Scribe{})
}

// Create creates a new producer based on the current scribe producer.
func (prod Scribe) Create(conf shared.PluginConfig) (shared.Producer, error) {

	err := prod.configureStandardProducer(conf)
	if err != nil {
		return nil, err
	}

	host := conf.GetString("Host", "localhost")
	port := conf.GetInt("Port", 1463)
	batchSizeThreshold := conf.GetInt("BatchSizeThreshold", 8388608)

	prod.category = make(map[shared.MessageStreamID]string, 0)
	prod.batchSize = conf.GetInt("BatchSize", 8192)
	prod.batchTimeoutSec = conf.GetInt("BatchTimeoutSec", 5)
	prod.batch = createScribeMessageBuffer(batchSizeThreshold, prod.flags)
	prod.bufferSizeKB = conf.GetInt("BufferSizeKB", 1<<10) // 1 MB
	prod.sendLock = new(sync.Mutex)

	// Read stream to category mapping

	defaultMapping := make(map[interface{}]interface{})
	defaultMapping[shared.WildcardStream] = "default"

	categoryMap := conf.GetValue("Category", defaultMapping).(map[interface{}]interface{})
	for stream, category := range categoryMap {
		prod.category[shared.GetStreamID(stream.(string))] = category.(string)
	}

	prod.defaultCategory = "default"

	wildcardCategory, wildcardCategorySet := prod.category[shared.WildcardStreamID]
	if wildcardCategorySet {
		prod.defaultCategory = wildcardCategory
	}

	// Initialize scribe connection

	prod.socket, err = thrift.NewTSocket(host + ":" + strconv.Itoa(port))
	if err != nil {
		shared.Log.Error("Scribe socket error:", err)
		return nil, err
	}

	prod.transport = thrift.NewTFramedTransport(prod.socket)
	protocolFactory := thrift.NewTBinaryProtocolFactory(false, false)

	prod.scribe = scribe.NewScribeClientFactory(prod.transport, protocolFactory)
	return prod, nil
}

func (prod Scribe) send() {
	prod.sendLock.Lock()
	defer prod.sendLock.Unlock()

	if !prod.transport.IsOpen() {
		err := prod.transport.Open()
		if err != nil {
			shared.Log.Error("Scribe connection error:", err)
		} else {
			prod.socket.Conn().(bufferedConn).SetWriteBuffer(prod.bufferSizeKB << 10)
		}
	}

	if prod.transport.IsOpen() {
		err := prod.batch.flush(prod.scribe)

		if err != nil {
			shared.Log.Error("Scribe log error: ", err)
			prod.transport.Close()
		}
	}
}

func (prod Scribe) sendMessage(message shared.Message) {
	category, exists := prod.category[message.PinnedStream]
	if !exists {
		category = prod.defaultCategory
	}

	prod.batch.appendAndRelease(message, category)
	if prod.batch.reachedSizeThreshold(prod.batchSize) {
		prod.send()
	}
}

func (prod Scribe) flush() {
	for {
		select {
		case message := <-prod.messages:
			prod.sendMessage(message)
		default:
			prod.send()
			return
		}
	}
}

// Produce writes to a buffer that is sent to scribe.
func (prod Scribe) Produce(threads *sync.WaitGroup) {
	threads.Add(1)

	defer func() {
		prod.flush()
		prod.transport.Close()
		prod.socket.Close()
		threads.Done()
	}()

	flushTicker := time.NewTicker(time.Duration(prod.batchTimeoutSec) * time.Second)

	for {
		select {
		case message := <-prod.messages:
			prod.sendMessage(message)

		case command := <-prod.control:
			if command == shared.ProducerControlStop {
				return // ### return, done ###
			}

		case <-flushTicker.C:
			if prod.batch.reachedTimeThreshold(prod.batchTimeoutSec) {
				prod.send()
			}
		}
	}
}
<commit_msg>documentation and fragments<commit_after>package producer

import (
	"github.com/artyom/scribe"
	"github.com/artyom/thrift"
	"github.com/trivago/gollum/shared"
	"strconv"
	"sync"
	"time"
)

// Scribe producer plugin
// Configuration example
//
// - "producer.Scribe":
//   Enable: true
//   Host: "192.168.222.30"
//   Port: 1463
//   BufferSizeKB: 4096
//   BatchSize: 4096
//   BatchSizeThreshold: 16777216
//   BatchTimeoutSec: 2
//   Stream:
//     - "console"
//     - "_GOLLUM_"
//   Category:
//     "console" : "default"
//     "_GOLLUM_"  : "default"
//
// Host and Port should be clear
//
// Category maps a stream to a specific scribe category. You can define the
// wildcard stream (*) here, too. All streams that do not have a specific
// mapping will go to this stream (including _GOLLUM_).
// If no category mappings are set all messages will be send to "default".
//
// BufferSizeKB sets the connection buffer size in KB. By default this is set to
// 1024, i.e. 1 MB buffer.
//
// BatchSize defines the number of bytes to be buffered before they are written
// to scribe. By default this is set to 8KB.
//
// BatchSizeThreshold defines the maximum number of bytes to buffer before
// messages get dropped. If a message crosses the threshold it is still buffered
// but additional messages will be dropped. By default this is set to 8MB.
//
// BatchTimeoutSec defines the maximum number of seconds to wait after the last
// message arrived before a batch is flushed automatically. By default this is
// set to 5.
type Scribe struct {
	standardProducer
	scribe          *scribe.ScribeClient
	transport       *thrift.TFramedTransport
	socket          *thrift.TSocket
	batch           *scribeMessageBuffer
	category        map[shared.MessageStreamID]string
	batchSize       int
	batchTimeoutSec int
	bufferSizeKB    int
	defaultCategory string
}

func init() {
	shared.Plugin.Register(Scribe{})
}

// Create creates a new producer based on the current scribe producer.
func (prod Scribe) Create(conf shared.PluginConfig) (shared.Producer, error) {

	err := prod.configureStandardProducer(conf)
	if err != nil {
		return nil, err
	}

	host := conf.GetString("Host", "localhost")
	port := conf.GetInt("Port", 1463)
	batchSizeThreshold := conf.GetInt("BatchSizeThreshold", 8388608)

	prod.category = make(map[shared.MessageStreamID]string, 0)
	prod.batchSize = conf.GetInt("BatchSize", 8192)
	prod.batchTimeoutSec = conf.GetInt("BatchTimeoutSec", 5)
	prod.batch = createScribeMessageBuffer(batchSizeThreshold, prod.flags)
	prod.bufferSizeKB = conf.GetInt("BufferSizeKB", 1<<10) // 1 MB

	// Read stream to category mapping

	defaultMapping := make(map[interface{}]interface{})
	defaultMapping[shared.WildcardStream] = "default"

	categoryMap := conf.GetValue("Category", defaultMapping).(map[interface{}]interface{})
	for stream, category := range categoryMap {
		prod.category[shared.GetStreamID(stream.(string))] = category.(string)
	}

	prod.defaultCategory = "default"

	wildcardCategory, wildcardCategorySet := prod.category[shared.WildcardStreamID]
	if wildcardCategorySet {
		prod.defaultCategory = wildcardCategory
	}

	// Initialize scribe connection

	prod.socket, err = thrift.NewTSocket(host + ":" + strconv.Itoa(port))
	if err != nil {
		shared.Log.Error("Scribe socket error:", err)
		return nil, err
	}

	prod.transport = thrift.NewTFramedTransport(prod.socket)
	protocolFactory := thrift.NewTBinaryProtocolFactory(false, false)

	prod.scribe = scribe.NewScribeClientFactory(prod.transport, protocolFactory)
	return prod, nil
}

func (prod Scribe) send() {
	if !prod.transport.IsOpen() {
		err := prod.transport.Open()
		if err != nil {
			shared.Log.Error("Scribe connection error:", err)
		} else {
			prod.socket.Conn().(bufferedConn).SetWriteBuffer(prod.bufferSizeKB << 10)
		}
	}

	if prod.transport.IsOpen() {
		err := prod.batch.flush(prod.scribe)

		if err != nil {
			shared.Log.Error("Scribe log error: ", err)
			prod.transport.Close()
		}
	}
}

func (prod Scribe) sendMessage(message shared.Message) {
	category, exists := prod.category[message.PinnedStream]
	if !exists {
		category = prod.defaultCategory
	}

	prod.batch.appendAndRelease(message, category)
	if prod.batch.reachedSizeThreshold(prod.batchSize) {
		prod.send()
	}
}

func (prod Scribe) flush() {
	for {
		select {
		case message := <-prod.messages:
			prod.sendMessage(message)
		default:
			prod.send()
			return
		}
	}
}

// Produce writes to a buffer that is sent to scribe.
func (prod Scribe) Produce(threads *sync.WaitGroup) {
	threads.Add(1)

	defer func() {
		prod.flush()
		prod.transport.Close()
		prod.socket.Close()
		threads.Done()
	}()

	flushTicker := time.NewTicker(time.Duration(prod.batchTimeoutSec) * time.Second)

	for {
		select {
		case message := <-prod.messages:
			prod.sendMessage(message)

		case command := <-prod.control:
			if command == shared.ProducerControlStop {
				return // ### return, done ###
			}

		case <-flushTicker.C:
			if prod.batch.reachedTimeThreshold(prod.batchTimeoutSec) {
				prod.send()
			}
		}
	}
}
<|endoftext|>


<commit_before>package producer

import (
	"github.com/trivago/gollum/shared"
	"net"
	"strings"
	"sync"
	"time"
)

var fileSocketPrefix = "unix://"

// Socket producer plugin
// Configuration example
//
// - "producer.Socket":
//   Enable: true
//   Address: "unix:///var/gollum.socket"
//   BufferSizeKB: 4096
//   BatchSize: 4096
//   BatchSizeThreshold: 16777216
//   BatchTimeoutSec: 5
//
// Address stores the identifier to connect to.
// This can either be any ip address and port like "localhost:5880" or a file
// like "unix:///var/gollum.socket". By default this is set to ":5880".
//
// BufferSize sets the connection buffer size in KB. By default this is set to
// 1024, i.e. 1 MB buffer.
//
// BatchSize defines the number of bytes to be buffered before they are written
// to scribe. By default this is set to 8KB.
//
// BatchSizeThreshold defines the maximum number of bytes to buffer before
// messages get dropped. Any message that crosses the threshold is dropped.
// By default this is set to 8MB.
//
// BatchTimeoutSec defines the maximum number of seconds to wait after the last
// message arrived before a batch is flushed automatically. By default this is
// set to 5.
type Socket struct {
	standardProducer
	connection      net.Conn
	batch           *shared.MessageBuffer
	protocol        string
	address         string
	batchSize       int
	batchTimeoutSec int
	bufferSizeKB    int
}

type bufferedConn interface {
	SetWriteBuffer(bytes int) error
}

func init() {
	shared.Plugin.Register(Socket{})
}

// Create creates a new producer based on the current socket producer.
func (prod Socket) Create(conf shared.PluginConfig) (shared.Producer, error) {
	err := prod.configureStandardProducer(conf)
	if err != nil {
		return nil, err
	}

	batchSizeThreshold := conf.GetInt("BatchSizeThreshold", 8388608)

	prod.flags |= shared.MessageFormatNewLine
	prod.protocol = "tcp"
	prod.address = conf.GetString("Address", ":5880")
	prod.batchSize = conf.GetInt("BatchSize", 8192)
	prod.batchTimeoutSec = conf.GetInt("BatchTimeoutSec", 5)
	prod.batch = shared.CreateMessageBuffer(batchSizeThreshold, prod.flags)
	prod.bufferSizeKB = conf.GetInt("BufferSizeKB", 1<<10) // 1 MB

	if strings.HasPrefix(prod.address, fileSocketPrefix) {
		prod.address = prod.address[len(fileSocketPrefix):]
		prod.protocol = "unix"
	}

	return prod, nil
}

func (prod *Socket) send() {
	// If we have not yet connected or the connection dropped: connect.
	if prod.connection == nil {
		conn, err := net.Dial(prod.protocol, prod.address)

		if err != nil {
			shared.Log.Error("Socket connection error:", err)
		} else {
			conn.(bufferedConn).SetWriteBuffer(prod.bufferSizeKB << 10)
			prod.connection = conn
		}
	}

	// Flush the buffer to the connection if it is active
	if prod.connection != nil {
		err := prod.batch.Flush(prod.connection)
		if err != nil {
			shared.Log.Error("Socket error:", err)
			prod.connection.Close()
			prod.connection = nil
		}
	}
}

func (prod *Socket) sendMessage(message shared.Message) {
	prod.batch.AppendAndRelease(message)
	if prod.batch.ReachedSizeThreshold(prod.batchSize) {
		prod.send()
	}
}

func (prod *Socket) flush() {
	for {
		select {
		case message := <-prod.messages:
			prod.sendMessage(message)
		default:
			return
		}
	}
}

// Produce writes to a buffer that is sent to a given socket.
func (prod Socket) Produce(threads *sync.WaitGroup) {
	threads.Add(1)

	defer func() {
		prod.flush()
		if prod.connection != nil {
			prod.connection.Close()
		}
		threads.Done()
	}()

	flushTick := time.NewTicker(time.Duration(prod.batchTimeoutSec) * time.Second)

	for {
		select {
		case message := <-prod.messages:
			prod.sendMessage(message)

		case command := <-prod.control:
			if command == shared.ProducerControlStop {
				return // ### return, done ###
			}

		case <-flushTick.C:
			if prod.batch.ReachedTimeThreshold(prod.batchTimeoutSec) {
				prod.send()
			}
		}
	}
}
<commit_msg>documentation<commit_after>package producer

import (
	"github.com/trivago/gollum/shared"
	"net"
	"strings"
	"sync"
	"time"
)

var fileSocketPrefix = "unix://"

// Socket producer plugin
// Configuration example
//
// - "producer.Socket":
//   Enable: true
//   Address: "unix:///var/gollum.socket"
//   BufferSizeKB: 4096
//   BatchSize: 4096
//   BatchSizeThreshold: 16777216
//   BatchTimeoutSec: 5
//
// Address stores the identifier to connect to.
// This can either be any ip address and port like "localhost:5880" or a file
// like "unix:///var/gollum.socket". By default this is set to ":5880".
//
// BufferSizeKB sets the connection buffer size in KB. By default this is set to
// 1024, i.e. 1 MB buffer.
//
// BatchSize defines the number of bytes to be buffered before they are written
// to scribe. By default this is set to 8KB.
//
// BatchSizeThreshold defines the maximum number of bytes to buffer before
// messages get dropped. Any message that crosses the threshold is dropped.
// By default this is set to 8MB.
//
// BatchTimeoutSec defines the maximum number of seconds to wait after the last
// message arrived before a batch is flushed automatically. By default this is
// set to 5.
type Socket struct {
	standardProducer
	connection      net.Conn
	batch           *shared.MessageBuffer
	protocol        string
	address         string
	batchSize       int
	batchTimeoutSec int
	bufferSizeKB    int
}

type bufferedConn interface {
	SetWriteBuffer(bytes int) error
}

func init() {
	shared.Plugin.Register(Socket{})
}

// Create creates a new producer based on the current socket producer.
func (prod Socket) Create(conf shared.PluginConfig) (shared.Producer, error) {
	err := prod.configureStandardProducer(conf)
	if err != nil {
		return nil, err
	}

	batchSizeThreshold := conf.GetInt("BatchSizeThreshold", 8388608)

	prod.flags |= shared.MessageFormatNewLine
	prod.protocol = "tcp"
	prod.address = conf.GetString("Address", ":5880")
	prod.batchSize = conf.GetInt("BatchSize", 8192)
	prod.batchTimeoutSec = conf.GetInt("BatchTimeoutSec", 5)
	prod.batch = shared.CreateMessageBuffer(batchSizeThreshold, prod.flags)
	prod.bufferSizeKB = conf.GetInt("BufferSizeKB", 1<<10) // 1 MB

	if strings.HasPrefix(prod.address, fileSocketPrefix) {
		prod.address = prod.address[len(fileSocketPrefix):]
		prod.protocol = "unix"
	}

	return prod, nil
}

func (prod *Socket) send() {
	// If we have not yet connected or the connection dropped: connect.
	if prod.connection == nil {
		conn, err := net.Dial(prod.protocol, prod.address)

		if err != nil {
			shared.Log.Error("Socket connection error:", err)
		} else {
			conn.(bufferedConn).SetWriteBuffer(prod.bufferSizeKB << 10)
			prod.connection = conn
		}
	}

	// Flush the buffer to the connection if it is active
	if prod.connection != nil {
		err := prod.batch.Flush(prod.connection)
		if err != nil {
			shared.Log.Error("Socket error:", err)
			prod.connection.Close()
			prod.connection = nil
		}
	}
}

func (prod *Socket) sendMessage(message shared.Message) {
	prod.batch.AppendAndRelease(message)
	if prod.batch.ReachedSizeThreshold(prod.batchSize) {
		prod.send()
	}
}

func (prod *Socket) flush() {
	for {
		select {
		case message := <-prod.messages:
			prod.sendMessage(message)
		default:
			return
		}
	}
}

// Produce writes to a buffer that is sent to a given socket.
func (prod Socket) Produce(threads *sync.WaitGroup) {
	threads.Add(1)

	defer func() {
		prod.flush()
		if prod.connection != nil {
			prod.connection.Close()
		}
		threads.Done()
	}()

	flushTick := time.NewTicker(time.Duration(prod.batchTimeoutSec) * time.Second)

	for {
		select {
		case message := <-prod.messages:
			prod.sendMessage(message)

		case command := <-prod.control:
			if command == shared.ProducerControlStop {
				return // ### return, done ###
			}

		case <-flushTick.C:
			if prod.batch.ReachedTimeThreshold(prod.batchTimeoutSec) {
				prod.send()
			}
		}
	}
}
<|endoftext|>