<commit_before>package gitbackend

import (
	"bytes"
	"fmt"
	"github.com/libgit2/git2go"
	"io"
	"io/ioutil"
	"strings"
	"time"
)

type CommitInfo struct {
	authorName, authorEmail, message string
	time                             time.Time
}

func (c *CommitInfo) AuthorName() string {
	return c.authorName
}

func (c *CommitInfo) AuthorEmail() string {
	return c.authorEmail
}

func (c *CommitInfo) Time() time.Time {
	return c.time
}

func (c *CommitInfo) Message() string {
	return c.message
}

type FileInfo struct {
	name string // base name of the file
	// path  string
	// size  int64 // length in bytes for regular files; system-dependent for others
	// isDir bool  // abbreviation for Mode().IsDir()
}

func (f *FileInfo) Name() string {
	return f.name
}

type FileStore struct {
	repo *git.Repository
}

func (f *FileStore) ReadDir(path string) (list []FileInfo, err error) {
	if strings.Trim(path, "/ ") == "" {
		return f.readRootDir()
	} else {
		return f.readSubDir(path)
	}
}

func (f *FileStore) readRootDir() (list []FileInfo, err error) {
	tree, err, noHead := f.tree()
	if err != nil {
		// return empty list for newly initialized repository without proper HEAD
		// usually the first commit sets a proper HEAD
		// this is only necessary for the root directory since there are no files after init
		if noHead {
			err = nil
		}
		return
	}
	list = f.listTree(tree)
	return
}

func (f *FileStore) readSubDir(path string) (list []FileInfo, err error) {
	root, err, _ := f.tree()
	if err != nil {
		return
	}

	entry, err := root.EntryByPath(path)
	if err != nil {
		return
	}
	tree, err := f.repo.LookupTree(entry.Id)
	if err != nil {
		return
	}

	list = f.listTree(tree)
	return
}

func (f *FileStore) listTree(tree *git.Tree) (list []FileInfo) {
	var i uint64
	for i = 0; i < tree.EntryCount(); i++ {
		entry := tree.EntryByIndex(i)
		list = append(list, FileInfo{entry.Name})
	}
	return
}

func (f *FileStore) ReadFile(path string) (reader io.Reader, err error) {
	tree, err, _ := f.tree()
	if err != nil {
		fmt.Println(err)
		return
	}
	entry, err := tree.EntryByPath(path)
	if err != nil {
		return
	}
	blob, err := f.repo.LookupBlob(entry.Id)
	if err != nil {
		return
	}
	reader = bytes.NewBuffer(blob.Contents())
	return
}

func (f *FileStore) WriteFile(path string, reader io.Reader, commitInfo CommitInfo) (err error) {
	odb, err := f.repo.Odb()
	if err != nil {
		fmt.Println(err)
		return
	}

	data, err := ioutil.ReadAll(reader)
	if err != nil {
		fmt.Println(err)
		return
	}

	blobOid, err := odb.Write(data, git.ObjectBlob)
	if err != nil {
		fmt.Println(err)
		return
	}

	oldTree, err, _ := f.tree()
	if err != nil {
		fmt.Println(err)
		return
	}

	newTreeId, err := f.updateTree(oldTree, path, blobOid)
	if err != nil {
		fmt.Println(err)
		return
	}

	tree, err := f.repo.LookupTree(newTreeId)
	if err != nil {
		fmt.Println(err)
		return
	}

	sig := &git.Signature{
		Name:  commitInfo.AuthorName(),
		Email: commitInfo.AuthorEmail(),
		When:  commitInfo.Time(),
	}

	commit, err, _ := f.headCommit()
	if err != nil {
		return
	}

	_, err = f.repo.CreateCommit("HEAD", sig, sig, commitInfo.Message(), tree, commit)
	if err != nil {
		fmt.Println(err)
		return
	}
	return
}
func (f *FileStore) updateTree(oldParentTree *git.Tree, path string, blobOid *git.Oid) (oid *git.Oid, err error) {
	treebuilder, err := f.repo.TreeBuilderFromTree(oldParentTree)
	if err != nil {
		fmt.Println(err)
		return
	}
	parts := strings.SplitN(path, "/", 2)
	if len(parts) == 1 {
		err = treebuilder.Insert(parts[0], blobOid, int(git.FilemodeBlob))
		if err != nil {
			fmt.Println(err)
			return
		}
		return treebuilder.Write()
	}

	newTreeOid, err := treebuilder.Write()
	if err != nil {
		fmt.Println(err)
		return
	}
	newTree, err := f.repo.LookupTree(newTreeOid)
	if err != nil {
		fmt.Println(err)
		return
	}

	oldChildTreeTreeEntry := newTree.EntryByName(parts[0])
	if oldChildTreeTreeEntry == nil {
		err = fmt.Errorf("Could not find Entry by Name %s", parts[0])
		return
	}
	oldChildTree, err2 := f.repo.LookupTree(oldChildTreeTreeEntry.Id)
	if err2 != nil {
		fmt.Println(err2)
		err = err2
		return
	}
	childTreeOid, err2 := f.updateTree(oldChildTree, parts[1], blobOid)
	if err2 != nil {
		fmt.Println(err2)
		err = err2
		return
	}
	err = treebuilder.Insert(parts[0], childTreeOid, int(git.FilemodeTree))
	if err != nil {
		fmt.Println(err)
		return
	}

	oid, err = treebuilder.Write()
	return
}

func (f *FileStore) tree() (tree *git.Tree, err error, noHead bool) {
	commit, err, noHead := f.headCommit()
	if err != nil {
		return
	}
	tree, err = commit.Tree()
	return
}

func (f *FileStore) headCommit() (commit *git.Commit, err error, noHead bool) {
	oid, err, noHead := f.headCommitId()
	if err != nil {
		return
	}
	commit, err = f.repo.LookupCommit(oid)
	return
}

func (f *FileStore) headCommitId() (oid *git.Oid, err error, noHead bool) {
	headRef, err := f.repo.LookupReference("HEAD")
	if err != nil {
		return
	}
	ref, err := headRef.Resolve()
	if err != nil {
		noHead = true
		return
	}
	oid = ref.Target()
	if oid == nil {
		err = fmt.Errorf("Could not get Target for HEAD(%s)\n", oid.String())
	}
	return
}

func NewFileStore(path string, isBare bool) (fs FileStore, err error) {
	repo, err := git.InitRepository(path, isBare)
	if err != nil {
		return
	}
	fs.repo = repo
	return
}
<commit_msg>Rename tree() helper method headCommitTree().<commit_after>package gitbackend

import (
	"bytes"
	"fmt"
	"github.com/libgit2/git2go"
	"io"
	"io/ioutil"
	"strings"
	"time"
)

type CommitInfo struct {
	authorName, authorEmail, message string
	time                             time.Time
}

func (c *CommitInfo) AuthorName() string {
	return c.authorName
}

func (c *CommitInfo) AuthorEmail() string {
	return c.authorEmail
}

func (c *CommitInfo) Time() time.Time {
	return c.time
}

func (c *CommitInfo) Message() string {
	return c.message
}

type FileInfo struct {
	name string // base name of the file
	// path  string
	// size  int64 // length in bytes for regular files; system-dependent for others
	// isDir bool  // abbreviation for Mode().IsDir()
}

func (f *FileInfo) Name() string {
	return f.name
}

type FileStore struct {
	repo *git.Repository
}

func (f *FileStore) ReadDir(path string) (list []FileInfo, err error) {
	if strings.Trim(path, "/ ") == "" {
		return f.readRootDir()
	} else {
		return f.readSubDir(path)
	}
}

func (f *FileStore) readRootDir() (list []FileInfo, err error) {
	headCommitTree, err, noHead := f.headCommitTree()
	if err != nil {
		// return empty list for newly initialized repository without proper HEAD
		// usually the first commit sets a proper HEAD
		// this is only necessary for the root directory since there are no files after init
		if noHead {
			err = nil
		}
		return
	}
	list = f.listTree(headCommitTree)
	return
}

func (f *FileStore) readSubDir(path string) (list []FileInfo, err error) {
	headCommitTree, err, _ := f.headCommitTree()
	if err != nil {
		return
	}

	entry, err := headCommitTree.EntryByPath(path)
	if err != nil {
		return
	}
	tree, err := f.repo.LookupTree(entry.Id)
	if err != nil {
		return
	}

	list = f.listTree(tree)
	return
}

func (f *FileStore) listTree(tree *git.Tree) (list []FileInfo) {
	var i uint64
	for i = 0; i < tree.EntryCount(); i++ {
		entry := tree.EntryByIndex(i)
		list = append(list, FileInfo{entry.Name})
	}
	return
}

func (f *FileStore) ReadFile(path string) (reader io.Reader, err error) {
	headCommitTree, err, _ := f.headCommitTree()
	if err != nil {
		fmt.Println(err)
		return
	}
	entry, err := headCommitTree.EntryByPath(path)
	if err != nil {
		return
	}
	blob, err := f.repo.LookupBlob(entry.Id)
	if err != nil {
		return
	}
	reader = bytes.NewBuffer(blob.Contents())
	return
}

func (f *FileStore) WriteFile(path string, reader io.Reader, commitInfo CommitInfo) (err error) {
	odb, err := f.repo.Odb()
	if err != nil {
		fmt.Println(err)
		return
	}

	data, err := ioutil.ReadAll(reader)
	if err != nil {
		fmt.Println(err)
		return
	}

	blobOid, err := odb.Write(data, git.ObjectBlob)
	if err != nil {
		fmt.Println(err)
		return
	}

	oldTree, err, _ := f.headCommitTree()
	if err != nil {
		fmt.Println(err)
		return
	}

	newTreeId, err := f.updateTree(oldTree, path, blobOid)
	if err != nil {
		fmt.Println(err)
		return
	}

	tree, err := f.repo.LookupTree(newTreeId)
	if err != nil {
		fmt.Println(err)
		return
	}

	sig := &git.Signature{
		Name:  commitInfo.AuthorName(),
		Email: commitInfo.AuthorEmail(),
		When:  commitInfo.Time(),
	}

	commit, err, _ := f.headCommit()
	if err != nil {
		return
	}

	_, err = f.repo.CreateCommit("HEAD", sig, sig, commitInfo.Message(), tree, commit)
	if err != nil {
		fmt.Println(err)
		return
	}
	return
}
func (f *FileStore) updateTree(oldParentTree *git.Tree, path string, blobOid *git.Oid) (oid *git.Oid, err error) {
	treebuilder, err := f.repo.TreeBuilderFromTree(oldParentTree)
	if err != nil {
		fmt.Println(err)
		return
	}
	parts := strings.SplitN(path, "/", 2)
	if len(parts) == 1 {
		err = treebuilder.Insert(parts[0], blobOid, int(git.FilemodeBlob))
		if err != nil {
			fmt.Println(err)
			return
		}
		return treebuilder.Write()
	}

	newTreeOid, err := treebuilder.Write()
	if err != nil {
		fmt.Println(err)
		return
	}
	newTree, err := f.repo.LookupTree(newTreeOid)
	if err != nil {
		fmt.Println(err)
		return
	}

	oldChildTreeTreeEntry := newTree.EntryByName(parts[0])
	if oldChildTreeTreeEntry == nil {
		err = fmt.Errorf("Could not find Entry by Name %s", parts[0])
		return
	}
	oldChildTree, err2 := f.repo.LookupTree(oldChildTreeTreeEntry.Id)
	if err2 != nil {
		fmt.Println(err2)
		err = err2
		return
	}
	childTreeOid, err2 := f.updateTree(oldChildTree, parts[1], blobOid)
	if err2 != nil {
		fmt.Println(err2)
		err = err2
		return
	}
	err = treebuilder.Insert(parts[0], childTreeOid, int(git.FilemodeTree))
	if err != nil {
		fmt.Println(err)
		return
	}

	oid, err = treebuilder.Write()
	return
}

func (f *FileStore) headCommitTree() (tree *git.Tree, err error, noHead bool) {
	commit, err, noHead := f.headCommit()
	if err != nil {
		return
	}
	tree, err = commit.Tree()
	return
}

func (f *FileStore) headCommit() (commit *git.Commit, err error, noHead bool) {
	oid, err, noHead := f.headCommitId()
	if err != nil {
		return
	}
	commit, err = f.repo.LookupCommit(oid)
	return
}

func (f *FileStore) headCommitId() (oid *git.Oid, err error, noHead bool) {
	headRef, err := f.repo.LookupReference("HEAD")
	if err != nil {
		return
	}
	ref, err := headRef.Resolve()
	if err != nil {
		noHead = true
		return
	}
	oid = ref.Target()
	if oid == nil {
		err = fmt.Errorf("Could not get Target for HEAD(%s)\n", oid.String())
	}
	return
}

func NewFileStore(path string, isBare bool) (fs FileStore, err error) {
	repo, err := git.InitRepository(path, isBare)
	if err != nil {
		return
	}
	fs.repo = repo
	return
}
<|endoftext|>


<commit_before>/*
The memberlist package is used to provide a lightweight gossip based
mechanism for node membership and failure detection. It is loosely
based on the SWIM paper (Scalable Weakly-consistent Infection-style
process group Membership protocol). There are a few notable differences,
including the uses of additional fanout (instead of purely piggybacking on
failure detection) and the addition of a state push/pull mechanism.

A fanout mechanism is used because it allows for changes to be propogated
more quickly, and also enables us to dynamically increase the amount of
gossip in response to a surge of events. The fanout rate is provided as a tunable
parameter.

A Push/Pull mechanism is also included because it allows new nodes to
get an almost complete member list upon joining. It also is used as
a periodic anti-entropy mechanism to ensure very high convergence rates.
The frequency of this can be adjusted to change the overhead, or disabled
entirely.

*/
package memberlist

import (
	"fmt"
	"net"
	"os"
	"sync"
	"time"
)

type Config struct {
	Name           string        // Node name (FQDN)
	BindAddr       string        // Binding address
	UDPPort        int           // UDP port to listen on
	TCPPort        int           // TCP port to listen on
	Fanout         int           // Number of nodes to publish to per round
	IndirectChecks int           // Number of indirect checks to use
	RetransmitMult int           // Retransmits = RetransmitMult * log(N+1)
	SuspicionMult  int           // Suspicion time = SuspcicionMult * log(N+1) * Interval
	PushPullFreq   float32       // How often we do a Push/Pull update
	RTT            time.Duration // 99% precentile of round-trip-time
	ProbeInterval  time.Duration // Failure probing interval length
}

type Memberlist struct {
	config   *Config
	shutdown bool

	udpListener *net.UDPConn
	tcpListener *net.TCPListener

	notifyLock  sync.RWMutex
	notifyJoin  []chan<- *Node // Channels to notify on join
	notifyLeave []chan<- *Node // Channels to notify on leave

	sequenceNum uint32 // Local sequence number
	incarnation uint32 // Local incarnation number

	nodeLock sync.RWMutex
	nodes    []*NodeState          // Known nodes
	nodeMap  map[string]*NodeState // Maps Addr.String() -> NodeState

	tickerLock sync.Mutex
	ticker     *time.Ticker
	stopTick   chan struct{}
	tickIndex  int

	ackLock     sync.Mutex
	ackHandlers map[uint32]*ackHandler

	broadcastLock sync.Mutex
	bcQueue       broadcasts
}

func DefaultConfig() *Config {
	hostname, _ := os.Hostname()
	return &Config{
		hostname,
		"0.0.0.0",
		7946,
		7946,
		3,    // Fanout to 3 nodes
		3,    // Use 3 nodes for the indirect ping
		4,    // Retransmit a message 4 * log(N+1) nodes
		5,    // Suspect a node for 5 * log(N+1) * Interval
		0.05, // 5% frequency for push/pull
		20 * time.Millisecond, // Reasonable RTT time for LAN
		1 * time.Second,       // Failure check every second
	}
}

// newMemberlist creates the network listeners.
// Does not schedule exeuction of background maintenence.
func newMemberlist(conf *Config) (*Memberlist, error) {
	tcpAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.TCPPort)
	tcpLn, err := net.Listen("tcp", tcpAddr)
	if err != nil {
		return nil, fmt.Errorf("Failed to start TCP listener. Err: %s", err)
	}

	udpAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.UDPPort)
	udpLn, err := net.ListenPacket("udp", udpAddr)
	if err != nil {
		tcpLn.Close()
		return nil, fmt.Errorf("Failed to start UDP listener. Err: %s", err)
	}

	m := &Memberlist{config: conf,
		udpListener: udpLn.(*net.UDPConn),
		tcpListener: tcpLn.(*net.TCPListener),
		nodeMap:     make(map[string]*NodeState),
		stopTick:    make(chan struct{}, 1),
		ackHandlers: make(map[uint32]*ackHandler),
	}
	go m.tcpListen()
	go m.udpListen()
	return m, nil
}

// Create will start memberlist and create a new gossip pool, but
// will not connect to an existing node. This should only be used
// for the first node in the cluster.
func Create(conf *Config) (*Memberlist, error) {
	m, err := newMemberlist(conf)
	if err != nil {
		return nil, err
	}
	m.schedule()
	return m, nil
}

// Join will start memberlist and perform an initial push/pull with
// all the given hosts. If none of the existing hosts could be contacted,
// the join will fail.
func Join(conf *Config, existing []string) (*Memberlist, error) {
	m, err := newMemberlist(conf)
	if err != nil {
		return nil, err
	}

	// TODO: Attempt to join...

	// Schedule background work
	m.schedule()
	return m, nil
}

// NotifyJoin is used to subscribe a channel to join events
func (m *Memberlist) NotifyJoin(ch chan<- *Node) {
	m.notifyLock.Lock()
	defer m.notifyLock.Unlock()

	// Bail if the channel is already subscribed
	if channelIndex(m.notifyJoin, ch) >= 0 {
		return
	}
	m.notifyJoin = append(m.notifyJoin, ch)
}

// NotifyLeave is used to subscribe a channel to leave events
func (m *Memberlist) NotifyLeave(ch chan<- *Node) {
	m.notifyLock.Lock()
	defer m.notifyLock.Unlock()

	// Bail if the channel is already subscribed
	if channelIndex(m.notifyLeave, ch) >= 0 {
		return
	}
	m.notifyLeave = append(m.notifyLeave, ch)
}

// Stop is used to unsubscribe a channel from any notifications
func (m *Memberlist) Stop(ch chan<- *Node) {
	m.notifyLock.Lock()
	defer m.notifyLock.Unlock()

	idx := channelIndex(m.notifyJoin, ch)
	if idx >= 0 {
		m.notifyJoin = channelDelete(m.notifyJoin, idx)
	}

	idx = channelIndex(m.notifyLeave, ch)
	if idx >= 0 {
		m.notifyLeave = channelDelete(m.notifyLeave, idx)
	}
}

// Members is used to return a list of all known live nodes
func (m *Memberlist) Members() []*Node {
	nodes := make([]*Node, 0, len(m.nodes))
	m.nodeLock.RLock()
	defer m.nodeLock.RUnlock()
	for _, n := range m.nodes {
		if n.State != StateDead {
			nodes = append(nodes, &n.Node)
		}
	}
	return nodes
}

// Leave will broadcast a leave message but will not shutdown
// the memberlist background maintenence. This should be followed/
// by a Shutdown().
func (m *Memberlist) Leave() error {
	return nil
}

// Shutdown will stop the memberlist background maintenence
// but will not broadcast a leave message prior. If no prior
// leave was issued, other nodes will detect this as a failure.
func (m *Memberlist) Shutdown() error {
	m.shutdown = true
	m.deschedule()
	m.udpListener.Close()
	m.tcpListener.Close()
	return nil
}
<commit_msg>Adding a seperate gossip interval<commit_after>/*
The memberlist package is used to provide a lightweight gossip based
mechanism for node membership and failure detection. It is loosely
based on the SWIM paper (Scalable Weakly-consistent Infection-style
process group Membership protocol). There are a few notable differences,
including the uses of additional gossip (instead of purely piggybacking on
failure detection) and the addition of a state push/pull mechanism.

An independent gossip mechanism is used because it allows for changes to be propogated
more quickly, and also enables us to gossip at a different interval that we perform
failure checks. The gossip rate is tunable, and can be disabled.

A Push/Pull mechanism is also included because it allows new nodes to
get an almost complete member list upon joining. It also is used as
a periodic anti-entropy mechanism to ensure very high convergence rates.
The frequency of this can be adjusted to change the overhead, or disabled
entirely.

*/
package memberlist

import (
	"fmt"
	"net"
	"os"
	"sync"
	"time"
)

type Config struct {
	Name           string        // Node name (FQDN)
	BindAddr       string        // Binding address
	UDPPort        int           // UDP port to listen on
	TCPPort        int           // TCP port to listen on
	IndirectChecks int           // Number of indirect checks to use
	RetransmitMult int           // Retransmits = RetransmitMult * log(N+1)
	SuspicionMult  int           // Suspicion time = SuspcicionMult * log(N+1) * Interval
	PushPullFreq   float32       // How often we do a Push/Pull update
	RTT            time.Duration // 99% precentile of round-trip-time
	ProbeInterval  time.Duration // Failure probing interval length

	GossipNodes    int           // Number of nodes to gossip to per GossipInterval
	GossipInterval time.Duration // Gossip interval for non-piggyback messages (only if GossipNodes > 0)
}

type Memberlist struct {
	config   *Config
	shutdown bool

	udpListener *net.UDPConn
	tcpListener *net.TCPListener

	notifyLock  sync.RWMutex
	notifyJoin  []chan<- *Node // Channels to notify on join
	notifyLeave []chan<- *Node // Channels to notify on leave

	sequenceNum uint32 // Local sequence number
	incarnation uint32 // Local incarnation number

	nodeLock sync.RWMutex
	nodes    []*NodeState          // Known nodes
	nodeMap  map[string]*NodeState // Maps Addr.String() -> NodeState

	tickerLock sync.Mutex
	ticker     *time.Ticker
	stopTick   chan struct{}
	tickIndex  int

	ackLock     sync.Mutex
	ackHandlers map[uint32]*ackHandler

	broadcastLock sync.Mutex
	bcQueue       broadcasts
}

func DefaultConfig() *Config {
	hostname, _ := os.Hostname()
	return &Config{
		hostname,
		"0.0.0.0",
		7946,
		7946,
		3,    // Use 3 nodes for the indirect ping
		4,    // Retransmit a message 4 * log(N+1) nodes
		5,    // Suspect a node for 5 * log(N+1) * Interval
		0.05, // 5% frequency for push/pull
		20 * time.Millisecond, // Reasonable RTT time for LAN
		1 * time.Second,       // Failure check every second

		3, // Gossip to 3 nodes
		200 * time.Millisecond, // Gossip more rapidly
	}
}

// newMemberlist creates the network listeners.
// Does not schedule exeuction of background maintenence.
func newMemberlist(conf *Config) (*Memberlist, error) {
	tcpAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.TCPPort)
	tcpLn, err := net.Listen("tcp", tcpAddr)
	if err != nil {
		return nil, fmt.Errorf("Failed to start TCP listener. Err: %s", err)
	}

	udpAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.UDPPort)
	udpLn, err := net.ListenPacket("udp", udpAddr)
	if err != nil {
		tcpLn.Close()
		return nil, fmt.Errorf("Failed to start UDP listener. Err: %s", err)
	}

	m := &Memberlist{config: conf,
		udpListener: udpLn.(*net.UDPConn),
		tcpListener: tcpLn.(*net.TCPListener),
		nodeMap:     make(map[string]*NodeState),
		stopTick:    make(chan struct{}, 1),
		ackHandlers: make(map[uint32]*ackHandler),
	}
	go m.tcpListen()
	go m.udpListen()
	return m, nil
}

// Create will start memberlist and create a new gossip pool, but
// will not connect to an existing node. This should only be used
// for the first node in the cluster.
func Create(conf *Config) (*Memberlist, error) {
	m, err := newMemberlist(conf)
	if err != nil {
		return nil, err
	}
	m.schedule()
	return m, nil
}

// Join will start memberlist and perform an initial push/pull with
// all the given hosts. If none of the existing hosts could be contacted,
// the join will fail.
func Join(conf *Config, existing []string) (*Memberlist, error) {
	m, err := newMemberlist(conf)
	if err != nil {
		return nil, err
	}

	// TODO: Attempt to join...

	// Schedule background work
	m.schedule()
	return m, nil
}

// NotifyJoin is used to subscribe a channel to join events
func (m *Memberlist) NotifyJoin(ch chan<- *Node) {
	m.notifyLock.Lock()
	defer m.notifyLock.Unlock()

	// Bail if the channel is already subscribed
	if channelIndex(m.notifyJoin, ch) >= 0 {
		return
	}
	m.notifyJoin = append(m.notifyJoin, ch)
}

// NotifyLeave is used to subscribe a channel to leave events
func (m *Memberlist) NotifyLeave(ch chan<- *Node) {
	m.notifyLock.Lock()
	defer m.notifyLock.Unlock()

	// Bail if the channel is already subscribed
	if channelIndex(m.notifyLeave, ch) >= 0 {
		return
	}
	m.notifyLeave = append(m.notifyLeave, ch)
}

// Stop is used to unsubscribe a channel from any notifications
func (m *Memberlist) Stop(ch chan<- *Node) {
	m.notifyLock.Lock()
	defer m.notifyLock.Unlock()

	idx := channelIndex(m.notifyJoin, ch)
	if idx >= 0 {
		m.notifyJoin = channelDelete(m.notifyJoin, idx)
	}

	idx = channelIndex(m.notifyLeave, ch)
	if idx >= 0 {
		m.notifyLeave = channelDelete(m.notifyLeave, idx)
	}
}

// Members is used to return a list of all known live nodes
func (m *Memberlist) Members() []*Node {
	nodes := make([]*Node, 0, len(m.nodes))
	m.nodeLock.RLock()
	defer m.nodeLock.RUnlock()
	for _, n := range m.nodes {
		if n.State != StateDead {
			nodes = append(nodes, &n.Node)
		}
	}
	return nodes
}

// Leave will broadcast a leave message but will not shutdown
// the memberlist background maintenence. This should be followed/
// by a Shutdown().
func (m *Memberlist) Leave() error {
	return nil
}

// Shutdown will stop the memberlist background maintenence
// but will not broadcast a leave message prior. If no prior
// leave was issued, other nodes will detect this as a failure.
func (m *Memberlist) Shutdown() error {
	m.shutdown = true
	m.deschedule()
	m.udpListener.Close()
	m.tcpListener.Close()
	return nil
}
<|endoftext|>


<commit_before>/*
memberlist is a library that manages cluster
membership and member failure detection using a gossip based protocol.

The use cases for such a library are far-reaching: all distributed systems
require membership, and memberlist is a re-usable solution to managing
cluster membership and node failure detection.

memberlist is eventually consistent but converges quickly on average.
The speed at which it converges can be heavily tuned via various knobs
on the protocol. Node failures are detected and network partitions are partially
tolerated by attempting to communicate to potentially dead nodes through
multiple routes.
*/
package memberlist

import (
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"sync"
	"time"
)

type Config struct {
	// The name of this node. This must be unique in the cluster.
	Name string

	// Configuration related to what address to bind to and ports to
	// listen on. The ports used must match every node in the cluster,
	// since they'll be used for connecting as well as listening.
	BindAddr string
	UDPPort  int
	TCPPort  int

	// TCPTimeout is the timeout for establishing a TCP connection with
	// a remote node for a full state sync.
	TCPTimeout time.Duration

	// IndirectChecks is the number of nodes that will be asked to perform
	// an indirect probe of a node in the case a direct probe fails. Memberlist
	// waits for an ack from any single indirect node, so increasing this
	// number will increase the likelihood that an indirect probe will succeed
	// at the expense of bandwidth.
	IndirectChecks int

	// RetransmitMult is the multiplier for the number of retransmissions
	// that are attempted for messages broadcasted over gossip. The actual
	// count of retransmissions is calculated using the formula:
	//
	//   Retransmits = RetransmitMult * log(N+1)
	//
	// This allows the retransmits to scale properly with cluster size. The
	// higher the multiplier, the more likely a failed broadcast is to converge
	// at the expense of increased bandwidth.
	RetransmitMult int

	// SuspicionMult is the multiplier for determining the time an
	// inaccessible node is considered suspect before declaring it dead.
	// The actual timeout is calculated using the formula:
	//
	//   SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval
	//
	// This allows the timeout to scale properly with expected propagation
	// delay with a larger cluster size. The higher the multiplier, the longer
	// an inaccessible node is considered part of the cluster before declaring
	// it dead, giving that suspect node more time to refute if it is indeed
	// still alive.
	SuspicionMult int

	// PushPullInterval is the interval between complete state syncs.
	// Complete state syncs are done with a single node over TCP and are
	// quite expensive relative to standard gossiped messages. Setting this
	// to zero will disable state push/pull syncs completely.
	//
	// Setting this interval lower (more frequent) will increase convergence
	// speeds across larger clusters at the expense of increased bandwidth
	// usage.
	PushPullInterval time.Duration

	// ProbeInterval and ProbeTimeout are used to configure probing
	// behavior for memberlist.
	//
	// ProbeInterval is the interval between random node probes. Setting
	// this lower (more frequent) will cause the memberlist cluster to detect
	// failed nodes more quickly at the expense of increased bandwidth usage.
	//
	// ProbeTimeout is the timeout to wait for an ack from a probed node
	// before assuming it is unhealthy. This should be set to 99-percentile
	// of RTT (round-trip time) on your network.
	ProbeInterval time.Duration
	ProbeTimeout  time.Duration

	// GossipInterval and GossipNodes are used to configure the gossip
	// behavior of memberlist.
	//
	// GossipInterval is the interval between sending messages that need
	// to be gossiped that haven't been able to piggyback on probing messages.
	// If this is set to zero, non-piggyback gossip is disabled. By lowering
	// this value (more frequent) gossip messages are propagated across
	// the cluster more quickly at the expense of increased bandwidth.
	//
	// GossipNodes is the number of random nodes to send gossip messages to
	// per GossipInterval. Increasing this number causes the gossip messages
	// to propagate across the cluster more quickly at the expense of
	// increased bandwidth.
	GossipInterval time.Duration
	GossipNodes    int

	// Delegate and Events are delegates for receiving and providing
	// data to memberlist via callback mechanisms. For Delegate, see
	// the Delegate interface. For Events, see the EventDelegate interface.
	Delegate Delegate
	Events   EventDelegate

	// LogOutput is the writer where logs should be sent. If this is not
	// set, logging will go to stderr by default.
	LogOutput io.Writer
}

type Memberlist struct {
	config         *Config
	shutdown       bool
	leave          bool
	leaveBroadcast chan struct{}

	udpListener *net.UDPConn
	tcpListener *net.TCPListener

	sequenceNum uint32 // Local sequence number
	incarnation uint32 // Local incarnation number

	nodeLock sync.RWMutex
	nodes    []*nodeState          // Known nodes
	nodeMap  map[string]*nodeState // Maps Addr.String() -> NodeState

	tickerLock sync.Mutex
	tickers    []*time.Ticker
	stopTick   chan struct{}
	probeIndex int

	ackLock     sync.Mutex
	ackHandlers map[uint32]*ackHandler

	broadcasts *TransmitLimitedQueue

	startStopLock sync.Mutex

	logger *log.Logger
}

// DefaultConfig returns a sane set of configurations for Memberlist.
// It uses the hostname as the node name, and otherwise sets very conservative
// values that are sane for most LAN environments. The default configuration
// errs on the side on the side of caution, choosing values that are optimized
// for higher convergence at the cost of higher bandwidth usage. Regardless,
// these values are a good starting point when getting started with memberlist.
func DefaultConfig() *Config {
	hostname, _ := os.Hostname()
	return &Config{
		Name:             hostname,
		BindAddr:         "0.0.0.0",
		UDPPort:          7946,
		TCPPort:          7946,
		TCPTimeout:       30 * time.Second,       // Timeout after 10 seconds
		IndirectChecks:   3,                      // Use 3 nodes for the indirect ping
		RetransmitMult:   4,                      // Retransmit a message 4 * log(N+1) nodes
		SuspicionMult:    5,                      // Suspect a node for 5 * log(N+1) * Interval
		PushPullInterval: 30 * time.Second,       // Low frequency
		ProbeTimeout:     500 * time.Millisecond, // Reasonable RTT time for LAN
		ProbeInterval:    1 * time.Second,        // Failure check every second

		GossipNodes:    3,                      // Gossip to 3 nodes
		GossipInterval: 200 * time.Millisecond, // Gossip more rapidly
	}
}

// newMemberlist creates the network listeners.
// Does not schedule execution of background maintenence.
func newMemberlist(conf *Config) (*Memberlist, error) {
	tcpAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.TCPPort)
	tcpLn, err := net.Listen("tcp", tcpAddr)
	if err != nil {
		return nil, fmt.Errorf("Failed to start TCP listener. Err: %s", err)
	}

	udpAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.UDPPort)
	udpLn, err := net.ListenPacket("udp", udpAddr)
	if err != nil {
		tcpLn.Close()
		return nil, fmt.Errorf("Failed to start UDP listener. Err: %s", err)
	}

	// Set the UDP receive window size
	setUDPRecvBuf(udpLn.(*net.UDPConn))

	if conf.LogOutput == nil {
		conf.LogOutput = os.Stderr
	}

	m := &Memberlist{
		config:         conf,
		leaveBroadcast: make(chan struct{}, 1),
		udpListener:    udpLn.(*net.UDPConn),
		tcpListener:    tcpLn.(*net.TCPListener),
		nodeMap:        make(map[string]*nodeState),
		ackHandlers:    make(map[uint32]*ackHandler),
		broadcasts:     &TransmitLimitedQueue{RetransmitMult: conf.RetransmitMult},
		logger:         log.New(conf.LogOutput, "", log.LstdFlags),
	}
	m.broadcasts.NumNodes = func() int { return len(m.nodes) }
	go m.tcpListen()
	go m.udpListen()
	return m, nil
}

// Create will create a new Memberlist using the given configuration.
// This will not connect to any other node (see Join) yet, but will start
// all the listeners to allow other nodes to join this memberlist.
// After creating a Memberlist, the configuration given should not be
// modified by the user anymore.
func Create(conf *Config) (*Memberlist, error) {
	m, err := newMemberlist(conf)
	if err != nil {
		return nil, err
	}
	if err := m.setAlive(); err != nil {
		m.Shutdown()
		return nil, err
	}
	m.schedule()
	return m, nil
}

// Join is used to take an existing Memberlist and attempt to join a cluster
// by contacting all the given hosts and performing a state sync. Initially,
// the Memberlist only contains our own state, so doing this will cause
// remote nodes to become aware of the existence of this node, effectively
// joining the cluster.
//
// This returns the number of hosts successfully contacted and an error if
// none could be reached. If an error is returned, the node did not successfully
// join the cluster.
func (m *Memberlist) Join(existing []string) (int, error) {
	// Attempt to join any of them
	numSuccess := 0
	var retErr error
	for _, exist := range existing {
		addr, err := net.ResolveIPAddr("ip", exist)
		if err != nil {
			retErr = err
			continue
		}

		if err := m.pushPullNode(addr.IP); err != nil {
			retErr = err
			continue
		}

		numSuccess++
	}

	if numSuccess > 0 {
		retErr = nil
	}

	return numSuccess, retErr
}

// setAlive is used to mark this node as being alive. This is the same
// as if we received an alive notification our own network channel for
// ourself.
func (m *Memberlist) setAlive() error {
	// Pick a private IP address
	var ipAddr []byte
	if m.config.BindAddr == "0.0.0.0" {
		// We're not bound to a specific IP, so let's list the interfaces
		// on this machine and use the first private IP we find.
		addresses, err := net.InterfaceAddrs()
		if err != nil {
			return fmt.Errorf("Failed to get interface addresses! Err: %vn", err)
		}

		// Find private IPv4 address
		for _, addr := range addresses {
			ip, ok := addr.(*net.IPNet)
			if !ok {
				continue
			}
			if ip.IP.To4() == nil {
				continue
			}
			if !isPrivateIP(ip.IP.String()) {
				continue
			}
			ipAddr = ip.IP
			break
		}

		// Failed to find private IP, use loopback
		if ipAddr == nil {
			ipAddr = []byte{127, 0, 0, 1}
		}
	} else {
		// Use the IP that we're bound to.
		addr := m.tcpListener.Addr().(*net.TCPAddr)
		ipAddr = addr.IP
	}

	// Get the node meta data
	var meta []byte
	if m.config.Delegate != nil {
		meta = m.config.Delegate.NodeMeta(metaMaxSize)
		if len(meta) > metaMaxSize {
			panic("Node meta data provided is longer than the limit")
		}
	}

	a := alive{
		Incarnation: m.nextIncarnation(),
		Node:        m.config.Name,
		Addr:        ipAddr,
		Meta:        meta,
	}
	m.aliveNode(&a)

	return nil
}

// Members returns a list of all known live nodes. The node structures
// returned must not be modified. If you wish to modify a Node, make a
// copy first.
func (m *Memberlist) Members() []*Node {
	m.nodeLock.RLock()
	defer m.nodeLock.RUnlock()

	nodes := make([]*Node, 0, len(m.nodes))
	for _, n := range m.nodes {
		if n.State != stateDead {
			nodes = append(nodes, &n.Node)
		}
	}

	return nodes
}

// NumMembers returns the number of alive nodes currently known. Between
// the time of calling this and calling Members, the number of alive nodes
// may have changed, so this shouldn't be used to determine how many
// members will be returned by Members.
func (m *Memberlist) NumMembers() (alive int) {
	m.nodeLock.RLock()
	defer m.nodeLock.RUnlock()

	for _, n := range m.nodes {
		if n.State != stateDead {
			alive++
		}
	}

	return
}

// Leave will broadcast a leave message but will not shutdown the background
// listeners, meaning the node will continue participating in gossip and state
// updates.
//
// This will block until the leave message is successfully broadcasted to
// a member of the cluster, if any exist or until a specified timeout
// is reached.
//
// This method is safe to call multiple times, but must not be called
// after the cluster is already shut down.
func (m *Memberlist) Leave(timeout time.Duration) error {
	m.startStopLock.Lock()
	defer m.startStopLock.Unlock()

	if m.shutdown {
		panic("leave after shutdown")
	}

	if !m.leave {
		m.leave = true

		state, ok := m.nodeMap[m.config.Name]
		if !ok {
			m.logger.Println("[WARN] Leave but we're not in the node map.")
			return nil
		}

		d := dead{
			Incarnation: state.Incarnation,
			Node:        state.Name,
		}
		m.deadNode(&d)

		// Block until the broadcast goes out
		if len(m.nodes) > 1 {
			var timeoutCh <-chan time.Time
			if timeout > 0 {
				timeoutCh = time.After(timeout)
			}
			select {
			case <-m.leaveBroadcast:
			case <-timeoutCh:
				return fmt.Errorf("timeout waiting for leave broadcast")
			}
		}
	}

	return nil
}

// Shutdown will stop any background maintanence of network activity
// for this memberlist, causing it to appear "dead". A leave message
// will not be broadcasted prior, so the cluster being left will have
// to detect this node's shutdown using probing. If you wish to more
// gracefully exit the cluster, call Leave prior to shutting down.
//
// This method is safe to call multiple times.
func (m *Memberlist) Shutdown() error {
	m.startStopLock.Lock()
	defer m.startStopLock.Unlock()

	if !m.shutdown {
		m.shutdown = true
		m.deschedule()
		m.udpListener.Close()
		m.tcpListener.Close()
	}

	return nil
}
<commit_msg>check if any node is alive before broadcasting<commit_after>/*
memberlist is a library that manages cluster
membership and member failure detection using a gossip based protocol.

The use cases for such a library are far-reaching: all distributed systems
require membership, and memberlist is a re-usable solution to managing
cluster membership and node failure detection.

memberlist is eventually consistent but converges quickly on average.
The speed at which it converges can be heavily tuned via various knobs
on the protocol. Node failures are detected and network partitions are partially
tolerated by attempting to communicate to potentially dead nodes through
multiple routes.
*/
package memberlist

import (
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"sync"
	"time"
)

type Config struct {
	// The name of this node. This must be unique in the cluster.
	Name string

	// Configuration related to what address to bind to and ports to
	// listen on. The ports used must match every node in the cluster,
	// since they'll be used for connecting as well as listening.
	BindAddr string
	UDPPort  int
	TCPPort  int

	// TCPTimeout is the timeout for establishing a TCP connection with
	// a remote node for a full state sync.
	TCPTimeout time.Duration

	// IndirectChecks is the number of nodes that will be asked to perform
	// an indirect probe of a node in the case a direct probe fails. Memberlist
	// waits for an ack from any single indirect node, so increasing this
	// number will increase the likelihood that an indirect probe will succeed
	// at the expense of bandwidth.
	IndirectChecks int

	// RetransmitMult is the multiplier for the number of retransmissions
	// that are attempted for messages broadcasted over gossip. The actual
	// count of retransmissions is calculated using the formula:
	//
	//   Retransmits = RetransmitMult * log(N+1)
	//
	// This allows the retransmits to scale properly with cluster size. The
	// higher the multiplier, the more likely a failed broadcast is to converge
	// at the expense of increased bandwidth.
	RetransmitMult int

	// SuspicionMult is the multiplier for determining the time an
	// inaccessible node is considered suspect before declaring it dead.
	// The actual timeout is calculated using the formula:
	//
	//   SuspicionTimeout = SuspicionMult * log(N+1) * ProbeInterval
	//
	// This allows the timeout to scale properly with expected propagation
	// delay with a larger cluster size. The higher the multiplier, the longer
	// an inaccessible node is considered part of the cluster before declaring
	// it dead, giving that suspect node more time to refute if it is indeed
	// still alive.
	SuspicionMult int

	// PushPullInterval is the interval between complete state syncs.
	// Complete state syncs are done with a single node over TCP and are
	// quite expensive relative to standard gossiped messages. Setting this
	// to zero will disable state push/pull syncs completely.
	//
	// Setting this interval lower (more frequent) will increase convergence
	// speeds across larger clusters at the expense of increased bandwidth
	// usage.
	PushPullInterval time.Duration

	// ProbeInterval and ProbeTimeout are used to configure probing
	// behavior for memberlist.
	//
	// ProbeInterval is the interval between random node probes. Setting
	// this lower (more frequent) will cause the memberlist cluster to detect
	// failed nodes more quickly at the expense of increased bandwidth usage.
	//
	// ProbeTimeout is the timeout to wait for an ack from a probed node
	// before assuming it is unhealthy. This should be set to 99-percentile
	// of RTT (round-trip time) on your network.
	ProbeInterval time.Duration
	ProbeTimeout  time.Duration

	// GossipInterval and GossipNodes are used to configure the gossip
	// behavior of memberlist.
	//
	// GossipInterval is the interval between sending messages that need
	// to be gossiped that haven't been able to piggyback on probing messages.
	// If this is set to zero, non-piggyback gossip is disabled. By lowering
	// this value (more frequent) gossip messages are propagated across
	// the cluster more quickly at the expense of increased bandwidth.
	//
	// GossipNodes is the number of random nodes to send gossip messages to
	// per GossipInterval. Increasing this number causes the gossip messages
	// to propagate across the cluster more quickly at the expense of
	// increased bandwidth.
	GossipInterval time.Duration
	GossipNodes    int

	// Delegate and Events are delegates for receiving and providing
	// data to memberlist via callback mechanisms. For Delegate, see
	// the Delegate interface. For Events, see the EventDelegate interface.
	Delegate Delegate
	Events   EventDelegate

	// LogOutput is the writer where logs should be sent. If this is not
	// set, logging will go to stderr by default.
	LogOutput io.Writer
}

type Memberlist struct {
	config         *Config
	shutdown       bool
	leave          bool
	leaveBroadcast chan struct{}

	udpListener *net.UDPConn
	tcpListener *net.TCPListener

	sequenceNum uint32 // Local sequence number
	incarnation uint32 // Local incarnation number

	nodeLock sync.RWMutex
	nodes    []*nodeState          // Known nodes
	nodeMap  map[string]*nodeState // Maps Addr.String() -> NodeState

	tickerLock sync.Mutex
	tickers    []*time.Ticker
	stopTick   chan struct{}
	probeIndex int

	ackLock     sync.Mutex
	ackHandlers map[uint32]*ackHandler

	broadcasts *TransmitLimitedQueue

	startStopLock sync.Mutex

	logger *log.Logger
}

// DefaultConfig returns a sane set of configurations for Memberlist.
// It uses the hostname as the node name, and otherwise sets very conservative
// values that are sane for most LAN environments. The default configuration
// errs on the side on the side of caution, choosing values that are optimized
// for higher convergence at the cost of higher bandwidth usage. Regardless,
// these values are a good starting point when getting started with memberlist.
func DefaultConfig() *Config {
	hostname, _ := os.Hostname()
	return &Config{
		Name:             hostname,
		BindAddr:         "0.0.0.0",
		UDPPort:          7946,
		TCPPort:          7946,
		TCPTimeout:       30 * time.Second,       // Timeout after 10 seconds
		IndirectChecks:   3,                      // Use 3 nodes for the indirect ping
		RetransmitMult:   4,                      // Retransmit a message 4 * log(N+1) nodes
		SuspicionMult:    5,                      // Suspect a node for 5 * log(N+1) * Interval
		PushPullInterval: 30 * time.Second,       // Low frequency
		ProbeTimeout:     500 * time.Millisecond, // Reasonable RTT time for LAN
		ProbeInterval:    1 * time.Second,        // Failure check every second

		GossipNodes:    3,                      // Gossip to 3 nodes
		GossipInterval: 200 * time.Millisecond, // Gossip more rapidly
	}
}

// newMemberlist creates the network listeners.
// Does not schedule execution of background maintenence.
func newMemberlist(conf *Config) (*Memberlist, error) {
	tcpAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.TCPPort)
	tcpLn, err := net.Listen("tcp", tcpAddr)
	if err != nil {
		return nil, fmt.Errorf("Failed to start TCP listener. Err: %s", err)
	}

	udpAddr := fmt.Sprintf("%s:%d", conf.BindAddr, conf.UDPPort)
	udpLn, err := net.ListenPacket("udp", udpAddr)
	if err != nil {
		tcpLn.Close()
		return nil, fmt.Errorf("Failed to start UDP listener. Err: %s", err)
	}

	// Set the UDP receive window size
	setUDPRecvBuf(udpLn.(*net.UDPConn))

	if conf.LogOutput == nil {
		conf.LogOutput = os.Stderr
	}

	m := &Memberlist{
		config:         conf,
		leaveBroadcast: make(chan struct{}, 1),
		udpListener:    udpLn.(*net.UDPConn),
		tcpListener:    tcpLn.(*net.TCPListener),
		nodeMap:        make(map[string]*nodeState),
		ackHandlers:    make(map[uint32]*ackHandler),
		broadcasts:     &TransmitLimitedQueue{RetransmitMult: conf.RetransmitMult},
		logger:         log.New(conf.LogOutput, "", log.LstdFlags),
	}
	m.broadcasts.NumNodes = func() int { return len(m.nodes) }
	go m.tcpListen()
	go m.udpListen()
	return m, nil
}

// Create will create a new Memberlist using the given configuration.
// This will not connect to any other node (see Join) yet, but will start
// all the listeners to allow other nodes to join this memberlist.
// After creating a Memberlist, the configuration given should not be
// modified by the user anymore.
func Create(conf *Config) (*Memberlist, error) {
	m, err := newMemberlist(conf)
	if err != nil {
		return nil, err
	}
	if err := m.setAlive(); err != nil {
		m.Shutdown()
		return nil, err
	}
	m.schedule()
	return m, nil
}

// Join is used to take an existing Memberlist and attempt to join a cluster
// by contacting all the given hosts and performing a state sync. Initially,
// the Memberlist only contains our own state, so doing this will cause
// remote nodes to become aware of the existence of this node, effectively
// joining the cluster.
//
// This returns the number of hosts successfully contacted and an error if
// none could be reached. If an error is returned, the node did not successfully
// join the cluster.
func (m *Memberlist) Join(existing []string) (int, error) {
	// Attempt to join any of them
	numSuccess := 0
	var retErr error
	for _, exist := range existing {
		addr, err := net.ResolveIPAddr("ip", exist)
		if err != nil {
			retErr = err
			continue
		}

		if err := m.pushPullNode(addr.IP); err != nil {
			retErr = err
			continue
		}

		numSuccess++
	}

	if numSuccess > 0 {
		retErr = nil
	}

	return numSuccess, retErr
}

// setAlive is used to mark this node as being alive. This is the same
// as if we received an alive notification our own network channel for
// ourself.
func (m *Memberlist) setAlive() error {
	// Pick a private IP address
	var ipAddr []byte
	if m.config.BindAddr == "0.0.0.0" {
		// We're not bound to a specific IP, so let's list the interfaces
		// on this machine and use the first private IP we find.
		addresses, err := net.InterfaceAddrs()
		if err != nil {
			return fmt.Errorf("Failed to get interface addresses! Err: %vn", err)
		}

		// Find private IPv4 address
		for _, addr := range addresses {
			ip, ok := addr.(*net.IPNet)
			if !ok {
				continue
			}
			if ip.IP.To4() == nil {
				continue
			}
			if !isPrivateIP(ip.IP.String()) {
				continue
			}
			ipAddr = ip.IP
			break
		}

		// Failed to find private IP, use loopback
		if ipAddr == nil {
			ipAddr = []byte{127, 0, 0, 1}
		}
	} else {
		// Use the IP that we're bound to.
		addr := m.tcpListener.Addr().(*net.TCPAddr)
		ipAddr = addr.IP
	}

	// Get the node meta data
	var meta []byte
	if m.config.Delegate != nil {
		meta = m.config.Delegate.NodeMeta(metaMaxSize)
		if len(meta) > metaMaxSize {
			panic("Node meta data provided is longer than the limit")
		}
	}

	a := alive{
		Incarnation: m.nextIncarnation(),
		Node:        m.config.Name,
		Addr:        ipAddr,
		Meta:        meta,
	}
	m.aliveNode(&a)

	return nil
}

// Members returns a list of all known live nodes. The node structures
// returned must not be modified. If you wish to modify a Node, make a
// copy first.
func (m *Memberlist) Members() []*Node {
	m.nodeLock.RLock()
	defer m.nodeLock.RUnlock()

	nodes := make([]*Node, 0, len(m.nodes))
	for _, n := range m.nodes {
		if n.State != stateDead {
			nodes = append(nodes, &n.Node)
		}
	}

	return nodes
}

// NumMembers returns the number of alive nodes currently known. Between
// the time of calling this and calling Members, the number of alive nodes
// may have changed, so this shouldn't be used to determine how many
// members will be returned by Members.
func (m *Memberlist) NumMembers() (alive int) {
	m.nodeLock.RLock()
	defer m.nodeLock.RUnlock()

	for _, n := range m.nodes {
		if n.State != stateDead {
			alive++
		}
	}

	return
}

// Leave will broadcast a leave message but will not shutdown the background
// listeners, meaning the node will continue participating in gossip and state
// updates.
//
// This will block until the leave message is successfully broadcasted to
// a member of the cluster, if any exist or until a specified timeout
// is reached.
//
// This method is safe to call multiple times, but must not be called
// after the cluster is already shut down.
func (m *Memberlist) Leave(timeout time.Duration) error {
	m.startStopLock.Lock()
	defer m.startStopLock.Unlock()

	if m.shutdown {
		panic("leave after shutdown")
	}

	if !m.leave {
		m.leave = true

		state, ok := m.nodeMap[m.config.Name]
		if !ok {
			m.logger.Println("[WARN] Leave but we're not in the node map.")
			return nil
		}

		d := dead{
			Incarnation: state.Incarnation,
			Node:        state.Name,
		}
		m.deadNode(&d)

		// Check for any other alive node
		anyAlive := false
		for _, n := range m.nodes {
			if n.State != stateDead {
				anyAlive = true
				break
			}
		}

		// Block until the broadcast goes out
		if anyAlive {
			var timeoutCh <-chan time.Time
			if timeout > 0 {
				timeoutCh = time.After(timeout)
			}
			select {
			case <-m.leaveBroadcast:
			case <-timeoutCh:
				return fmt.Errorf("timeout waiting for leave broadcast")
			}
		}
	}

	return nil
}

// Shutdown will stop any background maintanence of network activity
// for this memberlist, causing it to appear "dead". A leave message
// will not be broadcasted prior, so the cluster being left will have
// to detect this node's shutdown using probing. If you wish to more
// gracefully exit the cluster, call Leave prior to shutting down.
//
// This method is safe to call multiple times.
func (m *Memberlist) Shutdown() error {
	m.startStopLock.Lock()
	defer m.startStopLock.Unlock()

	if !m.shutdown {
		m.shutdown = true
		m.deschedule()
		m.udpListener.Close()
		m.tcpListener.Close()
	}

	return nil
}
<|endoftext|>


<commit_before>package gorange

import (
	"fmt"
	"time"

	"github.com/karrick/goswarm"
)

// CachingClient memoizes responses from a Querier.
type CachingClient struct {
	cache *goswarm.Simple
}

type queryResult struct {
	value        []string
	err          error
	lastResponse time.Time
}

// NewCachingClient returns CachingClient that attempts to respond to Query methods by consulting
// its TTL cache, then directing the call to the underlying Querier if a valid response is not
// stored.
func newCachingClient(querier Querier, ttl, tte time.Duration) (*CachingClient, error) {
	if ttl < 0 {
		return nil, fmt.Errorf("cannot create CachingClient with negative TTL: %v", ttl)
	}
	cache, err := goswarm.NewSimple(&goswarm.Config{
		GoodStaleDuration:  ttl,
		GoodExpiryDuration: tte,
		BadStaleDuration:   1 * time.Minute,
		BadExpiryDuration:  5 * time.Minute,
		GCPeriodicity:      time.Hour,
		Lookup: func(url string) (interface{}, error) {
			raw, err := querier.Query(url)
			if err != nil {
				// don't return structs with error value
				return nil, err
			}
			return queryResult{
				value:        raw,
				lastResponse: time.Now(),
			}, nil
		},
	})
	if err != nil {
		return nil, err
	}
	return &CachingClient{cache}, nil
}

// Query returns the response of the query, first checking in the TTL cache, then by actually
// invoking the Query method on the underlying Querier.
func (c *CachingClient) Query(query string) ([]string, error) {
	raw, err := c.cache.Query(query)
	if err != nil {
		return nil, err
	}
	qr, ok := raw.(queryResult)
	if !ok {
		return nil, fmt.Errorf("expected queryResult; actual: %T", raw)
	}
	return qr.value, nil
}
<commit_msg>no longer stores error<commit_after>package gorange

import (
	"fmt"
	"time"

	"github.com/karrick/goswarm"
)

// CachingClient memoizes responses from a Querier.
type CachingClient struct {
	cache *goswarm.Simple
}

type queryResult struct {
	value        []string
	lastResponse time.Time
}

// NewCachingClient returns CachingClient that attempts to respond to Query methods by consulting
// its TTL cache, then directing the call to the underlying Querier if a valid response is not
// stored.
func newCachingClient(querier Querier, ttl, tte time.Duration) (*CachingClient, error) {
	if ttl < 0 {
		return nil, fmt.Errorf("cannot create CachingClient with negative TTL: %v", ttl)
	}
	cache, err := goswarm.NewSimple(&goswarm.Config{
		GoodStaleDuration:  ttl,
		GoodExpiryDuration: tte,
		BadStaleDuration:   1 * time.Minute,
		BadExpiryDuration:  5 * time.Minute,
		GCPeriodicity:      time.Hour,
		Lookup: func(url string) (interface{}, error) {
			raw, err := querier.Query(url)
			if err != nil {
				// don't return structs with error value
				return nil, err
			}
			return queryResult{
				value:        raw,
				lastResponse: time.Now(),
			}, nil
		},
	})
	if err != nil {
		return nil, err
	}
	return &CachingClient{cache}, nil
}

// Query returns the response of the query, first checking in the TTL cache, then by actually
// invoking the Query method on the underlying Querier.
func (c *CachingClient) Query(query string) ([]string, error) {
	raw, err := c.cache.Query(query)
	if err != nil {
		return nil, err
	}
	qr, ok := raw.(queryResult)
	if !ok {
		return nil, fmt.Errorf("expected queryResult; actual: %T", raw)
	}
	return qr.value, nil
}
<|endoftext|>


<commit_before>package fragbag

import (
	"encoding/json"
	"fmt"
	"io"

	"github.com/TuftsBCB/seq"
)

// SequenceHMM represents a Fragbag sequence fragment library.
// Fragbag fragment libraries are fixed both in the number of fragments and in
// the size of each fragment.
type SequenceHMM struct {
	Ident     string
	Fragments []SequenceHMMFrag
	FragSize  int
}

// NewSequenceHMM initializes a new Fragbag sequence library with the
// given name. It is not written to disk until Save is called.
func NewSequenceHMM(name string) *SequenceHMM {
	lib := new(SequenceHMM)
	lib.Ident = name
	return lib
}

// Add adds a sequence fragment to the library, where a sequence fragment
// corresponds to an HMM.
//
// The first time Add is called, the HMM may have any number of nodes. All
// subsequent calls to Add must supply an HMM with the number of nodes
// equal to the first HMM added.
func (lib *SequenceHMM) Add(hmm *seq.HMM) error {
	if lib.Fragments == nil || len(lib.Fragments) == 0 {
		frag := SequenceHMMFrag{0, hmm}
		lib.Fragments = append(lib.Fragments, frag)
		lib.FragSize = len(hmm.Nodes)
		return nil
	}

	frag := SequenceHMMFrag{len(lib.Fragments), hmm}
	if lib.FragSize != len(hmm.Nodes) {
		return fmt.Errorf("Fragment %d has length %d; expected length %d.",
			frag.Number(), len(hmm.Nodes), lib.FragSize)
	}
	lib.Fragments = append(lib.Fragments, frag)
	return nil
}

// Save saves the full fragment library to the writer provied.
func (lib *SequenceHMM) Save(w io.Writer) error {
	return saveLibrary(w, kindSequenceHMM, lib)
}

// Open loads an existing structure fragment library from the reader provided.
func openSequenceHMM(r io.Reader) (*SequenceHMM, error) {
	var lib *SequenceHMM
	dec := json.NewDecoder(r)
	if err := dec.Decode(&lib); err != nil {
		return nil, err
	}
	return lib, nil
}

// Size returns the number of fragments in the library.
func (lib *SequenceHMM) Size() int {
	return len(lib.Fragments)
}

// FragmentSize returns the size of every fragment in the library.
func (lib *SequenceHMM) FragmentSize() int {
	return lib.FragSize
}

// Fragment returns the ith fragment in this library (starting from 0).
func (lib *SequenceHMM) Fragment(i int) SequenceFragment {
	return lib.Fragments[i]
}

// String returns a string with the name of the library, the number of
// fragments in the library and the size of each fragment.
func (lib *SequenceHMM) String() string {
	return fmt.Sprintf("%s (%d, %d)",
		lib.Ident, len(lib.Fragments), lib.FragSize)
}

func (lib *SequenceHMM) Name() string {
	return lib.Ident
}

// Best returns the number of the fragment that best corresponds
// to the string of amino acids provided.
// The length of `sequence` must be equivalent to the fragment size.
//
// If no "good" fragments can be found, then `-1` is returned. This
// behavior will almost certainly change in the future.
func (lib *SequenceHMM) Best(s seq.Sequence) int {
	// Since fragments are guaranteed not to have gaps by construction,
	// we can do a straight-forward summation of the negative log-odds
	// probabilities corresponding to the residues in `s`.
	var testAlign seq.Prob
	bestAlign, bestFragNum := seq.MinProb, -1
	for _, frag := range lib.Fragments {
		testAlign = frag.AlignmentProb(s)
		if bestAlign.Less(testAlign) {
			bestAlign, bestFragNum = testAlign, frag.FragNumber
		}
	}
	return bestFragNum
}

// Fragment corresponds to a single sequence fragment in a fragment library.
// It holds the fragment number identifier and embeds an HMM.
type SequenceHMMFrag struct {
	FragNumber int
	*seq.HMM
}

// AlignmentProb computes the probability of the sequence `s` aligning
// with the HMM in `frag`. The sequence must have length equivalent
// to the fragment size.
func (frag SequenceHMMFrag) AlignmentProb(s seq.Sequence) seq.Prob {
	if s.Len() != len(frag.Nodes) {
		panic(fmt.Sprintf("Sequence length %d != fragment size %d",
			s.Len(), len(frag.Nodes)))
	}
	return frag.ViterbiScore(s)
}

func (frag SequenceHMMFrag) Number() int {
	return frag.FragNumber
}

func (frag SequenceHMMFrag) String() string {
	return fmt.Sprintf("> %d\n%s", frag.FragNumber, frag.HMM)
}
<commit_msg>Smarter memory allocation for Viterbi.<commit_after>package fragbag

import (
	"encoding/json"
	"fmt"
	"io"

	"github.com/TuftsBCB/seq"
)

// SequenceHMM represents a Fragbag sequence fragment library.
// Fragbag fragment libraries are fixed both in the number of fragments and in
// the size of each fragment.
type SequenceHMM struct {
	Ident     string
	Fragments []SequenceHMMFrag
	FragSize  int
}

// NewSequenceHMM initializes a new Fragbag sequence library with the
// given name. It is not written to disk until Save is called.
func NewSequenceHMM(name string) *SequenceHMM {
	lib := new(SequenceHMM)
	lib.Ident = name
	return lib
}

// Add adds a sequence fragment to the library, where a sequence fragment
// corresponds to an HMM.
//
// The first time Add is called, the HMM may have any number of nodes. All
// subsequent calls to Add must supply an HMM with the number of nodes
// equal to the first HMM added.
func (lib *SequenceHMM) Add(hmm *seq.HMM) error {
	if lib.Fragments == nil || len(lib.Fragments) == 0 {
		frag := SequenceHMMFrag{0, hmm}
		lib.Fragments = append(lib.Fragments, frag)
		lib.FragSize = len(hmm.Nodes)
		return nil
	}

	frag := SequenceHMMFrag{len(lib.Fragments), hmm}
	if lib.FragSize != len(hmm.Nodes) {
		return fmt.Errorf("Fragment %d has length %d; expected length %d.",
			frag.Number(), len(hmm.Nodes), lib.FragSize)
	}
	lib.Fragments = append(lib.Fragments, frag)
	return nil
}

// Save saves the full fragment library to the writer provied.
func (lib *SequenceHMM) Save(w io.Writer) error {
	return saveLibrary(w, kindSequenceHMM, lib)
}

// Open loads an existing structure fragment library from the reader provided.
func openSequenceHMM(r io.Reader) (*SequenceHMM, error) {
	var lib *SequenceHMM
	dec := json.NewDecoder(r)
	if err := dec.Decode(&lib); err != nil {
		return nil, err
	}
	return lib, nil
}

// Size returns the number of fragments in the library.
func (lib *SequenceHMM) Size() int {
	return len(lib.Fragments)
}

// FragmentSize returns the size of every fragment in the library.
func (lib *SequenceHMM) FragmentSize() int {
	return lib.FragSize
}

// Fragment returns the ith fragment in this library (starting from 0).
func (lib *SequenceHMM) Fragment(i int) SequenceFragment {
	return lib.Fragments[i]
}

// String returns a string with the name of the library, the number of
// fragments in the library and the size of each fragment.
func (lib *SequenceHMM) String() string {
	return fmt.Sprintf("%s (%d, %d)",
		lib.Ident, len(lib.Fragments), lib.FragSize)
}

func (lib *SequenceHMM) Name() string {
	return lib.Ident
}

// Best returns the number of the fragment that best corresponds
// to the string of amino acids provided.
// The length of `sequence` must be equivalent to the fragment size.
//
// If no "good" fragments can be found, then `-1` is returned. This
// behavior will almost certainly change in the future.
func (lib *SequenceHMM) Best(s seq.Sequence) int {
	if s.Len() != lib.FragmentSize() {
		panic(fmt.Sprintf("Sequence length %d != fragment size %d",
			s.Len(), lib.FragmentSize()))
	}
	var testAlign seq.Prob
	dynamicTable := seq.AllocTable(lib.FragmentSize(), s.Len())
	bestAlign, bestFragNum := seq.MinProb, -1
	for _, frag := range lib.Fragments {
		testAlign = frag.ViterbiScoreMem(s, dynamicTable)
		if bestAlign.Less(testAlign) {
			bestAlign, bestFragNum = testAlign, frag.FragNumber
		}
	}
	return bestFragNum
}

// Fragment corresponds to a single sequence fragment in a fragment library.
// It holds the fragment number identifier and embeds an HMM.
type SequenceHMMFrag struct {
	FragNumber int
	*seq.HMM
}

// AlignmentProb computes the probability of the sequence `s` aligning
// with the HMM in `frag`. The sequence must have length equivalent
// to the fragment size.
func (frag SequenceHMMFrag) AlignmentProb(s seq.Sequence) seq.Prob {
	if s.Len() != len(frag.Nodes) {
		panic(fmt.Sprintf("Sequence length %d != fragment size %d",
			s.Len(), len(frag.Nodes)))
	}
	return frag.ViterbiScore(s)
}

func (frag SequenceHMMFrag) Number() int {
	return frag.FragNumber
}

func (frag SequenceHMMFrag) String() string {
	return fmt.Sprintf("> %d\n%s", frag.FragNumber, frag.HMM)
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"encoding/xml"
	"flag"
	"io"
	"log"
	"net"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"syscall"
)

var (
	host = flag.String("host", "", "IP PC-104 will listen to")

	exitFuncs  = []func(){}
	wdir       = "."
	fcsVersion = "dev-SNAPSHOT"
)

func main() {

	defer func() {
		os.Stderr.Sync()
		os.Stdout.Sync()
		runExitFuncs()
	}()

	flag.Parse()
	if *host == "" {
		*host = getHostIP()
	}

	go func() {
		sigch := make(chan os.Signal)
		signal.Notify(sigch, os.Interrupt, os.Kill)
		for {
			select {
			case <-sigch:
				os.Stderr.Sync()
				os.Stdout.Sync()
				runExitFuncs()
				os.Stdout.Sync()
				os.Stderr.Sync()
			}
		}
	}()

	run()
}

func run() {
	var err error
	wdir, err = os.Getwd()
	if err != nil {
		fatalf("could not retrieve current work directory: %v\n", wdir)
	}

	errc := make(chan error)

	startCWrapper(errc)
	initProject()
	makeDistrib()
	setupEnv()
	dispatch(errc)

	select {
	case err := <-errc:
		if err != nil {
			fatalf("error: %v\n", err)
		}
	}
}

func atexit(f func()) {
	exitFuncs = append(exitFuncs, f)
}

func runExitFuncs() {
	log.Printf("running atexit-funcs...\n")
	for _, f := range exitFuncs {
		f()
	}
	log.Printf("running atexit-funcs... [done]\n")
}

func fatalf(format string, args ...interface{}) {
	runExitFuncs()
	log.Fatalf(format, args...)
}

func runCmd(cmd string, args ...string) error {
	exe := exec.Command(cmd, args...)
	exe.Stdin = os.Stdin
	exe.Stdout = os.Stdout
	exe.Stderr = os.Stderr
	atexit(func() {
		killProc(exe)
	})
	return exe.Run()
}

func killProc(cmd *exec.Cmd) {
	if cmd == nil || cmd.Process == nil {
		return
	}
	state := cmd.ProcessState
	if state != nil && state.Exited() {
		return
	}
	p := cmd.Process
	pgid, err := syscall.Getpgid(p.Pid)
	if err != nil {
		log.Printf("could not get process group-id of [%v]: %v\n",
			cmd, err,
		)
		return
	}

	err = syscall.Kill(-pgid, syscall.SIGKILL)
	if err != nil {
		log.Printf("could not kill process [%v]: %v\n",
			cmd, err,
		)
	}
}

func getHostIP() string {
	host, err := os.Hostname()
	if err != nil {
		fatalf("could not retrieve hostname: %v\n", err)
	}

	addrs, err := net.LookupIP(host)
	if err != nil {
		fatalf("could not lookup hostname IP: %v\n", err)
	}

	for _, addr := range addrs {
		ipv4 := addr.To4()
		if ipv4 == nil {
			continue
		}
		return ipv4.String()
	}

	fatalf("could not infer host IP")
	return ""
}

func startCWrapper(errc chan error) {
	log.Printf("Starting c-wrapper on PC-104... (listen for %q)\n", *host)

	cmd := exec.Command(
		"ssh",
		"-X",
		"root@clrlsstemb01.in2p3.fr",
		"startCWrapper --host="+*host,
	)
	cmd.Env = append(cmd.Env, "TERM=vt100")
	//cmd.Stdin = os.Stdin
	//cmd.Stdout = os.Stderr
	//cmd.Stderr = os.Stderr
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}

	atexit(func() {
		killProc(cmd)
	})

	log.Printf("c-wrapper command: %v\n", cmd.Args)

	go func() {
		err := cmd.Run()
		if err != nil {
			log.Printf("c-wrapper= %v\n", err)
			errc <- err
		}
	}()
}

func initProject() {
	fname := filepath.Join(wdir, "pom.xml")
	pom, err := os.Open(fname)
	if err != nil {
		fatalf("could not open [%s]: %v\n", fname, err)
	}
	defer pom.Close()

	type POM struct {
		XMLName xml.Name `xml:"project"`
		Name    string   `xml:"name"`
		Version string   `xml:"version"`
	}
	var data POM
	err = xml.NewDecoder(pom).Decode(&data)
	if err != nil {
		fatalf("error decoding pom.xml: %v\n", err)
	}

	log.Printf("pom-data: %#v\n", data)
	fcsVersion = data.Version
}

func makeDistrib() {
	dist := filepath.Join(wdir, "DISTRIB")
	log.Printf("creating DISTRIB [%s]...\n", dist)
	os.RemoveAll(dist)
	err := os.MkdirAll(dist, 0755)
	if err != nil {
		fatalf("error creating [%s]: %v\n", dist, err)
	}

	for _, d := range []string{"main", "gui"} {
		dir := filepath.Join(wdir, d)
		_, err = os.Stat(dir)
		if err != nil {
			fatalf("could not stat [%s]: %v\n", dir, err)
		}

		srcs, err := filepath.Glob(
			filepath.Join(dir, "target", "org-lsst-ccs-subsystem-fcs-"+d+"-*-dist.zip"),
		)
		if err != nil {
			fatalf("could not stat %s-*-dist.zip: %v\n", d, err)
		}
		switch len(srcs) {
		case 0:
			fatalf("no %s-*-dist.zip!", d)
		case 1:
			// ok
		default:
			fatalf("too many %s-*-dist.zip files (%d): %v\n", len(srcs), srcs)
		}

		err = unzip(dist, srcs[0])
		if err != nil {
			fatalf("could not unzip [%s] into [%s]: %v\n", srcs[0], dist, err)
		}
	}
	log.Printf("creating DISTRIB [%s]... [done]\n", dist)
}

func setupEnv() {
	for _, v := range [][2]string{
		{"FCS_VERSION", fcsVersion},
		{"LOCALDB_VERSION", "1.3.0-SNAPSHOT"},
		// logging
		{"P0", "org.lsst.ccs.utilities.logging.ConsoleHandlerN.level=ALL"},
		{"P1", "org.lsst.ccs.utilities.logging.FileHandlerN.formatter=org.lsst.ccs.utilities.logging.TextFormatter"},
		{"P2", ".level=WARNING"},
		{"P3", "org.lsst.ccs.level=ALL"},
		{"P4", "org.lsst.ccs.bus.level=INFO"},
		{"P5", "org.lsst.ccs.subsystems.fcs.level=ALL"},

		{"TEST_ENV_ROOT", filepath.Join(wdir, "DISTRIB")},
		{"TEST_ENV_BIN", filepath.Join("${TEST_ENV_ROOT}", "bin")},
		{"TEST_ENV_RESOURCES", filepath.Join("${TEST_ENV_ROOT}", "externalResources")},
		{"TEST_ENV_DIRVERS", filepath.Join("${TEST_ENV_ROOT}", "drivers")},

		// needed by CCSbootstrap.sh
		{"CCS_RESOURCE_PATH", "${TEST_ENV_RESOURCES}"},

		// make sure we use IPv4 in JAS
		{"JASJVM_OPTS", "-Djava.net.preferIPV4Stack=true"},

		{"DESCRIPTION_FILE", "testbenchLPC.groo"},
		{"CONFIGURATION_FILE", "testbenchLPC_XXXXX_.properties"},
		{"WORKDIR", filepath.Join(wdir, "work")},
		{"LOGFILENAME", "org.lsst.ccs.utilites.logging.FileHandlerN.pattern=%W/logs/ccs-logs-%A-testbenchLPC.log"},
	} {
		err := os.Setenv(v[0], v[1])
		if err != nil {
			fatalf("error calling os.Setenv(%q, %q): %v\n", v[0], v[1], err)
		}
	}
}

func dispatch(errc chan error) {
	switch flag.Arg(0) {
	case "lpc":
		runTestbench(errc)
	case "console":
		runConsole(errc)
	case "jas3":
		runJAS3(errc)
	default:
		fatalf("unknown command [%s]\n", flag.Arg(0))
	}
}

func runTestbench(errc chan error) {
	cmd := exec.Command(
		filepath.Join(
			os.Getenv("TEST_ENV_ROOT"),
			"org-lsst-ccs-subsystem-fcs-main-"+fcsVersion,
			"bin",
			"CCSbootstrap.sh",
		),
		"-app", "FcsSubsystem",
		"--description", "/org/lsst/ccs/subsystems/fcs/conf/"+os.Getenv("DESCRIPTION_FILE"),
		"-D", os.Getenv("P0"),
		"-D", os.Getenv("P1"),
		"-D", os.Getenv("P2"),
		"-D", os.Getenv("P3"),
		"-D", os.Getenv("P4"),
		"-D", os.Getenv("P5"),
		"-D", os.Getenv("WORKDIR"),
		"-D", os.Getenv("LOGFILENAME"),
	)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	atexit(func() {
		killProc(cmd)
	})

	go func() {
		errc <- cmd.Run()
	}()
}

func runConsole(errc chan error) {
	cmd := exec.Command(
		filepath.Join(
			os.Getenv("TEST_ENV_ROOT"),
			"org-lsst-ccs-subsystem-fcs-gui-"+fcsVersion,
			"bin",
			"CCSbootstrap.sh",
		),
		"-app", "ShellCommandConsole",
	)
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout

	atexit(func() {
		killProc(cmd)
	})

	go func() {
		errc <- cmd.Run()
	}()
}

func runJAS3(errc chan error) {
	cmd := exec.Command(
		filepath.Join(
			os.Getenv("TEST_ENV_ROOT"),
			"org-lsst-ccs-subsystem-fcs-gui-"+fcsVersion,
			"bin",
			"CCSbootstrap.sh",
		),
		"-app", "CCS-Console",
	)
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout

	atexit(func() {
		killProc(cmd)
	})

	go func() {
		errc <- cmd.Run()
	}()
}

func unzip(dest, src string) error {
	r, err := zip.OpenReader(src)
	if err != nil {
		return err
	}
	defer func() {
		if err := r.Close(); err != nil {
			fatalf("error closing zip archive: %v\n", err)
		}
	}()

	os.MkdirAll(dest, 0755)

	// Closure to address file descriptors issue with all the deferred .Close() methods
	extract := func(f *zip.File) error {
		rc, err := f.Open()
		if err != nil {
			return err
		}
		defer func() {
			if err := rc.Close(); err != nil {
				fatalf("error closing zip file: %v\n", err)
			}
		}()

		path := filepath.Join(dest, f.Name)

		if f.FileInfo().IsDir() {
			os.MkdirAll(path, f.Mode())
		} else {
			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
			if err != nil {
				return err
			}
			defer func() {
				if err := f.Close(); err != nil {
					fatalf("error closing zip output file: %v\n", err)
				}
			}()

			_, err = io.Copy(f, rc)
			if err != nil {
				return err
			}
		}
		return nil
	}

	for _, f := range r.File {
		err := extract(f)
		if err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>fcs-mgr: make functions blocking<commit_after>package main

import (
	"archive/zip"
	"encoding/xml"
	"flag"
	"io"
	"log"
	"net"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"syscall"
)

var (
	host = flag.String("host", "", "IP PC-104 will listen to")

	exitFuncs  = []func(){}
	wdir       = "."
	fcsVersion = "dev-SNAPSHOT"
)

func main() {

	defer func() {
		os.Stderr.Sync()
		os.Stdout.Sync()
		runExitFuncs()
	}()

	flag.Parse()
	if *host == "" {
		*host = getHostIP()
	}

	go func() {
		sigch := make(chan os.Signal)
		signal.Notify(sigch, os.Interrupt, os.Kill)
		for {
			select {
			case <-sigch:
				os.Stderr.Sync()
				os.Stdout.Sync()
				runExitFuncs()
				os.Stdout.Sync()
				os.Stderr.Sync()
			}
		}
	}()

	run()
}

func run() {
	var err error
	wdir, err = os.Getwd()
	if err != nil {
		fatalf("could not retrieve current work directory: %v\n", wdir)
	}

	errc := make(chan error)

	go startCWrapper(errc)
	initProject()
	makeDistrib()
	setupEnv()
	go dispatch(errc)

	select {
	case err := <-errc:
		if err != nil {
			fatalf("error: %v\n", err)
		}
	}
}

func atexit(f func()) {
	exitFuncs = append(exitFuncs, f)
}

func runExitFuncs() {
	log.Printf("running atexit-funcs...\n")
	for _, f := range exitFuncs {
		f()
	}
	log.Printf("running atexit-funcs... [done]\n")
}

func fatalf(format string, args ...interface{}) {
	runExitFuncs()
	log.Fatalf(format, args...)
}

func runCmd(cmd string, args ...string) error {
	exe := exec.Command(cmd, args...)
	exe.Stdin = os.Stdin
	exe.Stdout = os.Stdout
	exe.Stderr = os.Stderr
	atexit(func() {
		killProc(exe)
	})
	return exe.Run()
}

func killProc(cmd *exec.Cmd) {
	if cmd == nil || cmd.Process == nil {
		return
	}
	state := cmd.ProcessState
	if state != nil && state.Exited() {
		return
	}
	p := cmd.Process
	pgid, err := syscall.Getpgid(p.Pid)
	if err != nil {
		log.Printf("could not get process group-id of [%v]: %v\n",
			cmd, err,
		)
		return
	}

	err = syscall.Kill(-pgid, syscall.SIGKILL)
	if err != nil {
		log.Printf("could not kill process [%v]: %v\n",
			cmd, err,
		)
	}
}

func getHostIP() string {
	host, err := os.Hostname()
	if err != nil {
		fatalf("could not retrieve hostname: %v\n", err)
	}

	addrs, err := net.LookupIP(host)
	if err != nil {
		fatalf("could not lookup hostname IP: %v\n", err)
	}

	for _, addr := range addrs {
		ipv4 := addr.To4()
		if ipv4 == nil {
			continue
		}
		return ipv4.String()
	}

	fatalf("could not infer host IP")
	return ""
}

func startCWrapper(errc chan error) {
	log.Printf("Starting c-wrapper on PC-104... (listen for %q)\n", *host)

	cmd := exec.Command(
		"ssh",
		"-X",
		"root@clrlsstemb01.in2p3.fr",
		"startCWrapper --host="+*host,
	)
	cmd.Env = append(cmd.Env, "TERM=vt100")
	//cmd.Stdin = os.Stdin
	//cmd.Stdout = os.Stderr
	//cmd.Stderr = os.Stderr
	cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}

	atexit(func() {
		killProc(cmd)
	})

	log.Printf("c-wrapper command: %v\n", cmd.Args)

	err := cmd.Run()
	if err != nil {
		log.Printf("c-wrapper= %v\n", err)
		errc <- err
	}
}

func initProject() {
	fname := filepath.Join(wdir, "pom.xml")
	pom, err := os.Open(fname)
	if err != nil {
		fatalf("could not open [%s]: %v\n", fname, err)
	}
	defer pom.Close()

	type POM struct {
		XMLName xml.Name `xml:"project"`
		Name    string   `xml:"name"`
		Version string   `xml:"version"`
	}
	var data POM
	err = xml.NewDecoder(pom).Decode(&data)
	if err != nil {
		fatalf("error decoding pom.xml: %v\n", err)
	}

	log.Printf("pom-data: %#v\n", data)
	fcsVersion = data.Version
}

func makeDistrib() {
	dist := filepath.Join(wdir, "DISTRIB")
	log.Printf("creating DISTRIB [%s]...\n", dist)
	os.RemoveAll(dist)
	err := os.MkdirAll(dist, 0755)
	if err != nil {
		fatalf("error creating [%s]: %v\n", dist, err)
	}

	for _, d := range []string{"main", "gui"} {
		dir := filepath.Join(wdir, d)
		_, err = os.Stat(dir)
		if err != nil {
			fatalf("could not stat [%s]: %v\n", dir, err)
		}

		srcs, err := filepath.Glob(
			filepath.Join(dir, "target", "org-lsst-ccs-subsystem-fcs-"+d+"-*-dist.zip"),
		)
		if err != nil {
			fatalf("could not stat %s-*-dist.zip: %v\n", d, err)
		}
		switch len(srcs) {
		case 0:
			fatalf("no %s-*-dist.zip!", d)
		case 1:
			// ok
		default:
			fatalf("too many %s-*-dist.zip files (%d): %v\n", len(srcs), srcs)
		}

		err = unzip(dist, srcs[0])
		if err != nil {
			fatalf("could not unzip [%s] into [%s]: %v\n", srcs[0], dist, err)
		}
	}
	log.Printf("creating DISTRIB [%s]... [done]\n", dist)
}

func setupEnv() {
	for _, v := range [][2]string{
		{"FCS_VERSION", fcsVersion},
		{"LOCALDB_VERSION", "1.3.0-SNAPSHOT"},
		// logging
		{"P0", "org.lsst.ccs.utilities.logging.ConsoleHandlerN.level=ALL"},
		{"P1", "org.lsst.ccs.utilities.logging.FileHandlerN.formatter=org.lsst.ccs.utilities.logging.TextFormatter"},
		{"P2", ".level=WARNING"},
		{"P3", "org.lsst.ccs.level=ALL"},
		{"P4", "org.lsst.ccs.bus.level=INFO"},
		{"P5", "org.lsst.ccs.subsystems.fcs.level=ALL"},

		{"TEST_ENV_ROOT", filepath.Join(wdir, "DISTRIB")},
		{"TEST_ENV_BIN", filepath.Join("${TEST_ENV_ROOT}", "bin")},
		{"TEST_ENV_RESOURCES", filepath.Join("${TEST_ENV_ROOT}", "externalResources")},
		{"TEST_ENV_DIRVERS", filepath.Join("${TEST_ENV_ROOT}", "drivers")},

		// needed by CCSbootstrap.sh
		{"CCS_RESOURCE_PATH", "${TEST_ENV_RESOURCES}"},

		// make sure we use IPv4 in JAS
		{"JASJVM_OPTS", "-Djava.net.preferIPV4Stack=true"},

		{"DESCRIPTION_FILE", "testbenchLPC.groo"},
		{"CONFIGURATION_FILE", "testbenchLPC_XXXXX_.properties"},
		{"WORKDIR", filepath.Join(wdir, "work")},
		{"LOGFILENAME", "org.lsst.ccs.utilites.logging.FileHandlerN.pattern=%W/logs/ccs-logs-%A-testbenchLPC.log"},
	} {
		err := os.Setenv(v[0], v[1])
		if err != nil {
			fatalf("error calling os.Setenv(%q, %q): %v\n", v[0], v[1], err)
		}
	}
}

func dispatch(errc chan error) {
	switch flag.Arg(0) {
	case "lpc":
		runTestbench(errc)
	case "console":
		runConsole(errc)
	case "jas3":
		runJAS3(errc)
	default:
		fatalf("unknown command [%s]\n", flag.Arg(0))
	}
}

func runTestbench(errc chan error) {
	cmd := exec.Command(
		filepath.Join(
			os.Getenv("TEST_ENV_ROOT"),
			"org-lsst-ccs-subsystem-fcs-main-"+fcsVersion,
			"bin",
			"CCSbootstrap.sh",
		),
		"-app", "FcsSubsystem",
		"--description", "/org/lsst/ccs/subsystems/fcs/conf/"+os.Getenv("DESCRIPTION_FILE"),
		"-D", os.Getenv("P0"),
		"-D", os.Getenv("P1"),
		"-D", os.Getenv("P2"),
		"-D", os.Getenv("P3"),
		"-D", os.Getenv("P4"),
		"-D", os.Getenv("P5"),
		"-D", os.Getenv("WORKDIR"),
		"-D", os.Getenv("LOGFILENAME"),
	)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	atexit(func() {
		killProc(cmd)
	})

	errc <- cmd.Run()
}

func runConsole(errc chan error) {
	cmd := exec.Command(
		filepath.Join(
			os.Getenv("TEST_ENV_ROOT"),
			"org-lsst-ccs-subsystem-fcs-gui-"+fcsVersion,
			"bin",
			"CCSbootstrap.sh",
		),
		"-app", "ShellCommandConsole",
	)
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout

	atexit(func() {
		killProc(cmd)
	})

	errc <- cmd.Run()
}

func runJAS3(errc chan error) {
	cmd := exec.Command(
		filepath.Join(
			os.Getenv("TEST_ENV_ROOT"),
			"org-lsst-ccs-subsystem-fcs-gui-"+fcsVersion,
			"bin",
			"CCSbootstrap.sh",
		),
		"-app", "CCS-Console",
	)
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout

	atexit(func() {
		killProc(cmd)
	})

	errc <- cmd.Run()
}

func unzip(dest, src string) error {
	r, err := zip.OpenReader(src)
	if err != nil {
		return err
	}
	defer func() {
		if err := r.Close(); err != nil {
			fatalf("error closing zip archive: %v\n", err)
		}
	}()

	os.MkdirAll(dest, 0755)

	// Closure to address file descriptors issue with all the deferred .Close() methods
	extract := func(f *zip.File) error {
		rc, err := f.Open()
		if err != nil {
			return err
		}
		defer func() {
			if err := rc.Close(); err != nil {
				fatalf("error closing zip file: %v\n", err)
			}
		}()

		path := filepath.Join(dest, f.Name)

		if f.FileInfo().IsDir() {
			os.MkdirAll(path, f.Mode())
		} else {
			f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
			if err != nil {
				return err
			}
			defer func() {
				if err := f.Close(); err != nil {
					fatalf("error closing zip output file: %v\n", err)
				}
			}()

			_, err = io.Copy(f, rc)
			if err != nil {
				return err
			}
		}
		return nil
	}

	for _, f := range r.File {
		err := extract(f)
		if err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>//go:build !lambdabinary
// +build !lambdabinary

package sparta

import (
	"encoding/base64"
	"reflect"
	"strings"

	gof "github.com/awslabs/goformation/v5/cloudformation"
	gofintrinsics "github.com/awslabs/goformation/v5/intrinsics"
	"github.com/pkg/errors"
	"github.com/rs/zerolog"
)

type resourceRefType int

const (
	resourceLiteral resourceRefType = iota
	resourceRefFunc
	resourceGetAttrFunc
)

type resourceRef struct {
	RefType      resourceRefType
	ResourceName string
}

// resolvedResourceVisitor represents the signature of a function that
// visits
type resolvedResourceVisitor func(lambdaAWSInfo *LambdaAWSInfo,
	eventSourceMapping *EventSourceMapping,
	mappingIndex int,
	resource *resourceRef) error

// resolveResourceRef takes an interface representing an ARN (either static or explicit)
// and tries to determine the CloudFormation resource name it resolves to
func resolveResourceRef(expr string) (*resourceRef, error) {

	// Ref: https://github.com/awslabs/goformation/blob/346053f16b2c9aba3a050c6a2956e18fe3a3f56f/intrinsics/intrinsics.go#L76
	// Decode the expression, hand it off to goformation to unmarshall
	// with the nested Base64 strings inside.
	// then turn it into a Map, look at the key and determine what kind of reference it is.

	// Is there a chance it's a Base64 encoded string? That indicates
	// it's a goformation reference.

	base64Decoded, base64DecodedErr := base64.StdEncoding.DecodeString(expr)
	if base64DecodedErr != nil {
		// It's possible it's a plain old literal...test it
		// Ref: https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html
		if strings.HasPrefix(expr, "arn:aws") {
			return &resourceRef{
				RefType:      resourceLiteral,
				ResourceName: expr,
			}, nil
		}
	}

	// Setup the intrinsic handlers so that when we unmarshal
	// the encoded functions we keep track of what we found

	var hookedResourceRef *resourceRef

	hookedOverrides := map[string]gofintrinsics.IntrinsicHandler{
		"Ref": func(name string, input interface{}, template interface{}) interface{} {
			hookedResourceRef = &resourceRef{
				RefType:      resourceRefFunc,
				ResourceName: input.(string),
			}
			return nil
		},
		"Fn:GetAtt": func(name string, input interface{}, template interface{}) interface{} {
			// The input should be an array...
			inputArr, inputArrOk := input.([]interface{})
			if !inputArrOk {
				return nil
			}
			inputStringElemZero, inputStringElemZeroOk := inputArr[0].(string)
			if !inputStringElemZeroOk {
				return nil
			}
			hookedResourceRef = &resourceRef{
				RefType:      resourceGetAttrFunc,
				ResourceName: inputStringElemZero,
			}
			return nil
		},
	}

	procOptions := &gofintrinsics.ProcessorOptions{
		IntrinsicHandlerOverrides: hookedOverrides,
	}
	_, processedJSONErr := gofintrinsics.ProcessJSON(base64Decoded, procOptions)
	if processedJSONErr != nil {
		return nil, processedJSONErr
	}
	// Whatever we have at this point is what it is...
	return hookedResourceRef, nil
}

// isResolvedResourceType is a utility function to determine if a resolved
// reference is a given type. If it is a literal, the literalTokenIndicator
// substring match is used for the predicate. If it is a resource provisioned
// by this template, the &gocf.RESOURCE_TYPE{} will be used via reflection
// Example:
// isResolvedResourceType(resourceRef, template, ":dynamodb:", &gocf.DynamoDBTable{}) {
//
func isResolvedResourceType(resource *resourceRef,
	template *gof.Template,
	literalTokenIndicator string,
	templateType interface{}) bool {

	if resource.RefType == resourceLiteral {
		return strings.Contains(resource.ResourceName, literalTokenIndicator)
	}

	// Dynamically provisioned resource included in the template definition?
	existingResource, existingResourceExists := template.Resources[resource.ResourceName]
	if existingResourceExists {
		if reflect.TypeOf(existingResource) == reflect.TypeOf(templateType) {
			return true
		}
	}
	return false

}

// visitResolvedEventSourceMapping is a utility function that visits all
// the EventSourceMapping entries for the given lambdaAWSInfo struct
func visitResolvedEventSourceMapping(visitor resolvedResourceVisitor,
	lambdaAWSInfos []*LambdaAWSInfo,
	template *gof.Template,
	logger *zerolog.Logger) error {

	//
	// BEGIN
	// Inline closure to wrap the visitor function so that we can provide
	// specific error messages
	visitEventSourceMappingRef := func(lambdaAWSInfo *LambdaAWSInfo,
		eventSourceMapping *EventSourceMapping,
		mappingIndex int,
		resource *resourceRef) error {

		annotateStatementsErr := visitor(lambdaAWSInfo,
			eventSourceMapping,
			mappingIndex,
			resource)

		// Early exit?
		if annotateStatementsErr != nil {
			return errors.Wrapf(annotateStatementsErr,
				"Visiting event source mapping: %#v",
				eventSourceMapping)
		}
		return nil
	}
	//
	// END

	// Iterate through every lambda function. If there is an EventSourceMapping
	// that points to a piece of infastructure provisioned by this stack,
	// find the referred resource and supply it to the visitor
	for _, eachLambda := range lambdaAWSInfos {
		for eachIndex, eachEventSource := range eachLambda.EventSourceMappings {
			resourceRef, resourceRefErr := resolveResourceRef(eachEventSource.EventSourceArn)
			if resourceRefErr != nil {
				return errors.Wrapf(resourceRefErr,
					"Failed to resolve EventSourceArn: %#v", eachEventSource)
			}

			// At this point everything is a string, so we need to unmarshall
			// and see if the Arn is supplied by either a Ref or a GetAttr
			// function. In those cases, we need to look around in the template
			// to go from: EventMapping -> Type -> Lambda -> LambdaIAMRole
			// so that we can add the permissions
			if resourceRef != nil {
				annotationErr := visitEventSourceMappingRef(eachLambda,
					eachEventSource,
					eachIndex,
					resourceRef)
				// Anything go wrong?
				if annotationErr != nil {
					return errors.Wrapf(annotationErr,
						"Failed to annotate template for EventSourceMapping: %#v",
						eachEventSource)
				}
			}
		}
	}
	return nil
}
<commit_msg>Correct Fn:::GetAtt processor option<commit_after>//go:build !lambdabinary
// +build !lambdabinary

package sparta

import (
	"encoding/base64"
	"reflect"
	"strings"

	gof "github.com/awslabs/goformation/v5/cloudformation"
	gofintrinsics "github.com/awslabs/goformation/v5/intrinsics"
	"github.com/pkg/errors"
	"github.com/rs/zerolog"
)

type resourceRefType int

const (
	resourceLiteral resourceRefType = iota
	resourceRefFunc
	resourceGetAttrFunc
)

type resourceRef struct {
	RefType      resourceRefType
	ResourceName string
}

// resolvedResourceVisitor represents the signature of a function that
// visits
type resolvedResourceVisitor func(lambdaAWSInfo *LambdaAWSInfo,
	eventSourceMapping *EventSourceMapping,
	mappingIndex int,
	resource *resourceRef) error

// resolveResourceRef takes an interface representing an ARN (either static or explicit)
// and tries to determine the CloudFormation resource name it resolves to
func resolveResourceRef(expr string) (*resourceRef, error) {

	// Ref: https://github.com/awslabs/goformation/blob/346053f16b2c9aba3a050c6a2956e18fe3a3f56f/intrinsics/intrinsics.go#L76
	// Decode the expression, hand it off to goformation to unmarshall
	// with the nested Base64 strings inside.
	// then turn it into a Map, look at the key and determine what kind of reference it is.

	// Is there a chance it's a Base64 encoded string? That indicates
	// it's a goformation reference.

	base64Decoded, base64DecodedErr := base64.StdEncoding.DecodeString(expr)
	if base64DecodedErr != nil {
		// It's possible it's a plain old literal...test it
		// Ref: https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html
		if strings.HasPrefix(expr, "arn:aws") {
			return &resourceRef{
				RefType:      resourceLiteral,
				ResourceName: expr,
			}, nil
		}
	}

	// Setup the intrinsic handlers so that when we unmarshal
	// the encoded functions we keep track of what we found

	var hookedResourceRef *resourceRef

	hookedOverrides := map[string]gofintrinsics.IntrinsicHandler{
		"Ref": func(name string, input interface{}, template interface{}) interface{} {
			hookedResourceRef = &resourceRef{
				RefType:      resourceRefFunc,
				ResourceName: input.(string),
			}
			return nil
		},
		"Fn::GetAtt": func(name string, input interface{}, template interface{}) interface{} {
			// The input should be an array...
			inputArr, inputArrOk := input.([]interface{})
			if !inputArrOk {
				return nil
			}
			inputStringElemZero, inputStringElemZeroOk := inputArr[0].(string)
			if !inputStringElemZeroOk {
				return nil
			}
			hookedResourceRef = &resourceRef{
				RefType:      resourceGetAttrFunc,
				ResourceName: inputStringElemZero,
			}
			return nil
		},
	}

	procOptions := &gofintrinsics.ProcessorOptions{
		IntrinsicHandlerOverrides: hookedOverrides,
	}
	_, processedJSONErr := gofintrinsics.ProcessJSON(base64Decoded, procOptions)
	if processedJSONErr != nil {
		return nil, processedJSONErr
	}
	// Whatever we have at this point is what it is...
	return hookedResourceRef, nil
}

// isResolvedResourceType is a utility function to determine if a resolved
// reference is a given type. If it is a literal, the literalTokenIndicator
// substring match is used for the predicate. If it is a resource provisioned
// by this template, the &gocf.RESOURCE_TYPE{} will be used via reflection
// Example:
// isResolvedResourceType(resourceRef, template, ":dynamodb:", &gocf.DynamoDBTable{}) {
//
func isResolvedResourceType(resource *resourceRef,
	template *gof.Template,
	literalTokenIndicator string,
	templateType interface{}) bool {

	if resource.RefType == resourceLiteral {
		return strings.Contains(resource.ResourceName, literalTokenIndicator)
	}

	// Dynamically provisioned resource included in the template definition?
	existingResource, existingResourceExists := template.Resources[resource.ResourceName]
	if existingResourceExists {
		if reflect.TypeOf(existingResource) == reflect.TypeOf(templateType) {
			return true
		}
	}
	return false

}

// visitResolvedEventSourceMapping is a utility function that visits all
// the EventSourceMapping entries for the given lambdaAWSInfo struct
func visitResolvedEventSourceMapping(visitor resolvedResourceVisitor,
	lambdaAWSInfos []*LambdaAWSInfo,
	template *gof.Template,
	logger *zerolog.Logger) error {

	//
	// BEGIN
	// Inline closure to wrap the visitor function so that we can provide
	// specific error messages
	visitEventSourceMappingRef := func(lambdaAWSInfo *LambdaAWSInfo,
		eventSourceMapping *EventSourceMapping,
		mappingIndex int,
		resource *resourceRef) error {

		annotateStatementsErr := visitor(lambdaAWSInfo,
			eventSourceMapping,
			mappingIndex,
			resource)

		// Early exit?
		if annotateStatementsErr != nil {
			return errors.Wrapf(annotateStatementsErr,
				"Visiting event source mapping: %#v",
				eventSourceMapping)
		}
		return nil
	}
	//
	// END

	// Iterate through every lambda function. If there is an EventSourceMapping
	// that points to a piece of infastructure provisioned by this stack,
	// find the referred resource and supply it to the visitor
	for _, eachLambda := range lambdaAWSInfos {
		for eachIndex, eachEventSource := range eachLambda.EventSourceMappings {
			resourceRef, resourceRefErr := resolveResourceRef(eachEventSource.EventSourceArn)
			if resourceRefErr != nil {
				return errors.Wrapf(resourceRefErr,
					"Failed to resolve EventSourceArn: %#v", eachEventSource)
			}

			// At this point everything is a string, so we need to unmarshall
			// and see if the Arn is supplied by either a Ref or a GetAttr
			// function. In those cases, we need to look around in the template
			// to go from: EventMapping -> Type -> Lambda -> LambdaIAMRole
			// so that we can add the permissions
			if resourceRef != nil {
				annotationErr := visitEventSourceMappingRef(eachLambda,
					eachEventSource,
					eachIndex,
					resourceRef)
				// Anything go wrong?
				if annotationErr != nil {
					return errors.Wrapf(annotationErr,
						"Failed to annotate template for EventSourceMapping: %#v",
						eachEventSource)
				}
			}
		}
	}
	return nil
}
<|endoftext|>