<commit_before>// package bitswap implements the IPFS Exchange interface with the BitSwap
// bilateral exchange protocol.
package bitswap

import (
	"math"
	"sync"
	"time"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	blocks "github.com/jbenet/go-ipfs/blocks"
	blockstore "github.com/jbenet/go-ipfs/blocks/blockstore"
	exchange "github.com/jbenet/go-ipfs/exchange"
	decision "github.com/jbenet/go-ipfs/exchange/bitswap/decision"
	bsmsg "github.com/jbenet/go-ipfs/exchange/bitswap/message"
	bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network"
	notifications "github.com/jbenet/go-ipfs/exchange/bitswap/notifications"
	wantlist "github.com/jbenet/go-ipfs/exchange/bitswap/wantlist"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	"github.com/jbenet/go-ipfs/thirdparty/delay"
	eventlog "github.com/jbenet/go-ipfs/thirdparty/eventlog"
	u "github.com/jbenet/go-ipfs/util"
	errors "github.com/jbenet/go-ipfs/util/debugerror"
	pset "github.com/jbenet/go-ipfs/util/peerset" // TODO move this to peerstore
)

var log = eventlog.Logger("bitswap")

const (
	// maxProvidersPerRequest specifies the maximum number of providers desired
	// from the network. This value is specified because the network streams
	// results.
	// TODO: if a 'non-nice' strategy is implemented, consider increasing this value
	maxProvidersPerRequest = 3
	providerRequestTimeout = time.Second * 10
	hasBlockTimeout        = time.Second * 15
	sizeBatchRequestChan   = 32
	// kMaxPriority is the max priority as defined by the bitswap protocol
	kMaxPriority = math.MaxInt32
)

var (
	rebroadcastDelay = delay.Fixed(time.Second * 10)
)

// New initializes a BitSwap instance that communicates over the provided
// BitSwapNetwork. This function registers the returned instance as the network
// delegate.
// Runs until context is cancelled.
func New(parent context.Context, p peer.ID, network bsnet.BitSwapNetwork,
	bstore blockstore.Blockstore, nice bool) exchange.Interface {

	ctx, cancelFunc := context.WithCancel(parent)

	notif := notifications.New()
	go func() {
		<-ctx.Done()
		cancelFunc()
		notif.Shutdown()
	}()

	bs := &bitswap{
		self:          p,
		blockstore:    bstore,
		cancelFunc:    cancelFunc,
		notifications: notif,
		engine:        decision.NewEngine(ctx, bstore),
		network:       network,
		wantlist:      wantlist.NewThreadSafe(),
		batchRequests: make(chan []u.Key, sizeBatchRequestChan),
	}
	network.SetDelegate(bs)
	go bs.clientWorker(ctx)
	go bs.taskWorker(ctx)

	return bs
}

// bitswap instances implement the bitswap protocol.
type bitswap struct {

	// the ID of the peer to act on behalf of
	self peer.ID

	// network delivers messages on behalf of the session
	network bsnet.BitSwapNetwork

	// blockstore is the local database
	// NB: ensure threadsafety
	blockstore blockstore.Blockstore

	notifications notifications.PubSub

	// Requests for a set of related blocks
	// the assumption is made that the same peer is likely to
	// have more than a single block in the set
	batchRequests chan []u.Key

	engine *decision.Engine

	wantlist *wantlist.ThreadSafe

	// cancelFunc signals cancellation to the bitswap event loop
	cancelFunc func()
}

// GetBlock attempts to retrieve a particular block from peers within the
// deadline enforced by the context.
func (bs *bitswap) GetBlock(parent context.Context, k u.Key) (*blocks.Block, error) {

	// Any async work initiated by this function must end when this function
	// returns. To ensure this, derive a new context. Note that it is okay to
	// listen on parent in this scope, but NOT okay to pass |parent| to
	// functions called by this one. Otherwise those functions won't return
	// when this context's cancel func is executed. This is difficult to
	// enforce. May this comment keep you safe.

	ctx, cancelFunc := context.WithCancel(parent)

	ctx = eventlog.ContextWithLoggable(ctx, eventlog.Uuid("GetBlockRequest"))
	defer log.EventBegin(ctx, "GetBlockRequest", &k).Done()

	defer func() {
		cancelFunc()
	}()

	promise, err := bs.GetBlocks(ctx, []u.Key{k})
	if err != nil {
		return nil, err
	}

	select {
	case block := <-promise:
		return block, nil
	case <-parent.Done():
		return nil, parent.Err()
	}

}

// GetBlocks returns a channel where the caller may receive blocks that
// correspond to the provided |keys|. Returns an error if BitSwap is unable to
// begin this request within the deadline enforced by the context.
//
// NB: Your request remains open until the context expires. To conserve
// resources, provide a context with a reasonably short deadline (ie. not one
// that lasts throughout the lifetime of the server)
func (bs *bitswap) GetBlocks(ctx context.Context, keys []u.Key) (<-chan *blocks.Block, error) {

	promise := bs.notifications.Subscribe(ctx, keys...)
	select {
	case bs.batchRequests <- keys:
		return promise, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

// HasBlock announces the existance of a block to this bitswap service. The
// service will potentially notify its peers.
func (bs *bitswap) HasBlock(ctx context.Context, blk *blocks.Block) error {
	if err := bs.blockstore.Put(blk); err != nil {
		return err
	}
	bs.wantlist.Remove(blk.Key())
	bs.notifications.Publish(blk)
	return bs.network.Provide(ctx, blk.Key())
}

func (bs *bitswap) sendWantlistMsgToPeers(ctx context.Context, m bsmsg.BitSwapMessage, peers <-chan peer.ID) error {
	if peers == nil {
		panic("Cant send wantlist to nil peerchan")
	}

	set := pset.New()
	wg := sync.WaitGroup{}
	for peerToQuery := range peers {
		log.Event(ctx, "PeerToQuery", peerToQuery)

		if !set.TryAdd(peerToQuery) { //Do once per peer
			log.Debugf("%s skipped (already sent)", peerToQuery)
			continue
		}
		log.Debugf("%s sending", peerToQuery)

		wg.Add(1)
		go func(p peer.ID) {
			defer wg.Done()
			if err := bs.send(ctx, p, m); err != nil {
				log.Error(err) // TODO remove if too verbose
			}
		}(peerToQuery)
	}
	wg.Wait()
	return nil
}

func (bs *bitswap) sendWantlistToPeers(ctx context.Context, peers <-chan peer.ID) error {
	message := bsmsg.New()
	message.SetFull(true)
	for _, wanted := range bs.wantlist.Entries() {
		message.AddEntry(wanted.Key, wanted.Priority)
	}
	return bs.sendWantlistMsgToPeers(ctx, message, peers)
}

func (bs *bitswap) sendWantlistToProviders(ctx context.Context) {
	entries := bs.wantlist.Entries()
	if len(entries) == 0 {
		log.Debug("No entries in wantlist, skipping send routine.")
		return
	}

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	// prepare a channel to hand off to sendWantlistToPeers
	sendToPeers := make(chan peer.ID)

	// Get providers for all entries in wantlist (could take a while)
	wg := sync.WaitGroup{}
	for _, e := range entries {
		wg.Add(1)
		go func(k u.Key) {
			defer wg.Done()

			child, _ := context.WithTimeout(ctx, providerRequestTimeout)
			providers := bs.network.FindProvidersAsync(child, k, maxProvidersPerRequest)
			for prov := range providers {
				log.Debugf("dht returned provider %s. send wantlist", prov)
				sendToPeers <- prov
			}
		}(e.Key)
	}

	go func() {
		wg.Wait() // make sure all our children do finish.
		close(sendToPeers)
	}()

	err := bs.sendWantlistToPeers(ctx, sendToPeers)
	if err != nil {
		log.Errorf("sendWantlistToPeers error: %s", err)
	}
}

func (bs *bitswap) taskWorker(ctx context.Context) {
	for {
		select {
		case <-ctx.Done():
			log.Debugf("exiting")
			return
		case nextEnvelope := <-bs.engine.Outbox():
			select {
			case <-ctx.Done():
				return
			case envelope, ok := <-nextEnvelope:
				if !ok {
					continue
				}
				bs.send(ctx, envelope.Peer, envelope.Message)
			}
		}
	}
}

// TODO ensure only one active request per key
func (bs *bitswap) clientWorker(parent context.Context) {

	ctx, cancel := context.WithCancel(parent)

	broadcastSignal := time.After(rebroadcastDelay.Get())
	defer cancel()

	for {
		select {
		case <-broadcastSignal: // resend unfulfilled wantlist keys
			bs.sendWantlistToProviders(ctx)
			broadcastSignal = time.After(rebroadcastDelay.Get())
		case keys := <-bs.batchRequests:
			if len(keys) == 0 {
				log.Warning("Received batch request for zero blocks")
				continue
			}
			for i, k := range keys {
				bs.wantlist.Add(k, kMaxPriority-i)
			}
			// NB: Optimization. Assumes that providers of key[0] are likely to
			// be able to provide for all keys. This currently holds true in most
			// every situation. Later, this assumption may not hold as true.
			child, _ := context.WithTimeout(ctx, providerRequestTimeout)
			providers := bs.network.FindProvidersAsync(child, keys[0], maxProvidersPerRequest)
			err := bs.sendWantlistToPeers(ctx, providers)
			if err != nil {
				log.Errorf("error sending wantlist: %s", err)
			}
		case <-parent.Done():
			return
		}
	}
}

// TODO(brian): handle errors
func (bs *bitswap) ReceiveMessage(ctx context.Context, p peer.ID, incoming bsmsg.BitSwapMessage) (
	peer.ID, bsmsg.BitSwapMessage) {
	log.Debugf("ReceiveMessage from %s", p)

	if p == "" {
		log.Error("Received message from nil peer!")
		// TODO propagate the error upward
		return "", nil
	}
	if incoming == nil {
		log.Error("Got nil bitswap message!")
		// TODO propagate the error upward
		return "", nil
	}

	// This call records changes to wantlists, blocks received,
	// and number of bytes transfered.
	bs.engine.MessageReceived(p, incoming)
	// TODO: this is bad, and could be easily abused.
	// Should only track *useful* messages in ledger

	for _, block := range incoming.Blocks() {
		hasBlockCtx, _ := context.WithTimeout(ctx, hasBlockTimeout)
		if err := bs.HasBlock(hasBlockCtx, block); err != nil {
			log.Error(err)
		}
	}
	var keys []u.Key
	for _, block := range incoming.Blocks() {
		keys = append(keys, block.Key())
	}
	bs.cancelBlocks(ctx, keys)

	// TODO: consider changing this function to not return anything
	return "", nil
}

// Connected/Disconnected warns bitswap about peer connections
func (bs *bitswap) PeerConnected(p peer.ID) {
	// TODO: add to clientWorker??

	peers := make(chan peer.ID)
	err := bs.sendWantlistToPeers(context.TODO(), peers)
	if err != nil {
		log.Errorf("error sending wantlist: %s", err)
	}
	peers <- p
	close(peers)
}

// Connected/Disconnected warns bitswap about peer connections
func (bs *bitswap) PeerDisconnected(peer.ID) {
	// TODO: release resources.
}

func (bs *bitswap) cancelBlocks(ctx context.Context, bkeys []u.Key) {
	if len(bkeys) < 1 {
		return
	}
	message := bsmsg.New()
	message.SetFull(false)
	for _, k := range bkeys {
		message.Cancel(k)
	}
	for _, p := range bs.engine.Peers() {
		err := bs.send(ctx, p, message)
		if err != nil {
			log.Errorf("Error sending message: %s", err)
		}
	}
}

func (bs *bitswap) ReceiveError(err error) {
	log.Errorf("Bitswap ReceiveError: %s", err)
	// TODO log the network error
	// TODO bubble the network error up to the parent context/error logger
}

// send strives to ensure that accounting is always performed when a message is
// sent
func (bs *bitswap) send(ctx context.Context, p peer.ID, m bsmsg.BitSwapMessage) error {
	if err := bs.network.SendMessage(ctx, p, m); err != nil {
		return errors.Wrap(err)
	}
	return bs.engine.MessageSent(p, m)
}

func (bs *bitswap) Close() error {
	bs.cancelFunc()
	return nil // to conform to Closer interface
}
<commit_msg>revert bitswap network notification<commit_after>// package bitswap implements the IPFS Exchange interface with the BitSwap
// bilateral exchange protocol.
package bitswap

import (
	"math"
	"sync"
	"time"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"

	blocks "github.com/jbenet/go-ipfs/blocks"
	blockstore "github.com/jbenet/go-ipfs/blocks/blockstore"
	exchange "github.com/jbenet/go-ipfs/exchange"
	decision "github.com/jbenet/go-ipfs/exchange/bitswap/decision"
	bsmsg "github.com/jbenet/go-ipfs/exchange/bitswap/message"
	bsnet "github.com/jbenet/go-ipfs/exchange/bitswap/network"
	notifications "github.com/jbenet/go-ipfs/exchange/bitswap/notifications"
	wantlist "github.com/jbenet/go-ipfs/exchange/bitswap/wantlist"
	peer "github.com/jbenet/go-ipfs/p2p/peer"
	"github.com/jbenet/go-ipfs/thirdparty/delay"
	eventlog "github.com/jbenet/go-ipfs/thirdparty/eventlog"
	u "github.com/jbenet/go-ipfs/util"
	errors "github.com/jbenet/go-ipfs/util/debugerror"
	pset "github.com/jbenet/go-ipfs/util/peerset" // TODO move this to peerstore
)

var log = eventlog.Logger("bitswap")

const (
	// maxProvidersPerRequest specifies the maximum number of providers desired
	// from the network. This value is specified because the network streams
	// results.
	// TODO: if a 'non-nice' strategy is implemented, consider increasing this value
	maxProvidersPerRequest = 3
	providerRequestTimeout = time.Second * 10
	hasBlockTimeout        = time.Second * 15
	sizeBatchRequestChan   = 32
	// kMaxPriority is the max priority as defined by the bitswap protocol
	kMaxPriority = math.MaxInt32
)

var (
	rebroadcastDelay = delay.Fixed(time.Second * 10)
)

// New initializes a BitSwap instance that communicates over the provided
// BitSwapNetwork. This function registers the returned instance as the network
// delegate.
// Runs until context is cancelled.
func New(parent context.Context, p peer.ID, network bsnet.BitSwapNetwork,
	bstore blockstore.Blockstore, nice bool) exchange.Interface {

	ctx, cancelFunc := context.WithCancel(parent)

	notif := notifications.New()
	go func() {
		<-ctx.Done()
		cancelFunc()
		notif.Shutdown()
	}()

	bs := &bitswap{
		self:          p,
		blockstore:    bstore,
		cancelFunc:    cancelFunc,
		notifications: notif,
		engine:        decision.NewEngine(ctx, bstore),
		network:       network,
		wantlist:      wantlist.NewThreadSafe(),
		batchRequests: make(chan []u.Key, sizeBatchRequestChan),
	}
	network.SetDelegate(bs)
	go bs.clientWorker(ctx)
	go bs.taskWorker(ctx)

	return bs
}

// bitswap instances implement the bitswap protocol.
type bitswap struct {

	// the ID of the peer to act on behalf of
	self peer.ID

	// network delivers messages on behalf of the session
	network bsnet.BitSwapNetwork

	// blockstore is the local database
	// NB: ensure threadsafety
	blockstore blockstore.Blockstore

	notifications notifications.PubSub

	// Requests for a set of related blocks
	// the assumption is made that the same peer is likely to
	// have more than a single block in the set
	batchRequests chan []u.Key

	engine *decision.Engine

	wantlist *wantlist.ThreadSafe

	// cancelFunc signals cancellation to the bitswap event loop
	cancelFunc func()
}

// GetBlock attempts to retrieve a particular block from peers within the
// deadline enforced by the context.
func (bs *bitswap) GetBlock(parent context.Context, k u.Key) (*blocks.Block, error) {

	// Any async work initiated by this function must end when this function
	// returns. To ensure this, derive a new context. Note that it is okay to
	// listen on parent in this scope, but NOT okay to pass |parent| to
	// functions called by this one. Otherwise those functions won't return
	// when this context's cancel func is executed. This is difficult to
	// enforce. May this comment keep you safe.

	ctx, cancelFunc := context.WithCancel(parent)

	ctx = eventlog.ContextWithLoggable(ctx, eventlog.Uuid("GetBlockRequest"))
	defer log.EventBegin(ctx, "GetBlockRequest", &k).Done()

	defer func() {
		cancelFunc()
	}()

	promise, err := bs.GetBlocks(ctx, []u.Key{k})
	if err != nil {
		return nil, err
	}

	select {
	case block := <-promise:
		return block, nil
	case <-parent.Done():
		return nil, parent.Err()
	}

}

// GetBlocks returns a channel where the caller may receive blocks that
// correspond to the provided |keys|. Returns an error if BitSwap is unable to
// begin this request within the deadline enforced by the context.
//
// NB: Your request remains open until the context expires. To conserve
// resources, provide a context with a reasonably short deadline (ie. not one
// that lasts throughout the lifetime of the server)
func (bs *bitswap) GetBlocks(ctx context.Context, keys []u.Key) (<-chan *blocks.Block, error) {

	promise := bs.notifications.Subscribe(ctx, keys...)
	select {
	case bs.batchRequests <- keys:
		return promise, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

// HasBlock announces the existance of a block to this bitswap service. The
// service will potentially notify its peers.
func (bs *bitswap) HasBlock(ctx context.Context, blk *blocks.Block) error {
	if err := bs.blockstore.Put(blk); err != nil {
		return err
	}
	bs.wantlist.Remove(blk.Key())
	bs.notifications.Publish(blk)
	return bs.network.Provide(ctx, blk.Key())
}

func (bs *bitswap) sendWantlistMsgToPeers(ctx context.Context, m bsmsg.BitSwapMessage, peers <-chan peer.ID) error {
	if peers == nil {
		panic("Cant send wantlist to nil peerchan")
	}

	set := pset.New()
	wg := sync.WaitGroup{}
	for peerToQuery := range peers {
		log.Event(ctx, "PeerToQuery", peerToQuery)

		if !set.TryAdd(peerToQuery) { //Do once per peer
			log.Debugf("%s skipped (already sent)", peerToQuery)
			continue
		}
		log.Debugf("%s sending", peerToQuery)

		wg.Add(1)
		go func(p peer.ID) {
			defer wg.Done()
			if err := bs.send(ctx, p, m); err != nil {
				log.Error(err) // TODO remove if too verbose
			}
		}(peerToQuery)
	}
	wg.Wait()
	return nil
}

func (bs *bitswap) sendWantlistToPeers(ctx context.Context, peers <-chan peer.ID) error {
	message := bsmsg.New()
	message.SetFull(true)
	for _, wanted := range bs.wantlist.Entries() {
		message.AddEntry(wanted.Key, wanted.Priority)
	}
	return bs.sendWantlistMsgToPeers(ctx, message, peers)
}

func (bs *bitswap) sendWantlistToProviders(ctx context.Context) {
	entries := bs.wantlist.Entries()
	if len(entries) == 0 {
		log.Debug("No entries in wantlist, skipping send routine.")
		return
	}

	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	// prepare a channel to hand off to sendWantlistToPeers
	sendToPeers := make(chan peer.ID)

	// Get providers for all entries in wantlist (could take a while)
	wg := sync.WaitGroup{}
	for _, e := range entries {
		wg.Add(1)
		go func(k u.Key) {
			defer wg.Done()

			child, _ := context.WithTimeout(ctx, providerRequestTimeout)
			providers := bs.network.FindProvidersAsync(child, k, maxProvidersPerRequest)
			for prov := range providers {
				log.Debugf("dht returned provider %s. send wantlist", prov)
				sendToPeers <- prov
			}
		}(e.Key)
	}

	go func() {
		wg.Wait() // make sure all our children do finish.
		close(sendToPeers)
	}()

	err := bs.sendWantlistToPeers(ctx, sendToPeers)
	if err != nil {
		log.Errorf("sendWantlistToPeers error: %s", err)
	}
}

func (bs *bitswap) taskWorker(ctx context.Context) {
	for {
		select {
		case <-ctx.Done():
			log.Debugf("exiting")
			return
		case nextEnvelope := <-bs.engine.Outbox():
			select {
			case <-ctx.Done():
				return
			case envelope, ok := <-nextEnvelope:
				if !ok {
					continue
				}
				bs.send(ctx, envelope.Peer, envelope.Message)
			}
		}
	}
}

// TODO ensure only one active request per key
func (bs *bitswap) clientWorker(parent context.Context) {

	ctx, cancel := context.WithCancel(parent)

	broadcastSignal := time.After(rebroadcastDelay.Get())
	defer cancel()

	for {
		select {
		case <-broadcastSignal: // resend unfulfilled wantlist keys
			bs.sendWantlistToProviders(ctx)
			broadcastSignal = time.After(rebroadcastDelay.Get())
		case keys := <-bs.batchRequests:
			if len(keys) == 0 {
				log.Warning("Received batch request for zero blocks")
				continue
			}
			for i, k := range keys {
				bs.wantlist.Add(k, kMaxPriority-i)
			}
			// NB: Optimization. Assumes that providers of key[0] are likely to
			// be able to provide for all keys. This currently holds true in most
			// every situation. Later, this assumption may not hold as true.
			child, _ := context.WithTimeout(ctx, providerRequestTimeout)
			providers := bs.network.FindProvidersAsync(child, keys[0], maxProvidersPerRequest)
			err := bs.sendWantlistToPeers(ctx, providers)
			if err != nil {
				log.Errorf("error sending wantlist: %s", err)
			}
		case <-parent.Done():
			return
		}
	}
}

// TODO(brian): handle errors
func (bs *bitswap) ReceiveMessage(ctx context.Context, p peer.ID, incoming bsmsg.BitSwapMessage) (
	peer.ID, bsmsg.BitSwapMessage) {
	log.Debugf("ReceiveMessage from %s", p)

	if p == "" {
		log.Error("Received message from nil peer!")
		// TODO propagate the error upward
		return "", nil
	}
	if incoming == nil {
		log.Error("Got nil bitswap message!")
		// TODO propagate the error upward
		return "", nil
	}

	// This call records changes to wantlists, blocks received,
	// and number of bytes transfered.
	bs.engine.MessageReceived(p, incoming)
	// TODO: this is bad, and could be easily abused.
	// Should only track *useful* messages in ledger

	for _, block := range incoming.Blocks() {
		hasBlockCtx, _ := context.WithTimeout(ctx, hasBlockTimeout)
		if err := bs.HasBlock(hasBlockCtx, block); err != nil {
			log.Error(err)
		}
	}
	var keys []u.Key
	for _, block := range incoming.Blocks() {
		keys = append(keys, block.Key())
	}
	bs.cancelBlocks(ctx, keys)

	// TODO: consider changing this function to not return anything
	return "", nil
}

// Connected/Disconnected warns bitswap about peer connections
func (bs *bitswap) PeerConnected(p peer.ID) {
	// TODO: add to clientWorker??
}

// Connected/Disconnected warns bitswap about peer connections
func (bs *bitswap) PeerDisconnected(peer.ID) {
	// TODO: release resources.
}

func (bs *bitswap) cancelBlocks(ctx context.Context, bkeys []u.Key) {
	if len(bkeys) < 1 {
		return
	}
	message := bsmsg.New()
	message.SetFull(false)
	for _, k := range bkeys {
		message.Cancel(k)
	}
	for _, p := range bs.engine.Peers() {
		err := bs.send(ctx, p, message)
		if err != nil {
			log.Errorf("Error sending message: %s", err)
		}
	}
}

func (bs *bitswap) ReceiveError(err error) {
	log.Errorf("Bitswap ReceiveError: %s", err)
	// TODO log the network error
	// TODO bubble the network error up to the parent context/error logger
}

// send strives to ensure that accounting is always performed when a message is
// sent
func (bs *bitswap) send(ctx context.Context, p peer.ID, m bsmsg.BitSwapMessage) error {
	if err := bs.network.SendMessage(ctx, p, m); err != nil {
		return errors.Wrap(err)
	}
	return bs.engine.MessageSent(p, m)
}

func (bs *bitswap) Close() error {
	bs.cancelFunc()
	return nil // to conform to Closer interface
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/gitiles"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"golang.org/x/oauth2/google"
)

const (
	livenessMetric       = "k8s_deployer"
	gitstoreSubscriberID = "k8s-deployer"
)

func main() {
	configRepo := flag.String("config_repo", "https://skia.googlesource.com/k8s-config.git", "Repo containing Kubernetes configurations.")
	configSubdir := flag.String("config_subdir", "", "Subdirectory within the config repo to apply to this cluster.")
	configFiles := common.NewMultiStringFlag("config_file", nil, "Individual config files to apply. Supports regex. Incompatible with --prune.")
	interval := flag.Duration("interval", 10*time.Minute, "How often to re-apply configurations to the cluster")
	port := flag.String("port", ":8000", "HTTP service port for the web server (e.g., ':8000')")
	promPort := flag.String("prom_port", ":20000", "Metrics service address (e.g., ':20000')")
	prune := flag.Bool("prune", false, "Whether to run 'kubectl apply' with '--prune'")
	kubectl := flag.String("kubectl", "kubectl", "Path to the kubectl executable.")
	k8sServer := flag.String("k8s_server", "", "Address of the Kubernetes server.")

	common.InitWithMust("k8s_deployer", common.PrometheusOpt(promPort))
	defer sklog.Flush()

	if *configRepo == "" {
		sklog.Fatal("config_repo is required.")
	}
	if *configSubdir == "" {
		// Note: this wouldn't be required if we had separate config repos per
		// cluster.
		sklog.Fatal("config_subdir is required.")
	}
	var configFileRegexes []*regexp.Regexp
	if len(*configFiles) > 0 {
		if *prune {
			sklog.Fatal("--config_file is incompatible with --prune.")
		}
		for _, expr := range *configFiles {
			re, err := regexp.Compile(expr)
			if err != nil {
				sklog.Fatal(err)
			}
			configFileRegexes = append(configFileRegexes, re)
		}
	}

	ctx := context.Background()

	// OAuth2.0 TokenSource.
	ts, err := google.DefaultTokenSource(ctx, auth.ScopeUserinfoEmail, auth.ScopeGerrit)
	if err != nil {
		sklog.Fatal(err)
	}
	// Authenticated HTTP client.
	httpClient := httputils.DefaultClientConfig().WithTokenSource(ts).With2xxOnly().Client()

	// Gitiles repo.
	repo := gitiles.NewRepo(*configRepo, httpClient)

	// Apply configurations in a loop.  Note that we could respond directly to
	// commits in the repo via GitStore and PubSub, but that would require
	// access to BigTable, and with a relatively small interval we won't notice
	// too much of a delay.
	liveness := metrics2.NewLiveness(livenessMetric)
	go util.RepeatCtx(ctx, *interval, func(ctx context.Context) {
		if err := applyConfigs(ctx, repo, *kubectl, *k8sServer, *configSubdir, configFileRegexes, *prune); err != nil {
			sklog.Errorf("Failed to apply configs to cluster: %s", err)
		} else {
			liveness.Reset()
		}
	})

	// Run health check server.
	httputils.RunHealthCheckServer(*port)
}

func applyConfigs(ctx context.Context, repo *gitiles.Repo, kubectl, k8sServer, configSubdir string, configFileRegexes []*regexp.Regexp, prune bool) error {
	// Download the configs from Gitiles instead of maintaining a local Git
	// checkout, to avoid dealing with Git, persistent checkouts, etc.

	// Obtain the current set of configurations for the cluster.
	head, err := repo.Details(ctx, "main")
	if err != nil {
		return skerr.Wrapf(err, "failed to get most recent commit")
	}
	files, err := repo.ListFilesRecursiveAtRef(ctx, configSubdir, head.Hash)
	if err != nil {
		return skerr.Wrapf(err, "failed to list configs")
	}

	// Read the config contents in the given directory.
	eg := util.NewNamedErrGroup()
	contents := make(map[string][]byte, len(files))
	contentsMtx := sync.Mutex{}
	sklog.Infof("Downloading config files at %s", head.Hash)
	for _, file := range files {
		file := file // https://golang.org/doc/faq#closures_and_goroutines

		// Ensure that the file matches any provided regular expressions.
		if len(configFileRegexes) > 0 {
			sklog.Infof("Match: %q ?", file)
			match := false
			for _, re := range configFileRegexes {
				if re.MatchString(file) {
					sklog.Infof("  Matched %q", re.String())
					match = true
					break
				}
			}
			if !match {
				sklog.Infof("  No match.")
				continue
			}
		}

		// Download the file contents.
		eg.Go(file, func() error {
			fullPath := path.Join(configSubdir, file)
			sklog.Infof("  %s", fullPath)
			fileContents, err := repo.ReadFileAtRef(ctx, fullPath, head.Hash)
			if err != nil {
				return skerr.Wrapf(err, "failed to retrieve contents of %s", fullPath)
			}
			contentsMtx.Lock()
			defer contentsMtx.Unlock()
			contents[file] = fileContents
			return nil
		})
	}
	if err := eg.Wait(); err != nil {
		return skerr.Wrapf(err, "failed to download configs")
	}

	// Write the config contents to a temporary dir.
	tmp, err := ioutil.TempDir("", "")
	if err != nil {
		return skerr.Wrapf(err, "failed to create temp dir")
	}
	defer util.RemoveAll(tmp)

	for path, fileContents := range contents {
		fullPath := filepath.Join(tmp, path)
		dir := filepath.Dir(fullPath)
		if _, err := os.Stat(path); os.IsNotExist(err) {
			if err := os.MkdirAll(dir, os.ModePerm); err != nil {
				return skerr.Wrapf(err, "failed to create %s", dir)
			}
		}
		if err := ioutil.WriteFile(fullPath, fileContents, os.ModePerm); err != nil {
			return skerr.Wrapf(err, "failed to create %s", fullPath)
		}
	}

	// Apply the configs to the cluster.
	// Note: this is a very naive approach.  We rely on the Kubernetes server to
	// determine when changes actually need to be made.  We could instead use
	// the REST API to load the full set of configuration which is actually
	// running on the server, diff that against the checked-in config files,
	// and then explicitly make only the changes we want to make.  That would be
	// a much more complicated and error-prone approach, but it would allow us
	// to partially apply configurations in the case of a failure and to alert
	// on specific components which fail to apply for whatever reason.
	cmd := []string{kubectl, "apply"}
	if k8sServer != "" {
		cmd = append(cmd, "--server", k8sServer)
	}
	if prune {
		cmd = append(cmd, "--prune", "--all")
	}
	cmd = append(cmd, "-f", ".")
	output, err := exec.RunCwd(ctx, tmp, cmd...)
	if err != nil {
		return skerr.Wrapf(err, "failed to apply configs: %s", output)
	}
	sklog.Info("Output from kubectl")
	for _, line := range strings.Split(output, "\n") {
		sklog.Info(line)
	}
	return nil
}
<commit_msg>Revert "[k8s-deployer] Add logging to determine why some files aren't getting applied"<commit_after>package main

import (
	"context"
	"flag"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"
	"sync"
	"time"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/gitiles"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/metrics2"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"golang.org/x/oauth2/google"
)

const (
	livenessMetric       = "k8s_deployer"
	gitstoreSubscriberID = "k8s-deployer"
)

func main() {
	configRepo := flag.String("config_repo", "https://skia.googlesource.com/k8s-config.git", "Repo containing Kubernetes configurations.")
	configSubdir := flag.String("config_subdir", "", "Subdirectory within the config repo to apply to this cluster.")
	configFiles := common.NewMultiStringFlag("config_file", nil, "Individual config files to apply. Supports regex. Incompatible with --prune.")
	interval := flag.Duration("interval", 10*time.Minute, "How often to re-apply configurations to the cluster")
	port := flag.String("port", ":8000", "HTTP service port for the web server (e.g., ':8000')")
	promPort := flag.String("prom_port", ":20000", "Metrics service address (e.g., ':20000')")
	prune := flag.Bool("prune", false, "Whether to run 'kubectl apply' with '--prune'")
	kubectl := flag.String("kubectl", "kubectl", "Path to the kubectl executable.")
	k8sServer := flag.String("k8s_server", "", "Address of the Kubernetes server.")

	common.InitWithMust("k8s_deployer", common.PrometheusOpt(promPort))
	defer sklog.Flush()

	if *configRepo == "" {
		sklog.Fatal("config_repo is required.")
	}
	if *configSubdir == "" {
		// Note: this wouldn't be required if we had separate config repos per
		// cluster.
		sklog.Fatal("config_subdir is required.")
	}
	var configFileRegexes []*regexp.Regexp
	if len(*configFiles) > 0 {
		if *prune {
			sklog.Fatal("--config_file is incompatible with --prune.")
		}
		for _, expr := range *configFiles {
			re, err := regexp.Compile(expr)
			if err != nil {
				sklog.Fatal(err)
			}
			configFileRegexes = append(configFileRegexes, re)
		}
	}

	ctx := context.Background()

	// OAuth2.0 TokenSource.
	ts, err := google.DefaultTokenSource(ctx, auth.ScopeUserinfoEmail, auth.ScopeGerrit)
	if err != nil {
		sklog.Fatal(err)
	}
	// Authenticated HTTP client.
	httpClient := httputils.DefaultClientConfig().WithTokenSource(ts).With2xxOnly().Client()

	// Gitiles repo.
	repo := gitiles.NewRepo(*configRepo, httpClient)

	// Apply configurations in a loop.  Note that we could respond directly to
	// commits in the repo via GitStore and PubSub, but that would require
	// access to BigTable, and with a relatively small interval we won't notice
	// too much of a delay.
	liveness := metrics2.NewLiveness(livenessMetric)
	go util.RepeatCtx(ctx, *interval, func(ctx context.Context) {
		if err := applyConfigs(ctx, repo, *kubectl, *k8sServer, *configSubdir, configFileRegexes, *prune); err != nil {
			sklog.Errorf("Failed to apply configs to cluster: %s", err)
		} else {
			liveness.Reset()
		}
	})

	// Run health check server.
	httputils.RunHealthCheckServer(*port)
}

func applyConfigs(ctx context.Context, repo *gitiles.Repo, kubectl, k8sServer, configSubdir string, configFileRegexes []*regexp.Regexp, prune bool) error {
	// Download the configs from Gitiles instead of maintaining a local Git
	// checkout, to avoid dealing with Git, persistent checkouts, etc.

	// Obtain the current set of configurations for the cluster.
	head, err := repo.Details(ctx, "main")
	if err != nil {
		return skerr.Wrapf(err, "failed to get most recent commit")
	}
	files, err := repo.ListFilesRecursiveAtRef(ctx, configSubdir, head.Hash)
	if err != nil {
		return skerr.Wrapf(err, "failed to list configs")
	}

	// Read the config contents in the given directory.
	eg := util.NewNamedErrGroup()
	contents := make(map[string][]byte, len(files))
	contentsMtx := sync.Mutex{}
	sklog.Infof("Downloading config files at %s", head.Hash)
	for _, file := range files {
		file := file // https://golang.org/doc/faq#closures_and_goroutines

		// Ensure that the file matches any provided regular expressions.
		if len(configFileRegexes) > 0 {
			match := false
			for _, re := range configFileRegexes {
				if re.MatchString(file) {
					match = true
					break
				}
			}
			if !match {
				continue
			}
		}

		// Download the file contents.
		eg.Go(file, func() error {
			fullPath := path.Join(configSubdir, file)
			sklog.Infof("  %s", fullPath)
			fileContents, err := repo.ReadFileAtRef(ctx, fullPath, head.Hash)
			if err != nil {
				return skerr.Wrapf(err, "failed to retrieve contents of %s", fullPath)
			}
			contentsMtx.Lock()
			defer contentsMtx.Unlock()
			contents[file] = fileContents
			return nil
		})
	}
	if err := eg.Wait(); err != nil {
		return skerr.Wrapf(err, "failed to download configs")
	}

	// Write the config contents to a temporary dir.
	tmp, err := ioutil.TempDir("", "")
	if err != nil {
		return skerr.Wrapf(err, "failed to create temp dir")
	}
	defer util.RemoveAll(tmp)

	for path, fileContents := range contents {
		fullPath := filepath.Join(tmp, path)
		dir := filepath.Dir(fullPath)
		if _, err := os.Stat(path); os.IsNotExist(err) {
			if err := os.MkdirAll(dir, os.ModePerm); err != nil {
				return skerr.Wrapf(err, "failed to create %s", dir)
			}
		}
		if err := ioutil.WriteFile(fullPath, fileContents, os.ModePerm); err != nil {
			return skerr.Wrapf(err, "failed to create %s", fullPath)
		}
	}

	// Apply the configs to the cluster.
	// Note: this is a very naive approach.  We rely on the Kubernetes server to
	// determine when changes actually need to be made.  We could instead use
	// the REST API to load the full set of configuration which is actually
	// running on the server, diff that against the checked-in config files,
	// and then explicitly make only the changes we want to make.  That would be
	// a much more complicated and error-prone approach, but it would allow us
	// to partially apply configurations in the case of a failure and to alert
	// on specific components which fail to apply for whatever reason.
	cmd := []string{kubectl, "apply"}
	if k8sServer != "" {
		cmd = append(cmd, "--server", k8sServer)
	}
	if prune {
		cmd = append(cmd, "--prune", "--all")
	}
	cmd = append(cmd, "-f", ".")
	output, err := exec.RunCwd(ctx, tmp, cmd...)
	if err != nil {
		return skerr.Wrapf(err, "failed to apply configs: %s", output)
	}
	sklog.Info("Output from kubectl")
	for _, line := range strings.Split(output, "\n") {
		sklog.Info(line)
	}
	return nil
}
<|endoftext|>


<commit_before>package kubernetes

import (
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/validation"
)

func persistentVolumeClaimFields() map[string]*schema.Schema {
	return map[string]*schema.Schema{
		"metadata": namespacedMetadataSchema("persistent volume claim", true),
		"spec": {
			Type:        schema.TypeList,
			Description: "Spec defines the desired characteristics of a volume requested by a pod author. More info: http://kubernetes.io/docs/user-guide/persistent-volumes#persistentvolumeclaims",
			Required:    true,
			ForceNew:    true,
			MaxItems:    1,
			Elem: &schema.Resource{
				Schema: persistentVolumeClaimSpecFields(),
			},
		},
	}
}

func persistentVolumeClaimSpecFields() map[string]*schema.Schema {
	return map[string]*schema.Schema{
		"access_modes": {
			Type:        schema.TypeSet,
			Description: "A set of the desired access modes the volume should have. More info: http://kubernetes.io/docs/user-guide/persistent-volumes#access-modes-1",
			Required:    true,
			ForceNew:    true,
			Elem: &schema.Schema{
				Type: schema.TypeString,
				ValidateFunc: validation.StringInSlice([]string{
					"ReadWriteOnce",
					"ReadOnlyMany",
					"ReadWriteMany",
				}, false),
			},
			Set: schema.HashString,
		},
		"resources": {
			Type:        schema.TypeList,
			Description: "A list of the minimum resources the volume should have. More info: http://kubernetes.io/docs/user-guide/persistent-volumes#resources",
			Required:    true,
			ForceNew:    true,
			MaxItems:    1,
			Elem: &schema.Resource{
				Schema: map[string]*schema.Schema{
					"limits": {
						Type:        schema.TypeMap,
						Description: "Map describing the maximum amount of compute resources allowed. More info: http://kubernetes.io/docs/user-guide/compute-resources/",
						Optional:    true,
						ForceNew:    true,
					},
					"requests": {
						Type:        schema.TypeMap,
						Description: "Map describing the minimum amount of compute resources required. If this is omitted for a container, it defaults to `limits` if that is explicitly specified, otherwise to an implementation-defined value. More info: http://kubernetes.io/docs/user-guide/compute-resources/",
						Optional:    true,
						ForceNew:    true,
					},
				},
			},
		},
		"selector": {
			Type:        schema.TypeList,
			Description: "A label query over volumes to consider for binding.",
			Optional:    true,
			ForceNew:    true,
			MaxItems:    1,
			Elem: &schema.Resource{
				Schema: labelSelectorFields(),
			},
		},
		"volume_name": {
			Type:        schema.TypeString,
			Description: "The binding reference to the PersistentVolume backing this claim.",
			Optional:    true,
			ForceNew:    true,
			Computed:    true,
		},
		"storage_class_name": {
			Type:        schema.TypeString,
			Description: "Name of the storage class requested by the claim",
			Optional:    true,
			Computed:    true,
			ForceNew:    true,
		},
	}
}
<commit_msg>Bring back 'wait_until_bound' parameter on PVCs (#232)<commit_after>package kubernetes

import (
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/validation"
)

func persistentVolumeClaimFields() map[string]*schema.Schema {
	return map[string]*schema.Schema{
		"metadata": namespacedMetadataSchema("persistent volume claim", true),
		"spec": {
			Type:        schema.TypeList,
			Description: "Spec defines the desired characteristics of a volume requested by a pod author. More info: http://kubernetes.io/docs/user-guide/persistent-volumes#persistentvolumeclaims",
			Required:    true,
			ForceNew:    true,
			MaxItems:    1,
			Elem: &schema.Resource{
				Schema: persistentVolumeClaimSpecFields(),
			},
		},
		"wait_until_bound": {
			Type:        schema.TypeBool,
			Description: "Whether to wait for the claim to reach `Bound` state (to find volume in which to claim the space)",
			Optional:    true,
			Default:     true,
		},
	}
}

func persistentVolumeClaimSpecFields() map[string]*schema.Schema {
	return map[string]*schema.Schema{
		"access_modes": {
			Type:        schema.TypeSet,
			Description: "A set of the desired access modes the volume should have. More info: http://kubernetes.io/docs/user-guide/persistent-volumes#access-modes-1",
			Required:    true,
			ForceNew:    true,
			Elem: &schema.Schema{
				Type: schema.TypeString,
				ValidateFunc: validation.StringInSlice([]string{
					"ReadWriteOnce",
					"ReadOnlyMany",
					"ReadWriteMany",
				}, false),
			},
			Set: schema.HashString,
		},
		"resources": {
			Type:        schema.TypeList,
			Description: "A list of the minimum resources the volume should have. More info: http://kubernetes.io/docs/user-guide/persistent-volumes#resources",
			Required:    true,
			ForceNew:    true,
			MaxItems:    1,
			Elem: &schema.Resource{
				Schema: map[string]*schema.Schema{
					"limits": {
						Type:        schema.TypeMap,
						Description: "Map describing the maximum amount of compute resources allowed. More info: http://kubernetes.io/docs/user-guide/compute-resources/",
						Optional:    true,
						ForceNew:    true,
					},
					"requests": {
						Type:        schema.TypeMap,
						Description: "Map describing the minimum amount of compute resources required. If this is omitted for a container, it defaults to `limits` if that is explicitly specified, otherwise to an implementation-defined value. More info: http://kubernetes.io/docs/user-guide/compute-resources/",
						Optional:    true,
						ForceNew:    true,
					},
				},
			},
		},
		"selector": {
			Type:        schema.TypeList,
			Description: "A label query over volumes to consider for binding.",
			Optional:    true,
			ForceNew:    true,
			MaxItems:    1,
			Elem: &schema.Resource{
				Schema: labelSelectorFields(),
			},
		},
		"volume_name": {
			Type:        schema.TypeString,
			Description: "The binding reference to the PersistentVolume backing this claim.",
			Optional:    true,
			ForceNew:    true,
			Computed:    true,
		},
		"storage_class_name": {
			Type:        schema.TypeString,
			Description: "Name of the storage class requested by the claim",
			Optional:    true,
			Computed:    true,
			ForceNew:    true,
		},
	}
}
<|endoftext|>


<commit_before>package filer2

import (
	"context"
	"fmt"
	"io"
	"strings"
	"time"

	"github.com/golang/protobuf/proto"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/notification"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func (f *Filer) NotifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool) {
	var fullpath string
	if oldEntry != nil {
		fullpath = string(oldEntry.FullPath)
	} else if newEntry != nil {
		fullpath = string(newEntry.FullPath)
	} else {
		return
	}

	// println("fullpath:", fullpath)

	if strings.HasPrefix(fullpath, SystemLogDir) {
		return
	}

	newParentPath := ""
	if newEntry != nil {
		newParentPath, _ = newEntry.FullPath.DirAndName()
	}
	eventNotification := &filer_pb.EventNotification{
		OldEntry:           oldEntry.ToProtoEntry(),
		NewEntry:           newEntry.ToProtoEntry(),
		DeleteChunks:       deleteChunks,
		NewParentPath:      newParentPath,
		IsFromOtherCluster: isFromOtherCluster,
	}

	if notification.Queue != nil {
		glog.V(3).Infof("notifying entry update %v", fullpath)
		notification.Queue.SendMessage(fullpath, eventNotification)
	}

	f.logMetaEvent(ctx, fullpath, eventNotification)

}

func (f *Filer) logMetaEvent(ctx context.Context, fullpath string, eventNotification *filer_pb.EventNotification) {

	dir, _ := util.FullPath(fullpath).DirAndName()

	event := &filer_pb.SubscribeMetadataResponse{
		Directory:         dir,
		EventNotification: eventNotification,
		TsNs:              time.Now().UnixNano(),
	}
	data, err := proto.Marshal(event)
	if err != nil {
		glog.Errorf("failed to marshal filer_pb.SubscribeMetadataResponse %+v: %v", event, err)
		return
	}

	f.LocalMetaLogBuffer.AddToBuffer([]byte(dir), data)

}

func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) {

	targetFile := fmt.Sprintf("%s/%04d-%02d-%02d/%02d-%02d.segment", SystemLogDir,
		startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(),
		// startTime.Second(), startTime.Nanosecond(),
	)

	if err := f.appendToFile(targetFile, buf); err != nil {
		glog.V(0).Infof("log write failed %s: %v", targetFile, err)
	}
}

func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, err error) {

	startDate := fmt.Sprintf("%04d-%02d-%02d", startTime.Year(), startTime.Month(), startTime.Day())
	startHourMinute := fmt.Sprintf("%02d-%02d.segment", startTime.Hour(), startTime.Minute())

	sizeBuf := make([]byte, 4)
	startTsNs := startTime.UnixNano()

	dayEntries, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366)
	if listDayErr != nil {
		return lastTsNs, fmt.Errorf("fail to list log by day: %v", listDayErr)
	}
	for _, dayEntry := range dayEntries {
		// println("checking day", dayEntry.FullPath)
		hourMinuteEntries, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60)
		if listHourMinuteErr != nil {
			return lastTsNs, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr)
		}
		for _, hourMinuteEntry := range hourMinuteEntries {
			// println("checking hh-mm", hourMinuteEntry.FullPath)
			if dayEntry.Name() == startDate {
				if strings.Compare(hourMinuteEntry.Name(), startHourMinute) < 0 {
					continue
				}
			}
			// println("processing", hourMinuteEntry.FullPath)
			chunkedFileReader := NewChunkStreamReaderFromFiler(f.MasterClient, hourMinuteEntry.Chunks)
			if lastTsNs, err = ReadEachLogEntry(chunkedFileReader, sizeBuf, startTsNs, eachLogEntryFn); err != nil {
				chunkedFileReader.Close()
				if err == io.EOF {
					break
				}
				return lastTsNs, fmt.Errorf("reading %s: %v", hourMinuteEntry.FullPath, err)
			}
			chunkedFileReader.Close()
		}
	}

	return lastTsNs, nil
}

func ReadEachLogEntry(r io.Reader, sizeBuf []byte, ns int64, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, err error) {
	for {
		n, err := r.Read(sizeBuf)
		if err != nil {
			return lastTsNs, err
		}
		if n != 4 {
			return lastTsNs, fmt.Errorf("size %d bytes, expected 4 bytes", n)
		}
		size := util.BytesToUint32(sizeBuf)
		// println("entry size", size)
		entryData := make([]byte, size)
		n, err = r.Read(entryData)
		if err != nil {
			return lastTsNs, err
		}
		if n != int(size) {
			return lastTsNs, fmt.Errorf("entry data %d bytes, expected %d bytes", n, size)
		}
		logEntry := &filer_pb.LogEntry{}
		if err = proto.Unmarshal(entryData, logEntry); err != nil {
			return lastTsNs, err
		}
		if logEntry.TsNs <= ns {
			return lastTsNs, nil
		}
		// println("each log: ", logEntry.TsNs)
		if err := eachLogEntryFn(logEntry); err != nil {
			return lastTsNs, err
		} else {
			lastTsNs = logEntry.TsNs
		}
	}
}
<commit_msg>filer: ensure logs are persisted<commit_after>package filer2

import (
	"context"
	"fmt"
	"io"
	"strings"
	"time"

	"github.com/golang/protobuf/proto"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/notification"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func (f *Filer) NotifyUpdateEvent(ctx context.Context, oldEntry, newEntry *Entry, deleteChunks, isFromOtherCluster bool) {
	var fullpath string
	if oldEntry != nil {
		fullpath = string(oldEntry.FullPath)
	} else if newEntry != nil {
		fullpath = string(newEntry.FullPath)
	} else {
		return
	}

	// println("fullpath:", fullpath)

	if strings.HasPrefix(fullpath, SystemLogDir) {
		return
	}

	newParentPath := ""
	if newEntry != nil {
		newParentPath, _ = newEntry.FullPath.DirAndName()
	}
	eventNotification := &filer_pb.EventNotification{
		OldEntry:           oldEntry.ToProtoEntry(),
		NewEntry:           newEntry.ToProtoEntry(),
		DeleteChunks:       deleteChunks,
		NewParentPath:      newParentPath,
		IsFromOtherCluster: isFromOtherCluster,
	}

	if notification.Queue != nil {
		glog.V(3).Infof("notifying entry update %v", fullpath)
		notification.Queue.SendMessage(fullpath, eventNotification)
	}

	f.logMetaEvent(ctx, fullpath, eventNotification)

}

func (f *Filer) logMetaEvent(ctx context.Context, fullpath string, eventNotification *filer_pb.EventNotification) {

	dir, _ := util.FullPath(fullpath).DirAndName()

	event := &filer_pb.SubscribeMetadataResponse{
		Directory:         dir,
		EventNotification: eventNotification,
		TsNs:              time.Now().UnixNano(),
	}
	data, err := proto.Marshal(event)
	if err != nil {
		glog.Errorf("failed to marshal filer_pb.SubscribeMetadataResponse %+v: %v", event, err)
		return
	}

	f.LocalMetaLogBuffer.AddToBuffer([]byte(dir), data)

}

func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) {

	targetFile := fmt.Sprintf("%s/%04d-%02d-%02d/%02d-%02d.segment", SystemLogDir,
		startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(),
		// startTime.Second(), startTime.Nanosecond(),
	)

	for {
		if err := f.appendToFile(targetFile, buf); err != nil {
			glog.V(1).Infof("log write failed %s: %v", targetFile, err)
			time.Sleep(737 * time.Millisecond)
		} else {
			break
		}
	}
}

func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, err error) {

	startDate := fmt.Sprintf("%04d-%02d-%02d", startTime.Year(), startTime.Month(), startTime.Day())
	startHourMinute := fmt.Sprintf("%02d-%02d.segment", startTime.Hour(), startTime.Minute())

	sizeBuf := make([]byte, 4)
	startTsNs := startTime.UnixNano()

	dayEntries, listDayErr := f.ListDirectoryEntries(context.Background(), SystemLogDir, startDate, true, 366)
	if listDayErr != nil {
		return lastTsNs, fmt.Errorf("fail to list log by day: %v", listDayErr)
	}
	for _, dayEntry := range dayEntries {
		// println("checking day", dayEntry.FullPath)
		hourMinuteEntries, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60)
		if listHourMinuteErr != nil {
			return lastTsNs, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr)
		}
		for _, hourMinuteEntry := range hourMinuteEntries {
			// println("checking hh-mm", hourMinuteEntry.FullPath)
			if dayEntry.Name() == startDate {
				if strings.Compare(hourMinuteEntry.Name(), startHourMinute) < 0 {
					continue
				}
			}
			// println("processing", hourMinuteEntry.FullPath)
			chunkedFileReader := NewChunkStreamReaderFromFiler(f.MasterClient, hourMinuteEntry.Chunks)
			if lastTsNs, err = ReadEachLogEntry(chunkedFileReader, sizeBuf, startTsNs, eachLogEntryFn); err != nil {
				chunkedFileReader.Close()
				if err == io.EOF {
					break
				}
				return lastTsNs, fmt.Errorf("reading %s: %v", hourMinuteEntry.FullPath, err)
			}
			chunkedFileReader.Close()
		}
	}

	return lastTsNs, nil
}

func ReadEachLogEntry(r io.Reader, sizeBuf []byte, ns int64, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, err error) {
	for {
		n, err := r.Read(sizeBuf)
		if err != nil {
			return lastTsNs, err
		}
		if n != 4 {
			return lastTsNs, fmt.Errorf("size %d bytes, expected 4 bytes", n)
		}
		size := util.BytesToUint32(sizeBuf)
		// println("entry size", size)
		entryData := make([]byte, size)
		n, err = r.Read(entryData)
		if err != nil {
			return lastTsNs, err
		}
		if n != int(size) {
			return lastTsNs, fmt.Errorf("entry data %d bytes, expected %d bytes", n, size)
		}
		logEntry := &filer_pb.LogEntry{}
		if err = proto.Unmarshal(entryData, logEntry); err != nil {
			return lastTsNs, err
		}
		if logEntry.TsNs <= ns {
			return lastTsNs, nil
		}
		// println("each log: ", logEntry.TsNs)
		if err := eachLogEntryFn(logEntry); err != nil {
			return lastTsNs, err
		} else {
			lastTsNs = logEntry.TsNs
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Tobias Schottdorf

package terrafarm

import (
	"bufio"
	"bytes"
	"fmt"
	"os/exec"
	"strings"
)

func (f *Farmer) run(cmd string, args ...string) (_ string, _ string, err error) {
	c := exec.Command(cmd, args...)
	c.Dir = f.Cwd
	var outBuf, errBuf bytes.Buffer
	{
		p, err := c.StdinPipe()
		if err != nil {
			return "", "", err
		}
		_ = p.Close() // no input
	}
	f.logf("+ %s %s\n", cmd, strings.Join(args, " "))
	o, err := c.StdoutPipe()
	if err != nil {
		return "", "", err
	}
	e, err := c.StderrPipe()
	if err != nil {
		return "", "", err
	}
	scanO := bufio.NewScanner(o)
	scanE := bufio.NewScanner(e)
	go func() {
		for scanO.Scan() {
			line := scanO.Text() + "\n"
			_, _ = outBuf.WriteString(line)
			f.logf("%s", line)
		}
	}()
	go func() {
		for scanE.Scan() {
			line := scanE.Text() + "\n"
			_, _ = errBuf.WriteString(line)
			f.logf("%s", line)
		}
	}()

	if err := c.Run(); err != nil {
		return "", "", err
	}
	return outBuf.String(), errBuf.String(), err
}

func (f *Farmer) runErr(cmd string, args ...string) error {
	o, e, err := f.run(cmd, args...)
	if err != nil {
		return fmt.Errorf("failed: %s\nstdout: %s\nstderr: %s", o, e, err)
	}
	return nil
}

func (f *Farmer) appendDefaults(args []string) []string {
	return append(
		args,
		"-no-color",
		"-var=key_name="+f.KeyName,
		"-state="+f.StateFile,
		"-var=prefix="+f.Prefix)
}

func (f *Farmer) apply(args ...string) error {
	args = f.appendDefaults(append([]string{"apply"}, args...))
	if err := f.runErr("terraform", args...); err != nil {
		return err
	}
	f.refresh()
	return nil
}

func (f *Farmer) output(key string) []string {
	o, _, err := f.run("terraform", "output", "-state="+f.StateFile, "-no-color", key)
	if _, ok := err.(*exec.ExitError); err != nil && !ok {
		f.logf("%s", err)
		return nil
	}
	o = strings.TrimSpace(o)
	if len(o) == 0 {
		return nil
	}
	return strings.Split(o, ",")
}

func (f *Farmer) execSupervisor(host string, action string) (string, string, error) {
	cmd := "supervisorctl -c supervisor.conf " + action
	return f.ssh(host, f.defaultKeyFile(), cmd)
}
<commit_msg>acceptance: consistently apply a newline when logging<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Tobias Schottdorf

package terrafarm

import (
	"bufio"
	"bytes"
	"fmt"
	"os/exec"
	"strings"
)

func (f *Farmer) run(cmd string, args ...string) (_ string, _ string, err error) {
	c := exec.Command(cmd, args...)
	c.Dir = f.Cwd
	var outBuf, errBuf bytes.Buffer
	{
		p, err := c.StdinPipe()
		if err != nil {
			return "", "", err
		}
		_ = p.Close() // no input
	}
	f.logf("+ %s %s\n", cmd, strings.Join(args, " "))
	o, err := c.StdoutPipe()
	if err != nil {
		return "", "", err
	}
	e, err := c.StderrPipe()
	if err != nil {
		return "", "", err
	}
	scanO := bufio.NewScanner(o)
	scanE := bufio.NewScanner(e)
	go func() {
		for scanO.Scan() {
			line := scanO.Text() + "\n"
			_, _ = outBuf.WriteString(line)
			f.logf("%s", line)
		}
	}()
	go func() {
		for scanE.Scan() {
			line := scanE.Text() + "\n"
			_, _ = errBuf.WriteString(line)
			f.logf("%s", line)
		}
	}()

	if err := c.Run(); err != nil {
		return "", "", err
	}
	return outBuf.String(), errBuf.String(), err
}

func (f *Farmer) runErr(cmd string, args ...string) error {
	o, e, err := f.run(cmd, args...)
	if err != nil {
		return fmt.Errorf("failed: %s\nstdout: %s\nstderr: %s", o, e, err)
	}
	return nil
}

func (f *Farmer) appendDefaults(args []string) []string {
	return append(
		args,
		"-no-color",
		"-var=key_name="+f.KeyName,
		"-state="+f.StateFile,
		"-var=prefix="+f.Prefix)
}

func (f *Farmer) apply(args ...string) error {
	args = f.appendDefaults(append([]string{"apply"}, args...))
	if err := f.runErr("terraform", args...); err != nil {
		return err
	}
	f.refresh()
	return nil
}

func (f *Farmer) output(key string) []string {
	o, _, err := f.run("terraform", "output", "-state="+f.StateFile, "-no-color", key)
	if _, ok := err.(*exec.ExitError); err != nil && !ok {
		f.logf("%s\n", err)
		return nil
	}
	o = strings.TrimSpace(o)
	if len(o) == 0 {
		return nil
	}
	return strings.Split(o, ",")
}

func (f *Farmer) execSupervisor(host string, action string) (string, string, error) {
	cmd := "supervisorctl -c supervisor.conf " + action
	return f.ssh(host, f.defaultKeyFile(), cmd)
}
<|endoftext|>


<commit_before>package transition

import (
	. "yap/alg"
	"yap/alg/graph"
	. "yap/alg/transition"
	nlp "yap/nlp/types"
	"yap/util"

	"fmt"
	// "log"
	// "reflect"
	"strings"
	// "sync"
)

type SimpleConfiguration struct {
	InternalStack    Stack
	InternalQueue    Queue
	InternalArcs     ArcSet
	Nodes            []*ArcCachedDepNode
	InternalPrevious *SimpleConfiguration
	Last             Transition
	lastOpAssignment uint16
	// Pointers                                           int
	EWord, EPOS, EWPOS, EMHost, EMSuffix, ERel, ETrans *util.EnumSet
	// test zpar parity
	NumHeadStack int
}

func (c *SimpleConfiguration) State() byte {
	return TransitionType
}
func (c *SimpleConfiguration) Graph() nlp.LabeledDependencyGraph {
	return nlp.LabeledDependencyGraph(c)
}

// Verify that SimpleConfiguration is a Configuration
var _ DependencyConfiguration = &SimpleConfiguration{}
var _ nlp.DependencyGraph = &SimpleConfiguration{}

func (c *SimpleConfiguration) ID() int {
	return 0
}

func (c *SimpleConfiguration) Init(abstractSentence interface{}) {
	sent := abstractSentence.(nlp.EnumTaggedSentence)
	// var exists bool
	sentLength := len(sent.TaggedTokens())
	// Nodes is always the same slice to the same token array
	c.Nodes = make([]*ArcCachedDepNode, 0, sentLength)
	for i, enumToken := range sent.EnumTaggedTokens() {
		node := &TaggedDepNode{
			i,
			enumToken.EToken,
			enumToken.EPOS,
			enumToken.ETPOS,
			enumToken.EMHost,
			enumToken.EMSuffix,
			enumToken.Token,
			enumToken.Lemma,
			enumToken.POS,
		}
		c.Nodes = append(c.Nodes, NewArcCachedDepNode(nlp.DepNode(node)))
	}

	c.InternalStack = NewStackArray(sentLength)
	c.InternalQueue = NewQueueSlice(sentLength)
	// c.InternalQueue = NewStackArray(sentLength)

	c.InternalArcs = NewArcSetSimple(sentLength)

	// push index of ROOT node to Stack
	// c.Stack().Push(0) // TODO: note switch to zpar's PopRoot

	// push indexes of statement nodes to Queue, in reverse order (first word at the top of the queue)
	// for i := 0; i < sentLength; i++ {
	// 	c.Queue().Enqueue(i)
	// }
	for i := 0; i < sentLength; i++ {
		c.Queue().Enqueue(i)
	}
	// explicit resetting of zero-valued properties
	// in case of reuse
	c.Last = ConstTransition(0)
	c.InternalPrevious = nil
	c.NumHeadStack = 0
	// c.Pointers = 0
}

func (c *SimpleConfiguration) Clear() {
	// c.Lock()
	// defer c.Unlock()
	// if c.Pointers > 0 {
	// 	return
	// }
	// c.InternalStack = nil
	// c.InternalQueue = nil
	// c.InternalArcs = nil
	// if c.InternalPrevious != nil {
	// 	c.InternalPrevious.DecrementPointers()
	// 	c.InternalPrevious.Clear()
	// 	c.InternalPrevious = nil
	// }

}

func (c *SimpleConfiguration) Terminal() bool {
	// return c.Queue().Size() == 0 && c.Stack().Size() == 0
	// return c.Queue().Size() == 0 && c.Stack().Size() == 1
	return c.Queue().Size() == 0
}

func (c *SimpleConfiguration) Stack() Stack {
	return c.InternalStack
}

func (c *SimpleConfiguration) Queue() Queue {
	return c.InternalQueue
}

func (c *SimpleConfiguration) Arcs() ArcSet {
	return c.InternalArcs
}
func (c *SimpleConfiguration) Copy() Configuration {
	newConf := new(SimpleConfiguration)
	c.CopyTo(newConf)
	return newConf
}

func (c *SimpleConfiguration) CopyTo(target Configuration) {
	newConf, ok := target.(*SimpleConfiguration)
	if !ok {
		panic("Can't copy into non *SimpleConfiguration")
	}

	if c.Stack() != nil {
		newConf.InternalStack = c.Stack().Copy()
	}
	if c.Queue() != nil {
		newConf.InternalQueue = c.Queue().Copy()
	}
	if c.Arcs() != nil {
		newConf.InternalArcs = c.Arcs().Copy()
	}
	newConf.Nodes = make([]*ArcCachedDepNode, len(c.Nodes), cap(c.Nodes))
	copy(newConf.Nodes[0:len(c.Nodes)], c.Nodes[0:len(c.Nodes)])

	newConf.Last = c.Last
	newConf.NumHeadStack = c.NumHeadStack
	// store a pointer to the previous configuration
	newConf.InternalPrevious = c

	newConf.EWord, newConf.EPOS, newConf.EWPOS, newConf.ERel, newConf.ETrans, newConf.EMHost, newConf.EMSuffix = c.EWord, c.EPOS, c.EWPOS, c.ERel, c.ETrans, c.EMHost, c.EMSuffix
}

func (c *SimpleConfiguration) AddArc(arc *BasicDepArc) {
	c.Arcs().Add(arc)
	if c.Nodes[arc.Modifier].ELabel >= 0 {
		panic("Tried to change the label of a labeled node")
	}
	c.Nodes[arc.Modifier] = c.Nodes[arc.Modifier].Copy()
	c.Nodes[arc.Modifier].Head = arc.Head
	c.Nodes[arc.Modifier].ELabel = arc.Relation
	c.Nodes[arc.Modifier].ArcId = c.Arcs().Size() - 1
	c.Nodes[arc.Head] = c.Nodes[arc.Head].Copy()
	c.Nodes[arc.Head].AddModifier(arc.Modifier, arc.Relation, c.Nodes[arc.Modifier].Node.(*TaggedDepNode).POS)
}

func (c *SimpleConfiguration) Equal(otherEq util.Equaler) bool {
	// log.Println("Testing equality for")
	// log.Println("\t", c)
	// log.Println("\t", otherEq)
	if (otherEq == nil && c != nil) || (c == nil && otherEq != nil) {
		return false
	}
	switch other := otherEq.(type) {
	case *SimpleConfiguration:
		if (other == nil && c != nil) || (c == nil && other != nil) {
			return false
		}
		if !other.Last.Equal(c.Last) {
			return false
		}
		if c.InternalPrevious == nil && other.InternalPrevious == nil {
			return true
		}
		if c.InternalPrevious != nil && other.InternalPrevious != nil {
			return c.Previous().Equal(other.Previous())
		} else {
			return false
		}
		// return other.Last == c.Last &&
		// 	((c.InternalPrevious == nil && other.InternalPrevious == nil) ||
		// 		(c.InternalPrevious != nil && other.InternalPrevious != nil && c.Previous().Equal(other.Previous())))

		// return c.NumberOfArcs() == other.NumberOfArcs() &&
		// 	c.NumberOfNodes() == other.NumberOfNodes() &&
		// 	c.Stack().Equal(other.Stack()) &&
		// 	c.Queue().Equal(other.Queue()) &&
		// 	c.Arcs().Equal(other.Arcs()) &&
		// 	reflect.DeepEqual(c.Nodes, other.Nodes)
	case *BasicDepGraph:
		return other.Equal(c)
	}
	return false
}

func (c *SimpleConfiguration) Previous() Configuration {
	return c.InternalPrevious
}

func (c *SimpleConfiguration) SetPrevious(prev Configuration) {
	c.InternalPrevious = prev.(*SimpleConfiguration)
}

func (c *SimpleConfiguration) SetLastTransition(t Transition) {
	c.Last = t
}

func (c *SimpleConfiguration) GetLastTransition() Transition {
	return Transition(c.Last)
}

func (c *SimpleConfiguration) GetSequence() ConfigurationSequence {
	if c.Arcs() == nil {
		return make(ConfigurationSequence, 0)
	}
	retval := make(ConfigurationSequence, 0, c.Arcs().Size())
	currentConf := c
	for currentConf != nil {
		retval = append(retval, currentConf)
		currentConf = currentConf.InternalPrevious
	}
	return retval
}

// GRAPH FUNCTIONS
func (c *SimpleConfiguration) GetVertices() []int {
	return util.RangeInt(len(c.Nodes))
}

func (c *SimpleConfiguration) GetEdges() []int {
	// the + 1 is because there may be a missing edge, therefore an ID may be skipped over
	// leaving an off-by-one for the last edge ID
	return util.RangeInt(c.Arcs().Size() + 1)
}

func (c *SimpleConfiguration) GetVertex(vertexID int) graph.Vertex {
	return graph.Vertex(c.Nodes[vertexID])
}

func (c *SimpleConfiguration) GetEdge(edgeID int) graph.Edge {
	arcPtr := c.Arcs().Index(edgeID)
	return graph.Edge(arcPtr)
}

func (c *SimpleConfiguration) GetDirectedEdge(edgeID int) graph.DirectedEdge {
	arcPtr := c.Arcs().Index(edgeID)
	return graph.DirectedEdge(arcPtr)
}

func (c *SimpleConfiguration) NumberOfVertices() int {
	return c.NumberOfNodes()
}

func (c *SimpleConfiguration) NumberOfEdges() int {
	return c.NumberOfArcs()
}

func (c *SimpleConfiguration) NumberOfNodes() int {
	return len(c.Nodes)
}

func (c *SimpleConfiguration) NumberOfArcs() int {
	return c.Arcs().Size()
}

func (c *SimpleConfiguration) GetNode(nodeID int) nlp.DepNode {
	return c.Nodes[nodeID].Node
}

func (c *SimpleConfiguration) GetRawNode(nodeID int) *TaggedDepNode {
	return c.Nodes[nodeID].Node.(*TaggedDepNode)
}

func (c *SimpleConfiguration) GetArc(nodeID int) nlp.DepArc {
	if c.Nodes[nodeID].ArcId > -1 {
		arcPtr := c.Arcs().Index(c.Nodes[nodeID].ArcId)
		return nlp.DepArc(arcPtr)
	} else {
		return nil
	}
}

func (c *SimpleConfiguration) GetLabeledArc(nodeID int) nlp.LabeledDepArc {
	if nodeID < len(c.Nodes) && c.Nodes[nodeID].ArcId > -1 {
		arcPtr := c.Arcs().Index(c.Nodes[nodeID].ArcId)
		return nlp.LabeledDepArc(arcPtr)
	} else {
		return nil
	}
}

// OUTPUT FUNCTIONS

func (c *SimpleConfiguration) String() string {
	var (
		transitionVal string = ""
		transInt      int    = c.Last.Value()
	)
	if transInt >= 0 {
		transitionVal = c.ETrans.ValueOf(transInt).(string)
	}
	return fmt.Sprintf("%s\t=>([%s],\t[%s],\t%s)",
		transitionVal, c.StringStack(), c.StringQueue(),
		c.StringArcs())
}

func (c *SimpleConfiguration) StringStack() string {
	stackSize := c.Stack().Size()
	switch {
	case stackSize > 0 && stackSize <= 3:
		var stackStrings []string = make([]string, 0, 3)
		for i := c.Stack().Size() - 1; i >= 0; i-- {
			atI, _ := c.Stack().Index(i)
			stackStrings = append(stackStrings, c.GetRawNode(atI).RawToken)
		}
		return strings.Join(stackStrings, ",")
	case stackSize > 3:
		headID, _ := c.Stack().Index(0)
		tailID, _ := c.Stack().Index(c.Stack().Size() - 1)
		head := c.GetRawNode(headID)
		tail := c.GetRawNode(tailID)
		return strings.Join([]string{tail.RawToken, "...", head.RawToken}, ",")
	default:
		return ""
	}
}

func (c *SimpleConfiguration) StringQueue() string {
	queueSize := c.Queue().Size()
	switch {
	case queueSize > 0 && queueSize <= 3:
		var queueStrings []string = make([]string, 0, 3)
		for i := 0; i < c.Queue().Size(); i++ {
			atI, _ := c.Queue().Index(i)
			queueStrings = append(queueStrings, c.GetRawNode(atI).RawToken)
		}
		return strings.Join(queueStrings, ",")
	case queueSize > 3:
		headID, _ := c.Queue().Index(0)
		tailID, _ := c.Queue().Index(c.Queue().Size() - 1)
		head := c.GetRawNode(headID)
		tail := c.GetRawNode(tailID)
		return strings.Join([]string{head.RawToken, "...", tail.RawToken}, ",")
	default:
		return ""
	}
}

func (c *SimpleConfiguration) StringArcs() string {
	var transInt int = c.Last.Value()
	if transInt < 0 {
		return ""
	}
	last := c.ETrans.ValueOf(transInt).(string)
	if len(last) < 2 {
		return fmt.Sprintf("A%d", c.Arcs().Size())
	}
	switch last[:2] {
	case "LA", "RA":
		lastArc := c.Arcs().Last()
		head := c.GetRawNode(lastArc.GetHead())
		mod := c.GetRawNode(lastArc.GetModifier())
		arcStr := fmt.Sprintf("(%s,%s,%s)", head.RawToken, lastArc.GetRelation().String(), mod.RawToken)
		return fmt.Sprintf("A%d=A%d+{%s}", c.Arcs().Size(), c.Arcs().Size()-1, arcStr)
	default:
		return fmt.Sprintf("A%d", c.Arcs().Size())
	}
}

func (c *SimpleConfiguration) StringGraph() string {
	return fmt.Sprintf("%v %v", c.Nodes, c.InternalArcs)
}

func (c *SimpleConfiguration) Sentence() nlp.Sentence {
	return nlp.Sentence(c.TaggedSentence())
}

func (c *SimpleConfiguration) TaggedSentence() nlp.TaggedSentence {
	var sent nlp.BasicETaggedSentence = make([]nlp.EnumTaggedToken, c.NumberOfNodes()-1)
	for i, _ := range c.Nodes {
		taggedNode := c.GetRawNode(i)
		sent[i] = nlp.EnumTaggedToken{
			nlp.TaggedToken{taggedNode.RawToken, taggedNode.RawLemma, taggedNode.RawPOS},
			// TODO: add lemma enum
			taggedNode.Token, 0, taggedNode.POS, taggedNode.TokenPOS, taggedNode.MHost, taggedNode.MSuffix}
	}
	return sent
}

func (c *SimpleConfiguration) Len() int {
	if c == nil {
		return 0
	}
	if c.Previous() != nil {
		return 1 + c.Previous().Len()
	} else {
		return 1
	}
}

func (c *SimpleConfiguration) Assignment() uint16 {
	return c.lastOpAssignment
}

func (c *SimpleConfiguration) Assign(to uint16) {
	c.lastOpAssignment = to
}

func NewSimpleConfiguration() Configuration {
	return Configuration(new(SimpleConfiguration))
}
<commit_msg>Fix terminal condition for dependency configuration to match Arc (Z)Eager<commit_after>package transition

import (
	. "yap/alg"
	"yap/alg/graph"
	. "yap/alg/transition"
	nlp "yap/nlp/types"
	"yap/util"

	"fmt"
	// "log"
	// "reflect"
	"strings"
	// "sync"
)

type SimpleConfiguration struct {
	InternalStack    Stack
	InternalQueue    Queue
	InternalArcs     ArcSet
	Nodes            []*ArcCachedDepNode
	InternalPrevious *SimpleConfiguration
	Last             Transition
	lastOpAssignment uint16
	// Pointers                                           int
	EWord, EPOS, EWPOS, EMHost, EMSuffix, ERel, ETrans *util.EnumSet
	// test zpar parity
	NumHeadStack int
}

func (c *SimpleConfiguration) State() byte {
	return TransitionType
}
func (c *SimpleConfiguration) Graph() nlp.LabeledDependencyGraph {
	return nlp.LabeledDependencyGraph(c)
}

// Verify that SimpleConfiguration is a Configuration
var _ DependencyConfiguration = &SimpleConfiguration{}
var _ nlp.DependencyGraph = &SimpleConfiguration{}

func (c *SimpleConfiguration) ID() int {
	return 0
}

func (c *SimpleConfiguration) Init(abstractSentence interface{}) {
	sent := abstractSentence.(nlp.EnumTaggedSentence)
	// var exists bool
	sentLength := len(sent.TaggedTokens())
	// Nodes is always the same slice to the same token array
	c.Nodes = make([]*ArcCachedDepNode, 0, sentLength)
	for i, enumToken := range sent.EnumTaggedTokens() {
		node := &TaggedDepNode{
			i,
			enumToken.EToken,
			enumToken.EPOS,
			enumToken.ETPOS,
			enumToken.EMHost,
			enumToken.EMSuffix,
			enumToken.Token,
			enumToken.Lemma,
			enumToken.POS,
		}
		c.Nodes = append(c.Nodes, NewArcCachedDepNode(nlp.DepNode(node)))
	}

	c.InternalStack = NewStackArray(sentLength)
	c.InternalQueue = NewQueueSlice(sentLength)
	// c.InternalQueue = NewStackArray(sentLength)

	c.InternalArcs = NewArcSetSimple(sentLength)

	// push index of ROOT node to Stack
	// c.Stack().Push(0) // TODO: note switch to zpar's PopRoot

	// push indexes of statement nodes to Queue, in reverse order (first word at the top of the queue)
	// for i := 0; i < sentLength; i++ {
	// 	c.Queue().Enqueue(i)
	// }
	for i := 0; i < sentLength; i++ {
		c.Queue().Enqueue(i)
	}
	// explicit resetting of zero-valued properties
	// in case of reuse
	c.Last = ConstTransition(0)
	c.InternalPrevious = nil
	c.NumHeadStack = 0
	// c.Pointers = 0
}

func (c *SimpleConfiguration) Clear() {
	// c.Lock()
	// defer c.Unlock()
	// if c.Pointers > 0 {
	// 	return
	// }
	// c.InternalStack = nil
	// c.InternalQueue = nil
	// c.InternalArcs = nil
	// if c.InternalPrevious != nil {
	// 	c.InternalPrevious.DecrementPointers()
	// 	c.InternalPrevious.Clear()
	// 	c.InternalPrevious = nil
	// }

}

func (c *SimpleConfiguration) Terminal() bool {
	return c.Queue().Size() == 0 && c.Stack().Size() == 0
	// return c.Queue().Size() == 0 && c.Stack().Size() == 1
	// return c.Queue().Size() == 0
}

func (c *SimpleConfiguration) Stack() Stack {
	return c.InternalStack
}

func (c *SimpleConfiguration) Queue() Queue {
	return c.InternalQueue
}

func (c *SimpleConfiguration) Arcs() ArcSet {
	return c.InternalArcs
}
func (c *SimpleConfiguration) Copy() Configuration {
	newConf := new(SimpleConfiguration)
	c.CopyTo(newConf)
	return newConf
}

func (c *SimpleConfiguration) CopyTo(target Configuration) {
	newConf, ok := target.(*SimpleConfiguration)
	if !ok {
		panic("Can't copy into non *SimpleConfiguration")
	}

	if c.Stack() != nil {
		newConf.InternalStack = c.Stack().Copy()
	}
	if c.Queue() != nil {
		newConf.InternalQueue = c.Queue().Copy()
	}
	if c.Arcs() != nil {
		newConf.InternalArcs = c.Arcs().Copy()
	}
	newConf.Nodes = make([]*ArcCachedDepNode, len(c.Nodes), cap(c.Nodes))
	copy(newConf.Nodes[0:len(c.Nodes)], c.Nodes[0:len(c.Nodes)])

	newConf.Last = c.Last
	newConf.NumHeadStack = c.NumHeadStack
	// store a pointer to the previous configuration
	newConf.InternalPrevious = c

	newConf.EWord, newConf.EPOS, newConf.EWPOS, newConf.ERel, newConf.ETrans, newConf.EMHost, newConf.EMSuffix = c.EWord, c.EPOS, c.EWPOS, c.ERel, c.ETrans, c.EMHost, c.EMSuffix
}

func (c *SimpleConfiguration) AddArc(arc *BasicDepArc) {
	c.Arcs().Add(arc)
	if c.Nodes[arc.Modifier].ELabel >= 0 {
		panic("Tried to change the label of a labeled node")
	}
	c.Nodes[arc.Modifier] = c.Nodes[arc.Modifier].Copy()
	c.Nodes[arc.Modifier].Head = arc.Head
	c.Nodes[arc.Modifier].ELabel = arc.Relation
	c.Nodes[arc.Modifier].ArcId = c.Arcs().Size() - 1
	c.Nodes[arc.Head] = c.Nodes[arc.Head].Copy()
	c.Nodes[arc.Head].AddModifier(arc.Modifier, arc.Relation, c.Nodes[arc.Modifier].Node.(*TaggedDepNode).POS)
}

func (c *SimpleConfiguration) Equal(otherEq util.Equaler) bool {
	// log.Println("Testing equality for")
	// log.Println("\t", c)
	// log.Println("\t", otherEq)
	if (otherEq == nil && c != nil) || (c == nil && otherEq != nil) {
		return false
	}
	switch other := otherEq.(type) {
	case *SimpleConfiguration:
		if (other == nil && c != nil) || (c == nil && other != nil) {
			return false
		}
		if !other.Last.Equal(c.Last) {
			return false
		}
		if c.InternalPrevious == nil && other.InternalPrevious == nil {
			return true
		}
		if c.InternalPrevious != nil && other.InternalPrevious != nil {
			return c.Previous().Equal(other.Previous())
		} else {
			return false
		}
		// return other.Last == c.Last &&
		// 	((c.InternalPrevious == nil && other.InternalPrevious == nil) ||
		// 		(c.InternalPrevious != nil && other.InternalPrevious != nil && c.Previous().Equal(other.Previous())))

		// return c.NumberOfArcs() == other.NumberOfArcs() &&
		// 	c.NumberOfNodes() == other.NumberOfNodes() &&
		// 	c.Stack().Equal(other.Stack()) &&
		// 	c.Queue().Equal(other.Queue()) &&
		// 	c.Arcs().Equal(other.Arcs()) &&
		// 	reflect.DeepEqual(c.Nodes, other.Nodes)
	case *BasicDepGraph:
		return other.Equal(c)
	}
	return false
}

func (c *SimpleConfiguration) Previous() Configuration {
	return c.InternalPrevious
}

func (c *SimpleConfiguration) SetPrevious(prev Configuration) {
	c.InternalPrevious = prev.(*SimpleConfiguration)
}

func (c *SimpleConfiguration) SetLastTransition(t Transition) {
	c.Last = t
}

func (c *SimpleConfiguration) GetLastTransition() Transition {
	return Transition(c.Last)
}

func (c *SimpleConfiguration) GetSequence() ConfigurationSequence {
	if c.Arcs() == nil {
		return make(ConfigurationSequence, 0)
	}
	retval := make(ConfigurationSequence, 0, c.Arcs().Size())
	currentConf := c
	for currentConf != nil {
		retval = append(retval, currentConf)
		currentConf = currentConf.InternalPrevious
	}
	return retval
}

// GRAPH FUNCTIONS
func (c *SimpleConfiguration) GetVertices() []int {
	return util.RangeInt(len(c.Nodes))
}

func (c *SimpleConfiguration) GetEdges() []int {
	// the + 1 is because there may be a missing edge, therefore an ID may be skipped over
	// leaving an off-by-one for the last edge ID
	return util.RangeInt(c.Arcs().Size() + 1)
}

func (c *SimpleConfiguration) GetVertex(vertexID int) graph.Vertex {
	return graph.Vertex(c.Nodes[vertexID])
}

func (c *SimpleConfiguration) GetEdge(edgeID int) graph.Edge {
	arcPtr := c.Arcs().Index(edgeID)
	return graph.Edge(arcPtr)
}

func (c *SimpleConfiguration) GetDirectedEdge(edgeID int) graph.DirectedEdge {
	arcPtr := c.Arcs().Index(edgeID)
	return graph.DirectedEdge(arcPtr)
}

func (c *SimpleConfiguration) NumberOfVertices() int {
	return c.NumberOfNodes()
}

func (c *SimpleConfiguration) NumberOfEdges() int {
	return c.NumberOfArcs()
}

func (c *SimpleConfiguration) NumberOfNodes() int {
	return len(c.Nodes)
}

func (c *SimpleConfiguration) NumberOfArcs() int {
	return c.Arcs().Size()
}

func (c *SimpleConfiguration) GetNode(nodeID int) nlp.DepNode {
	return c.Nodes[nodeID].Node
}

func (c *SimpleConfiguration) GetRawNode(nodeID int) *TaggedDepNode {
	return c.Nodes[nodeID].Node.(*TaggedDepNode)
}

func (c *SimpleConfiguration) GetArc(nodeID int) nlp.DepArc {
	if c.Nodes[nodeID].ArcId > -1 {
		arcPtr := c.Arcs().Index(c.Nodes[nodeID].ArcId)
		return nlp.DepArc(arcPtr)
	} else {
		return nil
	}
}

func (c *SimpleConfiguration) GetLabeledArc(nodeID int) nlp.LabeledDepArc {
	if nodeID < len(c.Nodes) && c.Nodes[nodeID].ArcId > -1 {
		arcPtr := c.Arcs().Index(c.Nodes[nodeID].ArcId)
		return nlp.LabeledDepArc(arcPtr)
	} else {
		return nil
	}
}

// OUTPUT FUNCTIONS

func (c *SimpleConfiguration) String() string {
	var (
		transitionVal string = ""
		transInt      int    = c.Last.Value()
	)
	if transInt >= 0 {
		transitionVal = c.ETrans.ValueOf(transInt).(string)
	}
	return fmt.Sprintf("%s\t=>([%s],\t[%s],\t%s)",
		transitionVal, c.StringStack(), c.StringQueue(),
		c.StringArcs())
}

func (c *SimpleConfiguration) StringStack() string {
	stackSize := c.Stack().Size()
	switch {
	case stackSize > 0 && stackSize <= 3:
		var stackStrings []string = make([]string, 0, 3)
		for i := c.Stack().Size() - 1; i >= 0; i-- {
			atI, _ := c.Stack().Index(i)
			stackStrings = append(stackStrings, c.GetRawNode(atI).RawToken)
		}
		return strings.Join(stackStrings, ",")
	case stackSize > 3:
		headID, _ := c.Stack().Index(0)
		tailID, _ := c.Stack().Index(c.Stack().Size() - 1)
		head := c.GetRawNode(headID)
		tail := c.GetRawNode(tailID)
		return strings.Join([]string{tail.RawToken, "...", head.RawToken}, ",")
	default:
		return ""
	}
}

func (c *SimpleConfiguration) StringQueue() string {
	queueSize := c.Queue().Size()
	switch {
	case queueSize > 0 && queueSize <= 3:
		var queueStrings []string = make([]string, 0, 3)
		for i := 0; i < c.Queue().Size(); i++ {
			atI, _ := c.Queue().Index(i)
			queueStrings = append(queueStrings, c.GetRawNode(atI).RawToken)
		}
		return strings.Join(queueStrings, ",")
	case queueSize > 3:
		headID, _ := c.Queue().Index(0)
		tailID, _ := c.Queue().Index(c.Queue().Size() - 1)
		head := c.GetRawNode(headID)
		tail := c.GetRawNode(tailID)
		return strings.Join([]string{head.RawToken, "...", tail.RawToken}, ",")
	default:
		return ""
	}
}

func (c *SimpleConfiguration) StringArcs() string {
	var transInt int = c.Last.Value()
	if transInt < 0 {
		return ""
	}
	last := c.ETrans.ValueOf(transInt).(string)
	if len(last) < 2 {
		return fmt.Sprintf("A%d", c.Arcs().Size())
	}
	switch last[:2] {
	case "LA", "RA":
		lastArc := c.Arcs().Last()
		head := c.GetRawNode(lastArc.GetHead())
		mod := c.GetRawNode(lastArc.GetModifier())
		arcStr := fmt.Sprintf("(%s,%s,%s)", head.RawToken, lastArc.GetRelation().String(), mod.RawToken)
		return fmt.Sprintf("A%d=A%d+{%s}", c.Arcs().Size(), c.Arcs().Size()-1, arcStr)
	default:
		return fmt.Sprintf("A%d", c.Arcs().Size())
	}
}

func (c *SimpleConfiguration) StringGraph() string {
	return fmt.Sprintf("%v %v", c.Nodes, c.InternalArcs)
}

func (c *SimpleConfiguration) Sentence() nlp.Sentence {
	return nlp.Sentence(c.TaggedSentence())
}

func (c *SimpleConfiguration) TaggedSentence() nlp.TaggedSentence {
	var sent nlp.BasicETaggedSentence = make([]nlp.EnumTaggedToken, c.NumberOfNodes()-1)
	for i, _ := range c.Nodes {
		taggedNode := c.GetRawNode(i)
		sent[i] = nlp.EnumTaggedToken{
			nlp.TaggedToken{taggedNode.RawToken, taggedNode.RawLemma, taggedNode.RawPOS},
			// TODO: add lemma enum
			taggedNode.Token, 0, taggedNode.POS, taggedNode.TokenPOS, taggedNode.MHost, taggedNode.MSuffix}
	}
	return sent
}

func (c *SimpleConfiguration) Len() int {
	if c == nil {
		return 0
	}
	if c.Previous() != nil {
		return 1 + c.Previous().Len()
	} else {
		return 1
	}
}

func (c *SimpleConfiguration) Assignment() uint16 {
	return c.lastOpAssignment
}

func (c *SimpleConfiguration) Assign(to uint16) {
	c.lastOpAssignment = to
}

func NewSimpleConfiguration() Configuration {
	return Configuration(new(SimpleConfiguration))
}
<|endoftext|>


<commit_before>package dnsmasq

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/version"

	"golang.org/x/sys/unix"
)

// DHCPAllocation represents an IP allocation from dnsmasq.
type DHCPAllocation struct {
	IP     net.IP
	Name   string
	MAC    net.HardwareAddr
	Static bool
}

// ConfigMutex used to coordinate access to the dnsmasq config files.
var ConfigMutex sync.Mutex

// UpdateStaticEntry writes a single dhcp-host line for a network/instance combination.
func UpdateStaticEntry(network string, projectName string, instanceName string, netConfig map[string]string, hwaddr string, ipv4Address string, ipv6Address string) error {
	line := hwaddr

	// Generate the dhcp-host line
	if ipv4Address != "" {
		line += fmt.Sprintf(",%s", ipv4Address)
	}

	if ipv6Address != "" {
		line += fmt.Sprintf(",[%s]", ipv6Address)
	}

	if netConfig["dns.mode"] == "" || netConfig["dns.mode"] == "managed" {
		line += fmt.Sprintf(",%s", instanceName)
	}

	if line == hwaddr {
		return nil
	}

	err := ioutil.WriteFile(shared.VarPath("networks", network, "dnsmasq.hosts", project.Prefix(projectName, instanceName)), []byte(line+"\n"), 0644)
	if err != nil {
		return err
	}

	return nil
}

// RemoveStaticEntry removes a single dhcp-host line for a network/instance combination.
func RemoveStaticEntry(network string, projectName string, instanceName string) error {
	err := os.Remove(shared.VarPath("networks", network, "dnsmasq.hosts", project.Prefix(projectName, instanceName)))
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	return nil
}

// Kill kills dnsmasq for a particular network (or optionally reloads it).
func Kill(name string, reload bool) error {
	// Check if we have a running dnsmasq at all
	pidPath := shared.VarPath("networks", name, "dnsmasq.pid")
	if !shared.PathExists(pidPath) {
		if reload {
			return fmt.Errorf("dnsmasq isn't running")
		}

		return nil
	}

	// Grab the PID
	content, err := ioutil.ReadFile(pidPath)
	if err != nil {
		return err
	}
	pid := strings.TrimSpace(string(content))

	// Check for empty string
	if pid == "" {
		os.Remove(pidPath)

		if reload {
			return fmt.Errorf("dnsmasq isn't running")
		}

		return nil
	}

	// Check if the process still exists
	if !shared.PathExists(fmt.Sprintf("/proc/%s", pid)) {
		os.Remove(pidPath)

		if reload {
			return fmt.Errorf("dnsmasq isn't running")
		}

		return nil
	}

	// Check if it's dnsmasq
	cmdPath, err := os.Readlink(fmt.Sprintf("/proc/%s/exe", pid))
	if err != nil {
		cmdPath = ""
	}

	// Deal with deleted paths
	cmdName := filepath.Base(strings.Split(cmdPath, " ")[0])
	if cmdName != "dnsmasq" {
		if reload {
			return fmt.Errorf("dnsmasq isn't running")
		}

		os.Remove(pidPath)
		return nil
	}

	// Parse the pid
	pidInt, err := strconv.Atoi(pid)
	if err != nil {
		return err
	}

	// Actually kill the process
	if reload {
		err = unix.Kill(pidInt, unix.SIGHUP)
		if err != nil {
			return err
		}

		return nil
	}

	err = unix.Kill(pidInt, unix.SIGKILL)
	if err != nil {
		return err
	}

	// Cleanup
	os.Remove(pidPath)
	return nil
}

// GetVersion returns the version of dnsmasq.
func GetVersion() (*version.DottedVersion, error) {
	// Discard stderr on purpose (occasional linker errors)
	output, err := exec.Command("dnsmasq", "--version").Output()
	if err != nil {
		return nil, fmt.Errorf("Failed to check dnsmasq version: %v", err)
	}

	lines := strings.Split(string(output), " ")
	return version.NewDottedVersion(lines[2])
}

// DHCPStaticIPs retrieves the dnsmasq statically allocated IPs for a container.
// Returns IPv4 and IPv6 DHCPAllocation structs respectively.
func DHCPStaticIPs(network string, containerName string) (DHCPAllocation, DHCPAllocation, error) {
	var IPv4, IPv6 DHCPAllocation

	file, err := os.Open(shared.VarPath("networks", network, "dnsmasq.hosts") + "/" + containerName)
	if err != nil {
		return IPv4, IPv6, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		fields := strings.SplitN(scanner.Text(), ",", -1)
		for _, field := range fields {
			// Check if field is IPv4 or IPv6 address.
			if strings.Count(field, ".") == 3 {
				IP := net.ParseIP(field)
				if IP.To4() == nil {
					return IPv4, IPv6, fmt.Errorf("Error parsing IP address: %v", field)
				}
				IPv4 = DHCPAllocation{Name: containerName, Static: true, IP: IP.To4()}

			} else if strings.HasPrefix(field, "[") && strings.HasSuffix(field, "]") {
				IP := net.ParseIP(field[1 : len(field)-1])
				if IP == nil {
					return IPv4, IPv6, fmt.Errorf("Error parsing IP address: %v", field)
				}
				IPv6 = DHCPAllocation{Name: containerName, Static: true, IP: IP}
			}
		}
	}
	if err := scanner.Err(); err != nil {
		return IPv4, IPv6, err
	}

	return IPv4, IPv6, nil
}

// DHCPAllocatedIPs returns a map of IPs currently allocated (statically and dynamically)
// in dnsmasq for a specific network. The returned map is keyed by a 16 byte array representing
// the net.IP format. The value of each map item is a DHCPAllocation struct containing at least
// whether the allocation was static or dynamic and optionally container name or MAC address.
// MAC addresses are only included for dynamic IPv4 allocations (where name is not reliable).
// Static allocations are not overridden by dynamic allocations, allowing for container name to be
// included for static IPv6 allocations. IPv6 addresses that are dynamically assigned cannot be
// reliably linked to containers using either name or MAC because dnsmasq does not record the MAC
// address for these records, and the recorded host name can be set by the container if the dns.mode
// for the network is set to "dynamic" and so cannot be trusted, so in this case we do not return
// any identifying info.
func DHCPAllocatedIPs(network string) (map[[4]byte]DHCPAllocation, map[[16]byte]DHCPAllocation, error) {
	IPv4s := make(map[[4]byte]DHCPAllocation)
	IPv6s := make(map[[16]byte]DHCPAllocation)

	// First read all statically allocated IPs.
	files, err := ioutil.ReadDir(shared.VarPath("networks", network, "dnsmasq.hosts"))
	if err != nil {
		return IPv4s, IPv6s, err
	}

	for _, entry := range files {
		IPv4, IPv6, err := DHCPStaticIPs(network, entry.Name())
		if err != nil {
			return IPv4s, IPv6s, err
		}

		if IPv4.IP != nil {
			var IPKey [4]byte
			copy(IPKey[:], IPv4.IP.To4())
			IPv4s[IPKey] = IPv4
		}

		if IPv6.IP != nil {
			var IPKey [16]byte
			copy(IPKey[:], IPv6.IP.To16())
			IPv6s[IPKey] = IPv6
		}
	}

	// Next read all dynamic allocated IPs.
	file, err := os.Open(shared.VarPath("networks", network, "dnsmasq.leases"))
	if err != nil {
		return IPv4s, IPv6s, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		fields := strings.Fields(scanner.Text())
		if len(fields) == 5 {
			IP := net.ParseIP(fields[2])
			if IP == nil {
				return IPv4s, IPv6s, fmt.Errorf("Error parsing IP address: %v", fields[2])
			}

			// Handle IPv6 addresses.
			if IP.To4() == nil {
				var IPKey [16]byte
				copy(IPKey[:], IP.To16())

				// Don't replace IPs from static config as more reliable.
				if IPv6s[IPKey].Name != "" {
					continue
				}

				IPv6s[IPKey] = DHCPAllocation{
					Static: false,
					IP:     IP.To16(),
				}
			} else {
				// MAC only available in IPv4 leases.
				MAC, err := net.ParseMAC(fields[1])
				if err != nil {
					return IPv4s, IPv6s, err
				}

				var IPKey [4]byte
				copy(IPKey[:], IP.To4())

				// Don't replace IPs from static config as more reliable.
				if IPv4s[IPKey].Name != "" {
					continue
				}

				IPv4s[IPKey] = DHCPAllocation{
					MAC:    MAC,
					Static: false,
					IP:     IP.To4(),
				}
			}
		}
	}
	if err := scanner.Err(); err != nil {
		return IPv4s, IPv6s, err
	}

	return IPv4s, IPv6s, nil
}
<commit_msg>dnsmasq/dnsmasq: Updates version check to use shared.RunCommandCLocale<commit_after>package dnsmasq

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"

	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/version"

	"golang.org/x/sys/unix"
)

// DHCPAllocation represents an IP allocation from dnsmasq.
type DHCPAllocation struct {
	IP     net.IP
	Name   string
	MAC    net.HardwareAddr
	Static bool
}

// ConfigMutex used to coordinate access to the dnsmasq config files.
var ConfigMutex sync.Mutex

// UpdateStaticEntry writes a single dhcp-host line for a network/instance combination.
func UpdateStaticEntry(network string, projectName string, instanceName string, netConfig map[string]string, hwaddr string, ipv4Address string, ipv6Address string) error {
	line := hwaddr

	// Generate the dhcp-host line
	if ipv4Address != "" {
		line += fmt.Sprintf(",%s", ipv4Address)
	}

	if ipv6Address != "" {
		line += fmt.Sprintf(",[%s]", ipv6Address)
	}

	if netConfig["dns.mode"] == "" || netConfig["dns.mode"] == "managed" {
		line += fmt.Sprintf(",%s", instanceName)
	}

	if line == hwaddr {
		return nil
	}

	err := ioutil.WriteFile(shared.VarPath("networks", network, "dnsmasq.hosts", project.Prefix(projectName, instanceName)), []byte(line+"\n"), 0644)
	if err != nil {
		return err
	}

	return nil
}

// RemoveStaticEntry removes a single dhcp-host line for a network/instance combination.
func RemoveStaticEntry(network string, projectName string, instanceName string) error {
	err := os.Remove(shared.VarPath("networks", network, "dnsmasq.hosts", project.Prefix(projectName, instanceName)))
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	return nil
}

// Kill kills dnsmasq for a particular network (or optionally reloads it).
func Kill(name string, reload bool) error {
	// Check if we have a running dnsmasq at all
	pidPath := shared.VarPath("networks", name, "dnsmasq.pid")
	if !shared.PathExists(pidPath) {
		if reload {
			return fmt.Errorf("dnsmasq isn't running")
		}

		return nil
	}

	// Grab the PID
	content, err := ioutil.ReadFile(pidPath)
	if err != nil {
		return err
	}
	pid := strings.TrimSpace(string(content))

	// Check for empty string
	if pid == "" {
		os.Remove(pidPath)

		if reload {
			return fmt.Errorf("dnsmasq isn't running")
		}

		return nil
	}

	// Check if the process still exists
	if !shared.PathExists(fmt.Sprintf("/proc/%s", pid)) {
		os.Remove(pidPath)

		if reload {
			return fmt.Errorf("dnsmasq isn't running")
		}

		return nil
	}

	// Check if it's dnsmasq
	cmdPath, err := os.Readlink(fmt.Sprintf("/proc/%s/exe", pid))
	if err != nil {
		cmdPath = ""
	}

	// Deal with deleted paths
	cmdName := filepath.Base(strings.Split(cmdPath, " ")[0])
	if cmdName != "dnsmasq" {
		if reload {
			return fmt.Errorf("dnsmasq isn't running")
		}

		os.Remove(pidPath)
		return nil
	}

	// Parse the pid
	pidInt, err := strconv.Atoi(pid)
	if err != nil {
		return err
	}

	// Actually kill the process
	if reload {
		err = unix.Kill(pidInt, unix.SIGHUP)
		if err != nil {
			return err
		}

		return nil
	}

	err = unix.Kill(pidInt, unix.SIGKILL)
	if err != nil {
		return err
	}

	// Cleanup
	os.Remove(pidPath)
	return nil
}

// GetVersion returns the version of dnsmasq.
func GetVersion() (*version.DottedVersion, error) {
	output, err := shared.RunCommandCLocale("dnsmasq", "--version")
	if err != nil {
		return nil, fmt.Errorf("Failed to check dnsmasq version: %v", err)
	}

	lines := strings.Split(string(output), " ")
	return version.NewDottedVersion(lines[2])
}

// DHCPStaticIPs retrieves the dnsmasq statically allocated IPs for a container.
// Returns IPv4 and IPv6 DHCPAllocation structs respectively.
func DHCPStaticIPs(network string, containerName string) (DHCPAllocation, DHCPAllocation, error) {
	var IPv4, IPv6 DHCPAllocation

	file, err := os.Open(shared.VarPath("networks", network, "dnsmasq.hosts") + "/" + containerName)
	if err != nil {
		return IPv4, IPv6, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		fields := strings.SplitN(scanner.Text(), ",", -1)
		for _, field := range fields {
			// Check if field is IPv4 or IPv6 address.
			if strings.Count(field, ".") == 3 {
				IP := net.ParseIP(field)
				if IP.To4() == nil {
					return IPv4, IPv6, fmt.Errorf("Error parsing IP address: %v", field)
				}
				IPv4 = DHCPAllocation{Name: containerName, Static: true, IP: IP.To4()}

			} else if strings.HasPrefix(field, "[") && strings.HasSuffix(field, "]") {
				IP := net.ParseIP(field[1 : len(field)-1])
				if IP == nil {
					return IPv4, IPv6, fmt.Errorf("Error parsing IP address: %v", field)
				}
				IPv6 = DHCPAllocation{Name: containerName, Static: true, IP: IP}
			}
		}
	}
	if err := scanner.Err(); err != nil {
		return IPv4, IPv6, err
	}

	return IPv4, IPv6, nil
}

// DHCPAllocatedIPs returns a map of IPs currently allocated (statically and dynamically)
// in dnsmasq for a specific network. The returned map is keyed by a 16 byte array representing
// the net.IP format. The value of each map item is a DHCPAllocation struct containing at least
// whether the allocation was static or dynamic and optionally container name or MAC address.
// MAC addresses are only included for dynamic IPv4 allocations (where name is not reliable).
// Static allocations are not overridden by dynamic allocations, allowing for container name to be
// included for static IPv6 allocations. IPv6 addresses that are dynamically assigned cannot be
// reliably linked to containers using either name or MAC because dnsmasq does not record the MAC
// address for these records, and the recorded host name can be set by the container if the dns.mode
// for the network is set to "dynamic" and so cannot be trusted, so in this case we do not return
// any identifying info.
func DHCPAllocatedIPs(network string) (map[[4]byte]DHCPAllocation, map[[16]byte]DHCPAllocation, error) {
	IPv4s := make(map[[4]byte]DHCPAllocation)
	IPv6s := make(map[[16]byte]DHCPAllocation)

	// First read all statically allocated IPs.
	files, err := ioutil.ReadDir(shared.VarPath("networks", network, "dnsmasq.hosts"))
	if err != nil {
		return IPv4s, IPv6s, err
	}

	for _, entry := range files {
		IPv4, IPv6, err := DHCPStaticIPs(network, entry.Name())
		if err != nil {
			return IPv4s, IPv6s, err
		}

		if IPv4.IP != nil {
			var IPKey [4]byte
			copy(IPKey[:], IPv4.IP.To4())
			IPv4s[IPKey] = IPv4
		}

		if IPv6.IP != nil {
			var IPKey [16]byte
			copy(IPKey[:], IPv6.IP.To16())
			IPv6s[IPKey] = IPv6
		}
	}

	// Next read all dynamic allocated IPs.
	file, err := os.Open(shared.VarPath("networks", network, "dnsmasq.leases"))
	if err != nil {
		return IPv4s, IPv6s, err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		fields := strings.Fields(scanner.Text())
		if len(fields) == 5 {
			IP := net.ParseIP(fields[2])
			if IP == nil {
				return IPv4s, IPv6s, fmt.Errorf("Error parsing IP address: %v", fields[2])
			}

			// Handle IPv6 addresses.
			if IP.To4() == nil {
				var IPKey [16]byte
				copy(IPKey[:], IP.To16())

				// Don't replace IPs from static config as more reliable.
				if IPv6s[IPKey].Name != "" {
					continue
				}

				IPv6s[IPKey] = DHCPAllocation{
					Static: false,
					IP:     IP.To16(),
				}
			} else {
				// MAC only available in IPv4 leases.
				MAC, err := net.ParseMAC(fields[1])
				if err != nil {
					return IPv4s, IPv6s, err
				}

				var IPKey [4]byte
				copy(IPKey[:], IP.To4())

				// Don't replace IPs from static config as more reliable.
				if IPv4s[IPKey].Name != "" {
					continue
				}

				IPv4s[IPKey] = DHCPAllocation{
					MAC:    MAC,
					Static: false,
					IP:     IP.To4(),
				}
			}
		}
	}
	if err := scanner.Err(); err != nil {
		return IPv4s, IPv6s, err
	}

	return IPv4s, IPv6s, nil
}
<|endoftext|>