<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package message

import (
	"crypto/x509"
	"math"
	"net"
	"testing"
	"time"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/stretchr/testify/assert"

	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/utils"
	"github.com/ava-labs/avalanchego/utils/units"
)

func TestCodecPackInvalidOp(t *testing.T) {
	codec, err := NewCodecWithMemoryPool("", prometheus.NewRegistry(), 2*units.MiB)
	assert.NoError(t, err)

	_, err = codec.Pack(math.MaxUint8, make(map[Field]interface{}), false)
	assert.Error(t, err)

	_, err = codec.Pack(math.MaxUint8, make(map[Field]interface{}), true)
	assert.Error(t, err)
}

func TestCodecPackMissingField(t *testing.T) {
	codec, err := NewCodecWithMemoryPool("", prometheus.NewRegistry(), 2*units.MiB)
	assert.NoError(t, err)

	_, err = codec.Pack(Get, make(map[Field]interface{}), false)
	assert.Error(t, err)

	_, err = codec.Pack(Get, make(map[Field]interface{}), true)
	assert.Error(t, err)
}

func TestCodecParseInvalidOp(t *testing.T) {
	codec, err := NewCodecWithMemoryPool("", prometheus.NewRegistry(), 2*units.MiB)
	assert.NoError(t, err)

	_, err = codec.Parse([]byte{math.MaxUint8}, dummyNodeID, dummyOnFinishedHandling)
	assert.Error(t, err)
}

func TestCodecParseExtraSpace(t *testing.T) {
	codec, err := NewCodecWithMemoryPool("", prometheus.NewRegistry(), 2*units.MiB)
	assert.NoError(t, err)

	_, err = codec.Parse([]byte{byte(GetVersion), 0x00, 0x00}, dummyNodeID, dummyOnFinishedHandling)
	assert.Error(t, err)

	_, err = codec.Parse([]byte{byte(GetVersion), 0x00, 0x01}, dummyNodeID, dummyOnFinishedHandling)
	assert.Error(t, err)
}

// Test packing and then parsing messages
// when using a gzip compressor
func TestCodecPackParseGzip(t *testing.T) {
	c, err := NewCodecWithMemoryPool("", prometheus.DefaultRegisterer, 2*units.MiB)
	assert.NoError(t, err)
	id := ids.GenerateTestID()
	cert := &x509.Certificate{}

	msgs := []inboundMessage{
		{
			op:     GetVersion,
			fields: map[Field]interface{}{},
		},
		{
			op: Version,
			fields: map[Field]interface{}{
				NetworkID:      uint32(0),
				NodeID:         uint32(1337),
				MyTime:         uint64(time.Now().Unix()),
				IP:             utils.IPDesc{IP: net.IPv4(1, 2, 3, 4)},
				VersionStr:     "v1.2.3",
				VersionTime:    uint64(time.Now().Unix()),
				SigBytes:       []byte{'y', 'e', 'e', 't'},
				TrackedSubnets: [][]byte{id[:]},
			},
		},
		{
			op:     GetPeerList,
			fields: map[Field]interface{}{},
		},
		{
			op: PeerList,
			fields: map[Field]interface{}{
				SignedPeers: []utils.IPCertDesc{
					{
						Cert:      cert,
						IPDesc:    utils.IPDesc{IP: net.IPv4(1, 2, 3, 4)},
						Time:      uint64(time.Now().Unix()),
						Signature: make([]byte, 65),
					},
				},
			},
		},
		{
			op:     Ping,
			fields: map[Field]interface{}{},
		},
		{
			op:     Pong,
			fields: map[Field]interface{}{},
		},
		{
			op: UptimePong,
			fields: map[Field]interface{}{
				Uptime: 80,
			},
		},
		{
			op: GetAcceptedFrontier,
			fields: map[Field]interface{}{
				ChainID:   id[:],
				RequestID: uint32(1337),
				Deadline:  uint64(time.Now().Unix()),
			},
		},
		{
			op: AcceptedFrontier,
			fields: map[Field]interface{}{
				ChainID:      id[:],
				RequestID:    uint32(1337),
				ContainerIDs: [][]byte{id[:]},
			},
		},
		{
			op: GetAccepted,
			fields: map[Field]interface{}{
				ChainID:      id[:],
				RequestID:    uint32(1337),
				Deadline:     uint64(time.Now().Unix()),
				ContainerIDs: [][]byte{id[:]},
			},
		},
		{
			op: Accepted,
			fields: map[Field]interface{}{
				ChainID:      id[:],
				RequestID:    uint32(1337),
				ContainerIDs: [][]byte{id[:]},
			},
		},
		{
			op: MultiPut,
			fields: map[Field]interface{}{
				ChainID:             id[:],
				RequestID:           uint32(1337),
				MultiContainerBytes: [][]byte{id[:]},
			},
		},
		{
			op: Get,
			fields: map[Field]interface{}{
				ChainID:     id[:],
				RequestID:   uint32(1337),
				Deadline:    uint64(time.Now().Unix()),
				ContainerID: id[:],
			},
		},
		{
			op: Put,
			fields: map[Field]interface{}{
				ChainID:        id[:],
				RequestID:      uint32(1337),
				ContainerID:    id[:],
				ContainerBytes: make([]byte, 1024),
			},
		},
		{
			op: PushQuery,
			fields: map[Field]interface{}{
				ChainID:        id[:],
				RequestID:      uint32(1337),
				Deadline:       uint64(time.Now().Unix()),
				ContainerID:    id[:],
				ContainerBytes: make([]byte, 1024),
			},
		},
		{
			op: PullQuery,
			fields: map[Field]interface{}{
				ChainID:     id[:],
				RequestID:   uint32(1337),
				Deadline:    uint64(time.Now().Unix()),
				ContainerID: id[:],
			},
		},
		{
			op: Chits,
			fields: map[Field]interface{}{
				ChainID:      id[:],
				RequestID:    uint32(1337),
				ContainerIDs: [][]byte{id[:]},
			},
		},
	}
	for _, m := range msgs {
		packedIntf, err := c.Pack(m.op, m.fields, m.op.Compressable())
		assert.NoError(t, err, "failed to pack on operation %s", m.op)

		unpackedIntf, err := c.Parse(packedIntf.Bytes(), dummyNodeID, dummyOnFinishedHandling)
		assert.NoError(t, err, "failed to parse w/ compression on operation %s", m.op)

		unpacked := unpackedIntf.(*inboundMessage)

		assert.EqualValues(t, len(m.fields), len(unpacked.fields))
	}
}
<commit_msg>convert int to uint8<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package message

import (
	"crypto/x509"
	"math"
	"net"
	"testing"
	"time"

	"github.com/prometheus/client_golang/prometheus"

	"github.com/stretchr/testify/assert"

	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/utils"
	"github.com/ava-labs/avalanchego/utils/units"
)

func TestCodecPackInvalidOp(t *testing.T) {
	codec, err := NewCodecWithMemoryPool("", prometheus.NewRegistry(), 2*units.MiB)
	assert.NoError(t, err)

	_, err = codec.Pack(math.MaxUint8, make(map[Field]interface{}), false)
	assert.Error(t, err)

	_, err = codec.Pack(math.MaxUint8, make(map[Field]interface{}), true)
	assert.Error(t, err)
}

func TestCodecPackMissingField(t *testing.T) {
	codec, err := NewCodecWithMemoryPool("", prometheus.NewRegistry(), 2*units.MiB)
	assert.NoError(t, err)

	_, err = codec.Pack(Get, make(map[Field]interface{}), false)
	assert.Error(t, err)

	_, err = codec.Pack(Get, make(map[Field]interface{}), true)
	assert.Error(t, err)
}

func TestCodecParseInvalidOp(t *testing.T) {
	codec, err := NewCodecWithMemoryPool("", prometheus.NewRegistry(), 2*units.MiB)
	assert.NoError(t, err)

	_, err = codec.Parse([]byte{math.MaxUint8}, dummyNodeID, dummyOnFinishedHandling)
	assert.Error(t, err)
}

func TestCodecParseExtraSpace(t *testing.T) {
	codec, err := NewCodecWithMemoryPool("", prometheus.NewRegistry(), 2*units.MiB)
	assert.NoError(t, err)

	_, err = codec.Parse([]byte{byte(GetVersion), 0x00, 0x00}, dummyNodeID, dummyOnFinishedHandling)
	assert.Error(t, err)

	_, err = codec.Parse([]byte{byte(GetVersion), 0x00, 0x01}, dummyNodeID, dummyOnFinishedHandling)
	assert.Error(t, err)
}

// Test packing and then parsing messages
// when using a gzip compressor
func TestCodecPackParseGzip(t *testing.T) {
	c, err := NewCodecWithMemoryPool("", prometheus.DefaultRegisterer, 2*units.MiB)
	assert.NoError(t, err)
	id := ids.GenerateTestID()
	cert := &x509.Certificate{}

	msgs := []inboundMessage{
		{
			op:     GetVersion,
			fields: map[Field]interface{}{},
		},
		{
			op: Version,
			fields: map[Field]interface{}{
				NetworkID:      uint32(0),
				NodeID:         uint32(1337),
				MyTime:         uint64(time.Now().Unix()),
				IP:             utils.IPDesc{IP: net.IPv4(1, 2, 3, 4)},
				VersionStr:     "v1.2.3",
				VersionTime:    uint64(time.Now().Unix()),
				SigBytes:       []byte{'y', 'e', 'e', 't'},
				TrackedSubnets: [][]byte{id[:]},
			},
		},
		{
			op:     GetPeerList,
			fields: map[Field]interface{}{},
		},
		{
			op: PeerList,
			fields: map[Field]interface{}{
				SignedPeers: []utils.IPCertDesc{
					{
						Cert:      cert,
						IPDesc:    utils.IPDesc{IP: net.IPv4(1, 2, 3, 4)},
						Time:      uint64(time.Now().Unix()),
						Signature: make([]byte, 65),
					},
				},
			},
		},
		{
			op:     Ping,
			fields: map[Field]interface{}{},
		},
		{
			op:     Pong,
			fields: map[Field]interface{}{},
		},
		{
			op: UptimePong,
			fields: map[Field]interface{}{
				Uptime: uint8(80),
			},
		},
		{
			op: GetAcceptedFrontier,
			fields: map[Field]interface{}{
				ChainID:   id[:],
				RequestID: uint32(1337),
				Deadline:  uint64(time.Now().Unix()),
			},
		},
		{
			op: AcceptedFrontier,
			fields: map[Field]interface{}{
				ChainID:      id[:],
				RequestID:    uint32(1337),
				ContainerIDs: [][]byte{id[:]},
			},
		},
		{
			op: GetAccepted,
			fields: map[Field]interface{}{
				ChainID:      id[:],
				RequestID:    uint32(1337),
				Deadline:     uint64(time.Now().Unix()),
				ContainerIDs: [][]byte{id[:]},
			},
		},
		{
			op: Accepted,
			fields: map[Field]interface{}{
				ChainID:      id[:],
				RequestID:    uint32(1337),
				ContainerIDs: [][]byte{id[:]},
			},
		},
		{
			op: MultiPut,
			fields: map[Field]interface{}{
				ChainID:             id[:],
				RequestID:           uint32(1337),
				MultiContainerBytes: [][]byte{id[:]},
			},
		},
		{
			op: Get,
			fields: map[Field]interface{}{
				ChainID:     id[:],
				RequestID:   uint32(1337),
				Deadline:    uint64(time.Now().Unix()),
				ContainerID: id[:],
			},
		},
		{
			op: Put,
			fields: map[Field]interface{}{
				ChainID:        id[:],
				RequestID:      uint32(1337),
				ContainerID:    id[:],
				ContainerBytes: make([]byte, 1024),
			},
		},
		{
			op: PushQuery,
			fields: map[Field]interface{}{
				ChainID:        id[:],
				RequestID:      uint32(1337),
				Deadline:       uint64(time.Now().Unix()),
				ContainerID:    id[:],
				ContainerBytes: make([]byte, 1024),
			},
		},
		{
			op: PullQuery,
			fields: map[Field]interface{}{
				ChainID:     id[:],
				RequestID:   uint32(1337),
				Deadline:    uint64(time.Now().Unix()),
				ContainerID: id[:],
			},
		},
		{
			op: Chits,
			fields: map[Field]interface{}{
				ChainID:      id[:],
				RequestID:    uint32(1337),
				ContainerIDs: [][]byte{id[:]},
			},
		},
	}
	for _, m := range msgs {
		packedIntf, err := c.Pack(m.op, m.fields, m.op.Compressable())
		assert.NoError(t, err, "failed to pack on operation %s", m.op)

		unpackedIntf, err := c.Parse(packedIntf.Bytes(), dummyNodeID, dummyOnFinishedHandling)
		assert.NoError(t, err, "failed to parse w/ compression on operation %s", m.op)

		unpacked := unpackedIntf.(*inboundMessage)

		assert.EqualValues(t, len(m.fields), len(unpacked.fields))
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"fmt"
	"sort"

	"github.com/googlecloudplatform/gcsfuse/fs/inode"
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

// State required for reading from directories.
type dirHandle struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	in *inode.DirInode

	/////////////////////////
	// Mutable state
	/////////////////////////

	Mu syncutil.InvariantMutex

	// All entries in the directory. Populated the first time we need one.
	//
	// INVARIANT: For each i, entries[i+1].Offset == entries[i].Offset + 1
	//
	// GUARDED_BY(Mu)
	entries []fuseutil.Dirent

	// Has entries yet been populated?
	//
	// INVARIANT: If !entriesValid, then len(entries) == 0
	//
	// GUARDED_BY(Mu)
	entriesValid bool
}

// Create a directory handle that obtains listings from the supplied inode.
func newDirHandle(in *inode.DirInode) (dh *dirHandle) {
	// Set up the basic struct.
	dh = &dirHandle{
		in: in,
	}

	// Set up invariant checking.
	dh.Mu = syncutil.NewInvariantMutex(dh.checkInvariants)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Dirents, sorted by name.
type SortedDirents []fuseutil.Dirent

func (p SortedDirents) Len() int           { return len(p) }
func (p SortedDirents) Less(i, j int) bool { return p[i].Name < p[j].Name }
func (p SortedDirents) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func (dh *dirHandle) checkInvariants() {
	// INVARIANT: For each i, entries[i+1].Offset == entries[i].Offset + 1
	for i := 0; i < len(dh.entries)-1; i++ {
		if !(dh.entries[i+1].Offset == dh.entries[i].Offset+1) {
			panic(
				fmt.Sprintf(
					"Unexpected offset sequence: %v, %v",
					dh.entries[i].Offset,
					dh.entries[i+1].Offset))
		}
	}

	// INVARIANT: If !entriesValid, then len(entries) == 0
	if !dh.entriesValid && len(dh.entries) != 0 {
		panic("Unexpected non-empty entries slice")
	}
}

// Read some entries from the directory inode. Return newTok == "" (possibly
// with a non-empty list of entries) when the end of the directory has been
// hit.
//
// The contents of the entries' Offset fields are undefined.
func readSomeEntries(
	ctx context.Context,
	in *inode.DirInode,
	tok string) (entries []fuseutil.Dirent, newTok string, err error) {
	entries, newTok, err = in.ReadEntries(ctx, tok)
	if err != nil {
		err = fmt.Errorf("ReadEntries: %v", err)
		return
	}

	// Return a bogus inode ID for each entry, but not the root inode ID.
	//
	// NOTE(jacobsa): As far as I can tell this is harmless. Minting and
	// returning a real inode ID is difficult because fuse does not count
	// readdir as an operation that increases the inode ID's lookup count and
	// we therefore don't get a forget for it later, but we would like to not
	// have to remember every inode ID that we've ever minted for readdir.
	//
	// If it turns out this is not harmless, we'll need to switch to something
	// like inode IDs based on (object name, generation) hashes. But then what
	// about the birthday problem? And more importantly, what about our
	// semantic of not minting a new inode ID when the generation changes due
	// to a local action?
	for i, _ := range entries {
		entries[i].Inode = fuseops.RootInodeID + 1
	}

	return
}

// Call readSomeEntries in a loop until the directory is exhausted. Return
// contents sorted by name and with correct Offset fields.
func readAllEntries(
	ctx context.Context,
	in *inode.DirInode) (entries SortedDirents, err error) {
	// Ensure that our result is sorted, with correct offset fields.
	defer func() {
		sort.Sort(entries)

		for i := 0; i < len(entries); i++ {
			entries[i].Offset = fuseops.DirOffset(i) + 1
		}
	}()

	// Read in a loop.
	var tok string
	for {
		var batch []fuseutil.Dirent

		// Accumulate some more entries.
		batch, tok, err = readSomeEntries(ctx, in, tok)
		if err != nil {
			return
		}

		entries = append(entries, batch...)

		// Are we done?
		if tok == "" {
			break
		}
	}

	return
}

// Resolve name conflicts between file objects and directory objects (e.g. the
// objects "foo/bar" and "foo/bar/") by appending U+000A, which is illegal in
// GCS object names, to conflicting file names.
//
// Input must be sorted by name.
func fixConflictingNames(entries []fuseutil.Dirent) (err error) {
	// Sanity check.
	if !sort.IsSorted(SortedDirents(entries)) {
		err = fmt.Errorf("Expected sorted input")
		return
	}

	// Examine each adjacent pair of names.
	for i, _ := range entries {
		e := &entries[i]

		// Find the previous entry.
		if i == 0 {
			continue
		}

		prev := &entries[i-1]

		// Does the pair have matching names?
		if e.Name != prev.Name {
			continue
		}

		// Repair whichever is the file, remembering that there's no way to get any
		// other type in the mix.
		if e.Type == fuseutil.DT_File {
			e.Name += inode.ConflictingFileNameSuffix
		} else {
			if prev.Type != fuseutil.DT_File {
				panic(fmt.Sprintf("Unexpected type for entry: %v", prev))
			}
			prev.Name += inode.ConflictingFileNameSuffix
		}
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Handle a request to read from the directory, without responding.
//
// Special case: we assume that a zero offset indicates that rewinddir has been
// called (since fuse gives us no way to intercept and know for sure), and
// start the listing process over again.
//
// LOCKS_REQUIRED(dh.Mu)
func (dh *dirHandle) ReadDir(
	op *fuseops.ReadDirOp) (err error) {
	// If the request is for offset zero, we assume that either this is the first
	// call or rewinddir has been called. Reset state.
	if op.Offset == 0 {
		dh.entries = nil
		dh.entriesValid = false
	}

	// Do we need to read entries from GCS?
	if !dh.entriesValid {
		// Read entries.
		var entries []fuseutil.Dirent
		entries, err = readAllEntries(op.Context(), dh.in)
		if err != nil {
			err = fmt.Errorf("readAllEntries: %v", err)
			return
		}

		// Fix name conflicts.
		err = fixConflictingNames(entries)
		if err != nil {
			err = fmt.Errorf("fixConflictingNames: %v", err)
			return
		}

		// Update state.
		dh.entries = entries
		dh.entriesValid = true
	}

	// Is the offset past the end of what we have buffered? If so, this must be
	// an invalid seekdir according to posix.
	index := int(op.Offset)
	if index > len(dh.entries) {
		err = fuse.EINVAL
		return
	}

	// We copy out entries until we run out of entries or space.
	for i := index; i < len(dh.entries); i++ {
		op.Data = fuseutil.AppendDirent(op.Data, dh.entries[i])
		if len(op.Data) > op.Size {
			op.Data = op.Data[:op.Size]
			break
		}
	}

	return
}
<commit_msg>Refactored helpers in preparation for adding filtering.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"fmt"
	"sort"

	"github.com/googlecloudplatform/gcsfuse/fs/inode"
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

// State required for reading from directories.
type dirHandle struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	in *inode.DirInode

	/////////////////////////
	// Mutable state
	/////////////////////////

	Mu syncutil.InvariantMutex

	// All entries in the directory. Populated the first time we need one.
	//
	// INVARIANT: For each i, entries[i+1].Offset == entries[i].Offset + 1
	//
	// GUARDED_BY(Mu)
	entries []fuseutil.Dirent

	// Has entries yet been populated?
	//
	// INVARIANT: If !entriesValid, then len(entries) == 0
	//
	// GUARDED_BY(Mu)
	entriesValid bool
}

// Create a directory handle that obtains listings from the supplied inode.
func newDirHandle(in *inode.DirInode) (dh *dirHandle) {
	// Set up the basic struct.
	dh = &dirHandle{
		in: in,
	}

	// Set up invariant checking.
	dh.Mu = syncutil.NewInvariantMutex(dh.checkInvariants)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Dirents, sorted by name.
type SortedDirents []fuseutil.Dirent

func (p SortedDirents) Len() int           { return len(p) }
func (p SortedDirents) Less(i, j int) bool { return p[i].Name < p[j].Name }
func (p SortedDirents) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func (dh *dirHandle) checkInvariants() {
	// INVARIANT: For each i, entries[i+1].Offset == entries[i].Offset + 1
	for i := 0; i < len(dh.entries)-1; i++ {
		if !(dh.entries[i+1].Offset == dh.entries[i].Offset+1) {
			panic(
				fmt.Sprintf(
					"Unexpected offset sequence: %v, %v",
					dh.entries[i].Offset,
					dh.entries[i+1].Offset))
		}
	}

	// INVARIANT: If !entriesValid, then len(entries) == 0
	if !dh.entriesValid && len(dh.entries) != 0 {
		panic("Unexpected non-empty entries slice")
	}
}

// Read some entries from the directory inode. Return newTok == "" (possibly
// with a non-empty list of entries) when the end of the directory has been
// hit.
//
// The contents of the entries' Offset fields are undefined. This function
// always behaves as if implicit directories are defined; see notes on
// DirInode.ReadEntries.
func readSomeEntries(
	ctx context.Context,
	in *inode.DirInode,
	tok string) (entries []fuseutil.Dirent, newTok string, err error) {
	entries, newTok, err = in.ReadEntries(ctx, tok)
	if err != nil {
		err = fmt.Errorf("ReadEntries: %v", err)
		return
	}

	// Return a bogus inode ID for each entry, but not the root inode ID.
	//
	// NOTE(jacobsa): As far as I can tell this is harmless. Minting and
	// returning a real inode ID is difficult because fuse does not count
	// readdir as an operation that increases the inode ID's lookup count and
	// we therefore don't get a forget for it later, but we would like to not
	// have to remember every inode ID that we've ever minted for readdir.
	//
	// If it turns out this is not harmless, we'll need to switch to something
	// like inode IDs based on (object name, generation) hashes. But then what
	// about the birthday problem? And more importantly, what about our
	// semantic of not minting a new inode ID when the generation changes due
	// to a local action?
	for i, _ := range entries {
		entries[i].Inode = fuseops.RootInodeID + 1
	}

	return
}

// Read all entries for the directory, making no effort to deal with
// conflicting names or the lack of implicit directories (see notes on
// DirInode.ReadEntries).
//
// Write entries to the supplied channel, without closing. Entry Offset fields
// have unspecified contents.
func readEntries(
	ctx context.Context,
	in *inode.DirInode,
	entries chan<- fuseutil.Dirent) (err error) {
	var tok string
	for {
		// Read a batch.
		var batch []fuseutil.Dirent

		batch, tok, err = readSomeEntries(ctx, in, tok)
		if err != nil {
			return
		}

		// Write each to the channel.
		for _, e := range batch {
			select {
			case <-ctx.Done():
				err = ctx.Err()
				return

			case entries <- e:
			}
		}

		// Are we done?
		if tok == "" {
			break
		}
	}

	return
}

func readAllEntries(
	ctx context.Context,
	in *inode.DirInode) (entries SortedDirents, err error) {
	b := syncutil.NewBundle(ctx)

	// Ensure that our result is sorted, with correct offset fields.
	defer func() {
		sort.Sort(entries)

		for i := 0; i < len(entries); i++ {
			entries[i].Offset = fuseops.DirOffset(i) + 1
		}
	}()

	// Read into a channel.
	c := make(chan fuseutil.Dirent, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(c)
		err = readEntries(ctx, in, c)
		return
	})

	// Accumulate into the slice.
	b.Add(func(ctx context.Context) (err error) {
		for e := range c {
			entries = append(entries, e)
		}

		return
	})

	// Wait.
	err = b.Join()

	return
}

// Resolve name conflicts between file objects and directory objects (e.g. the
// objects "foo/bar" and "foo/bar/") by appending U+000A, which is illegal in
// GCS object names, to conflicting file names.
//
// Input must be sorted by name.
func fixConflictingNames(entries []fuseutil.Dirent) (err error) {
	// Sanity check.
	if !sort.IsSorted(SortedDirents(entries)) {
		err = fmt.Errorf("Expected sorted input")
		return
	}

	// Examine each adjacent pair of names.
	for i, _ := range entries {
		e := &entries[i]

		// Find the previous entry.
		if i == 0 {
			continue
		}

		prev := &entries[i-1]

		// Does the pair have matching names?
		if e.Name != prev.Name {
			continue
		}

		// Repair whichever is the file, remembering that there's no way to get any
		// other type in the mix.
		if e.Type == fuseutil.DT_File {
			e.Name += inode.ConflictingFileNameSuffix
		} else {
			if prev.Type != fuseutil.DT_File {
				panic(fmt.Sprintf("Unexpected type for entry: %v", prev))
			}
			prev.Name += inode.ConflictingFileNameSuffix
		}
	}

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Handle a request to read from the directory, without responding.
//
// Special case: we assume that a zero offset indicates that rewinddir has been
// called (since fuse gives us no way to intercept and know for sure), and
// start the listing process over again.
//
// LOCKS_REQUIRED(dh.Mu)
func (dh *dirHandle) ReadDir(
	op *fuseops.ReadDirOp) (err error) {
	// If the request is for offset zero, we assume that either this is the first
	// call or rewinddir has been called. Reset state.
	if op.Offset == 0 {
		dh.entries = nil
		dh.entriesValid = false
	}

	// Do we need to read entries from GCS?
	if !dh.entriesValid {
		// Read entries.
		var entries []fuseutil.Dirent
		entries, err = readAllEntries(op.Context(), dh.in)
		if err != nil {
			err = fmt.Errorf("readAllEntries: %v", err)
			return
		}

		// Fix name conflicts.
		err = fixConflictingNames(entries)
		if err != nil {
			err = fmt.Errorf("fixConflictingNames: %v", err)
			return
		}

		// Update state.
		dh.entries = entries
		dh.entriesValid = true
	}

	// Is the offset past the end of what we have buffered? If so, this must be
	// an invalid seekdir according to posix.
	index := int(op.Offset)
	if index > len(dh.entries) {
		err = fuse.EINVAL
		return
	}

	// We copy out entries until we run out of entries or space.
	for i := index; i < len(dh.entries); i++ {
		op.Data = fuseutil.AppendDirent(op.Data, dh.entries[i])
		if len(op.Data) > op.Size {
			op.Data = op.Data[:op.Size]
			break
		}
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/spf13/cobra"
)

var arch string
var supportedArch = []string{"amd64", "arm32v7", "arm64v8"}
var defaultArch = "amd64"

func init() {
	DockerBuildCmd.PersistentFlags().StringVar(&arch, "arch", defaultArch, "target architecture among: "+strings.Join(supportedArch, ", "))
	DockerPushCmd.PersistentFlags().StringVar(&arch, "arch", defaultArch, "target architecture among: "+strings.Join(supportedArch, ", "))

}

func checkArchIsSupported(arch string) {
	for _, a := range supportedArch {
		if arch == a {
			return
		}
	}
	log.Fatal("Architecture is not supported. Please select one of " + strings.Join(supportedArch, ", ") + ".")
}

func dockerBuildOfficialImage(arch string) error {
	docker := &Docker{}
	// Set default Architecture Dockerfile to amd64
	dockerfile := "Dockerfile"

	// If not the default value
	if arch != defaultArch {
		dockerfile = fmt.Sprintf("%s.%s", dockerfile, arch)
	}

	if arch == "arm32v7" {
		err := CommandWithStdout("docker", "run", "--rm", "--privileged", "multiarch/qemu-user-static", "--reset", "-p", "yes").Run()

		if err != nil {
			panic(err)
		}

		err = CommandWithStdout("bash", "-c", "wget https://github.com/multiarch/qemu-user-static/releases/download/v4.1.0-1/qemu-arm-static -O ./qemu-arm-static && chmod +x ./qemu-arm-static").Run()

		if err != nil {
			panic(err)
		}
	} else if arch == "arm64v8" {
		err := CommandWithStdout("docker", "run", "--rm", "--privileged", "multiarch/qemu-user-static", "--reset", "-p", "yes").Run()

		if err != nil {
			panic(err)
		}

		err = CommandWithStdout("bash", "-c", "wget https://github.com/multiarch/qemu-user-static/releases/download/v4.1.0-1/qemu-aarch64-static -O ./qemu-aarch64-static && chmod +x ./qemu-aarch64-static").Run()

		if err != nil {
			panic(err)
		}
	}

	return docker.Build(IntermediateDockerImageName, dockerfile, ".")
}

// DockerBuildCmd Command for building docker image of Authelia.
var DockerBuildCmd = &cobra.Command{
	Use:   "build",
	Short: "Build the docker image of Authelia",
	Run: func(cmd *cobra.Command, args []string) {
		checkArchIsSupported(arch)
		err := dockerBuildOfficialImage(arch)

		if err != nil {
			log.Fatal(err)
		}

		docker := &Docker{}
		err = docker.Tag(IntermediateDockerImageName, DockerImageName)

		if err != nil {
			panic(err)
		}
	},
}

// DockerPushCmd Command for pushing Authelia docker image to Dockerhub
var DockerPushCmd = &cobra.Command{
	Use:   "push-image",
	Short: "Publish Authelia docker image to Dockerhub",
	Run: func(cmd *cobra.Command, args []string) {
		checkArchIsSupported(arch)
		publishDockerImage(arch)
	},
}

// DockerManifestCmd Command for pushing Authelia docker manifest to Dockerhub
var DockerManifestCmd = &cobra.Command{
	Use:   "push-manifest",
	Short: "Publish Authelia docker manifest to Dockerhub",
	Run: func(cmd *cobra.Command, args []string) {
		publishDockerManifest()
	},
}

func login(docker *Docker) {
	username := os.Getenv("DOCKER_USERNAME")
	password := os.Getenv("DOCKER_PASSWORD")

	if username == "" {
		panic(errors.New("DOCKER_USERNAME is empty"))
	}

	if password == "" {
		panic(errors.New("DOCKER_PASSWORD is empty"))
	}

	fmt.Println("Login to dockerhub as " + username)
	err := docker.Login(username, password)

	if err != nil {
		fmt.Println("Login to dockerhub failed")
		panic(err)
	}
}

func deploy(docker *Docker, tag string) {
	imageWithTag := DockerImageName + ":" + tag
	fmt.Println("===================================================")
	fmt.Println("Docker image " + imageWithTag + " will be deployed on Dockerhub.")
	fmt.Println("===================================================")

	err := docker.Tag(DockerImageName, imageWithTag)

	if err != nil {
		panic(err)
	}

	err = docker.Push(imageWithTag)

	if err != nil {
		panic(err)
	}
}

func deployManifest(docker *Docker, tag string, amd64tag string, arm32v7tag string, arm64v8tag string) {
	imageWithTag := DockerImageName + ":" + tag
	fmt.Println("===================================================")
	fmt.Println("Docker manifest " + imageWithTag + " will be deployed on Dockerhub.")
	fmt.Println("===================================================")

	err := docker.Tag(DockerImageName, imageWithTag)

	if err != nil {
		panic(err)
	}

	err = docker.Manifest(imageWithTag, amd64tag, arm32v7tag, arm64v8tag)

	if err != nil {
		panic(err)
	}
}

func publishDockerImage(arch string) {
	docker := &Docker{}

	travisBranch := os.Getenv("TRAVIS_BRANCH")
	travisPullRequest := os.Getenv("TRAVIS_PULL_REQUEST")
	travisTag := os.Getenv("TRAVIS_TAG")

	if travisBranch == "master" && travisPullRequest == "false" {
		login(docker)
		deploy(docker, "master-"+arch)
	} else if travisTag != "" {
		login(docker)
		deploy(docker, travisTag+"-"+arch)
		deploy(docker, "latest-"+arch)
	} else {
		fmt.Println("Docker image will not be published")
	}
}

func publishDockerManifest() {
	docker := &Docker{}

	travisBranch := os.Getenv("TRAVIS_BRANCH")
	travisPullRequest := os.Getenv("TRAVIS_PULL_REQUEST")
	travisTag := os.Getenv("TRAVIS_TAG")

	if travisBranch == "master" && travisPullRequest == "false" {
		login(docker)
		deployManifest(docker, "master", "master-amd64", "master-arm32v7", "master-arm64v8")
	} else if travisTag != "" {
		login(docker)
		deployManifest(docker, travisTag, travisTag+"-amd64", travisTag+"-arm32v7", travisTag+"-arm64v8")
		deployManifest(docker, "latest", "latest-amd64", "latest-arm32v7", "latest-arm64v8")
	} else {
		fmt.Println("Docker manifest will not be published")
	}
}
<commit_msg>Fix docker manifest push<commit_after>package main

import (
	"errors"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/spf13/cobra"
)

var arch string
var supportedArch = []string{"amd64", "arm32v7", "arm64v8"}
var defaultArch = "amd64"

func init() {
	DockerBuildCmd.PersistentFlags().StringVar(&arch, "arch", defaultArch, "target architecture among: "+strings.Join(supportedArch, ", "))
	DockerPushCmd.PersistentFlags().StringVar(&arch, "arch", defaultArch, "target architecture among: "+strings.Join(supportedArch, ", "))

}

func checkArchIsSupported(arch string) {
	for _, a := range supportedArch {
		if arch == a {
			return
		}
	}
	log.Fatal("Architecture is not supported. Please select one of " + strings.Join(supportedArch, ", ") + ".")
}

func dockerBuildOfficialImage(arch string) error {
	docker := &Docker{}
	// Set default Architecture Dockerfile to amd64
	dockerfile := "Dockerfile"

	// If not the default value
	if arch != defaultArch {
		dockerfile = fmt.Sprintf("%s.%s", dockerfile, arch)
	}

	if arch == "arm32v7" {
		err := CommandWithStdout("docker", "run", "--rm", "--privileged", "multiarch/qemu-user-static", "--reset", "-p", "yes").Run()

		if err != nil {
			panic(err)
		}

		err = CommandWithStdout("bash", "-c", "wget https://github.com/multiarch/qemu-user-static/releases/download/v4.1.0-1/qemu-arm-static -O ./qemu-arm-static && chmod +x ./qemu-arm-static").Run()

		if err != nil {
			panic(err)
		}
	} else if arch == "arm64v8" {
		err := CommandWithStdout("docker", "run", "--rm", "--privileged", "multiarch/qemu-user-static", "--reset", "-p", "yes").Run()

		if err != nil {
			panic(err)
		}

		err = CommandWithStdout("bash", "-c", "wget https://github.com/multiarch/qemu-user-static/releases/download/v4.1.0-1/qemu-aarch64-static -O ./qemu-aarch64-static && chmod +x ./qemu-aarch64-static").Run()

		if err != nil {
			panic(err)
		}
	}

	return docker.Build(IntermediateDockerImageName, dockerfile, ".")
}

// DockerBuildCmd Command for building docker image of Authelia.
var DockerBuildCmd = &cobra.Command{
	Use:   "build",
	Short: "Build the docker image of Authelia",
	Run: func(cmd *cobra.Command, args []string) {
		checkArchIsSupported(arch)
		err := dockerBuildOfficialImage(arch)

		if err != nil {
			log.Fatal(err)
		}

		docker := &Docker{}
		err = docker.Tag(IntermediateDockerImageName, DockerImageName)

		if err != nil {
			panic(err)
		}
	},
}

// DockerPushCmd Command for pushing Authelia docker image to Dockerhub
var DockerPushCmd = &cobra.Command{
	Use:   "push-image",
	Short: "Publish Authelia docker image to Dockerhub",
	Run: func(cmd *cobra.Command, args []string) {
		checkArchIsSupported(arch)
		publishDockerImage(arch)
	},
}

// DockerManifestCmd Command for pushing Authelia docker manifest to Dockerhub
var DockerManifestCmd = &cobra.Command{
	Use:   "push-manifest",
	Short: "Publish Authelia docker manifest to Dockerhub",
	Run: func(cmd *cobra.Command, args []string) {
		publishDockerManifest()
	},
}

func login(docker *Docker) {
	username := os.Getenv("DOCKER_USERNAME")
	password := os.Getenv("DOCKER_PASSWORD")

	if username == "" {
		panic(errors.New("DOCKER_USERNAME is empty"))
	}

	if password == "" {
		panic(errors.New("DOCKER_PASSWORD is empty"))
	}

	fmt.Println("Login to dockerhub as " + username)
	err := docker.Login(username, password)

	if err != nil {
		fmt.Println("Login to dockerhub failed")
		panic(err)
	}
}

func deploy(docker *Docker, tag string) {
	imageWithTag := DockerImageName + ":" + tag
	fmt.Println("===================================================")
	fmt.Println("Docker image " + imageWithTag + " will be deployed on Dockerhub.")
	fmt.Println("===================================================")

	err := docker.Tag(DockerImageName, imageWithTag)

	if err != nil {
		panic(err)
	}

	err = docker.Push(imageWithTag)

	if err != nil {
		panic(err)
	}
}

func deployManifest(docker *Docker, tag string, amd64tag string, arm32v7tag string, arm64v8tag string) {
	imageWithTag := DockerImageName + ":" + tag
	fmt.Println("===================================================")
	fmt.Println("Docker manifest " + imageWithTag + " will be deployed on Dockerhub.")
	fmt.Println("===================================================")

	err := docker.Manifest(imageWithTag, amd64tag, arm32v7tag, arm64v8tag)

	if err != nil {
		panic(err)
	}
}

func publishDockerImage(arch string) {
	docker := &Docker{}

	travisBranch := os.Getenv("TRAVIS_BRANCH")
	travisPullRequest := os.Getenv("TRAVIS_PULL_REQUEST")
	travisTag := os.Getenv("TRAVIS_TAG")

	if travisBranch == "master" && travisPullRequest == "false" {
		login(docker)
		deploy(docker, "master-"+arch)
	} else if travisTag != "" {
		login(docker)
		deploy(docker, travisTag+"-"+arch)
		deploy(docker, "latest-"+arch)
	} else {
		fmt.Println("Docker image will not be published")
	}
}

func publishDockerManifest() {
	docker := &Docker{}

	travisBranch := os.Getenv("TRAVIS_BRANCH")
	travisPullRequest := os.Getenv("TRAVIS_PULL_REQUEST")
	travisTag := os.Getenv("TRAVIS_TAG")

	if travisBranch == "master" && travisPullRequest == "false" {
		login(docker)
		deployManifest(docker, "master", "master-amd64", "master-arm32v7", "master-arm64v8")
	} else if travisTag != "" {
		login(docker)
		deployManifest(docker, travisTag, travisTag+"-amd64", travisTag+"-arm32v7", travisTag+"-arm64v8")
		deployManifest(docker, "latest", "latest-amd64", "latest-arm32v7", "latest-arm64v8")
	} else {
		fmt.Println("Docker manifest will not be published")
	}
}
<|endoftext|>


<commit_before>package process

import (
    "bytes"
    "io"
    "os/exec"
    "github.com/johnny-morrice/pipeline"
)

// Config creates a new Info, given the args, and sends it to stdout.
func Config(stdout, stderr io.Writer, args []string) error {
    config := configbrot(args)
    return runPipeCmd(config, &bytes.Buffer{}, stdout, stderr)
}

// Render sends a new fractal image to the passed stdout pipe, corresponding to the Info
// serialized in stdin.
func Render(stdin io.Reader, stdout, stderr io.Writer) error {
    render := renderbrot()
    return runPipeCmd(render, stdin, stdout, stderr)
}

// ConfigRender sends a new fractal image to the passed stdout pipe, corresponding to configbrot's
// processing of the args slice.
func ConfigRender(stdout, stderr io.Writer, args []string) error {
    config := configbrot(args)
    render := renderbrot()

    pl := pipeline.New(&bytes.Buffer{}, stdout, stderr)
    pl.Chain(config, render)
    return pl.Exec()
}

// Zoom magnifies a section of the Info read from stdin, sending it to stdout.
func Zoom(stdin io.Reader, stdout, stderr io.Writer, args[]string) error {
    zoom := zoombrot(args)
    return runPipeCmd(zoom, stdin, stdout, stderr)
}

// Zoom reads Info from stdin, and sends a fractal to stdout, returning the magnified Info,
// serialized as a io.Reader.
func ZoomRender(stdin io.Reader, stdout, stderr io.Writer, args []string) (io.Reader, error) {
    zoomBuff := &bytes.Buffer{}
    zoomerr := Zoom(stdin, zoomBuff, stderr, args)
    if zoomerr != nil {
        return nil, zoomerr
    }

    outbuff := &bytes.Buffer{}
    rendin := io.TeeReader(zoomBuff, outbuff)

    err := Render(rendin, stdout, stderr)

    return outbuff, err
}

func zoombrot(args []string) *exec.Cmd {
    return exec.Command("zoombrot", args...)
}

func configbrot(args []string) *exec.Cmd {
    return exec.Command("configbrot", args...)
}

func renderbrot() *exec.Cmd {
    return exec.Command("renderbrot")
}

func runPipeCmd(cmd *exec.Cmd, stdin io.Reader, stdout, stderr io.Writer) error {
    cmd.Stdin = stdin
    cmd.Stdout = stdout
    cmd.Stderr = stderr
    return cmd.Run()
}<commit_msg>Add types to io streams<commit_after>package process

import (
    "bytes"
    "io"
    "os/exec"
    "github.com/johnny-morrice/pipeline"
)

type Infow io.Writer
type Infor io.Reader
type Pngw io.Writer
type Pngr io.Reader

func PngBuff() (Pngr, Pngw) {
    buff := &bytes.Buffer{}
    return Pngr(buff), Pngw(buff)
}

func InfoBuff() (Infor, Infow) {
    buff := &bytes.Buffer{}
    return Infor(buff), Infow(buff)
}

// Config creates a new Info, given the args, and sends it to stdout.
func Config(stdout Infow, stderr io.Writer, args []string) error {
    config := configbrot(args)
    return runPipeCmd(config, &bytes.Buffer{}, stdout, stderr)
}

// Render sends a new fractal image to the passed stdout pipe, corresponding to the Info
// serialized in stdin.
func Render(stdin Infor, stdout Pngw, stderr io.Writer) error {
    render := renderbrot()
    return runPipeCmd(render, stdin, stdout, stderr)
}

// ConfigRender sends a new fractal image to the passed stdout pipe, corresponding to configbrot's
// processing of the args slice.
func ConfigRender(stdout Pngw, stderr io.Writer, args []string) error {
    config := configbrot(args)
    render := renderbrot()

    pl := pipeline.New(&bytes.Buffer{}, stdout, stderr)
    pl.Chain(config, render)
    return pl.Exec()
}

// Zoom magnifies a section of the Info read from stdin, sending it to stdout.
func Zoom(stdin Infor, stdout Infow, stderr io.Writer, args[]string) error {
    zoom := zoombrot(args)
    return runPipeCmd(zoom, stdin, stdout, stderr)
}

// Zoom reads Info from stdin, and sends a fractal to stdout, returning the magnified Info,
// serialized as an Infor.
func ZoomRender(stdin Infor, stdout Pngw, stderr io.Writer, args []string) (Infor, error) {
    zoomBuff := &bytes.Buffer{}
    zoomerr := Zoom(stdin, zoomBuff, stderr, args)
    if zoomerr != nil {
        return nil, zoomerr
    }

    outbuff := &bytes.Buffer{}
    rendin := io.TeeReader(zoomBuff, outbuff)

    err := Render(rendin, stdout, stderr)

    return outbuff, err
}

func zoombrot(args []string) *exec.Cmd {
    return exec.Command("zoombrot", args...)
}

func configbrot(args []string) *exec.Cmd {
    return exec.Command("configbrot", args...)
}

func renderbrot() *exec.Cmd {
    return exec.Command("renderbrot")
}

func runPipeCmd(cmd *exec.Cmd, stdin io.Reader, stdout, stderr io.Writer) error {
    cmd.Stdin = stdin
    cmd.Stdout = stdout
    cmd.Stderr = stderr
    return cmd.Run()
}<|endoftext|>


<commit_before>package main

import (
	"log"
	"sync/atomic"
	"time"

	console "github.com/AsynkronIT/goconsole"
	"github.com/AsynkronIT/protoactor-go/actor"
	"github.com/AsynkronIT/protoactor-go/mailbox"
)

// sent to producer to request more work
type requestMoreWork struct {
	items int
}
type requestWorkBehavior struct {
	tokens   int64
	producer *actor.PID
}

func (m *requestWorkBehavior) MailboxStarted() {
	m.requestMore()
}
func (m *requestWorkBehavior) MessagePosted(msg interface{}) {

}
func (m *requestWorkBehavior) MessageReceived(msg interface{}) {
	atomic.AddInt64(&m.tokens, -1)
	if m.tokens == 0 {
		m.requestMore()
	}
}
func (m *requestWorkBehavior) MailboxEmpty() {
}

func (m *requestWorkBehavior) requestMore() {
	log.Println("Requesting more tokens")
	m.tokens = 50
	context.Send(m.producer, &requestMoreWork{items: 50})
}

type producer struct {
	requestedWork int
	worker        *actor.PID
}

func (p *producer) Receive(ctx actor.Context) {
	switch msg := ctx.Message().(type) {
	case *actor.Started:
		// spawn our worker
		workerProps := actor.PropsFromProducer(func() actor.Actor { return &worker{} }).WithMailbox(mailbox.Unbounded(&requestWorkBehavior{
			producer: ctx.Self(),
		}))
		p.worker = ctx.Spawn(workerProps)
	case *requestMoreWork:
		p.requestedWork += msg.items
		log.Println("Producer got a new work request")
		ctx.Send(ctx.Self(), &produce{})
	case *produce:
		// produce more work
		log.Println("Producer is producing work")
		ctx.Send(p.worker, &work{})

		// decrease our workload and tell ourselves to produce more work
		if p.requestedWork > 0 {
			p.requestedWork--
			ctx.Send(ctx.Self(), &produce{})
		}
	}
}

type produce struct{}

type worker struct {
}

func (w *worker) Receive(ctx actor.Context) {
	switch msg := ctx.Message().(type) {
	case *work:
		log.Printf("Worker is working %v", msg)
		time.Sleep(100 * time.Millisecond)
	}
}

type work struct {
}

var context *actor.RootContext

func main() {
	context := actor.EmptyRootContext
	producerProps := actor.PropsFromProducer(func() actor.Actor { return &producer{} })
	context.Spawn(producerProps)

	console.ReadLine()
}
<commit_msg>fix(examples/backpressure): initialize the global context insted local context<commit_after>package main

import (
	"log"
	"sync/atomic"
	"time"

	console "github.com/AsynkronIT/goconsole"
	"github.com/AsynkronIT/protoactor-go/actor"
	"github.com/AsynkronIT/protoactor-go/mailbox"
)

// sent to producer to request more work
type requestMoreWork struct {
	items int
}
type requestWorkBehavior struct {
	tokens   int64
	producer *actor.PID
}

func (m *requestWorkBehavior) MailboxStarted() {
	m.requestMore()
}
func (m *requestWorkBehavior) MessagePosted(msg interface{}) {

}
func (m *requestWorkBehavior) MessageReceived(msg interface{}) {
	atomic.AddInt64(&m.tokens, -1)
	if m.tokens == 0 {
		m.requestMore()
	}
}
func (m *requestWorkBehavior) MailboxEmpty() {
}

func (m *requestWorkBehavior) requestMore() {
	log.Println("Requesting more tokens")
	m.tokens = 50
	context.Send(m.producer, &requestMoreWork{items: 50})
}

type producer struct {
	requestedWork int
	worker        *actor.PID
}

func (p *producer) Receive(ctx actor.Context) {
	switch msg := ctx.Message().(type) {
	case *actor.Started:
		// spawn our worker
		workerProps := actor.PropsFromProducer(func() actor.Actor { return &worker{} }).WithMailbox(mailbox.Unbounded(&requestWorkBehavior{
			producer: ctx.Self(),
		}))
		p.worker = ctx.Spawn(workerProps)
	case *requestMoreWork:
		p.requestedWork += msg.items
		log.Println("Producer got a new work request")
		ctx.Send(ctx.Self(), &produce{})
	case *produce:
		// produce more work
		log.Println("Producer is producing work")
		ctx.Send(p.worker, &work{})

		// decrease our workload and tell ourselves to produce more work
		if p.requestedWork > 0 {
			p.requestedWork--
			ctx.Send(ctx.Self(), &produce{})
		}
	}
}

type produce struct{}

type worker struct {
}

func (w *worker) Receive(ctx actor.Context) {
	switch msg := ctx.Message().(type) {
	case *work:
		log.Printf("Worker is working %v", msg)
		time.Sleep(100 * time.Millisecond)
	}
}

type work struct {
}

var context *actor.RootContext

func main() {
	context = actor.EmptyRootContext
	producerProps := actor.PropsFromProducer(func() actor.Actor { return &producer{} })
	context.Spawn(producerProps)

	console.ReadLine()
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"regexp"
	"strings"
	"time"

	"github.com/flynn/flynn/Godeps/_workspace/src/github.com/flynn/go-docopt"
	"github.com/flynn/flynn/controller/client"
	ct "github.com/flynn/flynn/controller/types"
	"github.com/flynn/flynn/host/types"
	"github.com/flynn/flynn/pkg/cluster"
	"github.com/flynn/flynn/pkg/exec"
	"github.com/flynn/flynn/pkg/random"
	"github.com/flynn/flynn/pkg/version"
)

var clusterc = cluster.NewClient()

func init() {
	log.SetFlags(0)
}

var typesPattern = regexp.MustCompile("types.* -> (.+)\n")

const blobstoreURL = "http://blobstore.discoverd"
const scaleTimeout = 20 * time.Second

func parsePairs(args *docopt.Args, str string) (map[string]string, error) {
	pairs := args.All[str].([]string)
	item := make(map[string]string, len(pairs))
	for _, s := range pairs {
		v := strings.SplitN(s, "=", 2)
		if len(v) != 2 {
			return nil, fmt.Errorf("invalid var format: %q", s)
		}
		item[v[0]] = v[1]
	}
	return item, nil
}

func main() {
	client, err := controller.NewClient("", os.Getenv("CONTROLLER_KEY"))
	if err != nil {
		log.Fatalln("Unable to connect to controller:", err)
	}

	usage := `
Usage: flynn-receiver <app> <rev> [-e <var>=<val>]... [-m <key>=<val>]...

Options:
	-e,--env <var>=<val>
	-m,--meta <key>=<val>
`[1:]
	args, _ := docopt.Parse(usage, nil, true, version.String(), false)

	appName := args.String["<app>"]
	env, err := parsePairs(args, "--env")
	if err != nil {
		log.Fatal(err)
	}
	meta, err := parsePairs(args, "--meta")
	if err != nil {
		log.Fatal(err)
	}

	app, err := client.GetApp(appName)
	if err == controller.ErrNotFound {
		log.Fatalf("Unknown app %q", appName)
	} else if err != nil {
		log.Fatalln("Error retrieving app:", err)
	}
	prevRelease, err := client.GetAppRelease(app.Name)
	if err == controller.ErrNotFound {
		prevRelease = &ct.Release{}
	} else if err != nil {
		log.Fatalln("Error getting current app release:", err)
	}

	fmt.Printf("-----> Building %s...\n", app.Name)

	jobEnv := make(map[string]string)
	jobEnv["BUILD_CACHE_URL"] = fmt.Sprintf("%s/%s-cache.tgz", blobstoreURL, app.ID)
	if buildpackURL, ok := env["BUILDPACK_URL"]; ok {
		jobEnv["BUILDPACK_URL"] = buildpackURL
	} else if buildpackURL, ok := prevRelease.Env["BUILDPACK_URL"]; ok {
		jobEnv["BUILDPACK_URL"] = buildpackURL
	}
	for _, k := range []string{"SSH_CLIENT_KEY", "SSH_CLIENT_HOSTS"} {
		if v := os.Getenv(k); v != "" {
			jobEnv[k] = v
		}
	}
	slugURL := fmt.Sprintf("%s/%s/slug.tgz", blobstoreURL, random.UUID())

	job := &host.Job{
		Config: host.ContainerConfig{
			Cmd:        []string{slugURL},
			Env:        jobEnv,
			Stdin:      true,
			DisableLog: true,
		},
		Partition: "background",
		Metadata: map[string]string{
			"flynn-controller.app":      app.ID,
			"flynn-controller.app_name": app.Name,
			"flynn-controller.release":  prevRelease.ID,
			"flynn-controller.type":     "slugbuilder",
		},
	}
	if sb, ok := prevRelease.Processes["slugbuilder"]; ok {
		job.Resources = sb.Resources
	}

	cmd := exec.Job(exec.DockerImage(os.Getenv("SLUGBUILDER_IMAGE_URI")), job)
	var output bytes.Buffer
	cmd.Stdout = io.MultiWriter(os.Stdout, &output)
	cmd.Stderr = os.Stderr

	if len(prevRelease.Env) > 0 {
		stdin, err := cmd.StdinPipe()
		if err != nil {
			log.Fatalln(err)
		}
		go appendEnvDir(os.Stdin, stdin, prevRelease.Env)
	} else {
		cmd.Stdin = os.Stdin
	}

	if err := cmd.Run(); err != nil {
		log.Fatalln("Build failed:", err)
	}

	var types []string
	if match := typesPattern.FindSubmatch(output.Bytes()); match != nil {
		types = strings.Split(string(match[1]), ", ")
	}

	fmt.Printf("-----> Creating release...\n")

	artifact := &ct.Artifact{Type: host.ArtifactTypeDocker, URI: os.Getenv("SLUGRUNNER_IMAGE_URI")}
	if err := client.CreateArtifact(artifact); err != nil {
		log.Fatalln("Error creating image artifact:", err)
	}

	slugArtifact := &ct.Artifact{
		Type: host.ArtifactTypeFile,
		URI:  slugURL,
		Meta: map[string]string{"blobstore": "true"},
	}
	if err := client.CreateArtifact(slugArtifact); err != nil {
		log.Fatalln("Error creating slug artifact:", err)
	}

	release := &ct.Release{
		ArtifactIDs: []string{artifact.ID, slugArtifact.ID},
		Env:         prevRelease.Env,
		Meta:        prevRelease.Meta,
	}
	if release.Meta == nil {
		release.Meta = make(map[string]string, len(meta))
	}
	if release.Env == nil {
		release.Env = make(map[string]string, len(env))
	}
	for k, v := range env {
		release.Env[k] = v
	}
	for k, v := range meta {
		release.Meta[k] = v
	}
	procs := make(map[string]ct.ProcessType)
	for _, t := range types {
		proc := prevRelease.Processes[t]
		proc.Cmd = []string{"start", t}
		if t == "web" || strings.HasSuffix(t, "-web") {
			proc.Service = app.Name + "-" + t
			proc.Ports = []ct.Port{{
				Port:  8080,
				Proto: "tcp",
				Service: &host.Service{
					Name:   proc.Service,
					Create: true,
					Check:  &host.HealthCheck{Type: "tcp"},
				},
			}}
		}
		procs[t] = proc
	}
	release.Processes = procs

	if err := client.CreateRelease(release); err != nil {
		log.Fatalln("Error creating release:", err)
	}
	if err := client.DeployAppRelease(app.Name, release.ID); err != nil {
		log.Fatalln("Error deploying app release:", err)
	}

	fmt.Println("=====> Application deployed")

	if needsDefaultScale(app.ID, prevRelease.ID, procs, client) {
		formation := &ct.Formation{
			AppID:     app.ID,
			ReleaseID: release.ID,
			Processes: map[string]int{"web": 1},
		}

		watcher, err := client.WatchJobEvents(app.ID, release.ID)
		if err != nil {
			log.Fatalln("Error streaming job events", err)
			return
		}
		defer watcher.Close()

		if err := client.PutFormation(formation); err != nil {
			log.Fatalln("Error putting formation:", err)
		}
		fmt.Println("=====> Waiting for web job to start...")

		err = watcher.WaitFor(ct.JobEvents{"web": ct.JobUpEvents(1)}, scaleTimeout, func(e *ct.Job) error {
			switch e.State {
			case ct.JobStateUp:
				fmt.Println("=====> Default web formation scaled to 1")
			case ct.JobStateDown:
				return fmt.Errorf("Failed to scale web process type")
			}
			return nil
		})
		if err != nil {
			log.Fatalln(err.Error())
		}
	}
}

// needsDefaultScale indicates whether a release needs a default scale based on
// whether it has a web process type and either has no previous release or no
// previous scale.
func needsDefaultScale(appID, prevReleaseID string, procs map[string]ct.ProcessType, client controller.Client) bool {
	if _, ok := procs["web"]; !ok {
		return false
	}
	if prevReleaseID == "" {
		return true
	}
	_, err := client.GetFormation(appID, prevReleaseID)
	return err == controller.ErrNotFound
}

func appendEnvDir(stdin io.Reader, pipe io.WriteCloser, env map[string]string) {
	defer pipe.Close()
	tr := tar.NewReader(stdin)
	tw := tar.NewWriter(pipe)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			log.Fatalln(err)
		}
		hdr.Name = path.Join("app", hdr.Name)
		if err := tw.WriteHeader(hdr); err != nil {
			log.Fatalln(err)
		}
		if _, err := io.Copy(tw, tr); err != nil {
			log.Fatalln(err)
		}
	}
	// append env dir
	for key, value := range env {
		hdr := &tar.Header{
			Name:    path.Join("env", key),
			Mode:    0644,
			ModTime: time.Now(),
			Size:    int64(len(value)),
		}

		if err := tw.WriteHeader(hdr); err != nil {
			log.Fatalln(err)
		}
		if _, err := tw.Write([]byte(value)); err != nil {
			log.Fatalln(err)
		}
	}
	hdr := &tar.Header{
		Name:    ".ENV_DIR_bdca46b87df0537eaefe79bb632d37709ff1df18",
		Mode:    0400,
		ModTime: time.Now(),
		Size:    0,
	}
	if err := tw.WriteHeader(hdr); err != nil {
		log.Fatalln(err)
	}
}
<commit_msg>gitreceive/receiver: Use deploy timeout when waiting for web job<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"regexp"
	"strings"
	"time"

	"github.com/flynn/flynn/Godeps/_workspace/src/github.com/flynn/go-docopt"
	"github.com/flynn/flynn/controller/client"
	ct "github.com/flynn/flynn/controller/types"
	"github.com/flynn/flynn/host/types"
	"github.com/flynn/flynn/pkg/cluster"
	"github.com/flynn/flynn/pkg/exec"
	"github.com/flynn/flynn/pkg/random"
	"github.com/flynn/flynn/pkg/version"
)

var clusterc = cluster.NewClient()

func init() {
	log.SetFlags(0)
}

var typesPattern = regexp.MustCompile("types.* -> (.+)\n")

const blobstoreURL = "http://blobstore.discoverd"

func parsePairs(args *docopt.Args, str string) (map[string]string, error) {
	pairs := args.All[str].([]string)
	item := make(map[string]string, len(pairs))
	for _, s := range pairs {
		v := strings.SplitN(s, "=", 2)
		if len(v) != 2 {
			return nil, fmt.Errorf("invalid var format: %q", s)
		}
		item[v[0]] = v[1]
	}
	return item, nil
}

func main() {
	client, err := controller.NewClient("", os.Getenv("CONTROLLER_KEY"))
	if err != nil {
		log.Fatalln("Unable to connect to controller:", err)
	}

	usage := `
Usage: flynn-receiver <app> <rev> [-e <var>=<val>]... [-m <key>=<val>]...

Options:
	-e,--env <var>=<val>
	-m,--meta <key>=<val>
`[1:]
	args, _ := docopt.Parse(usage, nil, true, version.String(), false)

	appName := args.String["<app>"]
	env, err := parsePairs(args, "--env")
	if err != nil {
		log.Fatal(err)
	}
	meta, err := parsePairs(args, "--meta")
	if err != nil {
		log.Fatal(err)
	}

	app, err := client.GetApp(appName)
	if err == controller.ErrNotFound {
		log.Fatalf("Unknown app %q", appName)
	} else if err != nil {
		log.Fatalln("Error retrieving app:", err)
	}
	prevRelease, err := client.GetAppRelease(app.Name)
	if err == controller.ErrNotFound {
		prevRelease = &ct.Release{}
	} else if err != nil {
		log.Fatalln("Error getting current app release:", err)
	}

	fmt.Printf("-----> Building %s...\n", app.Name)

	jobEnv := make(map[string]string)
	jobEnv["BUILD_CACHE_URL"] = fmt.Sprintf("%s/%s-cache.tgz", blobstoreURL, app.ID)
	if buildpackURL, ok := env["BUILDPACK_URL"]; ok {
		jobEnv["BUILDPACK_URL"] = buildpackURL
	} else if buildpackURL, ok := prevRelease.Env["BUILDPACK_URL"]; ok {
		jobEnv["BUILDPACK_URL"] = buildpackURL
	}
	for _, k := range []string{"SSH_CLIENT_KEY", "SSH_CLIENT_HOSTS"} {
		if v := os.Getenv(k); v != "" {
			jobEnv[k] = v
		}
	}
	slugURL := fmt.Sprintf("%s/%s/slug.tgz", blobstoreURL, random.UUID())

	job := &host.Job{
		Config: host.ContainerConfig{
			Cmd:        []string{slugURL},
			Env:        jobEnv,
			Stdin:      true,
			DisableLog: true,
		},
		Partition: "background",
		Metadata: map[string]string{
			"flynn-controller.app":      app.ID,
			"flynn-controller.app_name": app.Name,
			"flynn-controller.release":  prevRelease.ID,
			"flynn-controller.type":     "slugbuilder",
		},
	}
	if sb, ok := prevRelease.Processes["slugbuilder"]; ok {
		job.Resources = sb.Resources
	}

	cmd := exec.Job(exec.DockerImage(os.Getenv("SLUGBUILDER_IMAGE_URI")), job)
	var output bytes.Buffer
	cmd.Stdout = io.MultiWriter(os.Stdout, &output)
	cmd.Stderr = os.Stderr

	if len(prevRelease.Env) > 0 {
		stdin, err := cmd.StdinPipe()
		if err != nil {
			log.Fatalln(err)
		}
		go appendEnvDir(os.Stdin, stdin, prevRelease.Env)
	} else {
		cmd.Stdin = os.Stdin
	}

	if err := cmd.Run(); err != nil {
		log.Fatalln("Build failed:", err)
	}

	var types []string
	if match := typesPattern.FindSubmatch(output.Bytes()); match != nil {
		types = strings.Split(string(match[1]), ", ")
	}

	fmt.Printf("-----> Creating release...\n")

	artifact := &ct.Artifact{Type: host.ArtifactTypeDocker, URI: os.Getenv("SLUGRUNNER_IMAGE_URI")}
	if err := client.CreateArtifact(artifact); err != nil {
		log.Fatalln("Error creating image artifact:", err)
	}

	slugArtifact := &ct.Artifact{
		Type: host.ArtifactTypeFile,
		URI:  slugURL,
		Meta: map[string]string{"blobstore": "true"},
	}
	if err := client.CreateArtifact(slugArtifact); err != nil {
		log.Fatalln("Error creating slug artifact:", err)
	}

	release := &ct.Release{
		ArtifactIDs: []string{artifact.ID, slugArtifact.ID},
		Env:         prevRelease.Env,
		Meta:        prevRelease.Meta,
	}
	if release.Meta == nil {
		release.Meta = make(map[string]string, len(meta))
	}
	if release.Env == nil {
		release.Env = make(map[string]string, len(env))
	}
	for k, v := range env {
		release.Env[k] = v
	}
	for k, v := range meta {
		release.Meta[k] = v
	}
	procs := make(map[string]ct.ProcessType)
	for _, t := range types {
		proc := prevRelease.Processes[t]
		proc.Cmd = []string{"start", t}
		if t == "web" || strings.HasSuffix(t, "-web") {
			proc.Service = app.Name + "-" + t
			proc.Ports = []ct.Port{{
				Port:  8080,
				Proto: "tcp",
				Service: &host.Service{
					Name:   proc.Service,
					Create: true,
					Check:  &host.HealthCheck{Type: "tcp"},
				},
			}}
		}
		procs[t] = proc
	}
	release.Processes = procs

	if err := client.CreateRelease(release); err != nil {
		log.Fatalln("Error creating release:", err)
	}
	if err := client.DeployAppRelease(app.Name, release.ID); err != nil {
		log.Fatalln("Error deploying app release:", err)
	}

	fmt.Println("=====> Application deployed")

	if needsDefaultScale(app.ID, prevRelease.ID, procs, client) {
		formation := &ct.Formation{
			AppID:     app.ID,
			ReleaseID: release.ID,
			Processes: map[string]int{"web": 1},
		}

		watcher, err := client.WatchJobEvents(app.ID, release.ID)
		if err != nil {
			log.Fatalln("Error streaming job events", err)
			return
		}
		defer watcher.Close()

		if err := client.PutFormation(formation); err != nil {
			log.Fatalln("Error putting formation:", err)
		}
		fmt.Println("=====> Waiting for web job to start...")

		err = watcher.WaitFor(ct.JobEvents{"web": ct.JobUpEvents(1)}, time.Duration(app.DeployTimeout)*time.Second, func(e *ct.Job) error {
			switch e.State {
			case ct.JobStateUp:
				fmt.Println("=====> Default web formation scaled to 1")
			case ct.JobStateDown:
				return fmt.Errorf("Failed to scale web process type")
			}
			return nil
		})
		if err != nil {
			log.Fatalln(err.Error())
		}
	}
}

// needsDefaultScale indicates whether a release needs a default scale based on
// whether it has a web process type and either has no previous release or no
// previous scale.
func needsDefaultScale(appID, prevReleaseID string, procs map[string]ct.ProcessType, client controller.Client) bool {
	if _, ok := procs["web"]; !ok {
		return false
	}
	if prevReleaseID == "" {
		return true
	}
	_, err := client.GetFormation(appID, prevReleaseID)
	return err == controller.ErrNotFound
}

func appendEnvDir(stdin io.Reader, pipe io.WriteCloser, env map[string]string) {
	defer pipe.Close()
	tr := tar.NewReader(stdin)
	tw := tar.NewWriter(pipe)
	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			// end of tar archive
			break
		}
		if err != nil {
			log.Fatalln(err)
		}
		hdr.Name = path.Join("app", hdr.Name)
		if err := tw.WriteHeader(hdr); err != nil {
			log.Fatalln(err)
		}
		if _, err := io.Copy(tw, tr); err != nil {
			log.Fatalln(err)
		}
	}
	// append env dir
	for key, value := range env {
		hdr := &tar.Header{
			Name:    path.Join("env", key),
			Mode:    0644,
			ModTime: time.Now(),
			Size:    int64(len(value)),
		}

		if err := tw.WriteHeader(hdr); err != nil {
			log.Fatalln(err)
		}
		if _, err := tw.Write([]byte(value)); err != nil {
			log.Fatalln(err)
		}
	}
	hdr := &tar.Header{
		Name:    ".ENV_DIR_bdca46b87df0537eaefe79bb632d37709ff1df18",
		Mode:    0400,
		ModTime: time.Now(),
		Size:    0,
	}
	if err := tw.WriteHeader(hdr); err != nil {
		log.Fatalln(err)
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2013-2017 The btcsuite developers
// Copyright (c) 2015-2016 The Decred developers
// Heavily inspired by https://github.com/btcsuite/btcd/blob/master/version.go
// Copyright (C) 2015-2017 The Lightning Network Developers

package build

import (
	"fmt"
	"strings"
)

var (
	// Commit stores the current commit of this build, which includes the
	// most recent tag, the number of commits since that tag (if non-zero),
	// the commit hash, and a dirty marker. This should be set using the
	// -ldflags during compilation.
	Commit string

	// CommitHash stores the current commit hash of this build, this should
	// be set using the -ldflags during compilation.
	CommitHash string

	// RawTags contains the raw set of build tags, separated by commas. This
	// should be set using -ldflags during compilation.
	RawTags string

	// GoVersion stores the go version that the executable was compiled
	// with. This hsould be set using -ldflags during compilation.
	GoVersion string
)

// semanticAlphabet is the set of characters that are permitted for use in an
// AppPreRelease.
const semanticAlphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-."

// These constants define the application version and follow the semantic
// versioning 2.0.0 spec (http://semver.org/).
const (
	// AppMajor defines the major version of this binary.
	AppMajor uint = 0

	// AppMinor defines the minor version of this binary.
	AppMinor uint = 13

	// AppPatch defines the application patch for this binary.
	AppPatch uint = 0

	// AppPreRelease MUST only contain characters from semanticAlphabet
	// per the semantic versioning spec.
	AppPreRelease = "beta.rc3"
)

func init() {
	// Assert that AppPreRelease is valid according to the semantic
	// versioning guidelines for pre-release version and build metadata
	// strings. In particular it MUST only contain characters in
	// semanticAlphabet.
	for _, r := range AppPreRelease {
		if !strings.ContainsRune(semanticAlphabet, r) {
			panic(fmt.Errorf("rune: %v is not in the semantic "+
				"alphabet", r))
		}
	}
}

// Version returns the application version as a properly formed string per the
// semantic versioning 2.0.0 spec (http://semver.org/).
func Version() string {
	// Start with the major, minor, and patch versions.
	version := fmt.Sprintf("%d.%d.%d", AppMajor, AppMinor, AppPatch)

	// Append pre-release version if there is one. The hyphen called for by
	// the semantic versioning spec is automatically appended and should not
	// be contained in the pre-release string.
	if AppPreRelease != "" {
		version = fmt.Sprintf("%s-%s", version, AppPreRelease)
	}

	return version
}

// Tags returns the list of build tags that were compiled into the executable.
func Tags() []string {
	if len(RawTags) == 0 {
		return nil
	}

	return strings.Split(RawTags, ",")
}
<commit_msg>build: bump version to v0.13.0-beta.rc4<commit_after>// Copyright (c) 2013-2017 The btcsuite developers
// Copyright (c) 2015-2016 The Decred developers
// Heavily inspired by https://github.com/btcsuite/btcd/blob/master/version.go
// Copyright (C) 2015-2017 The Lightning Network Developers

package build

import (
	"fmt"
	"strings"
)

var (
	// Commit stores the current commit of this build, which includes the
	// most recent tag, the number of commits since that tag (if non-zero),
	// the commit hash, and a dirty marker. This should be set using the
	// -ldflags during compilation.
	Commit string

	// CommitHash stores the current commit hash of this build, this should
	// be set using the -ldflags during compilation.
	CommitHash string

	// RawTags contains the raw set of build tags, separated by commas. This
	// should be set using -ldflags during compilation.
	RawTags string

	// GoVersion stores the go version that the executable was compiled
	// with. This hsould be set using -ldflags during compilation.
	GoVersion string
)

// semanticAlphabet is the set of characters that are permitted for use in an
// AppPreRelease.
const semanticAlphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz-."

// These constants define the application version and follow the semantic
// versioning 2.0.0 spec (http://semver.org/).
const (
	// AppMajor defines the major version of this binary.
	AppMajor uint = 0

	// AppMinor defines the minor version of this binary.
	AppMinor uint = 13

	// AppPatch defines the application patch for this binary.
	AppPatch uint = 0

	// AppPreRelease MUST only contain characters from semanticAlphabet
	// per the semantic versioning spec.
	AppPreRelease = "beta.rc4"
)

func init() {
	// Assert that AppPreRelease is valid according to the semantic
	// versioning guidelines for pre-release version and build metadata
	// strings. In particular it MUST only contain characters in
	// semanticAlphabet.
	for _, r := range AppPreRelease {
		if !strings.ContainsRune(semanticAlphabet, r) {
			panic(fmt.Errorf("rune: %v is not in the semantic "+
				"alphabet", r))
		}
	}
}

// Version returns the application version as a properly formed string per the
// semantic versioning 2.0.0 spec (http://semver.org/).
func Version() string {
	// Start with the major, minor, and patch versions.
	version := fmt.Sprintf("%d.%d.%d", AppMajor, AppMinor, AppPatch)

	// Append pre-release version if there is one. The hyphen called for by
	// the semantic versioning spec is automatically appended and should not
	// be contained in the pre-release string.
	if AppPreRelease != "" {
		version = fmt.Sprintf("%s-%s", version, AppPreRelease)
	}

	return version
}

// Tags returns the list of build tags that were compiled into the executable.
func Tags() []string {
	if len(RawTags) == 0 {
		return nil
	}

	return strings.Split(RawTags, ",")
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"strings"

	"github.com/mikedanese/gazel/third_party/go/path/filepath"

	bzl "github.com/bazelbuild/buildifier/core"
	"github.com/golang/glog"
)

var kubeRoot = flag.String("root", "", "root of kubernetes source")
var dryRun = flag.Bool("dry-run", false, "run in dry mode")

func main() {
	flag.Parse()
	flag.Set("alsologtostderr", "true")
	if *kubeRoot == "" {
		glog.Fatalf("-root argument is required")
	}
	v := Venderor{
		ctx: context(),
	}
	if len(flag.Args()) == 1 {
		v.updateSinglePkg(flag.Args()[0])
	} else {
		v.walkVendor()
		if err := v.walkRepo(); err != nil {
			glog.Fatalf("err: %v", err)
		}
	}
}

type Venderor struct {
	ctx *build.Context
}

func writeHeaders(file *bzl.File) {
	pkgRule := bzl.Rule{
		&bzl.CallExpr{
			X: &bzl.LiteralExpr{Token: "package"},
		},
	}
	pkgRule.SetAttr("default_visibility", asExpr([]string{"//visibility:public"}))

	file.Stmt = append(file.Stmt,
		[]bzl.Expr{
			pkgRule.Call,
			&bzl.CallExpr{
				X:    &bzl.LiteralExpr{Token: "licenses"},
				List: []bzl.Expr{asExpr([]string{"notice"})},
			},
			&bzl.CallExpr{
				X: &bzl.LiteralExpr{Token: "load"},
				List: asExpr([]string{
					"@io_bazel_rules_go//go:def.bzl",
					"go_binary",
					"go_library",
					"go_test",
					"cgo_library",
				}).(*bzl.ListExpr).List,
			},
		}...,
	)
}

func writeRules(file *bzl.File, rules []*bzl.Rule) {
	for _, rule := range rules {
		file.Stmt = append(file.Stmt, rule.Call)
	}
}

func (v *Venderor) resolve(ipath string) Label {
	if strings.HasPrefix(ipath, "k8s.io/kubernetes") {
		return Label{
			pkg: strings.TrimPrefix(ipath, "k8s.io/kubernetes/"),
			tag: "go_default_library",
		}
	}
	return Label{
		pkg: "vendor",
		tag: ipath,
	}
}

func (v *Venderor) walk(root string, f func(path, ipath string, pkg *build.Package) error) error {
	return sfilepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		ipath, err := filepath.Rel(root, path)
		if err != nil {
			return err
		}
		if !info.IsDir() {
			return nil
		}
		pkg, err := v.ctx.ImportDir(filepath.Join(*kubeRoot, path), build.ImportComment)
		if err != nil {
			if _, ok := err.(*build.NoGoError); err != nil && ok {
				return nil
			} else {
				return err
			}
		}

		return f(path, ipath, pkg)
	})
}

func (v *Venderor) walkRepo() error {
	for _, root := range []string{
		"./pkg",
		"./cmd",
		"./third_party",
		"./plugin",
		"./test",
		"./federation",
	} {
		if err := v.walk(root, v.updatePkg); err != nil {
			return err
		}
	}
	return nil
}

func (v *Venderor) updateSinglePkg(path string) error {
	pkg, err := v.ctx.ImportDir("./"+path, build.ImportComment)
	if err != nil {
		if _, ok := err.(*build.NoGoError); err != nil && ok {
			return nil
		} else {
			return err
		}
	}
	return v.updatePkg(path, "", pkg)
}

func (v *Venderor) updatePkg(path, _ string, pkg *build.Package) error {
	var rules []*bzl.Rule

	var attrs Attrs = make(Attrs)
	srcs := asExpr(merge(pkg.GoFiles, pkg.SFiles)).(*bzl.ListExpr)

	deps := v.extractDeps(pkg.Imports)

	if len(srcs.List) == 0 {
		return nil
	}
	attrs.Set("srcs", srcs)

	if len(deps.List) > 0 {
		attrs.Set("deps", deps)
	}

	if pkg.IsCommand() {
		rules = append(rules, newRule("go_binary", filepath.Base(pkg.Dir), attrs))
	} else {
		rules = append(rules, newRule("go_library", "go_default_library", attrs))
		if len(pkg.TestGoFiles) != 0 {
			rules = append(rules, newRule("go_test", "go_default_test", map[string]bzl.Expr{
				"srcs":    asExpr(pkg.TestGoFiles),
				"deps":    v.extractDeps(pkg.TestImports),
				"library": asExpr("go_default_library"),
			}))
		}
	}

	if len(pkg.XTestGoFiles) != 0 {
		rules = append(rules, newRule("go_test", "go_default_xtest", map[string]bzl.Expr{
			"srcs": asExpr(pkg.XTestGoFiles),
			"deps": v.extractDeps(pkg.XTestImports),
		}))
	}

	wrote, err := ReconcileRules(filepath.Join(path, "BUILD"), rules)
	if err != nil {
		return err
	}
	if wrote {
		fmt.Fprintf(os.Stderr, "wrote BUILD for %q\n", pkg.Dir)
	}
	return nil
}

func (v *Venderor) walkVendor() {
	var rules []*bzl.Rule
	if err := v.walk("./vendor", func(path, ipath string, pkg *build.Package) error {
		var attrs Attrs = make(Attrs)

		srcs := asExpr(
			apply(
				merge(pkg.GoFiles, pkg.SFiles),
				mapper(func(s string) string {
					return strings.TrimPrefix(filepath.Join(path, s), "vendor/")
				}),
			),
		).(*bzl.ListExpr)

		cgoSrcs := asExpr(
			apply(
				merge(pkg.CgoFiles, pkg.CFiles, pkg.CXXFiles, pkg.HFiles),
				mapper(func(s string) string {
					return strings.TrimPrefix(filepath.Join(path, s), "vendor/")
				}),
			),
		).(*bzl.ListExpr)

		deps := v.extractDeps(pkg.Imports)
		attrs.Set("srcs", srcs)

		if len(deps.List) > 0 {
			attrs.Set("deps", deps)
		}

		if pkg.IsCommand() {
			rules = append(rules, newRule("go_binary", v.resolve(ipath).tag, attrs))
		} else {
			if len(cgoSrcs.List) != 0 {
				cgoPname := v.resolve(ipath).tag + "_cgo"
				cgoDeps := v.extractDeps(pkg.TestImports)
				cgoRule := newRule("cgo_library", cgoPname, map[string]bzl.Expr{
					"srcs":      cgoSrcs,
					"clinkopts": asExpr([]string{"-ldl", "-lz", "-lm", "-lpthread", "-ldl"}),
				})
				rules = append(rules, cgoRule)
				if len(cgoDeps.List) != 0 {
					cgoRule.SetAttr("deps", cgoDeps)
				}
				attrs["library"] = asExpr(cgoPname)
			}
			rules = append(rules, newRule("go_library", v.resolve(ipath).tag, attrs))
		}
		return nil
	}); err != nil {
		glog.Fatalf("err: %v", err)
	}
	if _, err := ReconcileRules("./vendor/BUILD", rules); err != nil {
		glog.Fatalf("err: %v", err)
	}
}

func (v *Venderor) extractDeps(deps []string) *bzl.ListExpr {
	return asExpr(
		apply(
			merge(deps),
			filterer(func(s string) bool {
				pkg, err := v.ctx.Import(s, *kubeRoot, build.ImportComment)
				if err != nil {
					if !strings.Contains(err.Error(), `cannot find package "C"`) {
						fmt.Fprintf(os.Stderr, "extract err: %v\n", err)
					}
					return false
				}
				if pkg.Goroot {
					return false
				}
				return true
			}),
			mapper(func(s string) string {
				return v.resolve(s).String()
			}),
		),
	).(*bzl.ListExpr)
}

type Attrs map[string]bzl.Expr

func (a Attrs) Set(name string, expr bzl.Expr) {
	a[name] = expr
}

type Label struct {
	pkg, tag string
}

func (l Label) String() string {
	return fmt.Sprintf("//%v:%v", l.pkg, l.tag)
}

func asExpr(e interface{}) bzl.Expr {
	rv := reflect.ValueOf(e)
	switch rv.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		return &bzl.LiteralExpr{Token: fmt.Sprintf("%d", e)}
	case reflect.Float32, reflect.Float64:
		return &bzl.LiteralExpr{Token: fmt.Sprintf("%f", e)}
	case reflect.String:
		return &bzl.StringExpr{Value: e.(string)}
	case reflect.Slice, reflect.Array:
		var list []bzl.Expr
		for i := 0; i < rv.Len(); i++ {
			list = append(list, asExpr(rv.Index(i).Interface()))
		}
		return &bzl.ListExpr{List: list}
	default:
		glog.Fatalf("Uh oh")
		return nil
	}
}

type Sed func(s []string) []string

func mapString(in []string, f func(string) string) []string {
	var out []string
	for _, s := range in {
		out = append(out, f(s))
	}
	return out
}

func mapper(f func(string) string) Sed {
	return func(in []string) []string {
		return mapString(in, f)
	}
}

func filterString(in []string, f func(string) bool) []string {
	var out []string
	for _, s := range in {
		if f(s) {
			out = append(out, s)
		}
	}
	return out
}

func filterer(f func(string) bool) Sed {
	return func(in []string) []string {
		return filterString(in, f)
	}
}

func apply(stream []string, seds ...Sed) []string {
	for _, sed := range seds {
		stream = sed(stream)
	}
	return stream
}

func merge(streams ...[]string) []string {
	var out []string
	for _, stream := range streams {
		out = append(out, stream...)
	}
	return out
}

func newRule(kind, name string, attrs map[string]bzl.Expr) *bzl.Rule {
	rule := &bzl.Rule{
		Call: &bzl.CallExpr{
			X: &bzl.LiteralExpr{Token: kind},
		},
	}
	rule.SetAttr("name", asExpr(name))
	for k, v := range attrs {
		rule.SetAttr(k, v)
	}
	rule.SetAttr("tags", asExpr([]string{"automanaged"}))
	return rule
}

func ReconcileRules(path string, rules []*bzl.Rule) (bool, error) {
	info, err := os.Stat(path)
	if err != nil && os.IsNotExist(err) {
		f := &bzl.File{}
		writeHeaders(f)
		writeRules(f, rules)
		return writeFile(path, f, false)
	} else if err != nil {
		return false, err
	}
	if info.IsDir() {
		return false, fmt.Errorf("%q cannot be a directory", path)
	}
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return false, err
	}
	f, err := bzl.Parse(path, b)
	if err != nil {
		return false, err
	}
	oldRules := make(map[string]*bzl.Rule)
	for _, r := range f.Rules("") {
		oldRules[r.Name()] = r
	}
	for _, r := range rules {
		o, ok := oldRules[r.Name()]
		if !ok {
			f.Stmt = append(f.Stmt, r.Call)
			continue
		}
		if !RuleIsManaged(o) {
			continue
		}
		reconcileAttr := func(o, n *bzl.Rule, name string) {
			if e := n.Attr(name); e != nil {
				o.SetAttr(name, e)
			} else {
				o.DelAttr(name)
			}
		}
		reconcileAttr(o, r, "srcs")
		reconcileAttr(o, r, "deps")
		reconcileAttr(o, r, "library")
		delete(oldRules, r.Name())
	}
	for _, r := range oldRules {
		if !RuleIsManaged(r) {
			continue
		}
		f.DelRules(r.Kind(), r.Name())
	}
	return writeFile(path, f, true)
}

func RuleIsManaged(r *bzl.Rule) bool {
	var automanaged bool
	for _, tag := range r.AttrStrings("tags") {
		if tag == "automanaged" {
			automanaged = true
			break
		}
	}
	return automanaged
}

func writeFile(path string, f *bzl.File, exists bool) (bool, error) {
	var info bzl.RewriteInfo
	bzl.Rewrite(f, &info)
	out := bzl.Format(f)
	if exists {
		orig, err := ioutil.ReadFile(path)
		if err != nil {
			return false, err
		}
		if bytes.Compare(out, orig) == 0 {
			return false, nil
		}
	}
	if *dryRun {
		return true, nil
	}
	return true, ioutil.WriteFile(path, out, 0644)

}

func context() *build.Context {
	return &build.Context{
		GOARCH:      "amd64",
		GOOS:        "linux",
		GOROOT:      build.Default.GOROOT,
		GOPATH:      build.Default.GOPATH,
		ReleaseTags: []string{"go1.1", "go1.2", "go1.3", "go1.4", "go1.5", "go1.6", "go1.7"},
		Compiler:    runtime.Compiler,
		CgoEnabled:  true,
	}
}

func walk(root string, walkFn filepath.WalkFunc) error {
	return nil
}
<commit_msg>Check filepath error while walking directories<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/build"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"strings"

	"github.com/mikedanese/gazel/third_party/go/path/filepath"

	bzl "github.com/bazelbuild/buildifier/core"
	"github.com/golang/glog"
)

var kubeRoot = flag.String("root", "", "root of kubernetes source")
var dryRun = flag.Bool("dry-run", false, "run in dry mode")

func main() {
	flag.Parse()
	flag.Set("alsologtostderr", "true")
	if *kubeRoot == "" {
		glog.Fatalf("-root argument is required")
	}
	v := Venderor{
		ctx: context(),
	}
	if len(flag.Args()) == 1 {
		v.updateSinglePkg(flag.Args()[0])
	} else {
		v.walkVendor()
		if err := v.walkRepo(); err != nil {
			glog.Fatalf("err: %v", err)
		}
	}
}

type Venderor struct {
	ctx *build.Context
}

func writeHeaders(file *bzl.File) {
	pkgRule := bzl.Rule{
		&bzl.CallExpr{
			X: &bzl.LiteralExpr{Token: "package"},
		},
	}
	pkgRule.SetAttr("default_visibility", asExpr([]string{"//visibility:public"}))

	file.Stmt = append(file.Stmt,
		[]bzl.Expr{
			pkgRule.Call,
			&bzl.CallExpr{
				X:    &bzl.LiteralExpr{Token: "licenses"},
				List: []bzl.Expr{asExpr([]string{"notice"})},
			},
			&bzl.CallExpr{
				X: &bzl.LiteralExpr{Token: "load"},
				List: asExpr([]string{
					"@io_bazel_rules_go//go:def.bzl",
					"go_binary",
					"go_library",
					"go_test",
					"cgo_library",
				}).(*bzl.ListExpr).List,
			},
		}...,
	)
}

func writeRules(file *bzl.File, rules []*bzl.Rule) {
	for _, rule := range rules {
		file.Stmt = append(file.Stmt, rule.Call)
	}
}

func (v *Venderor) resolve(ipath string) Label {
	if strings.HasPrefix(ipath, "k8s.io/kubernetes") {
		return Label{
			pkg: strings.TrimPrefix(ipath, "k8s.io/kubernetes/"),
			tag: "go_default_library",
		}
	}
	return Label{
		pkg: "vendor",
		tag: ipath,
	}
}

func (v *Venderor) walk(root string, f func(path, ipath string, pkg *build.Package) error) error {
	return sfilepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		ipath, err := filepath.Rel(root, path)
		if err != nil {
			return err
		}
		if !info.IsDir() {
			return nil
		}
		pkg, err := v.ctx.ImportDir(filepath.Join(*kubeRoot, path), build.ImportComment)
		if err != nil {
			if _, ok := err.(*build.NoGoError); err != nil && ok {
				return nil
			} else {
				return err
			}
		}

		return f(path, ipath, pkg)
	})
}

func (v *Venderor) walkRepo() error {
	for _, root := range []string{
		"./pkg",
		"./cmd",
		"./third_party",
		"./plugin",
		"./test",
		"./federation",
	} {
		if err := v.walk(root, v.updatePkg); err != nil {
			return err
		}
	}
	return nil
}

func (v *Venderor) updateSinglePkg(path string) error {
	pkg, err := v.ctx.ImportDir("./"+path, build.ImportComment)
	if err != nil {
		if _, ok := err.(*build.NoGoError); err != nil && ok {
			return nil
		} else {
			return err
		}
	}
	return v.updatePkg(path, "", pkg)
}

func (v *Venderor) updatePkg(path, _ string, pkg *build.Package) error {
	var rules []*bzl.Rule

	var attrs Attrs = make(Attrs)
	srcs := asExpr(merge(pkg.GoFiles, pkg.SFiles)).(*bzl.ListExpr)

	deps := v.extractDeps(pkg.Imports)

	if len(srcs.List) == 0 {
		return nil
	}
	attrs.Set("srcs", srcs)

	if len(deps.List) > 0 {
		attrs.Set("deps", deps)
	}

	if pkg.IsCommand() {
		rules = append(rules, newRule("go_binary", filepath.Base(pkg.Dir), attrs))
	} else {
		rules = append(rules, newRule("go_library", "go_default_library", attrs))
		if len(pkg.TestGoFiles) != 0 {
			rules = append(rules, newRule("go_test", "go_default_test", map[string]bzl.Expr{
				"srcs":    asExpr(pkg.TestGoFiles),
				"deps":    v.extractDeps(pkg.TestImports),
				"library": asExpr("go_default_library"),
			}))
		}
	}

	if len(pkg.XTestGoFiles) != 0 {
		rules = append(rules, newRule("go_test", "go_default_xtest", map[string]bzl.Expr{
			"srcs": asExpr(pkg.XTestGoFiles),
			"deps": v.extractDeps(pkg.XTestImports),
		}))
	}

	wrote, err := ReconcileRules(filepath.Join(path, "BUILD"), rules)
	if err != nil {
		return err
	}
	if wrote {
		fmt.Fprintf(os.Stderr, "wrote BUILD for %q\n", pkg.Dir)
	}
	return nil
}

func (v *Venderor) walkVendor() {
	var rules []*bzl.Rule
	if err := v.walk("./vendor", func(path, ipath string, pkg *build.Package) error {
		var attrs Attrs = make(Attrs)

		srcs := asExpr(
			apply(
				merge(pkg.GoFiles, pkg.SFiles),
				mapper(func(s string) string {
					return strings.TrimPrefix(filepath.Join(path, s), "vendor/")
				}),
			),
		).(*bzl.ListExpr)

		cgoSrcs := asExpr(
			apply(
				merge(pkg.CgoFiles, pkg.CFiles, pkg.CXXFiles, pkg.HFiles),
				mapper(func(s string) string {
					return strings.TrimPrefix(filepath.Join(path, s), "vendor/")
				}),
			),
		).(*bzl.ListExpr)

		deps := v.extractDeps(pkg.Imports)
		attrs.Set("srcs", srcs)

		if len(deps.List) > 0 {
			attrs.Set("deps", deps)
		}

		if pkg.IsCommand() {
			rules = append(rules, newRule("go_binary", v.resolve(ipath).tag, attrs))
		} else {
			if len(cgoSrcs.List) != 0 {
				cgoPname := v.resolve(ipath).tag + "_cgo"
				cgoDeps := v.extractDeps(pkg.TestImports)
				cgoRule := newRule("cgo_library", cgoPname, map[string]bzl.Expr{
					"srcs":      cgoSrcs,
					"clinkopts": asExpr([]string{"-ldl", "-lz", "-lm", "-lpthread", "-ldl"}),
				})
				rules = append(rules, cgoRule)
				if len(cgoDeps.List) != 0 {
					cgoRule.SetAttr("deps", cgoDeps)
				}
				attrs["library"] = asExpr(cgoPname)
			}
			rules = append(rules, newRule("go_library", v.resolve(ipath).tag, attrs))
		}
		return nil
	}); err != nil {
		glog.Fatalf("err: %v", err)
	}
	if _, err := ReconcileRules("./vendor/BUILD", rules); err != nil {
		glog.Fatalf("err: %v", err)
	}
}

func (v *Venderor) extractDeps(deps []string) *bzl.ListExpr {
	return asExpr(
		apply(
			merge(deps),
			filterer(func(s string) bool {
				pkg, err := v.ctx.Import(s, *kubeRoot, build.ImportComment)
				if err != nil {
					if !strings.Contains(err.Error(), `cannot find package "C"`) {
						fmt.Fprintf(os.Stderr, "extract err: %v\n", err)
					}
					return false
				}
				if pkg.Goroot {
					return false
				}
				return true
			}),
			mapper(func(s string) string {
				return v.resolve(s).String()
			}),
		),
	).(*bzl.ListExpr)
}

type Attrs map[string]bzl.Expr

func (a Attrs) Set(name string, expr bzl.Expr) {
	a[name] = expr
}

type Label struct {
	pkg, tag string
}

func (l Label) String() string {
	return fmt.Sprintf("//%v:%v", l.pkg, l.tag)
}

func asExpr(e interface{}) bzl.Expr {
	rv := reflect.ValueOf(e)
	switch rv.Kind() {
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
		return &bzl.LiteralExpr{Token: fmt.Sprintf("%d", e)}
	case reflect.Float32, reflect.Float64:
		return &bzl.LiteralExpr{Token: fmt.Sprintf("%f", e)}
	case reflect.String:
		return &bzl.StringExpr{Value: e.(string)}
	case reflect.Slice, reflect.Array:
		var list []bzl.Expr
		for i := 0; i < rv.Len(); i++ {
			list = append(list, asExpr(rv.Index(i).Interface()))
		}
		return &bzl.ListExpr{List: list}
	default:
		glog.Fatalf("Uh oh")
		return nil
	}
}

type Sed func(s []string) []string

func mapString(in []string, f func(string) string) []string {
	var out []string
	for _, s := range in {
		out = append(out, f(s))
	}
	return out
}

func mapper(f func(string) string) Sed {
	return func(in []string) []string {
		return mapString(in, f)
	}
}

func filterString(in []string, f func(string) bool) []string {
	var out []string
	for _, s := range in {
		if f(s) {
			out = append(out, s)
		}
	}
	return out
}

func filterer(f func(string) bool) Sed {
	return func(in []string) []string {
		return filterString(in, f)
	}
}

func apply(stream []string, seds ...Sed) []string {
	for _, sed := range seds {
		stream = sed(stream)
	}
	return stream
}

func merge(streams ...[]string) []string {
	var out []string
	for _, stream := range streams {
		out = append(out, stream...)
	}
	return out
}

func newRule(kind, name string, attrs map[string]bzl.Expr) *bzl.Rule {
	rule := &bzl.Rule{
		Call: &bzl.CallExpr{
			X: &bzl.LiteralExpr{Token: kind},
		},
	}
	rule.SetAttr("name", asExpr(name))
	for k, v := range attrs {
		rule.SetAttr(k, v)
	}
	rule.SetAttr("tags", asExpr([]string{"automanaged"}))
	return rule
}

func ReconcileRules(path string, rules []*bzl.Rule) (bool, error) {
	info, err := os.Stat(path)
	if err != nil && os.IsNotExist(err) {
		f := &bzl.File{}
		writeHeaders(f)
		writeRules(f, rules)
		return writeFile(path, f, false)
	} else if err != nil {
		return false, err
	}
	if info.IsDir() {
		return false, fmt.Errorf("%q cannot be a directory", path)
	}
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return false, err
	}
	f, err := bzl.Parse(path, b)
	if err != nil {
		return false, err
	}
	oldRules := make(map[string]*bzl.Rule)
	for _, r := range f.Rules("") {
		oldRules[r.Name()] = r
	}
	for _, r := range rules {
		o, ok := oldRules[r.Name()]
		if !ok {
			f.Stmt = append(f.Stmt, r.Call)
			continue
		}
		if !RuleIsManaged(o) {
			continue
		}
		reconcileAttr := func(o, n *bzl.Rule, name string) {
			if e := n.Attr(name); e != nil {
				o.SetAttr(name, e)
			} else {
				o.DelAttr(name)
			}
		}
		reconcileAttr(o, r, "srcs")
		reconcileAttr(o, r, "deps")
		reconcileAttr(o, r, "library")
		delete(oldRules, r.Name())
	}
	for _, r := range oldRules {
		if !RuleIsManaged(r) {
			continue
		}
		f.DelRules(r.Kind(), r.Name())
	}
	return writeFile(path, f, true)
}

func RuleIsManaged(r *bzl.Rule) bool {
	var automanaged bool
	for _, tag := range r.AttrStrings("tags") {
		if tag == "automanaged" {
			automanaged = true
			break
		}
	}
	return automanaged
}

func writeFile(path string, f *bzl.File, exists bool) (bool, error) {
	var info bzl.RewriteInfo
	bzl.Rewrite(f, &info)
	out := bzl.Format(f)
	if exists {
		orig, err := ioutil.ReadFile(path)
		if err != nil {
			return false, err
		}
		if bytes.Compare(out, orig) == 0 {
			return false, nil
		}
	}
	if *dryRun {
		return true, nil
	}
	return true, ioutil.WriteFile(path, out, 0644)

}

func context() *build.Context {
	return &build.Context{
		GOARCH:      "amd64",
		GOOS:        "linux",
		GOROOT:      build.Default.GOROOT,
		GOPATH:      build.Default.GOPATH,
		ReleaseTags: []string{"go1.1", "go1.2", "go1.3", "go1.4", "go1.5", "go1.6", "go1.7"},
		Compiler:    runtime.Compiler,
		CgoEnabled:  true,
	}
}

func walk(root string, walkFn filepath.WalkFunc) error {
	return nil
}
<|endoftext|>


<commit_before>package agent

import (
	"reflect"
	"sync"
	"time"

	log "github.com/coreos/fleet/Godeps/_workspace/src/github.com/golang/glog"

	"github.com/coreos/fleet/machine"
	"github.com/coreos/fleet/registry"
	"github.com/coreos/fleet/unit"
)

func NewUnitStatePublisher(mgr unit.UnitManager, reg registry.Registry, mach machine.Machine) *UnitStatePublisher {
	return &UnitStatePublisher{
		mgr:   mgr,
		reg:   reg,
		mach:  mach,
		mutex: sync.RWMutex{},
		cache: make(map[string]*unit.UnitState),
	}
}

type UnitStatePublisher struct {
	mgr  unit.UnitManager
	reg  registry.Registry
	mach machine.Machine

	mutex sync.RWMutex
	cache map[string]*unit.UnitState
}

// Run caches all of the heartbeat objects from the provided channel, publishing
// them to the Registry every 5s. Heartbeat objects are also published as they
// are received on the channel.
func (p *UnitStatePublisher) Run(beatchan <-chan *unit.UnitStateHeartbeat, stop chan bool) {
	go func() {
		tick := time.Tick(5 * time.Second)
		for {
			select {
			case <-stop:
				return
			case <-tick:
				p.publishAll()
			}
		}
	}()

	machID := p.mach.State().ID

	for {
		select {
		case <-stop:
			return
		case bt := <-beatchan:
			if bt.State != nil {
				bt.State.MachineID = machID
			}

			p.addToCache(bt)
		}
	}
}

func (p *UnitStatePublisher) publishAll() {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	cache := make(map[string]*unit.UnitState)
	for name, us := range p.cache {
		p.publishOne(name, us)
		if us != nil {
			cache[name] = us
		}
	}

	p.cache = cache
}

func (p *UnitStatePublisher) publishOne(name string, us *unit.UnitState) {
	if us == nil {
		log.Infof("Destroying UnitState(%s) in Registry", name)
		err := p.reg.RemoveUnitState(name)
		if err != nil {
			log.Errorf("Failed to destroy UnitState(%s) in Registry: %v", name, err)
		}
	} else {
		// Sanity check - don't want to publish incomplete UnitStates
		// TODO(jonboulle): consider teasing apart a separate UnitState-like struct
		// so we can rely on a UnitState always being fully hydrated?
		if len(us.UnitHash) == 0 || len(us.MachineID) == 0 {
			log.Infof("Refusing to push UnitState(%s): %#v", name, us)
		} else {
			log.Infof("Pushing UnitState(%s) to Registry: %#v", name, us)
			p.reg.SaveUnitState(name, us)
		}
	}
}

func (p *UnitStatePublisher) addToCache(update *unit.UnitStateHeartbeat) {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	last := p.cache[update.Name]
	p.cache[update.Name] = update.State

	// As an optimization, publish changes as they flow in
	if !reflect.DeepEqual(last, update.State) {
		go p.publishOne(update.Name, update.State)
	}
}
<commit_msg>agent: tweak UnitStatePublisher logs<commit_after>package agent

import (
	"reflect"
	"sync"
	"time"

	log "github.com/coreos/fleet/Godeps/_workspace/src/github.com/golang/glog"

	"github.com/coreos/fleet/machine"
	"github.com/coreos/fleet/registry"
	"github.com/coreos/fleet/unit"
)

func NewUnitStatePublisher(mgr unit.UnitManager, reg registry.Registry, mach machine.Machine) *UnitStatePublisher {
	return &UnitStatePublisher{
		mgr:   mgr,
		reg:   reg,
		mach:  mach,
		mutex: sync.RWMutex{},
		cache: make(map[string]*unit.UnitState),
	}
}

type UnitStatePublisher struct {
	mgr  unit.UnitManager
	reg  registry.Registry
	mach machine.Machine

	mutex sync.RWMutex
	cache map[string]*unit.UnitState
}

// Run caches all of the heartbeat objects from the provided channel, publishing
// them to the Registry every 5s. Heartbeat objects are also published as they
// are received on the channel.
func (p *UnitStatePublisher) Run(beatchan <-chan *unit.UnitStateHeartbeat, stop chan bool) {
	go func() {
		tick := time.Tick(5 * time.Second)
		for {
			select {
			case <-stop:
				return
			case <-tick:
				p.publishAll()
			}
		}
	}()

	machID := p.mach.State().ID

	for {
		select {
		case <-stop:
			return
		case bt := <-beatchan:
			if bt.State != nil {
				bt.State.MachineID = machID
			}

			p.addToCache(bt)
		}
	}
}

func (p *UnitStatePublisher) publishAll() {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	cache := make(map[string]*unit.UnitState)
	for name, us := range p.cache {
		p.publishOne(name, us)
		if us != nil {
			cache[name] = us
		}
	}

	p.cache = cache
}

func (p *UnitStatePublisher) publishOne(name string, us *unit.UnitState) {
	if us == nil {
		log.V(1).Infof("Destroying UnitState(%s) in Registry", name)
		err := p.reg.RemoveUnitState(name)
		if err != nil {
			log.Errorf("Failed to destroy UnitState(%s) in Registry: %v", name, err)
		}
	} else {
		// Sanity check - don't want to publish incomplete UnitStates
		// TODO(jonboulle): consider teasing apart a separate UnitState-like struct
		// so we can rely on a UnitState always being fully hydrated?
		if len(us.UnitHash) == 0 {
			log.Errorf("Refusing to push UnitState(%s), no UnitHash: %#v", name, us)
		} else if len(us.MachineID) == 0 {
			log.Errorf("Refusing to push UnitState(%s), no MachineID: %#v", name, us)
		} else {
			log.V(1).Infof("Pushing UnitState(%s) to Registry: %#v", name, us)
			p.reg.SaveUnitState(name, us)
		}
	}
}

func (p *UnitStatePublisher) addToCache(update *unit.UnitStateHeartbeat) {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	last := p.cache[update.Name]
	p.cache[update.Name] = update.State

	// As an optimization, publish changes as they flow in
	if !reflect.DeepEqual(last, update.State) {
		go p.publishOne(update.Name, update.State)
	}
}
<|endoftext|>