<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"sort"

	"github.com/golang/snappy"
)

// TODO: Tests schreiben (leere dateien, blockgröße -1, +0, +1 etc.)
// TODO: linter durchlaufen lassen.
// TODO: Seek.

type reader struct {
	rawR    io.ReadSeeker
	zipR    io.Reader
	index   []Block
	readBuf *bytes.Buffer
	trailer *Trailer
}

func (r *reader) Seek(offset int64, whence int) (int64, error) {
	return offset, nil
}

// Return start (prev offset) and end (curr offset) of the block currOff is
// located in. If currOff is 0, the startoffset of the first and second block is
// returned. If currOff is at the end of file the end offset of the last block
// is returned twice.  The difference between prev block and curr block is then
// equal to 0.
func (r *reader) blockLookup(currOff int64) (*Block, *Block) {
	i := sort.Search(len(r.index), func(i int) bool {
		return r.index[i].zipOff > currOff
	})
	// Beginning of the file, first block: prev offset is 0, curr offset is 1
	if i == 0 {
		return &r.index[i], &r.index[i+1]
	}
	// End of the file, last block: prev and curr offset is the last index.
	if i == len(r.index) {
		return &r.index[i-1], &r.index[i-1]
	}
	return &r.index[i-1], &r.index[i]
}

//TODO: Clean code.
func (r *reader) parseHeaderIfNeeded() error {
	if r.trailer != nil {
		return nil
	}

	if _, err := r.rawR.Seek(-TrailerSize, os.SEEK_END); err != nil {
		return err
	}

	buf := [TrailerSize]byte{}
	if n, err := r.rawR.Read(buf[:]); err != nil || n != TrailerSize {
		return err
	}

	r.trailer = &Trailer{}
	r.trailer.unmarshal(buf[:])
	trailerBuf := make([]byte, r.trailer.indexSize)

	var err error
	seekIdx := -(int64(r.trailer.indexSize) + TrailerSize)
	if _, err = r.rawR.Seek(seekIdx, os.SEEK_END); err != nil {
		fmt.Println(err)
		return err
	}

	if _, err := r.rawR.Read(trailerBuf); err != nil {
		fmt.Println(err)
		return err
	}

	//Build Index
	prevBlock := Block{-1, -1}
	for i := uint64(0); i < (r.trailer.indexSize / IndexBlockSize); i++ {
		currBlock := Block{}
		currBlock.unmarshal(trailerBuf)

		if prevBlock.rawOff >= currBlock.rawOff && prevBlock.zipOff >= currBlock.zipOff {
			return ErrBadIndex
		}

		r.index = append(r.index, currBlock)
		trailerBuf = trailerBuf[IndexBlockSize:]
	}

	if _, err := r.rawR.Seek(0, os.SEEK_SET); err != nil {
		return err
	}

	return nil
}

func (r *reader) Read(p []byte) (int, error) {
	if err := r.parseHeaderIfNeeded(); err != nil {
		fmt.Println(err)
		return 0, err
	}

	read := 0
	for {
		fmt.Println("READBUF LEN:", r.readBuf.Len())
		if r.readBuf.Len() != 0 {
			n := copy(p, r.readBuf.Next(len(p)))
			read += n
			p = p[n:]
			fmt.Println("P:", len(p))
		}
		if len(p) == 0 {
			break
		}

		if _, err := r.readBlockBuffered(); err != nil {
			fmt.Println("EOF?", read, err)
			return read, err
		}
	}
	fmt.Println("END:", read)
	return read, nil
}

// TODO: Ist das noch "buffered"?
func (r *reader) readBlockBuffered() (int64, error) {
	// Get current raw position
	currOff, err := r.rawR.Seek(0, os.SEEK_CUR)
	if err != nil {
		return 0, err
	}

	// Get compressed offset and set cursor to that position.
	prevBlock, currBlock := r.blockLookup(currOff)
	fmt.Println("#########################", currOff, prevBlock, currBlock)
	if currBlock == nil || prevBlock == nil {
		return 0, ErrBadIndex
	}

	// Blocksize should only be 0 on empty file or at the end of file.
	blockSize := currBlock.rawOff - prevBlock.rawOff
	if blockSize == 0 {
		return 0, io.EOF
	}

	currZipOff := prevBlock.zipOff
	if _, err = r.rawR.Seek(currZipOff, os.SEEK_SET); err != nil {
		return 0, err
	}

	return io.CopyN(r.readBuf, r.zipR, blockSize)
}

func NewReader(r io.ReadSeeker) io.ReadSeeker {
	return &reader{
		rawR:    r,
		zipR:    snappy.NewReader(r),
		readBuf: &bytes.Buffer{},
	}
}
<commit_msg>reader.go: Code cleanup. Block redefined to Record.<commit_after>package main

import (
	"bytes"
	"io"
	"os"
	"sort"

	"github.com/golang/snappy"
)

// TODO: Tests schreiben (leere dateien, blockgröße -1, +0, +1 etc.)
// TODO: linter durchlaufen lassen.
// TODO: Seek.

type reader struct {
	rawR    io.ReadSeeker
	zipR    io.Reader
	index   []Record
	readBuf *bytes.Buffer
	trailer *Trailer
}

func (r *reader) Seek(offset int64, whence int) (int64, error) {
	return offset, nil
}

// Return start (prev offset) and end (curr offset) of the block currOff is
// located in. If currOff is 0, the startoffset of the first and second record is
// returned. If currOff is at the end of file the end offset of the last block
// is returned twice.  The difference between prev record and curr block is then
// equal to 0.
func (r *reader) blockLookup(currOff int64) (*Record, *Record) {
	i := sort.Search(len(r.index), func(i int) bool {
		return r.index[i].zipOff > currOff
	})

	// Beginning of the file, first block: prev offset is 0, curr offset is 1
	if i == 0 {
		return &r.index[i], &r.index[i+1]
	}

	// End of the file, last block: prev and curr offset is the last index.
	if i == len(r.index) {
		return &r.index[i-1], &r.index[i-1]
	}
	return &r.index[i-1], &r.index[i]
}

func (r *reader) parseHeaderIfNeeded() error {
	if r.trailer != nil {
		return nil
	}

	if _, err := r.rawR.Seek(-TrailerSize, os.SEEK_END); err != nil {
		return err
	}

	buf := [TrailerSize]byte{}
	if n, err := r.rawR.Read(buf[:]); err != nil || n != TrailerSize {
		return err
	}
	r.trailer = &Trailer{}
	r.trailer.unmarshal(buf[:])

	seekIdx := -(int64(r.trailer.indexSize) + TrailerSize)
	if _, err := r.rawR.Seek(seekIdx, os.SEEK_END); err != nil {
		return err
	}

	trailerBuf := make([]byte, r.trailer.indexSize)
	if _, err := r.rawR.Read(trailerBuf); err != nil {
		return err
	}

	// Build index with Records. A record encapsulates a raw offset and the
	// compressed offset it is mapped to.
	prevRecord := Record{-1, -1}
	for i := uint64(0); i < (r.trailer.indexSize / IndexBlockSize); i++ {
		currRecord := Record{}
		currRecord.unmarshal(trailerBuf)

		if prevRecord.rawOff >= currRecord.rawOff && prevRecord.zipOff >= currRecord.zipOff {
			return ErrBadIndex
		}

		r.index = append(r.index, currRecord)
		trailerBuf = trailerBuf[IndexBlockSize:]
	}

	if _, err := r.rawR.Seek(0, os.SEEK_SET); err != nil {
		return err
	}
	return nil
}

func (r *reader) Read(p []byte) (int, error) {
	if err := r.parseHeaderIfNeeded(); err != nil {
		return 0, err
	}

	read := 0
	for {
		if r.readBuf.Len() != 0 {
			n := copy(p, r.readBuf.Next(len(p)))
			read += n
			p = p[n:]
		}

		if len(p) == 0 {
			break
		}

		if _, err := r.readBlock(); err != nil {
			return read, err
		}
	}

	return read, nil
}

func (r *reader) readBlock() (int64, error) {
	// Get current position of reader (offset of the compressed file).
	currOff, err := r.rawR.Seek(0, os.SEEK_CUR)
	if err != nil {
		return 0, err
	}

	// Get the start and end record of the block currOff is located in
	prevRecord, currRecord := r.blockLookup(currOff)
	if currRecord == nil || prevRecord == nil {
		return 0, ErrBadIndex
	}

	// Blocksize should only be 0 on empty file or at the end of file.
	blockSize := currRecord.rawOff - prevRecord.rawOff
	if blockSize == 0 {
		return 0, io.EOF
	}

	currZipOff := prevRecord.zipOff
	if _, err = r.rawR.Seek(currZipOff, os.SEEK_SET); err != nil {
		return 0, err
	}

	return io.CopyN(r.readBuf, r.zipR, blockSize)
}

// Return a new ReadSeeker with compression support. As random access is the
// purpose of this layer, a ReadSeeker is required as parameter. The used
// compression algorithm is chosen based on trailer information.
func NewReader(r io.ReadSeeker) io.ReadSeeker {
	return &reader{
		rawR:    r,
		zipR:    snappy.NewReader(r),
		readBuf: &bytes.Buffer{},
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package migrationmaster_test

import (
	"time"

	"github.com/juju/errors"
	jujutesting "github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	apitesting "github.com/juju/juju/api/base/testing"
	"github.com/juju/juju/api/migrationmaster"
	"github.com/juju/juju/apiserver/params"
	coretesting "github.com/juju/juju/testing"
	"github.com/juju/juju/worker"
)

type ClientSuite struct {
	jujutesting.IsolationSuite
}

var _ = gc.Suite(&ClientSuite{})

func (s *ClientSuite) TestWatch(c *gc.C) {
	var stub jujutesting.Stub
	apiCaller := apitesting.APICallerFunc(func(objType string, version int, id, request string, arg, result interface{}) error {
		stub.AddCall("call", objType, version, id, request, arg)
		switch request {
		case "Watch":
			*(result.(*params.NotifyWatchResult)) = params.NotifyWatchResult{
				NotifyWatcherId: "abc",
			}
		case "Next":
			// The full success case is tested in api/watcher.
			return errors.New("boom")
		case "Stop":
		}
		return nil
	})

	client := migrationmaster.NewClient(apiCaller)
	w, err := client.Watch()
	c.Assert(err, jc.ErrorIsNil)
	defer worker.Stop(w)

	errC := make(chan error)
	go func() {
		errC <- w.Wait()
	}()

	select {
	case err := <-errC:
		c.Assert(err, gc.ErrorMatches, "boom")
		stub.CheckCalls(c, []jujutesting.StubCall{
			{"call", []interface{}{"MigrationMaster", 0, "", "Watch", nil}},
			{"call", []interface{}{"MigrationMasterWatcher", 0, "abc", "Next", nil}},
			{"call", []interface{}{"MigrationMasterWatcher", 0, "abc", "Stop", nil}},
		})
	case <-time.After(coretesting.LongWait):
		c.Fatal("timed out waiting for watcher to die")
	}
}

func (s *ClientSuite) TestWatchErr(c *gc.C) {
	apiCaller := apitesting.APICallerFunc(func(objType string, version int, id, request string, arg, result interface{}) error {
		return errors.New("boom")
	})

	client := migrationmaster.NewClient(apiCaller)
	_, err := client.Watch()
	c.Assert(err, gc.ErrorMatches, "boom")
}
<commit_msg>api/migrationmaster: Remove version from stub calls.<commit_after>// Copyright 2016 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package migrationmaster_test

import (
	"time"

	"github.com/juju/errors"
	jujutesting "github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"

	apitesting "github.com/juju/juju/api/base/testing"
	"github.com/juju/juju/api/migrationmaster"
	"github.com/juju/juju/apiserver/params"
	coretesting "github.com/juju/juju/testing"
	"github.com/juju/juju/worker"
)

type ClientSuite struct {
	jujutesting.IsolationSuite
}

var _ = gc.Suite(&ClientSuite{})

func (s *ClientSuite) TestWatch(c *gc.C) {
	var stub jujutesting.Stub
	apiCaller := apitesting.APICallerFunc(func(objType string, version int, id, request string, arg, result interface{}) error {
		stub.AddCall("call", objType, id, request, arg)
		switch request {
		case "Watch":
			*(result.(*params.NotifyWatchResult)) = params.NotifyWatchResult{
				NotifyWatcherId: "abc",
			}
		case "Next":
			// The full success case is tested in api/watcher.
			return errors.New("boom")
		case "Stop":
		}
		return nil
	})

	client := migrationmaster.NewClient(apiCaller)
	w, err := client.Watch()
	c.Assert(err, jc.ErrorIsNil)
	defer worker.Stop(w)

	errC := make(chan error)
	go func() {
		errC <- w.Wait()
	}()

	select {
	case err := <-errC:
		c.Assert(err, gc.ErrorMatches, "boom")
		stub.CheckCalls(c, []jujutesting.StubCall{
			{"call", []interface{}{"MigrationMaster", "", "Watch", nil}},
			{"call", []interface{}{"MigrationMasterWatcher", "abc", "Next", nil}},
			{"call", []interface{}{"MigrationMasterWatcher", "abc", "Stop", nil}},
		})
	case <-time.After(coretesting.LongWait):
		c.Fatal("timed out waiting for watcher to die")
	}
}

func (s *ClientSuite) TestWatchErr(c *gc.C) {
	apiCaller := apitesting.APICallerFunc(func(objType string, version int, id, request string, arg, result interface{}) error {
		return errors.New("boom")
	})

	client := migrationmaster.NewClient(apiCaller)
	_, err := client.Watch()
	c.Assert(err, gc.ErrorMatches, "boom")
}
<|endoftext|>


<commit_before>package build

import (
	"io"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/backend"
	"golang.org/x/net/context"
)

// Backend abstracts an image builder whose only purpose is to build an image referenced by an imageID.
type Backend interface {
	// Build builds a Docker image referenced by an imageID string.
	//
	// Note: Tagging an image should not be done by a Builder, it should instead be done
	// by the caller.
	//
	// TODO: make this return a reference instead of string
	BuildFromContext(ctx context.Context, src io.ReadCloser, remote string, buildOptions *types.ImageBuildOptions, pg backend.ProgressWriter) (string, error)
}
<commit_msg>fix func name "BuildFromContext" in comment<commit_after>package build

import (
	"io"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/backend"
	"golang.org/x/net/context"
)

// Backend abstracts an image builder whose only purpose is to build an image referenced by an imageID.
type Backend interface {
	// BuildFromContext builds a Docker image referenced by an imageID string.
	//
	// Note: Tagging an image should not be done by a Builder, it should instead be done
	// by the caller.
	//
	// TODO: make this return a reference instead of string
	BuildFromContext(ctx context.Context, src io.ReadCloser, remote string, buildOptions *types.ImageBuildOptions, pg backend.ProgressWriter) (string, error)
}
<|endoftext|>


<commit_before>package report

import (
	"net"
	"strings"
)

// TheInternet is used as a node ID to indicate a remote IP.
const TheInternet = "theinternet"

// Delimiters are used to separate parts of node IDs, to guarantee uniqueness
// in particular contexts.
const (
	// ScopeDelim is a general-purpose delimiter used within node IDs to
	// separate different contextual scopes. Different topologies have
	// different key structures.
	ScopeDelim = ";"

	// EdgeDelim separates two node IDs when they need to exist in the same key.
	// Concretely, it separates node IDs in keys that represent edges.
	EdgeDelim = "|"

	// Key added to nodes to prevent them being joined with conntracked connections
	DoesNotMakeConnections = "does_not_make_connections"

	// WeaveOverlayPeerPrefix is the prefix for weave peers in the overlay network
	WeaveOverlayPeerPrefix = ""

	// DockerOverlayPeerPrefix is the prefix for docker peers in the overlay network
	DockerOverlayPeerPrefix = "docker_peer_"
)

// MakeEndpointNodeID produces an endpoint node ID from its composite parts.
func MakeEndpointNodeID(hostID, namespaceID, address, port string) string {
	return makeAddressID(hostID, namespaceID, address) + ScopeDelim + port
}

// MakeAddressNodeID produces an address node ID from its composite parts.
func MakeAddressNodeID(hostID, address string) string {
	return makeAddressID(hostID, "", address)
}

func makeAddressID(hostID, namespaceID, address string) string {
	var scope string

	// Loopback addresses and addresses explicitly marked as local get
	// scoped by hostID
	// Loopback addresses are also scoped by the networking
	// namespace if available, since they can clash.
	addressIP := net.ParseIP(address)
	if addressIP != nil && LocalNetworks.Contains(addressIP) {
		scope = hostID
	} else if IsLoopback(address) {
		scope = hostID
		if namespaceID != "" {
			scope += "-" + namespaceID
		}
	}

	return scope + ScopeDelim + address
}

// MakeScopedEndpointNodeID is like MakeEndpointNodeID, but it always
// prefixes the ID with a scope.
func MakeScopedEndpointNodeID(scope, address, port string) string {
	return scope + ScopeDelim + address + ScopeDelim + port
}

// MakeScopedAddressNodeID is like MakeAddressNodeID, but it always
// prefixes the ID witha scope.
func MakeScopedAddressNodeID(scope, address string) string {
	return scope + ScopeDelim + address
}

// MakeProcessNodeID produces a process node ID from its composite parts.
func MakeProcessNodeID(hostID, pid string) string {
	return hostID + ScopeDelim + pid
}

var (
	// MakeHostNodeID produces a host node ID from its composite parts.
	MakeHostNodeID = makeSingleComponentID("host")

	// ParseHostNodeID parses a host node ID
	ParseHostNodeID = parseSingleComponentID("host")

	// MakeContainerNodeID produces a container node ID from its composite parts.
	MakeContainerNodeID = makeSingleComponentID("container")

	// ParseContainerNodeID parses a container node ID
	ParseContainerNodeID = parseSingleComponentID("container")

	// MakeContainerImageNodeID produces a container image node ID from its composite parts.
	MakeContainerImageNodeID = makeSingleComponentID("container_image")

	// ParseContainerImageNodeID parses a container image node ID
	ParseContainerImageNodeID = parseSingleComponentID("container_image")

	// MakePodNodeID produces a pod node ID from its composite parts.
	MakePodNodeID = makeSingleComponentID("pod")

	// ParsePodNodeID parses a pod node ID
	ParsePodNodeID = parseSingleComponentID("pod")

	// MakeServiceNodeID produces a service node ID from its composite parts.
	MakeServiceNodeID = makeSingleComponentID("service")

	// ParseServiceNodeID parses a service node ID
	ParseServiceNodeID = parseSingleComponentID("service")

	// MakeDeploymentNodeID produces a deployment node ID from its composite parts.
	MakeDeploymentNodeID = makeSingleComponentID("deployment")

	// ParseDeploymentNodeID parses a deployment node ID
	ParseDeploymentNodeID = parseSingleComponentID("deployment")

	// MakeReplicaSetNodeID produces a replica set node ID from its composite parts.
	MakeReplicaSetNodeID = makeSingleComponentID("replica_set")

	// ParseReplicaSetNodeID parses a replica set node ID
	ParseReplicaSetNodeID = parseSingleComponentID("replica_set")
)

// makeSingleComponentID makes a single-component node id encoder
func makeSingleComponentID(tag string) func(string) string {
	return func(id string) string {
		return id + ScopeDelim + "<" + tag + ">"
	}
}

// parseSingleComponentID makes a single-component node id decoder
func parseSingleComponentID(tag string) func(string) (string, bool) {
	return func(id string) (string, bool) {
		fields := strings.SplitN(id, ScopeDelim, 2)
		if len(fields) != 2 || fields[1] != "<"+tag+">" {
			return "", false
		}
		return fields[0], true
	}
}

// MakeOverlayNodeID produces an overlay topology node ID from a router peer's
// prefix and name, which is assumed to be globally unique.
func MakeOverlayNodeID(peerPrefix, peerName string) string {
	return "#" + peerPrefix + peerName
}

// ParseOverlayNodeID produces the overlay type and peer name.
func ParseOverlayNodeID(id string) (overlayPrefix string, peerName string) {

	if !strings.HasPrefix(id, "#") {
		// Best we can do
		return "", ""
	}

	id = id[1:]

	if strings.HasPrefix(id, DockerOverlayPeerPrefix) {
		return DockerOverlayPeerPrefix, id[len(DockerOverlayPeerPrefix):]
	}

	return WeaveOverlayPeerPrefix, peerName
}

// ParseNodeID produces the host ID and remainder (typically an address) from
// a node ID. Note that hostID may be blank.
func ParseNodeID(nodeID string) (hostID string, remainder string, ok bool) {
	fields := strings.SplitN(nodeID, ScopeDelim, 2)
	if len(fields) != 2 {
		return "", "", false
	}
	return fields[0], fields[1], true
}

// ParseEndpointNodeID produces the scope, address, and port and remainder.
// Note that hostID may be blank.
func ParseEndpointNodeID(endpointNodeID string) (scope, address, port string, ok bool) {
	fields := strings.SplitN(endpointNodeID, ScopeDelim, 3)
	if len(fields) != 3 {
		return "", "", "", false
	}

	return fields[0], fields[1], fields[2], true
}

// ParseAddressNodeID produces the host ID, address from an address node ID.
func ParseAddressNodeID(addressNodeID string) (hostID, address string, ok bool) {
	fields := strings.SplitN(addressNodeID, ScopeDelim, 2)
	if len(fields) != 2 {
		return "", "", false
	}
	return fields[0], fields[1], true
}

// ExtractHostID extracts the host id from Node
func ExtractHostID(m Node) string {
	hostNodeID, _ := m.Latest.Lookup(HostNodeID)
	hostID, _, _ := ParseNodeID(hostNodeID)
	return hostID
}

// IsLoopback ascertains if an address comes from a loopback interface.
func IsLoopback(address string) bool {
	ip := net.ParseIP(address)
	return ip != nil && ip.IsLoopback()
}
<commit_msg>Fix bug when parsing peer names<commit_after>package report

import (
	"net"
	"strings"
)

// TheInternet is used as a node ID to indicate a remote IP.
const TheInternet = "theinternet"

// Delimiters are used to separate parts of node IDs, to guarantee uniqueness
// in particular contexts.
const (
	// ScopeDelim is a general-purpose delimiter used within node IDs to
	// separate different contextual scopes. Different topologies have
	// different key structures.
	ScopeDelim = ";"

	// EdgeDelim separates two node IDs when they need to exist in the same key.
	// Concretely, it separates node IDs in keys that represent edges.
	EdgeDelim = "|"

	// Key added to nodes to prevent them being joined with conntracked connections
	DoesNotMakeConnections = "does_not_make_connections"

	// WeaveOverlayPeerPrefix is the prefix for weave peers in the overlay network
	WeaveOverlayPeerPrefix = ""

	// DockerOverlayPeerPrefix is the prefix for docker peers in the overlay network
	DockerOverlayPeerPrefix = "docker_peer_"
)

// MakeEndpointNodeID produces an endpoint node ID from its composite parts.
func MakeEndpointNodeID(hostID, namespaceID, address, port string) string {
	return makeAddressID(hostID, namespaceID, address) + ScopeDelim + port
}

// MakeAddressNodeID produces an address node ID from its composite parts.
func MakeAddressNodeID(hostID, address string) string {
	return makeAddressID(hostID, "", address)
}

func makeAddressID(hostID, namespaceID, address string) string {
	var scope string

	// Loopback addresses and addresses explicitly marked as local get
	// scoped by hostID
	// Loopback addresses are also scoped by the networking
	// namespace if available, since they can clash.
	addressIP := net.ParseIP(address)
	if addressIP != nil && LocalNetworks.Contains(addressIP) {
		scope = hostID
	} else if IsLoopback(address) {
		scope = hostID
		if namespaceID != "" {
			scope += "-" + namespaceID
		}
	}

	return scope + ScopeDelim + address
}

// MakeScopedEndpointNodeID is like MakeEndpointNodeID, but it always
// prefixes the ID with a scope.
func MakeScopedEndpointNodeID(scope, address, port string) string {
	return scope + ScopeDelim + address + ScopeDelim + port
}

// MakeScopedAddressNodeID is like MakeAddressNodeID, but it always
// prefixes the ID witha scope.
func MakeScopedAddressNodeID(scope, address string) string {
	return scope + ScopeDelim + address
}

// MakeProcessNodeID produces a process node ID from its composite parts.
func MakeProcessNodeID(hostID, pid string) string {
	return hostID + ScopeDelim + pid
}

var (
	// MakeHostNodeID produces a host node ID from its composite parts.
	MakeHostNodeID = makeSingleComponentID("host")

	// ParseHostNodeID parses a host node ID
	ParseHostNodeID = parseSingleComponentID("host")

	// MakeContainerNodeID produces a container node ID from its composite parts.
	MakeContainerNodeID = makeSingleComponentID("container")

	// ParseContainerNodeID parses a container node ID
	ParseContainerNodeID = parseSingleComponentID("container")

	// MakeContainerImageNodeID produces a container image node ID from its composite parts.
	MakeContainerImageNodeID = makeSingleComponentID("container_image")

	// ParseContainerImageNodeID parses a container image node ID
	ParseContainerImageNodeID = parseSingleComponentID("container_image")

	// MakePodNodeID produces a pod node ID from its composite parts.
	MakePodNodeID = makeSingleComponentID("pod")

	// ParsePodNodeID parses a pod node ID
	ParsePodNodeID = parseSingleComponentID("pod")

	// MakeServiceNodeID produces a service node ID from its composite parts.
	MakeServiceNodeID = makeSingleComponentID("service")

	// ParseServiceNodeID parses a service node ID
	ParseServiceNodeID = parseSingleComponentID("service")

	// MakeDeploymentNodeID produces a deployment node ID from its composite parts.
	MakeDeploymentNodeID = makeSingleComponentID("deployment")

	// ParseDeploymentNodeID parses a deployment node ID
	ParseDeploymentNodeID = parseSingleComponentID("deployment")

	// MakeReplicaSetNodeID produces a replica set node ID from its composite parts.
	MakeReplicaSetNodeID = makeSingleComponentID("replica_set")

	// ParseReplicaSetNodeID parses a replica set node ID
	ParseReplicaSetNodeID = parseSingleComponentID("replica_set")
)

// makeSingleComponentID makes a single-component node id encoder
func makeSingleComponentID(tag string) func(string) string {
	return func(id string) string {
		return id + ScopeDelim + "<" + tag + ">"
	}
}

// parseSingleComponentID makes a single-component node id decoder
func parseSingleComponentID(tag string) func(string) (string, bool) {
	return func(id string) (string, bool) {
		fields := strings.SplitN(id, ScopeDelim, 2)
		if len(fields) != 2 || fields[1] != "<"+tag+">" {
			return "", false
		}
		return fields[0], true
	}
}

// MakeOverlayNodeID produces an overlay topology node ID from a router peer's
// prefix and name, which is assumed to be globally unique.
func MakeOverlayNodeID(peerPrefix, peerName string) string {
	return "#" + peerPrefix + peerName
}

// ParseOverlayNodeID produces the overlay type and peer name.
func ParseOverlayNodeID(id string) (overlayPrefix string, peerName string) {

	if !strings.HasPrefix(id, "#") {
		// Best we can do
		return "", ""
	}

	id = id[1:]

	if strings.HasPrefix(id, DockerOverlayPeerPrefix) {
		return DockerOverlayPeerPrefix, id[len(DockerOverlayPeerPrefix):]
	}

	return WeaveOverlayPeerPrefix, id
}

// ParseNodeID produces the host ID and remainder (typically an address) from
// a node ID. Note that hostID may be blank.
func ParseNodeID(nodeID string) (hostID string, remainder string, ok bool) {
	fields := strings.SplitN(nodeID, ScopeDelim, 2)
	if len(fields) != 2 {
		return "", "", false
	}
	return fields[0], fields[1], true
}

// ParseEndpointNodeID produces the scope, address, and port and remainder.
// Note that hostID may be blank.
func ParseEndpointNodeID(endpointNodeID string) (scope, address, port string, ok bool) {
	fields := strings.SplitN(endpointNodeID, ScopeDelim, 3)
	if len(fields) != 3 {
		return "", "", "", false
	}

	return fields[0], fields[1], fields[2], true
}

// ParseAddressNodeID produces the host ID, address from an address node ID.
func ParseAddressNodeID(addressNodeID string) (hostID, address string, ok bool) {
	fields := strings.SplitN(addressNodeID, ScopeDelim, 2)
	if len(fields) != 2 {
		return "", "", false
	}
	return fields[0], fields[1], true
}

// ExtractHostID extracts the host id from Node
func ExtractHostID(m Node) string {
	hostNodeID, _ := m.Latest.Lookup(HostNodeID)
	hostID, _, _ := ParseNodeID(hostNodeID)
	return hostID
}

// IsLoopback ascertains if an address comes from a loopback interface.
func IsLoopback(address string) bool {
	ip := net.ParseIP(address)
	return ip != nil && ip.IsLoopback()
}
<|endoftext|>


<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"reflect"
	"testing"
)

func TestNewManagerRESTRouter(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	ring, err := NewMsgRing(nil, 1)

	cfg := NewCfgMem()
	mgr := NewManager(VERSION, cfg, NewUUID(), nil, "", 1, ":1000",
		emptyDir, "some-datasource", nil)
	r, err := NewManagerRESTRouter(mgr, emptyDir, ring)
	if r == nil || err != nil {
		t.Errorf("expected no errors")
	}

	mgr = NewManager(VERSION, cfg, NewUUID(), []string{"queryer", "anotherTag"},
		"", 1, ":1000", emptyDir, "some-datasource", nil)
	r, err = NewManagerRESTRouter(mgr, emptyDir, ring)
	if r == nil || err != nil {
		t.Errorf("expected no errors")
	}
}

func TestHandlers(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := NewCfgMem()
	meh := &TestMEH{}
	mgr := NewManager(VERSION, cfg, NewUUID(),
		nil, "", 1, ":1000", emptyDir, "some-datasource", meh)
	mgr.Start("wanted")
	mgr.Kick("test-start-kick")

	mr, _ := NewMsgRing(os.Stderr, 1000)

	router, err := NewManagerRESTRouter(mgr, "static", mr)
	if err != nil || router == nil {
		t.Errorf("no mux router")
	}

	tests := []struct {
		Desc          string
		Path          string
		Method        string
		Params        url.Values
		Body          []byte
		Status        int
		ResponseBody  []byte
		ResponseMatch map[string]bool
	}{
		{
			Desc:         "log",
			Path:         "/api/log",
			Method:       "GET",
			Params:       nil,
			Body:         nil,
			Status:       http.StatusOK,
			ResponseBody: []byte(`{"messages":[]}`),
		},
		{
			Desc:   "cfg",
			Path:   "/api/cfg",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`"status":"ok"`:       true,
				`"indexDefs":null`:    true,
				`"nodeDefsKnown":{`:   true,
				`"nodeDefsWanted":{`:  true,
				`"planPIndexes":null`: true,
			},
		},
		{
			Desc:   "cfg refresh",
			Path:   "/api/cfgRefresh",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok"}`: true,
			},
		},
		{
			Desc:   "manager kick",
			Path:   "/api/managerKick",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok"}`: true,
			},
		},
		{
			Desc:   "manager meta",
			Path:   "/api/managerMeta",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`"status":"ok"`:    true,
				`"startSamples":{`: true,
			},
		},
		{
			Desc:         "list empty indexes",
			Path:         "/api/index",
			Method:       "GET",
			Params:       nil,
			Body:         nil,
			Status:       http.StatusOK,
			ResponseBody: []byte(`{"status":"ok","indexDefs":null}`),
		},
		{
			Desc:         "try to get a nonexistent index",
			Path:         "/api/index/NOT-AN-INDEX",
			Method:       "GET",
			Params:       nil,
			Body:         nil,
			Status:       400,
			ResponseBody: []byte(`not an index`),
		},
		{
			Desc:   "try to create a default index with bad server",
			Path:   "/api/index/index-on-a-bad-server",
			Method: "PUT",
			Params: nil,
			Body:   nil,
			Status: 500,
			ResponseMatch: map[string]bool{
				`failed to connect`: true,
			},
		},
		{
			Desc:   "try to delete a nonexistent index when no indexes",
			Path:   "/api/index/NOT-AN-INDEX",
			Method: "DELETE",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`indexes do not exist`: true,
			},
		},
		{
			Desc:   "try to count a nonexistent index when no indexes",
			Path:   "/api/index/NOT-AN-INDEX/count",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`could not get indexDefs`: true,
			},
		},
		{
			Desc:   "try to query a nonexistent index when no indexes",
			Path:   "/api/index/NOT-AN-INDEX/query",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`could not get indexDefs`: true,
			},
		},
	}

	for _, test := range tests {
		record := httptest.NewRecorder()
		req := &http.Request{
			Method: test.Method,
			URL:    &url.URL{Path: test.Path},
			Form:   test.Params,
			Body:   ioutil.NopCloser(bytes.NewBuffer(test.Body)),
		}
		router.ServeHTTP(record, req)
		if got, want := record.Code, test.Status; got != want {
			t.Errorf("%s: response code = %d, want %d", test.Desc, got, want)
			t.Errorf("%s: response body = %s", test.Desc, record.Body)
		}

		got := bytes.TrimRight(record.Body.Bytes(), "\n")
		if test.ResponseBody != nil {
			if !reflect.DeepEqual(got, test.ResponseBody) {
				t.Errorf("%s: expected: '%s', got: '%s'",
					test.Desc, test.ResponseBody, got)
			}
		}
		for pattern, shouldMatch := range test.ResponseMatch {
			didMatch := bytes.Contains(got, []byte(pattern))
			if didMatch != shouldMatch {
				t.Errorf("%s: expected match %t for pattern %s, got %t",
					test.Desc, shouldMatch, pattern, didMatch)
				t.Errorf("%s: response body was: %s", test.Desc, got)
			}
		}
	}
}
<commit_msg>test /api/feedStats REST endpoint<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package main

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"reflect"
	"testing"
)

func TestNewManagerRESTRouter(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	ring, err := NewMsgRing(nil, 1)

	cfg := NewCfgMem()
	mgr := NewManager(VERSION, cfg, NewUUID(), nil, "", 1, ":1000",
		emptyDir, "some-datasource", nil)
	r, err := NewManagerRESTRouter(mgr, emptyDir, ring)
	if r == nil || err != nil {
		t.Errorf("expected no errors")
	}

	mgr = NewManager(VERSION, cfg, NewUUID(), []string{"queryer", "anotherTag"},
		"", 1, ":1000", emptyDir, "some-datasource", nil)
	r, err = NewManagerRESTRouter(mgr, emptyDir, ring)
	if r == nil || err != nil {
		t.Errorf("expected no errors")
	}
}

func TestHandlers(t *testing.T) {
	emptyDir, _ := ioutil.TempDir("./tmp", "test")
	defer os.RemoveAll(emptyDir)

	cfg := NewCfgMem()
	meh := &TestMEH{}
	mgr := NewManager(VERSION, cfg, NewUUID(),
		nil, "", 1, ":1000", emptyDir, "some-datasource", meh)
	mgr.Start("wanted")
	mgr.Kick("test-start-kick")

	mr, _ := NewMsgRing(os.Stderr, 1000)

	router, err := NewManagerRESTRouter(mgr, "static", mr)
	if err != nil || router == nil {
		t.Errorf("no mux router")
	}

	tests := []struct {
		Desc          string
		Path          string
		Method        string
		Params        url.Values
		Body          []byte
		Status        int
		ResponseBody  []byte
		ResponseMatch map[string]bool
	}{
		{
			Desc:         "log on empty msg ring",
			Path:         "/api/log",
			Method:       "GET",
			Params:       nil,
			Body:         nil,
			Status:       http.StatusOK,
			ResponseBody: []byte(`{"messages":[]}`),
		},
		{
			Desc:   "cfg on empty manaager",
			Path:   "/api/cfg",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`"status":"ok"`:       true,
				`"indexDefs":null`:    true,
				`"nodeDefsKnown":{`:   true,
				`"nodeDefsWanted":{`:  true,
				`"planPIndexes":null`: true,
			},
		},
		{
			Desc:   "cfg refresh on empty, unchanged manager",
			Path:   "/api/cfgRefresh",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok"}`: true,
			},
		},
		{
			Desc:   "manager kick on empty, unchanged manager",
			Path:   "/api/managerKick",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`{"status":"ok"}`: true,
			},
		},
		{
			Desc:   "manager meta",
			Path:   "/api/managerMeta",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`"status":"ok"`:    true,
				`"startSamples":{`: true,
			},
		},
		{
			Desc:   "feed stats when no feeds",
			Path:   "/api/feedStats",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: http.StatusOK,
			ResponseMatch: map[string]bool{
				`[`: true,
				`]`: true,
			},
		},
		{
			Desc:         "list empty indexes",
			Path:         "/api/index",
			Method:       "GET",
			Params:       nil,
			Body:         nil,
			Status:       http.StatusOK,
			ResponseBody: []byte(`{"status":"ok","indexDefs":null}`),
		},
		{
			Desc:         "try to get a nonexistent index",
			Path:         "/api/index/NOT-AN-INDEX",
			Method:       "GET",
			Params:       nil,
			Body:         nil,
			Status:       400,
			ResponseBody: []byte(`not an index`),
		},
		{
			Desc:   "try to create a default index with bad server",
			Path:   "/api/index/index-on-a-bad-server",
			Method: "PUT",
			Params: nil,
			Body:   nil,
			Status: 500,
			ResponseMatch: map[string]bool{
				`failed to connect`: true,
			},
		},
		{
			Desc:   "try to delete a nonexistent index when no indexes",
			Path:   "/api/index/NOT-AN-INDEX",
			Method: "DELETE",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`indexes do not exist`: true,
			},
		},
		{
			Desc:   "try to count a nonexistent index when no indexes",
			Path:   "/api/index/NOT-AN-INDEX/count",
			Method: "GET",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`could not get indexDefs`: true,
			},
		},
		{
			Desc:   "try to query a nonexistent index when no indexes",
			Path:   "/api/index/NOT-AN-INDEX/query",
			Method: "POST",
			Params: nil,
			Body:   nil,
			Status: 400,
			ResponseMatch: map[string]bool{
				`could not get indexDefs`: true,
			},
		},
	}

	for _, test := range tests {
		record := httptest.NewRecorder()
		req := &http.Request{
			Method: test.Method,
			URL:    &url.URL{Path: test.Path},
			Form:   test.Params,
			Body:   ioutil.NopCloser(bytes.NewBuffer(test.Body)),
		}
		router.ServeHTTP(record, req)
		if got, want := record.Code, test.Status; got != want {
			t.Errorf("%s: response code = %d, want %d", test.Desc, got, want)
			t.Errorf("%s: response body = %s", test.Desc, record.Body)
		}

		got := bytes.TrimRight(record.Body.Bytes(), "\n")
		if test.ResponseBody != nil {
			if !reflect.DeepEqual(got, test.ResponseBody) {
				t.Errorf("%s: expected: '%s', got: '%s'",
					test.Desc, test.ResponseBody, got)
			}
		}
		for pattern, shouldMatch := range test.ResponseMatch {
			didMatch := bytes.Contains(got, []byte(pattern))
			if didMatch != shouldMatch {
				t.Errorf("%s: expected match %t for pattern %s, got %t",
					test.Desc, shouldMatch, pattern, didMatch)
				t.Errorf("%s: response body was: %s", test.Desc, got)
			}
		}
	}
}
<|endoftext|>


<commit_before>// The command line tool for running Revel apps.
package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"strings"
	"text/template"
)

// Cribbed from the genius organization of the "go" command.
type Command struct {
	Run                    func(args []string)
	UsageLine, Short, Long string
}

func (cmd *Command) Name() string {
	name := cmd.UsageLine
	i := strings.Index(name, " ")
	if i >= 0 {
		name = name[:i]
	}
	return name
}

var commands = []*Command{
	cmdNew,
	cmdRun,
	cmdBuild,
	cmdPackage,
	cmdClean,
	cmdTest,
}

func main() {
	fmt.Fprintf(os.Stdout, header)
	flag.Parse()
	args := flag.Args()

	if len(args) < 1 || args[0] == "help" {
		if len(args) == 1 {
			usage(0)
		}
		if len(args) > 1 {
			for _, cmd := range commands {
				if cmd.Name() == args[1] {
					tmpl(os.Stdout, helpTemplate, cmd)
					return
				}
			}
		}
		usage(2)
	}

	// Commands use panic to abort execution when something goes wrong.
	// Panics are logged at the point of error.  Ignore those.
	defer func() {
		if err := recover(); err != nil {
			if _, ok := err.(LoggedError); !ok {
				// This panic was not expected / logged.
				panic(err)
			}
			os.Exit(1)
		}
	}()

	for _, cmd := range commands {
		if cmd.Name() == args[0] {
			cmd.Run(args[1:])
			return
		}
	}

	errorf("unknown command %q\nRun 'revel help' for usage.\n", args[0])
}

func errorf(format string, args ...interface{}) {
	// Ensure the user's command prompt starts on the next line.
	if !strings.HasSuffix(format, "\n") {
		format += "\n"
	}
	fmt.Fprintf(os.Stderr, format, args...)
	panic(LoggedError{}) // Panic instead of os.Exit so that deferred will run.
}

const header = `~
~ revel! http://robfig.github.com/revel
~
`

const usageTemplate = `usage: revel command [arguments]

The commands are:
{{range .}}
    {{.Name | printf "%-11s"}} {{.Short}}{{end}}

Use "revel help [command]" for more information.
`

var helpTemplate = `usage: revel {{.UsageLine}}
{{.Long}}
`

func usage(exitCode int) {
	tmpl(os.Stderr, usageTemplate, commands)
	os.Exit(exitCode)
}

func tmpl(w io.Writer, text string, data interface{}) {
	t := template.New("top")
	template.Must(t.Parse(text))
	if err := t.Execute(w, data); err != nil {
		panic(err)
	}
}
<commit_msg>Added flag.Usage again<commit_after>// The command line tool for running Revel apps.
package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"strings"
	"text/template"
)

// Cribbed from the genius organization of the "go" command.
type Command struct {
	Run                    func(args []string)
	UsageLine, Short, Long string
}

func (cmd *Command) Name() string {
	name := cmd.UsageLine
	i := strings.Index(name, " ")
	if i >= 0 {
		name = name[:i]
	}
	return name
}

var commands = []*Command{
	cmdNew,
	cmdRun,
	cmdBuild,
	cmdPackage,
	cmdClean,
	cmdTest,
}

func main() {
	fmt.Fprintf(os.Stdout, header)
	flag.Usage = func() { usage(1) }
	flag.Parse()
	args := flag.Args()

	if len(args) < 1 || args[0] == "help" {
		if len(args) == 1 {
			usage(0)
		}
		if len(args) > 1 {
			for _, cmd := range commands {
				if cmd.Name() == args[1] {
					tmpl(os.Stdout, helpTemplate, cmd)
					return
				}
			}
		}
		usage(2)
	}

	// Commands use panic to abort execution when something goes wrong.
	// Panics are logged at the point of error.  Ignore those.
	defer func() {
		if err := recover(); err != nil {
			if _, ok := err.(LoggedError); !ok {
				// This panic was not expected / logged.
				panic(err)
			}
			os.Exit(1)
		}
	}()

	for _, cmd := range commands {
		if cmd.Name() == args[0] {
			cmd.Run(args[1:])
			return
		}
	}

	errorf("unknown command %q\nRun 'revel help' for usage.\n", args[0])
}

func errorf(format string, args ...interface{}) {
	// Ensure the user's command prompt starts on the next line.
	if !strings.HasSuffix(format, "\n") {
		format += "\n"
	}
	fmt.Fprintf(os.Stderr, format, args...)
	panic(LoggedError{}) // Panic instead of os.Exit so that deferred will run.
}

const header = `~
~ revel! http://robfig.github.com/revel
~
`

const usageTemplate = `usage: revel command [arguments]

The commands are:
{{range .}}
    {{.Name | printf "%-11s"}} {{.Short}}{{end}}

Use "revel help [command]" for more information.
`

var helpTemplate = `usage: revel {{.UsageLine}}
{{.Long}}
`

func usage(exitCode int) {
	tmpl(os.Stderr, usageTemplate, commands)
	os.Exit(exitCode)
}

func tmpl(w io.Writer, text string, data interface{}) {
	t := template.New("top")
	template.Must(t.Parse(text))
	if err := t.Execute(w, data); err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package reviewdog

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"os"

	"github.com/haya14busa/reviewdog/diff"
)

// Version is version of reviewdog CLI.
const Version = "0.9.8"

// Reviewdog represents review dog application which parses result of compiler
// or linter, get diff and filter the results by diff, and report filtered
// results.
type Reviewdog struct {
	toolname string
	p        Parser
	c        CommentService
	d        DiffService
}

// NewReviewdog returns a new Reviewdog.
func NewReviewdog(toolname string, p Parser, c CommentService, d DiffService) *Reviewdog {
	return &Reviewdog{p: p, c: c, d: d, toolname: toolname}
}

func RunFromResult(ctx context.Context, c CommentService, results []*CheckResult,
	filediffs []*diff.FileDiff, strip int, toolname string) error {
	return (&Reviewdog{c: c, toolname: toolname}).runFromResult(ctx, results, filediffs, strip)
}

// CheckResult represents a checked result of static analysis tools.
// :h error-file-format
type CheckResult struct {
	Path    string   // relative file path
	Lnum    int      // line number
	Col     int      // column number (1 <tab> == 1 character column)
	Message string   // error message
	Lines   []string // Original error lines (often one line)
}

// Parser is an interface which parses compilers, linters, or any tools
// results.
type Parser interface {
	Parse(r io.Reader) ([]*CheckResult, error)
}

// Comment represents a reported result as a comment.
type Comment struct {
	*CheckResult
	Body     string
	LnumDiff int
	ToolName string
}

// CommentService is an interface which posts Comment.
type CommentService interface {
	Post(context.Context, *Comment) error
}

// BulkCommentService posts comments all at once when Flush() is called.
// Flush() will be called at the end of reviewdog run.
type BulkCommentService interface {
	CommentService
	Flush(context.Context) error
}

// DiffService is an interface which get diff.
type DiffService interface {
	Diff(context.Context) ([]byte, error)
	Strip() int
}

func (w *Reviewdog) runFromResult(ctx context.Context, results []*CheckResult,
	filediffs []*diff.FileDiff, strip int) error {
	wd, err := os.Getwd()
	if err != nil {
		return err
	}

	checks := FilterCheck(results, filediffs, strip, wd)
	for _, check := range checks {
		if !check.InDiff {
			continue
		}
		comment := &Comment{
			CheckResult: check.CheckResult,
			Body:        check.Message, // TODO: format message
			LnumDiff:    check.LnumDiff,
			ToolName:    w.toolname,
		}
		if err := w.c.Post(ctx, comment); err != nil {
			return err
		}
	}

	if bulk, ok := w.c.(BulkCommentService); ok {
		return bulk.Flush(ctx)
	}

	return nil
}

// Run runs Reviewdog application.
func (w *Reviewdog) Run(ctx context.Context, r io.Reader) error {
	results, err := w.p.Parse(r)
	if err != nil {
		return fmt.Errorf("parse error: %v", err)
	}

	d, err := w.d.Diff(ctx)
	if err != nil {
		return fmt.Errorf("fail to get diff: %v", err)
	}

	filediffs, err := diff.ParseMultiFile(bytes.NewReader(d))
	if err != nil {
		return fmt.Errorf("fail to parse diff: %v", err)
	}

	return w.runFromResult(ctx, results, filediffs, w.d.Strip())
}
<commit_msg>Bump up version to 0.9.9<commit_after>package reviewdog

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"os"

	"github.com/haya14busa/reviewdog/diff"
)

// Version is version of reviewdog CLI.
const Version = "0.9.9"

// Reviewdog represents review dog application which parses result of compiler
// or linter, get diff and filter the results by diff, and report filtered
// results.
type Reviewdog struct {
	toolname string
	p        Parser
	c        CommentService
	d        DiffService
}

// NewReviewdog returns a new Reviewdog.
func NewReviewdog(toolname string, p Parser, c CommentService, d DiffService) *Reviewdog {
	return &Reviewdog{p: p, c: c, d: d, toolname: toolname}
}

func RunFromResult(ctx context.Context, c CommentService, results []*CheckResult,
	filediffs []*diff.FileDiff, strip int, toolname string) error {
	return (&Reviewdog{c: c, toolname: toolname}).runFromResult(ctx, results, filediffs, strip)
}

// CheckResult represents a checked result of static analysis tools.
// :h error-file-format
type CheckResult struct {
	Path    string   // relative file path
	Lnum    int      // line number
	Col     int      // column number (1 <tab> == 1 character column)
	Message string   // error message
	Lines   []string // Original error lines (often one line)
}

// Parser is an interface which parses compilers, linters, or any tools
// results.
type Parser interface {
	Parse(r io.Reader) ([]*CheckResult, error)
}

// Comment represents a reported result as a comment.
type Comment struct {
	*CheckResult
	Body     string
	LnumDiff int
	ToolName string
}

// CommentService is an interface which posts Comment.
type CommentService interface {
	Post(context.Context, *Comment) error
}

// BulkCommentService posts comments all at once when Flush() is called.
// Flush() will be called at the end of reviewdog run.
type BulkCommentService interface {
	CommentService
	Flush(context.Context) error
}

// DiffService is an interface which get diff.
type DiffService interface {
	Diff(context.Context) ([]byte, error)
	Strip() int
}

func (w *Reviewdog) runFromResult(ctx context.Context, results []*CheckResult,
	filediffs []*diff.FileDiff, strip int) error {
	wd, err := os.Getwd()
	if err != nil {
		return err
	}

	checks := FilterCheck(results, filediffs, strip, wd)
	for _, check := range checks {
		if !check.InDiff {
			continue
		}
		comment := &Comment{
			CheckResult: check.CheckResult,
			Body:        check.Message, // TODO: format message
			LnumDiff:    check.LnumDiff,
			ToolName:    w.toolname,
		}
		if err := w.c.Post(ctx, comment); err != nil {
			return err
		}
	}

	if bulk, ok := w.c.(BulkCommentService); ok {
		return bulk.Flush(ctx)
	}

	return nil
}

// Run runs Reviewdog application.
func (w *Reviewdog) Run(ctx context.Context, r io.Reader) error {
	results, err := w.p.Parse(r)
	if err != nil {
		return fmt.Errorf("parse error: %v", err)
	}

	d, err := w.d.Diff(ctx)
	if err != nil {
		return fmt.Errorf("fail to get diff: %v", err)
	}

	filediffs, err := diff.ParseMultiFile(bytes.NewReader(d))
	if err != nil {
		return fmt.Errorf("fail to parse diff: %v", err)
	}

	return w.runFromResult(ctx, results, filediffs, w.d.Strip())
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestRootAccess(t *testing.T) {
	response := httptest.NewRecorder()

	n := setUpServer()

	req, err := http.NewRequest("GET", "http://localhost:8080/", nil)
	if err != nil {
		t.Error(err)
	}
	n.ServeHTTP(response, req)
	if response.Code != http.StatusOK {
		t.Errorf("Got error for GET ruquest to /")
	}
	body := string(response.Body.Bytes())
	expectedBody := "{\"status\":\"ok\"}"
	if body != expectedBody {
		t.Errorf("Got empty body for GET request to /\n Got: %s, Expected: %s", body, expectedBody)
	}
}
<commit_msg>Add create test base<commit_after>package main

import (
	"bytes"
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestRootAccess(t *testing.T) {
	response := httptest.NewRecorder()

	n := setUpServer()

	req, err := http.NewRequest("GET", "http://localhost:8080/", nil)
	if err != nil {
		t.Error(err)
	}
	n.ServeHTTP(response, req)
	if response.Code != http.StatusOK {
		t.Errorf("Got error for GET ruquest to /")
	}
	body := string(response.Body.Bytes())
	expectedBody := "{\"status\":\"ok\"}"
	if body != expectedBody {
		t.Errorf("Got empty body for GET request to /\n Got: %s, Expected: %s", body, expectedBody)
	}
}

func TestBuildFlow(t *testing.T) {
	response := httptest.NewRecorder()

	n := setUpServer()

	requestParams := `{
		"source_repo": "wantedly/risu",
		"source_revision": "ada9ce1829fab49e605e5a563dbf91274f64e923",
		"name": "quay.io/wantedly/risu:latest",
		"dockerfile": "Dockerfile.dev"
	}`

	// Create
	req, err := http.NewRequest("POST", "http://localhost:8080/builds", bytes.NewBuffer([]byte(requestParams)))
	if err != nil {
		t.Error(err)
	}
	n.ServeHTTP(response, req)
	if response.Code != http.StatusOK {
		t.Errorf("Got error for GET ruquest to /")
	}
	body := string(response.Body.Bytes())
	if body != "hoge" {
		t.Errorf("Got: %v", body)
	}

	// dec := json.NewDecoder(response.Body)
	// var build schema.Build
	// dec.Decode(&build)
	// expectedBuild := schema.Build{
	// 	ID:             uuid.NewUUID(),
	// 	SourceRepo:     "opts.SourceRepo",
	// 	SourceRevision: "opts.SourceRevision",
	// 	Name:           "opts.Name",
	// 	Dockerfile:     "Dockerfile",
	// 	Status:         "building",
	// 	CreatedAt:      time.Now(),
	// 	UpdatedAt:      time.Now(),
	// }
	//
	// if !build.Equals(expectedBuild) {
	// 	t.Errorf("Got empty body for GET request to /\n Got: %v\nExpected: %v", build, expectedBuild)
	// }
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/jroimartin/gocui"
	"log"

	"math/rand"
	"time"
	//"net/url"

	"github.com/gorilla/websocket"

	"crypto/tls"
	b "github.com/engineerbeard/barrenschat/shared"
	"net/url"
	"strings"
)

const CHATWINDOW = "CHATWINDOW"
const ONLINEWINDOW = "ONLINEWINDOW"
const ROOMWINDOW = "ROOMWINDOW"

type server struct{}

func init() {
	rand.Seed(time.Now().UnixNano())
	BClient = b.BChatClient{}

}

var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
var BClient b.BChatClient

func RandStringRunes(n int) string {
	a := make([]rune, n)
	for i := range a {
		a[i] = letterRunes[rand.Intn(len(letterRunes))]
	}
	return string(a)
}
func main() {
	// Setup Ws connection
	u := url.URL{Scheme: "wss", Host: "https://damp-springs-83733.herokuapp.com:5000", Path: "/bchatws"}
	d := websocket.Dialer{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	c, _, err := d.Dial(u.String(), nil)
	if err != nil {
		log.Fatalln(err)
	}

	//c.WriteJSON()

	//bhelpersb..BMessage{MsgType:B_CONNECT, Uid:RandStringRunes(32)}
	BClient = b.BChatClient{WsConn: c, Uid: RandStringRunes(32), Name: "Anon", Room: b.MAIN_ROOM}
	BClient.SendMessage(b.BMessage{MsgType: b.B_CONNECT, Uid: RandStringRunes(32), Payload: BClient.Name})

	// Setup CUI
	g, err := gocui.NewGui(gocui.OutputNormal)
	if err != nil {
		log.Panicln(err)
	}
	defer g.Close()

	g.Highlight = true
	g.Cursor = true

	g.SetManagerFunc(setLayout)

	if err := g.SetKeybinding("", gocui.KeyCtrlC, gocui.ModNone, quit); err != nil {
		log.Panicln(err)
	}
	if err := g.SetKeybinding("", gocui.KeyEnter, gocui.ModNone, onEnterEvt(c)); err != nil {
		log.Panicln(err)
	}

	go handleConnection(c, g)

	if err := g.MainLoop(); err != nil && err != gocui.ErrQuit {
		log.Panicln(err)
	}
}
func handleConnection(c *websocket.Conn, g *gocui.Gui) {
	var bMessage b.BMessage
	c.ReadJSON(&bMessage)
	processMsg(bMessage, g)
	for {
		err := c.ReadJSON(&bMessage)
		if err != nil {
			log.Fatal(err)
		}
		processMsg(bMessage, g)
	}
}

func processMsg(msg b.BMessage, g *gocui.Gui) {
	g.Update(func(g *gocui.Gui) error {

		v, _ := g.View(CHATWINDOW)
		if msg.MsgType == b.B_CONNECT || msg.MsgType == b.B_DISCONNECT || msg.MsgType == b.B_ROOMCHANGE {
			o, _ := g.View(ONLINEWINDOW)
			o.Clear()
			fmt.Fprint(o, msg.OnlineData)

			o, _ = g.View(ROOMWINDOW)
			o.Clear()
			fmt.Fprintf(o, msg.RoomData)
		}
		if msg.MsgType == b.B_NAMECHANGE {
			o, _ := g.View(ONLINEWINDOW)
			o.Clear()
			fmt.Fprint(o, msg.OnlineData)
		}

		fmt.Fprintln(v, fmt.Sprintf("\u001b[33m%s\u001b[0m (%s) %s", msg.TimeStamp.Format("2006-01-02 15:04"), msg.Name, msg.Payload))
		return nil
	})
}

func setActiveView(g *gocui.Gui, name string) (*gocui.View, error) {
	if _, err := g.SetCurrentView(name); err != nil {
		return nil, err
	}
	return g.SetViewOnTop(name)
}

func onEnterEvt(c *websocket.Conn) func(g *gocui.Gui, v *gocui.View) error {
	return func(g *gocui.Gui, v *gocui.View) error {
		buf := strings.Replace(v.Buffer(), "\n", "", -1)
		if len(buf) < 1 {
			v.SetCursor(0, 0)
			return nil
		}
		msgType := b.B_MESSAGE
		if strings.Contains(buf, "/name") && len(strings.SplitAfter(buf, "/name")) > 1 {
			newName := strings.SplitAfter(buf, "/name")[1]
			newName = strings.TrimSpace(newName)
			buf = fmt.Sprintf("%s changed name to %s", BClient.Name, newName)
			BClient.Name = newName
			msgType = b.B_NAMECHANGE

		} else if strings.Contains(buf, "/room") && len(strings.SplitAfter(buf, "/room")) > 1 {
			msgType = b.B_ROOMCHANGE
			newRoom := strings.SplitAfter(buf, "/room")[1]
			newRoom = strings.TrimSpace(newRoom)
			BClient.Room = newRoom
			//buf = fmt.Sprintf("%s left room", BClient.Name)
		}
		err := c.WriteJSON(b.BMessage{
			MsgType:   msgType,
			TimeStamp: time.Now(),
			Name:      BClient.Name,
			Room:      BClient.Room,
			Uid:       BClient.Uid,
			Payload:   buf,
		})
		if err != nil {
			log.Println(err)
		}

		v.Clear()
		v.SetCursor(0, 0)
		return nil
	}

}
func setLayout(g *gocui.Gui) error {
	maxX, maxY := g.Size()
	if v, err := g.SetView(ONLINEWINDOW, 0, 0, 20, 14); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Online"
		v.Autoscroll = true
		fmt.Fprintln(v, "")
	}

	if v, err := g.SetView(ROOMWINDOW, 0, 15, 20, maxY-1); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Rooms"
		fmt.Fprintln(v, "")
	}

	if v, err := g.SetView("input", 21, maxY-3, maxX-1, maxY-1); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Autoscroll = true
		v.Title = "Type To Chat"
		v.Editable = true
		v.Wrap = true
		//fmt.Fprintf(v, "H")
	}

	if v, err := g.SetView(CHATWINDOW, 21, 0, maxX-1, maxY-4); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Autoscroll = true
		v.Title = "BChats"
		v.Editable = false
		v.Wrap = true
	}

	if _, err := setActiveView(g, "input"); err != nil {
		return err
	}
	return nil
}
func quit(g *gocui.Gui, v *gocui.View) error {
	return gocui.ErrQuit
}
<commit_msg>heroku updates<commit_after>package main

import (
	"fmt"
	"github.com/jroimartin/gocui"
	"log"

	"math/rand"
	"time"
	//"net/url"

	"github.com/gorilla/websocket"

	"crypto/tls"
	b "github.com/engineerbeard/barrenschat/shared"
	"net/url"
	"strings"
)

const CHATWINDOW = "CHATWINDOW"
const ONLINEWINDOW = "ONLINEWINDOW"
const ROOMWINDOW = "ROOMWINDOW"

type server struct{}

func init() {
	rand.Seed(time.Now().UnixNano())
	BClient = b.BChatClient{}

}

var letterRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
var BClient b.BChatClient

func RandStringRunes(n int) string {
	a := make([]rune, n)
	for i := range a {
		a[i] = letterRunes[rand.Intn(len(letterRunes))]
	}
	return string(a)
}
func main() {
	// Setup Ws connection
	u := url.URL{Scheme: "wss", Host: "damp-springs-83733.herokuapp.com", Path: "/bchatws"}
	d := websocket.Dialer{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
	c, _, err := d.Dial(u.String(), nil)
	if err != nil {
		log.Fatalln(err)
	}

	//c.WriteJSON()

	//bhelpersb..BMessage{MsgType:B_CONNECT, Uid:RandStringRunes(32)}
	BClient = b.BChatClient{WsConn: c, Uid: RandStringRunes(32), Name: "Anon", Room: b.MAIN_ROOM}
	BClient.SendMessage(b.BMessage{MsgType: b.B_CONNECT, Uid: RandStringRunes(32), Payload: BClient.Name})

	// Setup CUI
	g, err := gocui.NewGui(gocui.OutputNormal)
	if err != nil {
		log.Panicln(err)
	}
	defer g.Close()

	g.Highlight = true
	g.Cursor = true

	g.SetManagerFunc(setLayout)

	if err := g.SetKeybinding("", gocui.KeyCtrlC, gocui.ModNone, quit); err != nil {
		log.Panicln(err)
	}
	if err := g.SetKeybinding("", gocui.KeyEnter, gocui.ModNone, onEnterEvt(c)); err != nil {
		log.Panicln(err)
	}

	go handleConnection(c, g)

	if err := g.MainLoop(); err != nil && err != gocui.ErrQuit {
		log.Panicln(err)
	}
}
func handleConnection(c *websocket.Conn, g *gocui.Gui) {
	var bMessage b.BMessage
	c.ReadJSON(&bMessage)
	processMsg(bMessage, g)
	for {
		err := c.ReadJSON(&bMessage)
		if err != nil {
			log.Fatal(err)
		}
		processMsg(bMessage, g)
	}
}

func processMsg(msg b.BMessage, g *gocui.Gui) {
	g.Update(func(g *gocui.Gui) error {

		v, _ := g.View(CHATWINDOW)
		if msg.MsgType == b.B_CONNECT || msg.MsgType == b.B_DISCONNECT || msg.MsgType == b.B_ROOMCHANGE {
			o, _ := g.View(ONLINEWINDOW)
			o.Clear()
			fmt.Fprint(o, msg.OnlineData)

			o, _ = g.View(ROOMWINDOW)
			o.Clear()
			fmt.Fprintf(o, msg.RoomData)
		}
		if msg.MsgType == b.B_NAMECHANGE {
			o, _ := g.View(ONLINEWINDOW)
			o.Clear()
			fmt.Fprint(o, msg.OnlineData)
		}

		fmt.Fprintln(v, fmt.Sprintf("\u001b[33m%s\u001b[0m (%s) %s", msg.TimeStamp.Format("2006-01-02 15:04"), msg.Name, msg.Payload))
		return nil
	})
}

func setActiveView(g *gocui.Gui, name string) (*gocui.View, error) {
	if _, err := g.SetCurrentView(name); err != nil {
		return nil, err
	}
	return g.SetViewOnTop(name)
}

func onEnterEvt(c *websocket.Conn) func(g *gocui.Gui, v *gocui.View) error {
	return func(g *gocui.Gui, v *gocui.View) error {
		buf := strings.Replace(v.Buffer(), "\n", "", -1)
		if len(buf) < 1 {
			v.SetCursor(0, 0)
			return nil
		}
		msgType := b.B_MESSAGE
		if strings.Contains(buf, "/name") && len(strings.SplitAfter(buf, "/name")) > 1 {
			newName := strings.SplitAfter(buf, "/name")[1]
			newName = strings.TrimSpace(newName)
			buf = fmt.Sprintf("%s changed name to %s", BClient.Name, newName)
			BClient.Name = newName
			msgType = b.B_NAMECHANGE

		} else if strings.Contains(buf, "/room") && len(strings.SplitAfter(buf, "/room")) > 1 {
			msgType = b.B_ROOMCHANGE
			newRoom := strings.SplitAfter(buf, "/room")[1]
			newRoom = strings.TrimSpace(newRoom)
			BClient.Room = newRoom
			//buf = fmt.Sprintf("%s left room", BClient.Name)
		}
		err := c.WriteJSON(b.BMessage{
			MsgType:   msgType,
			TimeStamp: time.Now(),
			Name:      BClient.Name,
			Room:      BClient.Room,
			Uid:       BClient.Uid,
			Payload:   buf,
		})
		if err != nil {
			log.Println(err)
		}

		v.Clear()
		v.SetCursor(0, 0)
		return nil
	}

}
func setLayout(g *gocui.Gui) error {
	maxX, maxY := g.Size()
	if v, err := g.SetView(ONLINEWINDOW, 0, 0, 20, 14); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Online"
		v.Autoscroll = true
		fmt.Fprintln(v, "")
	}

	if v, err := g.SetView(ROOMWINDOW, 0, 15, 20, maxY-1); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Title = "Rooms"
		fmt.Fprintln(v, "")
	}

	if v, err := g.SetView("input", 21, maxY-3, maxX-1, maxY-1); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Autoscroll = true
		v.Title = "Type To Chat"
		v.Editable = true
		v.Wrap = true
		//fmt.Fprintf(v, "H")
	}

	if v, err := g.SetView(CHATWINDOW, 21, 0, maxX-1, maxY-4); err != nil {
		if err != gocui.ErrUnknownView {
			return err
		}
		v.Autoscroll = true
		v.Title = "BChats"
		v.Editable = false
		v.Wrap = true
	}

	if _, err := setActiveView(g, "input"); err != nil {
		return err
	}
	return nil
}
func quit(g *gocui.Gui, v *gocui.View) error {
	return gocui.ErrQuit
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"reflect"
	"runtime"
	"sync"
	"text/template"
	"time"

	"github.com/armon/consul-api"
)

const (
	// failSleep controls how long to sleep on a failure
	failSleep = 5 * time.Second

	// maxFailures controls the maximum number of failures
	// before we limit the sleep value
	maxFailures = 3

	// waitTime is used to control how long we do a blocking
	// query for
	waitTime = 60 * time.Second
)

type backendData struct {
	sync.Mutex

	// Client is a shared Consul client
	Client *consulapi.Client

	// Servers maps each watch path to a list of entries
	Servers map[*WatchPath][]*consulapi.ServiceEntry

	// Backends maps a backend to a list of watch paths used
	// to build up the server list
	Backends map[string][]*WatchPath

	// ChangeCh is used to inform of an update
	ChangeCh chan struct{}

	// StopCh is used to trigger a stop
	StopCh chan struct{}
}

// watch is used to start a long running watcher to handle updates.
// Returns a stopCh, and a finishCh.
func watch(conf *Config) (chan struct{}, chan struct{}) {
	stopCh := make(chan struct{})
	finishCh := make(chan struct{})
	go runWatch(conf, stopCh, finishCh)
	return stopCh, finishCh
}

// runWatch is a long running routine that watches with a
// given configuration
func runWatch(conf *Config, stopCh, doneCh chan struct{}) {
	defer close(doneCh)

	// Create the consul client
	consulConf := consulapi.DefaultConfig()
	if conf.Address != "" {
		consulConf.Address = conf.Address
	}

	// Attempt to contact the agent
	client, err := consulapi.NewClient(consulConf)
	if err != nil {
		log.Printf("[ERR] Failed to initialize consul client: %v", err)
		return
	}
	if _, err := client.Agent().NodeName(); err != nil {
		log.Printf("[ERR] Failed to contact consul agent: %v", err)
		return
	}

	// Create a backend store
	data := &backendData{
		Client:   client,
		Servers:  make(map[*WatchPath][]*consulapi.ServiceEntry),
		Backends: make(map[string][]*WatchPath),
		ChangeCh: make(chan struct{}, 1),
		StopCh:   stopCh,
	}

	// Start the watches
	data.Lock()
	for idx, watch := range conf.watches {
		data.Backends[watch.Backend] = append(data.Backends[watch.Backend], watch)
		go runSingleWatch(conf, data, idx, watch)
	}
	data.Unlock()

	// Monitor for changes or stop
	for {
		select {
		case <-data.ChangeCh:
			if maybeRefresh(conf, data) {
				return
			}

		case <-stopCh:
			return
		}
	}
}

// maybeRefresh is used to handle a potential config update
func maybeRefresh(conf *Config, data *backendData) (exit bool) {
	// Ignore initial updates until all the data is ready
	data.Lock()
	num := len(data.Servers)
	data.Unlock()
	if num < len(conf.watches) {
		return
	}

	// Merge the data for each backend
	backendServers := make(map[string][]*consulapi.ServiceEntry)
	data.Lock()
	for backend, watches := range data.Backends {
		var all []*consulapi.ServiceEntry
		for _, watch := range watches {
			entries := data.Servers[watch]
			all = append(all, entries...)
		}
		backendServers[backend] = all
	}
	data.Unlock()

	// Format the output
	outVars := formatOutput(backendServers)

	// Read the template
	raw, err := ioutil.ReadFile(conf.Template)
	if err != nil {
		log.Printf("[ERR] Failed to read template: %v", err)
		return true
	}

	// Create the template
	templ, err := template.New("output").Parse(string(raw))
	if err != nil {
		log.Printf("[ERR] Failed to parse the template: %v", err)
		return true
	}

	// Generate the output
	var output bytes.Buffer
	if err := templ.Execute(&output, outVars); err != nil {
		log.Printf("[ERR] Failed to generate the template: %v", err)
		return true
	}

	// Check for a dry run
	if conf.DryRun {
		fmt.Printf("%s\n", output.Bytes())
		return true
	}

	// Write out the configuration
	if err := ioutil.WriteFile(conf.Path, output.Bytes(), 0660); err != nil {
		log.Printf("[ERR] Failed to write config file: %v", err)
		return true
	}
	log.Printf("[INFO] Updated configuration file at %s", conf.Path)

	// Invoke the reload hook
	if err := reload(conf); err != nil {
		log.Printf("[ERR] Failed to reload: %v", err)
	} else {
		log.Printf("[INFO] Completed reload")
	}
	return
}

// runSingleWatch is used to query a single watch path for changes
func runSingleWatch(conf *Config, data *backendData, idx int, watch *WatchPath) {
	health := data.Client.Health()
	opts := &consulapi.QueryOptions{
		WaitTime: waitTime,
	}
	if watch.Datacenter != "" {
		opts.Datacenter = watch.Datacenter
	}

	failures := 0
	for {
		if shouldStop(data.StopCh) {
			return
		}
		entries, qm, err := health.Service(watch.Service, watch.Tag, true, opts)
		if err != nil {
			log.Printf("[ERR] Failed to fetch service nodes: %v", err)
		}

		// Patch the entries as necessary
		for _, entry := range entries {
			// Modify the node name to prefix with the watch ID. This
			// prevents a name conflict on duplicate names
			entry.Node.Node = fmt.Sprintf("%d_%s", idx, entry.Node.Node)

			// Patch the port if provided and the service hasn't registered
			if watch.Port != 0 && entry.Service.Port == 0 {
				entry.Service.Port = watch.Port
			}
		}

		// Update the entries. If this is the first read, do it on error
		data.Lock()
		old, ok := data.Servers[watch]
		if !ok || (err == nil && !reflect.DeepEqual(old, entries)) {
			data.Servers[watch] = entries
			asyncNotify(data.ChangeCh)
			if !conf.DryRun {
				log.Printf("[DEBUG] Updated nodes for %v", watch.Spec)
			}
		}
		data.Unlock()

		// Stop immediately on a dry run
		if conf.DryRun {
			return
		}

		// Check for an error
		if err != nil {
			failures = min(failures+1, maxFailures)
			time.Sleep(backoff(failSleep, failures))
		} else {
			failures = 0
			opts.WaitIndex = qm.LastIndex
		}
	}
}

// reload is used to invoke the reload command
func reload(conf *Config) error {
	// Determine the shell invocation based on OS
	var shell, flag string
	if runtime.GOOS == "windows" {
		shell = "cmd"
		flag = "/C"
	} else {
		shell = "/bin/sh"
		flag = "-c"
	}

	// Create and invoke the command
	cmd := exec.Command(shell, flag, conf.ReloadCommand)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

// shouldStop checks for a closed control channel
func shouldStop(ch chan struct{}) bool {
	select {
	case <-ch:
		return true
	default:
		return false
	}
}

// asyncNotify is used to notify a channel
func asyncNotify(ch chan struct{}) {
	select {
	case ch <- struct{}{}:
	default:
	}
}

// min returns the min of two ints
func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

// backoff is used to compute an exponential backoff
func backoff(interval time.Duration, times int) time.Duration {
	base := interval
	for ; times > 1; times-- {
		base *= interval
	}
	return interval
}

// formatOutput converts the service entries into a format
// suitable for templating into the HAProxy file
func formatOutput(inp map[string][]*consulapi.ServiceEntry) map[string][]string {
	out := make(map[string][]string)
	for backend, entries := range inp {
		servers := make([]string, len(entries))
		for idx, entry := range entries {
			name := fmt.Sprintf("%s_%s", entry.Node.Node, entry.Service.ID)
			ip := net.ParseIP(entry.Node.Address)
			addr := &net.TCPAddr{IP: ip, Port: entry.Service.Port}
			servers[idx] = fmt.Sprintf("server %s %s", name, addr)
		}
		out[backend] = servers
	}
	return out
}
<commit_msg>Refactor watch<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"reflect"
	"runtime"
	"sync"
	"text/template"
	"time"

	"github.com/armon/consul-api"
)

const (
	// failSleep controls how long to sleep on a failure
	failSleep = 5 * time.Second

	// maxFailures controls the maximum number of failures
	// before we limit the sleep value
	maxFailures = 3

	// waitTime is used to control how long we do a blocking
	// query for
	waitTime = 60 * time.Second
)

type backendData struct {
	sync.Mutex

	// Client is a shared Consul client
	Client *consulapi.Client

	// Servers maps each watch path to a list of entries
	Servers map[*WatchPath][]*consulapi.ServiceEntry

	// Backends maps a backend to a list of watch paths used
	// to build up the server list
	Backends map[string][]*WatchPath

	// ChangeCh is used to inform of an update
	ChangeCh chan struct{}

	// StopCh is used to trigger a stop
	StopCh chan struct{}
}

// watch is used to start a long running watcher to handle updates.
// Returns a stopCh, and a finishCh.
func watch(conf *Config) (chan struct{}, chan struct{}) {
	stopCh := make(chan struct{})
	finishCh := make(chan struct{})
	go runWatch(conf, stopCh, finishCh)
	return stopCh, finishCh
}

// runWatch is a long running routine that watches with a
// given configuration
func runWatch(conf *Config, stopCh, doneCh chan struct{}) {
	defer close(doneCh)

	// Create the consul client
	consulConf := consulapi.DefaultConfig()
	if conf.Address != "" {
		consulConf.Address = conf.Address
	}

	// Attempt to contact the agent
	client, err := consulapi.NewClient(consulConf)
	if err != nil {
		log.Printf("[ERR] Failed to initialize consul client: %v", err)
		return
	}
	if _, err := client.Agent().NodeName(); err != nil {
		log.Printf("[ERR] Failed to contact consul agent: %v", err)
		return
	}

	// Create a backend store
	data := &backendData{
		Client:   client,
		Servers:  make(map[*WatchPath][]*consulapi.ServiceEntry),
		Backends: make(map[string][]*WatchPath),
		ChangeCh: make(chan struct{}, 1),
		StopCh:   stopCh,
	}

	// Start the watches
	data.Lock()
	for idx, watch := range conf.watches {
		data.Backends[watch.Backend] = append(data.Backends[watch.Backend], watch)
		go runSingleWatch(conf, data, idx, watch)
	}
	data.Unlock()

	// Monitor for changes or stop
	for {
		select {
		case <-data.ChangeCh:
			if maybeRefresh(conf, data) {
				return
			}

		case <-stopCh:
			return
		}
	}
}

// maybeRefresh is used to handle a potential config update
func maybeRefresh(conf *Config, data *backendData) (exit bool) {
	// Ignore initial updates until all the data is ready
	if !allWatchesReturned(conf, data) {
		return
	}

	// Merge the data for each backend
	backendServers := aggregateServers(data)

	// Build the output template
	output, err := buildTemplate(conf, backendServers)
	if err != nil {
		log.Printf("[ERR] %v", err)
		return true
	}

	// Check for a dry run
	if conf.DryRun {
		fmt.Printf("%s\n", output)
		return true
	}

	// Write out the configuration
	if err := ioutil.WriteFile(conf.Path, output, 0660); err != nil {
		log.Printf("[ERR] Failed to write config file: %v", err)
		return true
	}
	log.Printf("[INFO] Updated configuration file at %s", conf.Path)

	// Invoke the reload hook
	if err := reload(conf); err != nil {
		log.Printf("[ERR] Failed to reload: %v", err)
	} else {
		log.Printf("[INFO] Completed reload")
	}
	return
}

// allWatchesReturned checks if all the watches have some
// data registered. Prevents early template generation.
func allWatchesReturned(conf *Config, data *backendData) bool {
	data.Lock()
	defer data.Unlock()
	return len(data.Servers) >= len(conf.watches)
}

// aggregateServers merges the watches belonging to each
// backend together to prepare for template generation
func aggregateServers(data *backendData) map[string][]*consulapi.ServiceEntry {
	backendServers := make(map[string][]*consulapi.ServiceEntry)
	data.Lock()
	defer data.Unlock()
	for backend, watches := range data.Backends {
		var all []*consulapi.ServiceEntry
		for _, watch := range watches {
			entries := data.Servers[watch]
			all = append(all, entries...)
		}
		backendServers[backend] = all
	}
	return backendServers
}

// buildTemplate is used to build the output template
// from the configuration and server list
func buildTemplate(conf *Config,
	servers map[string][]*consulapi.ServiceEntry) ([]byte, error) {
	// Format the output
	outVars := formatOutput(servers)

	// Read the template
	raw, err := ioutil.ReadFile(conf.Template)
	if err != nil {
		return nil, fmt.Errorf("Failed to read template: %v", err)
	}

	// Create the template
	templ, err := template.New("output").Parse(string(raw))
	if err != nil {
		return nil, fmt.Errorf("Failed to parse the template: %v", err)
	}

	// Generate the output
	var output bytes.Buffer
	if err := templ.Execute(&output, outVars); err != nil {
		return nil, fmt.Errorf("Failed to generate the template: %v", err)
	}
	return output.Bytes(), nil
}

// runSingleWatch is used to query a single watch path for changes
func runSingleWatch(conf *Config, data *backendData, idx int, watch *WatchPath) {
	health := data.Client.Health()
	opts := &consulapi.QueryOptions{
		WaitTime: waitTime,
	}
	if watch.Datacenter != "" {
		opts.Datacenter = watch.Datacenter
	}

	failures := 0
	for {
		if shouldStop(data.StopCh) {
			return
		}
		entries, qm, err := health.Service(watch.Service, watch.Tag, true, opts)
		if err != nil {
			log.Printf("[ERR] Failed to fetch service nodes: %v", err)
		}

		// Patch the entries as necessary
		for _, entry := range entries {
			// Modify the node name to prefix with the watch ID. This
			// prevents a name conflict on duplicate names
			entry.Node.Node = fmt.Sprintf("%d_%s", idx, entry.Node.Node)

			// Patch the port if provided and the service hasn't registered
			if watch.Port != 0 && entry.Service.Port == 0 {
				entry.Service.Port = watch.Port
			}
		}

		// Update the entries. If this is the first read, do it on error
		data.Lock()
		old, ok := data.Servers[watch]
		if !ok || (err == nil && !reflect.DeepEqual(old, entries)) {
			data.Servers[watch] = entries
			asyncNotify(data.ChangeCh)
			if !conf.DryRun {
				log.Printf("[DEBUG] Updated nodes for %v", watch.Spec)
			}
		}
		data.Unlock()

		// Stop immediately on a dry run
		if conf.DryRun {
			return
		}

		// Check for an error
		if err != nil {
			failures = min(failures+1, maxFailures)
			time.Sleep(backoff(failSleep, failures))
		} else {
			failures = 0
			opts.WaitIndex = qm.LastIndex
		}
	}
}

// reload is used to invoke the reload command
func reload(conf *Config) error {
	// Determine the shell invocation based on OS
	var shell, flag string
	if runtime.GOOS == "windows" {
		shell = "cmd"
		flag = "/C"
	} else {
		shell = "/bin/sh"
		flag = "-c"
	}

	// Create and invoke the command
	cmd := exec.Command(shell, flag, conf.ReloadCommand)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	return cmd.Run()
}

// shouldStop checks for a closed control channel
func shouldStop(ch chan struct{}) bool {
	select {
	case <-ch:
		return true
	default:
		return false
	}
}

// asyncNotify is used to notify a channel
func asyncNotify(ch chan struct{}) {
	select {
	case ch <- struct{}{}:
	default:
	}
}

// min returns the min of two ints
func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

// backoff is used to compute an exponential backoff
func backoff(interval time.Duration, times int) time.Duration {
	base := interval
	for ; times > 1; times-- {
		base *= interval
	}
	return interval
}

// formatOutput converts the service entries into a format
// suitable for templating into the HAProxy file
func formatOutput(inp map[string][]*consulapi.ServiceEntry) map[string][]string {
	out := make(map[string][]string)
	for backend, entries := range inp {
		servers := make([]string, len(entries))
		for idx, entry := range entries {
			name := fmt.Sprintf("%s_%s", entry.Node.Node, entry.Service.ID)
			ip := net.ParseIP(entry.Node.Address)
			addr := &net.TCPAddr{IP: ip, Port: entry.Service.Port}
			servers[idx] = fmt.Sprintf("server %s %s", name, addr)
		}
		out[backend] = servers
	}
	return out
}
<|endoftext|>


<commit_before>package kafkamdm

import (
	"bytes"
	"encoding/binary"
	"flag"
	"fmt"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/raintank/worldping-api/pkg/log"
	"github.com/rakyll/globalconf"

	"github.com/raintank/met"
	"github.com/raintank/metrictank/idx"
	"github.com/raintank/metrictank/in"
	"github.com/raintank/metrictank/mdata"
	"github.com/raintank/metrictank/usage"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"github.com/syndtr/goleveldb/leveldb/storage"
)

type KafkaMdm struct {
	in.In
	consumer sarama.Consumer
	client   sarama.Client
	stats    met.Backend

	wg sync.WaitGroup
	// read from this channel to block until consumer is cleanly stopped
	StopChan chan int

	// signal to PartitionConsumers to shutdown
	stopConsuming chan struct{}
}

var LogLevel int
var Enabled bool
var brokerStr string
var brokers []string
var topicStr string
var topics []string
var offset string
var dataDir string
var config *sarama.Config
var channelBufferSize int
var consumerFetchMin int
var consumerFetchDefault int
var consumerMaxWaitTime time.Duration
var consumerMaxProcessingTime time.Duration
var netMaxOpenRequests int
var offsetMgr *OffsetMgr
var offsetDuration time.Duration
var offsetCommitInterval time.Duration

func ConfigSetup() {
	inKafkaMdm := flag.NewFlagSet("kafka-mdm-in", flag.ExitOnError)
	inKafkaMdm.BoolVar(&Enabled, "enabled", false, "")
	inKafkaMdm.StringVar(&brokerStr, "brokers", "kafka:9092", "tcp address for kafka (may be be given multiple times as a comma-separated list)")
	inKafkaMdm.StringVar(&topicStr, "topics", "mdm", "kafka topic (may be given multiple times as a comma-separated list)")
	inKafkaMdm.StringVar(&offset, "offset", "last", "Set the offset to start consuming from. Can be one of newest, oldest,last or a time duration")
	inKafkaMdm.DurationVar(&offsetCommitInterval, "offset-commit-interval", time.Second*5, "Interval at which offsets should be saved.")
	inKafkaMdm.StringVar(&dataDir, "data-dir", "", "Directory to store partition offsets index")
	inKafkaMdm.IntVar(&channelBufferSize, "channel-buffer-size", 1000000, "The number of metrics to buffer in internal and external channels")
	inKafkaMdm.IntVar(&consumerFetchMin, "consumer-fetch-min", 1024000, "The minimum number of message bytes to fetch in a request")
	inKafkaMdm.IntVar(&consumerFetchDefault, "consumer-fetch-default", 4096000, "The default number of message bytes to fetch in a request")
	inKafkaMdm.DurationVar(&consumerMaxWaitTime, "consumer-max-wait-time", time.Second, "The maximum amount of time the broker will wait for Consumer.Fetch.Min bytes to become available before it returns fewer than that anyway")
	inKafkaMdm.DurationVar(&consumerMaxProcessingTime, "consumer-max-processing-time", time.Second, "The maximum amount of time the consumer expects a message takes to process")
	inKafkaMdm.IntVar(&netMaxOpenRequests, "net-max-open-requests", 100, "How many outstanding requests a connection is allowed to have before sending on it blocks")
	globalconf.Register("kafka-mdm-in", inKafkaMdm)
}

func ConfigProcess(instance string) {
	if !Enabled {
		return
	}

	if offsetCommitInterval == 0 {
		log.Fatal("kafkamdm: offset-commit-interval must be greater then 0")
	}
	if consumerMaxWaitTime == 0 {
		log.Fatal("kafkamdm: consumer-max-wait-time must be greater then 0")
	}
	if consumerMaxProcessingTime == 0 {
		log.Fatal("kafkamdm: consumer-max-processing-time must be greater then 0")
	}

	switch offset {
	case "last":
	case "oldest":
	case "newest":
	default:
		offsetDuration, err = time.ParseDuration(offset)
		if err != nil {
			log.Fatal(4, "kafkamdm: invalid offest format. %s", err)
		}
	}

	var err error
	offsetMgr, err = NewOffsetMgr(dataDir)
	if err != nil {
		log.Fatal(4, "kafka-mdm couldnt create offsetMgr. %s", err)
	}
	brokers = strings.Split(brokerStr, ",")
	topics = strings.Split(topicStr, ",")

	config = sarama.NewConfig()

	config.ClientID = instance + "-mdm"
	config.ChannelBufferSize = channelBufferSize
	config.Consumer.Fetch.Min = int32(consumerFetchMin)
	config.Consumer.Fetch.Default = int32(consumerFetchDefault)
	config.Consumer.MaxWaitTime = consumerMaxWaitTime
	config.Consumer.MaxProcessingTime = consumerMaxProcessingTime
	config.Net.MaxOpenRequests = netMaxOpenRequests
	config.Version = sarama.V0_10_0_0
	err = config.Validate()
	if err != nil {
		log.Fatal(2, "kafka-mdm invalid config: %s", err)
	}
}

func New(stats met.Backend) *KafkaMdm {
	client, err := sarama.NewClient(brokers, config)
	if err != nil {
		log.Fatal(4, "kafka-mdm failed to create client. %s", err)
	}
	consumer, err := sarama.NewConsumerFromClient(client)
	if err != nil {
		log.Fatal(2, "kafka-mdm failed to create consumer: %s", err)
	}
	log.Info("kafka-mdm consumer created without error")
	k := KafkaMdm{
		consumer:      consumer,
		client:        client,
		stats:         stats,
		StopChan:      make(chan int),
		stopConsuming: make(chan struct{}),
	}

	return &k
}

func (k *KafkaMdm) Start(metrics mdata.Metrics, metricIndex idx.MetricIndex, usg *usage.Usage) {
	k.In = in.New(metrics, metricIndex, usg, "kafka-mdm", k.stats)
	for _, topic := range topics {
		// get partitions.
		partitions, err := k.consumer.Partitions(topic)
		if err != nil {
			log.Fatal(4, "kafka-mdm: Faild to get partitions for topic %s. %s", topic, err)
		}
		for _, partition := range partitions {
			switch offset {
			case "oldest":
				go k.consumePartition(topic, partition, -2)
			case "newest":
				go k.consumePartition(topic, partition, -1)
			case "last":
				o, err := offsetMgr.Last(topic, partition)
				if err != nil {
					log.Fatal(4, "kafka-mdm: Failed to get offset for %s:%d. %s", topic, partition, err)
				}
				go k.consumePartition(topic, partition, o)
			default:
				o, err := k.client.GetOffset(topic, partition, time.Now().Add(-1*offsetDuration).UnixNano()/int64(time.Millisecond))
				if err != nil {
					log.Fatal(4, "kafka-mdm: failed to get offset for %s:%d.  %s", topic, partition, err)
				}
				go k.consumePartition(topic, partition, o)
			}
		}
	}
}

// this will continually consume from the topic until k.stopConsuming is triggered.
func (k *KafkaMdm) consumePartition(topic string, partition int32, partitionOffset int64) {
	k.wg.Add(1)
	defer k.wg.Done()

	pc, err := k.consumer.ConsumePartition(topic, partition, partitionOffset)
	if err != nil {
		log.Fatal(4, "kafka-mdm: failed to start partitionConsumer for %s:%d. %s", topic, partition, err)
	}
	log.Info("kafka-mdm: consuming from %s:%d from offset %d", topic, partition, partitionOffset)
	currentOffset := partitionOffset
	messages := pc.Messages()
	ticker := time.NewTicker(offsetCommitInterval)
	for {
		select {
		case msg := <-messages:
			if LogLevel < 2 {
				log.Debug("kafka-mdm received message: Topic %s, Partition: %d, Offset: %d, Key: %x", msg.Topic, msg.Partition, msg.Offset, msg.Key)
			}
			k.In.Handle(msg.Value)
			currentOffset = msg.Offset
		case <-ticker.C:
			if err := offsetMgr.Commit(topic, partition, currentOffset); err != nil {
				log.Error(3, "kafka-mdm failed to commit offset for %s:%d, %s", topic, partition, err)
			}
		case <-k.stopConsuming:
			pc.Close()
			offsetMgr.Commit(topic, partition, currentOffset)
			log.Info("kafka-mdm consumer for %s:%d ended.", topic, partition)
			return
		}
	}
}

// Stop will initiate a graceful stop of the Consumer (permanent)
//
// NOTE: receive on StopChan to block until this process completes
func (k *KafkaMdm) Stop() {
	// closes notifications and messages channels, amongst others
	close(k.stopConsuming)
	go func() {
		k.wg.Wait()
		offsetMgr.Close()
		close(k.StopChan)
	}()
}

type OffsetMgr struct {
	db *leveldb.DB
}

func NewOffsetMgr(dir string) (*OffsetMgr, error) {
	dbFile := filepath.Join(dir, "partitionOffsets.db")
	db, err := leveldb.OpenFile(dbFile, &opt.Options{})
	if err != nil {
		if _, ok := err.(*storage.ErrCorrupted); ok {
			log.Warn("partitionOffsets.db is corrupt. Recovering.")
			db, err = leveldb.RecoverFile(dbFile, &opt.Options{})
			if err != nil {
				return nil, err
			}
		} else {
			return nil, err
		}
	}
	log.Info("Opened %s", dbFile)
	return &OffsetMgr{
		db: db,
	}, nil
}

func (o *OffsetMgr) Close() {
	log.Info("Closing partitionsOffset DB.")
	o.db.Close()
}

func (o *OffsetMgr) Commit(topic string, partition int32, offset int64) error {
	key := new(bytes.Buffer)
	key.WriteString(fmt.Sprintf("T:%s-P:%d", topic, partition))
	data := new(bytes.Buffer)
	if err := binary.Write(data, binary.LittleEndian, offset); err != nil {
		return err
	}
	log.Debug("commiting offset %d for %s:%d to partitionsOffset.db", offset, topic, partition)
	return o.db.Put(key.Bytes(), data.Bytes(), &opt.WriteOptions{Sync: true})
}

func (o *OffsetMgr) Last(topic string, partition int32) (int64, error) {
	key := new(bytes.Buffer)
	key.WriteString(fmt.Sprintf("T:%s-P:%d", topic, partition))
	data, err := o.db.Get(key.Bytes(), nil)
	if err != nil {
		if err == leveldb.ErrNotFound {
			log.Debug("no offset recorded for %s:%d", topic, partition)
			return -1, nil
		}
		return 0, err
	}
	var offset int64
	err = binary.Read(bytes.NewBuffer(data), binary.LittleEndian, &offset)
	if err != nil {
		return 0, err
	}
	log.Debug("found saved offset %d for %s:%d", offset, topic, partition)
	return offset, nil
}
<commit_msg>write error log if we cant commit offset to index during shutdown<commit_after>package kafkamdm

import (
	"bytes"
	"encoding/binary"
	"flag"
	"fmt"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"github.com/Shopify/sarama"
	"github.com/raintank/worldping-api/pkg/log"
	"github.com/rakyll/globalconf"

	"github.com/raintank/met"
	"github.com/raintank/metrictank/idx"
	"github.com/raintank/metrictank/in"
	"github.com/raintank/metrictank/mdata"
	"github.com/raintank/metrictank/usage"
	"github.com/syndtr/goleveldb/leveldb"
	"github.com/syndtr/goleveldb/leveldb/opt"
	"github.com/syndtr/goleveldb/leveldb/storage"
)

type KafkaMdm struct {
	in.In
	consumer sarama.Consumer
	client   sarama.Client
	stats    met.Backend

	wg sync.WaitGroup
	// read from this channel to block until consumer is cleanly stopped
	StopChan chan int

	// signal to PartitionConsumers to shutdown
	stopConsuming chan struct{}
}

var LogLevel int
var Enabled bool
var brokerStr string
var brokers []string
var topicStr string
var topics []string
var offset string
var dataDir string
var config *sarama.Config
var channelBufferSize int
var consumerFetchMin int
var consumerFetchDefault int
var consumerMaxWaitTime time.Duration
var consumerMaxProcessingTime time.Duration
var netMaxOpenRequests int
var offsetMgr *OffsetMgr
var offsetDuration time.Duration
var offsetCommitInterval time.Duration

func ConfigSetup() {
	inKafkaMdm := flag.NewFlagSet("kafka-mdm-in", flag.ExitOnError)
	inKafkaMdm.BoolVar(&Enabled, "enabled", false, "")
	inKafkaMdm.StringVar(&brokerStr, "brokers", "kafka:9092", "tcp address for kafka (may be be given multiple times as a comma-separated list)")
	inKafkaMdm.StringVar(&topicStr, "topics", "mdm", "kafka topic (may be given multiple times as a comma-separated list)")
	inKafkaMdm.StringVar(&offset, "offset", "last", "Set the offset to start consuming from. Can be one of newest, oldest,last or a time duration")
	inKafkaMdm.DurationVar(&offsetCommitInterval, "offset-commit-interval", time.Second*5, "Interval at which offsets should be saved.")
	inKafkaMdm.StringVar(&dataDir, "data-dir", "", "Directory to store partition offsets index")
	inKafkaMdm.IntVar(&channelBufferSize, "channel-buffer-size", 1000000, "The number of metrics to buffer in internal and external channels")
	inKafkaMdm.IntVar(&consumerFetchMin, "consumer-fetch-min", 1024000, "The minimum number of message bytes to fetch in a request")
	inKafkaMdm.IntVar(&consumerFetchDefault, "consumer-fetch-default", 4096000, "The default number of message bytes to fetch in a request")
	inKafkaMdm.DurationVar(&consumerMaxWaitTime, "consumer-max-wait-time", time.Second, "The maximum amount of time the broker will wait for Consumer.Fetch.Min bytes to become available before it returns fewer than that anyway")
	inKafkaMdm.DurationVar(&consumerMaxProcessingTime, "consumer-max-processing-time", time.Second, "The maximum amount of time the consumer expects a message takes to process")
	inKafkaMdm.IntVar(&netMaxOpenRequests, "net-max-open-requests", 100, "How many outstanding requests a connection is allowed to have before sending on it blocks")
	globalconf.Register("kafka-mdm-in", inKafkaMdm)
}

func ConfigProcess(instance string) {
	if !Enabled {
		return
	}

	if offsetCommitInterval == 0 {
		log.Fatal("kafkamdm: offset-commit-interval must be greater then 0")
	}
	if consumerMaxWaitTime == 0 {
		log.Fatal("kafkamdm: consumer-max-wait-time must be greater then 0")
	}
	if consumerMaxProcessingTime == 0 {
		log.Fatal("kafkamdm: consumer-max-processing-time must be greater then 0")
	}

	switch offset {
	case "last":
	case "oldest":
	case "newest":
	default:
		offsetDuration, err = time.ParseDuration(offset)
		if err != nil {
			log.Fatal(4, "kafkamdm: invalid offest format. %s", err)
		}
	}

	var err error
	offsetMgr, err = NewOffsetMgr(dataDir)
	if err != nil {
		log.Fatal(4, "kafka-mdm couldnt create offsetMgr. %s", err)
	}
	brokers = strings.Split(brokerStr, ",")
	topics = strings.Split(topicStr, ",")

	config = sarama.NewConfig()

	config.ClientID = instance + "-mdm"
	config.ChannelBufferSize = channelBufferSize
	config.Consumer.Fetch.Min = int32(consumerFetchMin)
	config.Consumer.Fetch.Default = int32(consumerFetchDefault)
	config.Consumer.MaxWaitTime = consumerMaxWaitTime
	config.Consumer.MaxProcessingTime = consumerMaxProcessingTime
	config.Net.MaxOpenRequests = netMaxOpenRequests
	config.Version = sarama.V0_10_0_0
	err = config.Validate()
	if err != nil {
		log.Fatal(2, "kafka-mdm invalid config: %s", err)
	}
}

func New(stats met.Backend) *KafkaMdm {
	client, err := sarama.NewClient(brokers, config)
	if err != nil {
		log.Fatal(4, "kafka-mdm failed to create client. %s", err)
	}
	consumer, err := sarama.NewConsumerFromClient(client)
	if err != nil {
		log.Fatal(2, "kafka-mdm failed to create consumer: %s", err)
	}
	log.Info("kafka-mdm consumer created without error")
	k := KafkaMdm{
		consumer:      consumer,
		client:        client,
		stats:         stats,
		StopChan:      make(chan int),
		stopConsuming: make(chan struct{}),
	}

	return &k
}

func (k *KafkaMdm) Start(metrics mdata.Metrics, metricIndex idx.MetricIndex, usg *usage.Usage) {
	k.In = in.New(metrics, metricIndex, usg, "kafka-mdm", k.stats)
	for _, topic := range topics {
		// get partitions.
		partitions, err := k.consumer.Partitions(topic)
		if err != nil {
			log.Fatal(4, "kafka-mdm: Faild to get partitions for topic %s. %s", topic, err)
		}
		for _, partition := range partitions {
			switch offset {
			case "oldest":
				go k.consumePartition(topic, partition, -2)
			case "newest":
				go k.consumePartition(topic, partition, -1)
			case "last":
				o, err := offsetMgr.Last(topic, partition)
				if err != nil {
					log.Fatal(4, "kafka-mdm: Failed to get offset for %s:%d. %s", topic, partition, err)
				}
				go k.consumePartition(topic, partition, o)
			default:
				o, err := k.client.GetOffset(topic, partition, time.Now().Add(-1*offsetDuration).UnixNano()/int64(time.Millisecond))
				if err != nil {
					log.Fatal(4, "kafka-mdm: failed to get offset for %s:%d.  %s", topic, partition, err)
				}
				go k.consumePartition(topic, partition, o)
			}
		}
	}
}

// this will continually consume from the topic until k.stopConsuming is triggered.
func (k *KafkaMdm) consumePartition(topic string, partition int32, partitionOffset int64) {
	k.wg.Add(1)
	defer k.wg.Done()

	pc, err := k.consumer.ConsumePartition(topic, partition, partitionOffset)
	if err != nil {
		log.Fatal(4, "kafka-mdm: failed to start partitionConsumer for %s:%d. %s", topic, partition, err)
	}
	log.Info("kafka-mdm: consuming from %s:%d from offset %d", topic, partition, partitionOffset)
	currentOffset := partitionOffset
	messages := pc.Messages()
	ticker := time.NewTicker(offsetCommitInterval)
	for {
		select {
		case msg := <-messages:
			if LogLevel < 2 {
				log.Debug("kafka-mdm received message: Topic %s, Partition: %d, Offset: %d, Key: %x", msg.Topic, msg.Partition, msg.Offset, msg.Key)
			}
			k.In.Handle(msg.Value)
			currentOffset = msg.Offset
		case <-ticker.C:
			if err := offsetMgr.Commit(topic, partition, currentOffset); err != nil {
				log.Error(3, "kafka-mdm failed to commit offset for %s:%d, %s", topic, partition, err)
			}
		case <-k.stopConsuming:
			pc.Close()
			if err := offsetMgr.Commit(topic, partition, currentOffset); err != nil {
				log.Error(3, "kafka-mdm failed to commit offset for %s:%d, %s", topic, partition, err)
			}
			log.Info("kafka-mdm consumer for %s:%d ended.", topic, partition)
			return
		}
	}
}

// Stop will initiate a graceful stop of the Consumer (permanent)
//
// NOTE: receive on StopChan to block until this process completes
func (k *KafkaMdm) Stop() {
	// closes notifications and messages channels, amongst others
	close(k.stopConsuming)
	go func() {
		k.wg.Wait()
		offsetMgr.Close()
		close(k.StopChan)
	}()
}

type OffsetMgr struct {
	db *leveldb.DB
}

func NewOffsetMgr(dir string) (*OffsetMgr, error) {
	dbFile := filepath.Join(dir, "partitionOffsets.db")
	db, err := leveldb.OpenFile(dbFile, &opt.Options{})
	if err != nil {
		if _, ok := err.(*storage.ErrCorrupted); ok {
			log.Warn("partitionOffsets.db is corrupt. Recovering.")
			db, err = leveldb.RecoverFile(dbFile, &opt.Options{})
			if err != nil {
				return nil, err
			}
		} else {
			return nil, err
		}
	}
	log.Info("Opened %s", dbFile)
	return &OffsetMgr{
		db: db,
	}, nil
}

func (o *OffsetMgr) Close() {
	log.Info("Closing partitionsOffset DB.")
	o.db.Close()
}

func (o *OffsetMgr) Commit(topic string, partition int32, offset int64) error {
	key := new(bytes.Buffer)
	key.WriteString(fmt.Sprintf("T:%s-P:%d", topic, partition))
	data := new(bytes.Buffer)
	if err := binary.Write(data, binary.LittleEndian, offset); err != nil {
		return err
	}
	log.Debug("commiting offset %d for %s:%d to partitionsOffset.db", offset, topic, partition)
	return o.db.Put(key.Bytes(), data.Bytes(), &opt.WriteOptions{Sync: true})
}

func (o *OffsetMgr) Last(topic string, partition int32) (int64, error) {
	key := new(bytes.Buffer)
	key.WriteString(fmt.Sprintf("T:%s-P:%d", topic, partition))
	data, err := o.db.Get(key.Bytes(), nil)
	if err != nil {
		if err == leveldb.ErrNotFound {
			log.Debug("no offset recorded for %s:%d", topic, partition)
			return -1, nil
		}
		return 0, err
	}
	var offset int64
	err = binary.Read(bytes.NewBuffer(data), binary.LittleEndian, &offset)
	if err != nil {
		return 0, err
	}
	log.Debug("found saved offset %d for %s:%d", offset, topic, partition)
	return offset, nil
}
<|endoftext|>