<commit_before>package internal

import (
	"fmt"
	"go/types"
	"path/filepath"
	"reflect"
	"strings"

	"github.com/segmentio/glue/provider"
)

// IsExportedOrBuiltin returns true if a type is either exported or primitive.
//
// Note(tejasmanohar): If the type is a map, both the key and value must be exported
// since they're both necessary to represent the type.
func IsExportedOrBuiltin(_t types.Type) bool {
	for _, t := range Unpack(_t) {
		if _, isPrimitive := t.(*types.Basic); isPrimitive {
			return true
		}

		namedType, ok := t.(*types.Named)
		if !ok {
			return false
		}

		obj := namedType.Obj()
		if obj == nil {
			return false
		}

		if !obj.Exported() {
			return false
		}
	}

	return true
}

// Unpack unpacks the most basic, underlying types from pointers, slices, maps, etc.
// There can be multiple types in the case of maps (key and value).
// Note(tejasmanohar): Are there any other structures that have multiple types
// that we need to validate are exported? Structs don't have to have all fields exported
// as we can just not encode the ones that aren't.
func Unpack(t types.Type) []types.Type {
	ret := make([]types.Type, 0, 1)

	if ptr, ok := t.(*types.Pointer); ok {
		ret = append(ret, Unpack(ptr.Elem())...)
	}

	if slice, ok := t.(*types.Slice); ok {
		ret = append(ret, Unpack(slice.Elem())...)
	}

	if array, ok := t.(*types.Array); ok {
		ret = append(ret, Unpack(array.Elem())...)
	}

	return ret
}

func GetTypeInfo(t types.Type) provider.TypeInfo {
	var ret provider.TypeInfo

	switch specific := t.(type) {
	case *types.Pointer:
		ret = GetTypeInfo(specific.Elem())
	case *types.Named:
		obj := specific.Obj()
		ret = provider.TypeInfo{
			Identifier: fmt.Sprintf("%s.%s", obj.Pkg().Name(), obj.Name()),
			Imports:    []string{obj.Pkg().Path()},
		}
	case *types.Basic:
		ret = provider.TypeInfo{
			Identifier: specific.Name(),
		}
	case *types.Slice:
		underlying := GetTypeInfo(specific.Elem())
		ret = provider.TypeInfo{
			Identifier: fmt.Sprintf("[]%s", underlying.Identifier),
			Imports:    underlying.Imports,
		}
	case *types.Array:
		underlying := GetTypeInfo(specific.Elem())
		ret = provider.TypeInfo{
			Identifier: fmt.Sprintf("[]%s", underlying.Identifier),
			Imports:    underlying.Imports,
		}
	case *types.Map:
		key := GetTypeInfo(specific.Key())
		val := GetTypeInfo(specific.Elem())

		ret = provider.TypeInfo{
			Identifier: fmt.Sprintf("map[%s]%s", key.Identifier, val.Identifier),
			Imports:    append(key.Imports, val.Imports...),
		}
	case *types.Interface:
		if !specific.Empty() {
			// TODO(tejasmanohar): This shouldn't really be a "panic", as it's not a
			// developer error but a user one. Figure that out later.
			panic("non-empty interfaces are not cannot be JSON-encoded")
		}

		ret = provider.TypeInfo{Identifier: "interface{}"}
	default:
		panic(fmt.Errorf("unexpected type: %s", reflect.TypeOf(t)))
	}

	for i, pkg := range ret.Imports {
		ret.Imports[i] = stripVendor(pkg)
	}

	return ret
}

// github.com/x/y/vendor/github.com/a/b -> github.com/a/b
func stripVendor(path string) string {
	dirs := strings.Split(path, string(filepath.Separator))
	num := len(dirs)
	vendorIndex := -1
	for i := 1; i <= num; i++ {
		dir := dirs[num-i]
		if dir == "vendor" {
			vendorIndex = num - i
			break
		}
	}

	if vendorIndex == -1 {
		return path
	}

	return filepath.Join(dirs[vendorIndex+1:]...)
}
<commit_msg>fix array signature<commit_after>package internal

import (
	"fmt"
	"go/types"
	"path/filepath"
	"reflect"
	"strings"

	"github.com/segmentio/glue/provider"
)

// IsExportedOrBuiltin returns true if a type is either exported or primitive.
//
// Note(tejasmanohar): If the type is a map, both the key and value must be exported
// since they're both necessary to represent the type.
func IsExportedOrBuiltin(_t types.Type) bool {
	for _, t := range Unpack(_t) {
		if _, isPrimitive := t.(*types.Basic); isPrimitive {
			return true
		}

		namedType, ok := t.(*types.Named)
		if !ok {
			return false
		}

		obj := namedType.Obj()
		if obj == nil {
			return false
		}

		if !obj.Exported() {
			return false
		}
	}

	return true
}

// Unpack unpacks the most basic, underlying types from pointers, slices, maps, etc.
// There can be multiple types in the case of maps (key and value).
// Note(tejasmanohar): Are there any other structures that have multiple types
// that we need to validate are exported? Structs don't have to have all fields exported
// as we can just not encode the ones that aren't.
func Unpack(t types.Type) []types.Type {
	ret := make([]types.Type, 0, 1)

	if ptr, ok := t.(*types.Pointer); ok {
		ret = append(ret, Unpack(ptr.Elem())...)
	}

	if slice, ok := t.(*types.Slice); ok {
		ret = append(ret, Unpack(slice.Elem())...)
	}

	if array, ok := t.(*types.Array); ok {
		ret = append(ret, Unpack(array.Elem())...)
	}

	return ret
}

func GetTypeInfo(t types.Type) provider.TypeInfo {
	var ret provider.TypeInfo

	switch specific := t.(type) {
	case *types.Pointer:
		ret = GetTypeInfo(specific.Elem())
	case *types.Named:
		obj := specific.Obj()
		ret = provider.TypeInfo{
			Identifier: fmt.Sprintf("%s.%s", obj.Pkg().Name(), obj.Name()),
			Imports:    []string{obj.Pkg().Path()},
		}
	case *types.Basic:
		ret = provider.TypeInfo{
			Identifier: specific.Name(),
		}
	case *types.Slice:
		underlying := GetTypeInfo(specific.Elem())
		ret = provider.TypeInfo{
			Identifier: fmt.Sprintf("[]%s", underlying.Identifier),
			Imports:    underlying.Imports,
		}
	case *types.Array:
		underlying := GetTypeInfo(specific.Elem())
		ret = provider.TypeInfo{
			Identifier: fmt.Sprintf("[%d]%s", specific.Len(), underlying.Identifier),
			Imports:    underlying.Imports,
		}
	case *types.Map:
		key := GetTypeInfo(specific.Key())
		val := GetTypeInfo(specific.Elem())

		ret = provider.TypeInfo{
			Identifier: fmt.Sprintf("map[%s]%s", key.Identifier, val.Identifier),
			Imports:    append(key.Imports, val.Imports...),
		}
	case *types.Interface:
		if !specific.Empty() {
			// TODO(tejasmanohar): This shouldn't really be a "panic", as it's not a
			// developer error but a user one. Figure that out later.
			panic("non-empty interfaces are not cannot be JSON-encoded")
		}

		ret = provider.TypeInfo{Identifier: "interface{}"}
	default:
		panic(fmt.Errorf("unexpected type: %s", reflect.TypeOf(t)))
	}

	for i, pkg := range ret.Imports {
		ret.Imports[i] = stripVendor(pkg)
	}

	return ret
}

// github.com/x/y/vendor/github.com/a/b -> github.com/a/b
func stripVendor(path string) string {
	dirs := strings.Split(path, string(filepath.Separator))
	num := len(dirs)
	vendorIndex := -1
	for i := 1; i <= num; i++ {
		dir := dirs[num-i]
		if dir == "vendor" {
			vendorIndex = num - i
			break
		}
	}

	if vendorIndex == -1 {
		return path
	}

	return filepath.Join(dirs[vendorIndex+1:]...)
}
<|endoftext|>


<commit_before>package event

import (
	"fmt"
	"sort"
	"strings"
	"time"

	"encoding/json"
	"errors"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/update"
)

// These are all the types of events.
const (
	EventCommit       = "commit"
	EventSync         = "sync"
	EventRelease      = "release"
	EventAutoRelease  = "autorelease"
	EventAutomate     = "automate"
	EventDeautomate   = "deautomate"
	EventLock         = "lock"
	EventUnlock       = "unlock"
	EventUpdatePolicy = "update_policy"

	// This is used to label e.g., commits that we _don't_ consider an event in themselves.
	NoneOfTheAbove = "other"

	LogLevelDebug = "debug"
	LogLevelInfo  = "info"
	LogLevelWarn  = "warn"
	LogLevelError = "error"
)

type EventID int64

type Event struct {
	// ID is a UUID for this event. Will be auto-set when saving if blank.
	ID EventID `json:"id"`

	// ServiceIDs affected by this event.
	ServiceIDs []flux.ResourceID `json:"serviceIDs"`

	// Type is the type of event, usually "release" for now, but could be other
	// things later
	Type string `json:"type"`

	// StartedAt is the time the event began.
	StartedAt time.Time `json:"startedAt"`

	// EndedAt is the time the event ended. For instantaneous events, this will
	// be the same as StartedAt.
	EndedAt time.Time `json:"endedAt"`

	// LogLevel for this event. Used to indicate how important it is.
	// `debug|info|warn|error`
	LogLevel string `json:"logLevel"`

	// Message is a pre-formatted string for errors and other stuff. Included for
	// backwards-compatibility, and is now somewhat unnecessary. Should only be
	// used if metadata is empty.
	Message string `json:"message,omitempty"`

	// Metadata is Event.Type-specific metadata. If an event has no metadata,
	// this will be nil.
	Metadata EventMetadata `json:"metadata,omitempty"`
}

type EventWriter interface {
	// LogEvent records a message in the history.
	LogEvent(Event) error
}

func (e Event) ServiceIDStrings() []string {
	var strServiceIDs []string
	for _, serviceID := range e.ServiceIDs {
		strServiceIDs = append(strServiceIDs, serviceID.String())
	}
	sort.Strings(strServiceIDs)
	return strServiceIDs
}

func (e Event) String() string {
	if e.Message != "" {
		return e.Message
	}

	strServiceIDs := e.ServiceIDStrings()
	switch e.Type {
	case EventRelease:
		metadata := e.Metadata.(*ReleaseEventMetadata)
		strImageIDs := metadata.Result.ChangedImages()
		if len(strImageIDs) == 0 {
			strImageIDs = []string{"no image changes"}
		}
		if metadata.Spec.Type == "" || metadata.Spec.Type == ReleaseImageSpecType {
			for _, spec := range metadata.Spec.ReleaseImageSpec.ServiceSpecs {
				if spec == update.ResourceSpecAll {
					strServiceIDs = []string{"all services"}
					break
				}
			}
		}
		if len(strServiceIDs) == 0 {
			strServiceIDs = []string{"no services"}
		}
		var user string
		if metadata.Cause.User != "" {
			user = fmt.Sprintf(", by %s", metadata.Cause.User)
		}
		var msg string
		if metadata.Cause.Message != "" {
			msg = fmt.Sprintf(", with message %q", metadata.Cause.Message)
		}
		return fmt.Sprintf(
			"Released: %s to %s%s%s",
			strings.Join(strImageIDs, ", "),
			strings.Join(strServiceIDs, ", "),
			user,
			msg,
		)
	case EventAutoRelease:
		metadata := e.Metadata.(*AutoReleaseEventMetadata)
		strImageIDs := metadata.Result.ChangedImages()
		if len(strImageIDs) == 0 {
			strImageIDs = []string{"no image changes"}
		}
		return fmt.Sprintf(
			"Automated release of %s",
			strings.Join(strImageIDs, ", "),
		)
	case EventCommit:
		metadata := e.Metadata.(*CommitEventMetadata)
		svcStr := "<no changes>"
		if len(strServiceIDs) > 0 {
			svcStr = strings.Join(strServiceIDs, ", ")
		}
		return fmt.Sprintf("Commit: %s, %s", shortRevision(metadata.Revision), svcStr)
	case EventSync:
		metadata := e.Metadata.(*SyncEventMetadata)
		revStr := "<no revision>"
		if 0 < len(metadata.Commits) && len(metadata.Commits) <= 2 {
			revStr = shortRevision(metadata.Commits[0].Revision)
		} else if len(metadata.Commits) > 2 {
			revStr = fmt.Sprintf(
				"%s..%s",
				shortRevision(metadata.Commits[len(metadata.Commits)-1].Revision),
				shortRevision(metadata.Commits[0].Revision),
			)
		}
		svcStr := "no services changed"
		if len(strServiceIDs) > 0 {
			svcStr = strings.Join(strServiceIDs, ", ")
		}
		return fmt.Sprintf("Sync: %s, %s", revStr, svcStr)
	case EventAutomate:
		return fmt.Sprintf("Automated: %s", strings.Join(strServiceIDs, ", "))
	case EventDeautomate:
		return fmt.Sprintf("Deautomated: %s", strings.Join(strServiceIDs, ", "))
	case EventLock:
		return fmt.Sprintf("Locked: %s", strings.Join(strServiceIDs, ", "))
	case EventUnlock:
		return fmt.Sprintf("Unlocked: %s", strings.Join(strServiceIDs, ", "))
	case EventUpdatePolicy:
		return fmt.Sprintf("Updated policies: %s", strings.Join(strServiceIDs, ", "))
	default:
		return fmt.Sprintf("Unknown event: %s", e.Type)
	}
}

func shortRevision(rev string) string {
	if len(rev) <= 7 {
		return rev
	}
	return rev[:7]
}

// CommitEventMetadata is the metadata for when new git commits are created
type CommitEventMetadata struct {
	Revision string        `json:"revision,omitempty"`
	Spec     *update.Spec  `json:"spec"`
	Result   update.Result `json:"result,omitempty"`
}

func (c CommitEventMetadata) ShortRevision() string {
	return shortRevision(c.Revision)
}

// Commit represents the commit information in a sync event. We could
// use git.Commit, but that would lead to an import cycle, and may
// anyway represent coupling (of an internal API to serialised data)
// that we don't want.
type Commit struct {
	Revision string `json:"revision"`
	Message  string `json:"message"`
}

type ResourceError struct {
	ID    flux.ResourceID
	Path  string
	Error string
}

// SyncEventMetadata is the metadata for when new a commit is synced to the cluster
type SyncEventMetadata struct {
	// for parsing old events; Commits is now used in preference
	Revs    []string `json:"revisions,omitempty"`
	Commits []Commit `json:"commits,omitempty"`
	// Which "kinds" of commit this includes; release, autoreleases,
	// policy changes, and "other" (meaning things we didn't commit
	// ourselves)
	Includes map[string]bool `json:"includes,omitempty"`
	// Per-resource errors
	Errors []ResourceError `json:"errors,omitempty"`
	// `true` if we have no record of having synced before
	InitialSync bool `json:"initialSync,omitempty"`
}

// Account for old events, which used the revisions field rather than commits
func (ev *SyncEventMetadata) UnmarshalJSON(b []byte) error {
	type data SyncEventMetadata
	err := json.Unmarshal(b, (*data)(ev))
	if err != nil {
		return err
	}
	if ev.Commits == nil {
		ev.Commits = make([]Commit, len(ev.Revs))
		for i, rev := range ev.Revs {
			ev.Commits[i].Revision = rev
		}
	}
	return nil
}

type ReleaseEventCommon struct {
	Revision string        // the revision which has the changes for the release
	Result   update.Result `json:"result"`
	// Message of the error if there was one.
	Error string `json:"error,omitempty"`
}

const (
	// ReleaseImageSpecType is a type of release spec when there are update.Images
	ReleaseImageSpecType = "releaseImageSpecType"
	// ReleaseContainersSpecType is a type of release spec when there are update.Containers
	ReleaseContainersSpecType = "releaseContainersSpecType"
)

// ReleaseSpec is a spec for images and containers release
type ReleaseSpec struct {
	// Type is ReleaseImageSpecType or ReleaseContainersSpecType
	// if empty (for previous version), then use ReleaseImageSpecType
	Type                  string
	ReleaseImageSpec      *update.ReleaseImageSpec
	ReleaseContainersSpec *update.ReleaseContainersSpec
}

// UnmarshalJSON for old version of spec (update.ReleaseImageSpec) where Type is empty
func (s *ReleaseSpec) UnmarshalJSON(b []byte) error {
	type T ReleaseSpec
	t := (*T)(s)
	if err := json.Unmarshal(b, t); err != nil {
		return err
	}

	switch t.Type {
	case "":
		r := &update.ReleaseImageSpec{}
		if err := json.Unmarshal(b, r); err != nil {
			return err
		}
		s.Type = ReleaseImageSpecType
		s.ReleaseImageSpec = r

	case ReleaseImageSpecType, ReleaseContainersSpecType:
		// all good
	default:
		return errors.New("unknown ReleaseSpec type")
	}
	return nil
}

// ReleaseEventMetadata is the metadata for when service(s) are released
type ReleaseEventMetadata struct {
	ReleaseEventCommon
	Spec  ReleaseSpec  `json:"spec"`
	Cause update.Cause `json:"cause"`
}

// AutoReleaseEventMetadata is for when service(s) are released
// automatically because there's a new image or images
type AutoReleaseEventMetadata struct {
	ReleaseEventCommon
	Spec update.Automated `json:"spec"`
}

type UnknownEventMetadata map[string]interface{}

func (e *Event) UnmarshalJSON(in []byte) error {
	type alias Event
	var wireEvent struct {
		*alias
		MetadataBytes json.RawMessage `json:"metadata,omitempty"`
	}
	wireEvent.alias = (*alias)(e)

	// Now unmarshall custom wireEvent with RawMessage
	if err := json.Unmarshal(in, &wireEvent); err != nil {
		return err
	}
	if wireEvent.Type == "" {
		return errors.New("Event type is empty")
	}

	// The cases correspond to kinds of event that we care about
	// processing e.g., for notifications.
	switch wireEvent.Type {
	case EventRelease:
		var metadata ReleaseEventMetadata
		if err := json.Unmarshal(wireEvent.MetadataBytes, &metadata); err != nil {
			return err
		}
		e.Metadata = &metadata
		break
	case EventAutoRelease:
		var metadata AutoReleaseEventMetadata
		if err := json.Unmarshal(wireEvent.MetadataBytes, &metadata); err != nil {
			return err
		}
		e.Metadata = &metadata
		break
	case EventCommit:
		var metadata CommitEventMetadata
		if err := json.Unmarshal(wireEvent.MetadataBytes, &metadata); err != nil {
			return err
		}
		e.Metadata = &metadata
		break
	case EventSync:
		var metadata SyncEventMetadata
		if err := json.Unmarshal(wireEvent.MetadataBytes, &metadata); err != nil {
			return err
		}
		e.Metadata = &metadata
		break
	default:
		if len(wireEvent.MetadataBytes) > 0 {
			var metadata UnknownEventMetadata
			if err := json.Unmarshal(wireEvent.MetadataBytes, &metadata); err != nil {
				return err
			}
			e.Metadata = metadata
		}
	}

	// By default, leave the Event Metadata as map[string]interface{}
	return nil
}

// EventMetadata is a type safety trick used to make sure that Metadata field
// of Event is always a pointer, so that consumers can cast without being
// concerned about encountering a value type instead. It works by virtue of the
// fact that the method is only defined for pointer receivers; the actual
// method chosen is entirely arbitary.
type EventMetadata interface {
	Type() string
}

func (cem *CommitEventMetadata) Type() string {
	return EventCommit
}

func (cem *SyncEventMetadata) Type() string {
	return EventSync
}

func (rem *ReleaseEventMetadata) Type() string {
	return EventRelease
}

func (rem *AutoReleaseEventMetadata) Type() string {
	return EventAutoRelease
}

// Special exception from pointer receiver rule, as UnknownEventMetadata is a
// type alias for a map
func (uem UnknownEventMetadata) Type() string {
	return "unknown"
}
<commit_msg>event: Add IsKindExecute() method for ReleaseSpec<commit_after>package event

import (
	"fmt"
	"sort"
	"strings"
	"time"

	"encoding/json"

	"github.com/pkg/errors"
	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/update"
)

// These are all the types of events.
const (
	EventCommit       = "commit"
	EventSync         = "sync"
	EventRelease      = "release"
	EventAutoRelease  = "autorelease"
	EventAutomate     = "automate"
	EventDeautomate   = "deautomate"
	EventLock         = "lock"
	EventUnlock       = "unlock"
	EventUpdatePolicy = "update_policy"

	// This is used to label e.g., commits that we _don't_ consider an event in themselves.
	NoneOfTheAbove = "other"

	LogLevelDebug = "debug"
	LogLevelInfo  = "info"
	LogLevelWarn  = "warn"
	LogLevelError = "error"
)

type EventID int64

type Event struct {
	// ID is a UUID for this event. Will be auto-set when saving if blank.
	ID EventID `json:"id"`

	// ServiceIDs affected by this event.
	ServiceIDs []flux.ResourceID `json:"serviceIDs"`

	// Type is the type of event, usually "release" for now, but could be other
	// things later
	Type string `json:"type"`

	// StartedAt is the time the event began.
	StartedAt time.Time `json:"startedAt"`

	// EndedAt is the time the event ended. For instantaneous events, this will
	// be the same as StartedAt.
	EndedAt time.Time `json:"endedAt"`

	// LogLevel for this event. Used to indicate how important it is.
	// `debug|info|warn|error`
	LogLevel string `json:"logLevel"`

	// Message is a pre-formatted string for errors and other stuff. Included for
	// backwards-compatibility, and is now somewhat unnecessary. Should only be
	// used if metadata is empty.
	Message string `json:"message,omitempty"`

	// Metadata is Event.Type-specific metadata. If an event has no metadata,
	// this will be nil.
	Metadata EventMetadata `json:"metadata,omitempty"`
}

type EventWriter interface {
	// LogEvent records a message in the history.
	LogEvent(Event) error
}

func (e Event) ServiceIDStrings() []string {
	var strServiceIDs []string
	for _, serviceID := range e.ServiceIDs {
		strServiceIDs = append(strServiceIDs, serviceID.String())
	}
	sort.Strings(strServiceIDs)
	return strServiceIDs
}

func (e Event) String() string {
	if e.Message != "" {
		return e.Message
	}

	strServiceIDs := e.ServiceIDStrings()
	switch e.Type {
	case EventRelease:
		metadata := e.Metadata.(*ReleaseEventMetadata)
		strImageIDs := metadata.Result.ChangedImages()
		if len(strImageIDs) == 0 {
			strImageIDs = []string{"no image changes"}
		}
		if metadata.Spec.Type == "" || metadata.Spec.Type == ReleaseImageSpecType {
			for _, spec := range metadata.Spec.ReleaseImageSpec.ServiceSpecs {
				if spec == update.ResourceSpecAll {
					strServiceIDs = []string{"all services"}
					break
				}
			}
		}
		if len(strServiceIDs) == 0 {
			strServiceIDs = []string{"no services"}
		}
		var user string
		if metadata.Cause.User != "" {
			user = fmt.Sprintf(", by %s", metadata.Cause.User)
		}
		var msg string
		if metadata.Cause.Message != "" {
			msg = fmt.Sprintf(", with message %q", metadata.Cause.Message)
		}
		return fmt.Sprintf(
			"Released: %s to %s%s%s",
			strings.Join(strImageIDs, ", "),
			strings.Join(strServiceIDs, ", "),
			user,
			msg,
		)
	case EventAutoRelease:
		metadata := e.Metadata.(*AutoReleaseEventMetadata)
		strImageIDs := metadata.Result.ChangedImages()
		if len(strImageIDs) == 0 {
			strImageIDs = []string{"no image changes"}
		}
		return fmt.Sprintf(
			"Automated release of %s",
			strings.Join(strImageIDs, ", "),
		)
	case EventCommit:
		metadata := e.Metadata.(*CommitEventMetadata)
		svcStr := "<no changes>"
		if len(strServiceIDs) > 0 {
			svcStr = strings.Join(strServiceIDs, ", ")
		}
		return fmt.Sprintf("Commit: %s, %s", shortRevision(metadata.Revision), svcStr)
	case EventSync:
		metadata := e.Metadata.(*SyncEventMetadata)
		revStr := "<no revision>"
		if 0 < len(metadata.Commits) && len(metadata.Commits) <= 2 {
			revStr = shortRevision(metadata.Commits[0].Revision)
		} else if len(metadata.Commits) > 2 {
			revStr = fmt.Sprintf(
				"%s..%s",
				shortRevision(metadata.Commits[len(metadata.Commits)-1].Revision),
				shortRevision(metadata.Commits[0].Revision),
			)
		}
		svcStr := "no services changed"
		if len(strServiceIDs) > 0 {
			svcStr = strings.Join(strServiceIDs, ", ")
		}
		return fmt.Sprintf("Sync: %s, %s", revStr, svcStr)
	case EventAutomate:
		return fmt.Sprintf("Automated: %s", strings.Join(strServiceIDs, ", "))
	case EventDeautomate:
		return fmt.Sprintf("Deautomated: %s", strings.Join(strServiceIDs, ", "))
	case EventLock:
		return fmt.Sprintf("Locked: %s", strings.Join(strServiceIDs, ", "))
	case EventUnlock:
		return fmt.Sprintf("Unlocked: %s", strings.Join(strServiceIDs, ", "))
	case EventUpdatePolicy:
		return fmt.Sprintf("Updated policies: %s", strings.Join(strServiceIDs, ", "))
	default:
		return fmt.Sprintf("Unknown event: %s", e.Type)
	}
}

func shortRevision(rev string) string {
	if len(rev) <= 7 {
		return rev
	}
	return rev[:7]
}

// CommitEventMetadata is the metadata for when new git commits are created
type CommitEventMetadata struct {
	Revision string        `json:"revision,omitempty"`
	Spec     *update.Spec  `json:"spec"`
	Result   update.Result `json:"result,omitempty"`
}

func (c CommitEventMetadata) ShortRevision() string {
	return shortRevision(c.Revision)
}

// Commit represents the commit information in a sync event. We could
// use git.Commit, but that would lead to an import cycle, and may
// anyway represent coupling (of an internal API to serialised data)
// that we don't want.
type Commit struct {
	Revision string `json:"revision"`
	Message  string `json:"message"`
}

type ResourceError struct {
	ID    flux.ResourceID
	Path  string
	Error string
}

// SyncEventMetadata is the metadata for when new a commit is synced to the cluster
type SyncEventMetadata struct {
	// for parsing old events; Commits is now used in preference
	Revs    []string `json:"revisions,omitempty"`
	Commits []Commit `json:"commits,omitempty"`
	// Which "kinds" of commit this includes; release, autoreleases,
	// policy changes, and "other" (meaning things we didn't commit
	// ourselves)
	Includes map[string]bool `json:"includes,omitempty"`
	// Per-resource errors
	Errors []ResourceError `json:"errors,omitempty"`
	// `true` if we have no record of having synced before
	InitialSync bool `json:"initialSync,omitempty"`
}

// Account for old events, which used the revisions field rather than commits
func (ev *SyncEventMetadata) UnmarshalJSON(b []byte) error {
	type data SyncEventMetadata
	err := json.Unmarshal(b, (*data)(ev))
	if err != nil {
		return err
	}
	if ev.Commits == nil {
		ev.Commits = make([]Commit, len(ev.Revs))
		for i, rev := range ev.Revs {
			ev.Commits[i].Revision = rev
		}
	}
	return nil
}

type ReleaseEventCommon struct {
	Revision string        // the revision which has the changes for the release
	Result   update.Result `json:"result"`
	// Message of the error if there was one.
	Error string `json:"error,omitempty"`
}

const (
	// ReleaseImageSpecType is a type of release spec when there are update.Images
	ReleaseImageSpecType = "releaseImageSpecType"
	// ReleaseContainersSpecType is a type of release spec when there are update.Containers
	ReleaseContainersSpecType = "releaseContainersSpecType"
)

// ReleaseSpec is a spec for images and containers release
type ReleaseSpec struct {
	// Type is ReleaseImageSpecType or ReleaseContainersSpecType
	// if empty (for previous version), then use ReleaseImageSpecType
	Type                  string
	ReleaseImageSpec      *update.ReleaseImageSpec
	ReleaseContainersSpec *update.ReleaseContainersSpec
}

// IsKindExecute reports whether the release spec s has ReleaseImageSpec or ReleaseImageSpec with Kind execute
// or error if s has invalid Type
func (s ReleaseSpec) IsKindExecute() (bool, error) {
	switch s.Type {
	case ReleaseImageSpecType:
		if s.ReleaseImageSpec != nil && s.ReleaseImageSpec.Kind == update.ReleaseKindExecute {
			return true, nil
		}
	case ReleaseContainersSpecType:
		if s.ReleaseContainersSpec != nil && s.ReleaseImageSpec.Kind == update.ReleaseKindExecute {
			return true, nil
		}

	default:
		return false, errors.Errorf("unknown release spec type %s", s.Type)
	}
	return false, nil
}

// UnmarshalJSON for old version of spec (update.ReleaseImageSpec) where Type is empty
func (s *ReleaseSpec) UnmarshalJSON(b []byte) error {
	type T ReleaseSpec
	t := (*T)(s)
	if err := json.Unmarshal(b, t); err != nil {
		return err
	}

	switch t.Type {
	case "":
		r := &update.ReleaseImageSpec{}
		if err := json.Unmarshal(b, r); err != nil {
			return err
		}
		s.Type = ReleaseImageSpecType
		s.ReleaseImageSpec = r

	case ReleaseImageSpecType, ReleaseContainersSpecType:
		// all good
	default:
		return errors.New("unknown ReleaseSpec type")
	}
	return nil
}

// ReleaseEventMetadata is the metadata for when service(s) are released
type ReleaseEventMetadata struct {
	ReleaseEventCommon
	Spec  ReleaseSpec  `json:"spec"`
	Cause update.Cause `json:"cause"`
}

// AutoReleaseEventMetadata is for when service(s) are released
// automatically because there's a new image or images
type AutoReleaseEventMetadata struct {
	ReleaseEventCommon
	Spec update.Automated `json:"spec"`
}

type UnknownEventMetadata map[string]interface{}

func (e *Event) UnmarshalJSON(in []byte) error {
	type alias Event
	var wireEvent struct {
		*alias
		MetadataBytes json.RawMessage `json:"metadata,omitempty"`
	}
	wireEvent.alias = (*alias)(e)

	// Now unmarshall custom wireEvent with RawMessage
	if err := json.Unmarshal(in, &wireEvent); err != nil {
		return err
	}
	if wireEvent.Type == "" {
		return errors.New("Event type is empty")
	}

	// The cases correspond to kinds of event that we care about
	// processing e.g., for notifications.
	switch wireEvent.Type {
	case EventRelease:
		var metadata ReleaseEventMetadata
		if err := json.Unmarshal(wireEvent.MetadataBytes, &metadata); err != nil {
			return err
		}
		e.Metadata = &metadata
		break
	case EventAutoRelease:
		var metadata AutoReleaseEventMetadata
		if err := json.Unmarshal(wireEvent.MetadataBytes, &metadata); err != nil {
			return err
		}
		e.Metadata = &metadata
		break
	case EventCommit:
		var metadata CommitEventMetadata
		if err := json.Unmarshal(wireEvent.MetadataBytes, &metadata); err != nil {
			return err
		}
		e.Metadata = &metadata
		break
	case EventSync:
		var metadata SyncEventMetadata
		if err := json.Unmarshal(wireEvent.MetadataBytes, &metadata); err != nil {
			return err
		}
		e.Metadata = &metadata
		break
	default:
		if len(wireEvent.MetadataBytes) > 0 {
			var metadata UnknownEventMetadata
			if err := json.Unmarshal(wireEvent.MetadataBytes, &metadata); err != nil {
				return err
			}
			e.Metadata = metadata
		}
	}

	// By default, leave the Event Metadata as map[string]interface{}
	return nil
}

// EventMetadata is a type safety trick used to make sure that Metadata field
// of Event is always a pointer, so that consumers can cast without being
// concerned about encountering a value type instead. It works by virtue of the
// fact that the method is only defined for pointer receivers; the actual
// method chosen is entirely arbitary.
type EventMetadata interface {
	Type() string
}

func (cem *CommitEventMetadata) Type() string {
	return EventCommit
}

func (cem *SyncEventMetadata) Type() string {
	return EventSync
}

func (rem *ReleaseEventMetadata) Type() string {
	return EventRelease
}

func (rem *AutoReleaseEventMetadata) Type() string {
	return EventAutoRelease
}

// Special exception from pointer receiver rule, as UnknownEventMetadata is a
// type alias for a map
func (uem UnknownEventMetadata) Type() string {
	return "unknown"
}
<|endoftext|>


<commit_before>package qlib

import (
   "sync/atomic"
   "fmt"
   "io"
   "io/ioutil"
   "encoding/json"
   "net"
   "os"
   "sort"
   "strconv"
   "strings"
   "sync"
   "time"
)

const kLoginTimeout time.Duration =  5 * time.Second
const kQueueIdleMax time.Duration = 28 * time.Hour
const kStoreIdIncr = 1000

var sNode tNodes
var sStore tStore

func init() {
   sNode.list = make(tNodeMap)
}

type Link struct { // network client msg handler
   conn net.Conn // link to client
   connSet chan<- net.Conn // updates tQueue
   ack chan<- string // forwards client acks to queue thread
   uid string
}

func NewLink(iConn net.Conn) *Link {
   aL := &Link{conn:iConn}
   go runLink(aL)
   return aL
}

func runLink(o *Link) {
   o.conn.SetReadDeadline(time.Now().Add(kLoginTimeout))
   aBuf := make([]byte, 64)
   for {
      aLen, err := o.conn.Read(aBuf)
      if err != nil {
         //todo if recoverable continue
         if err.(net.Error).Timeout() {
            o.conn.Write([]byte("login timeout"))
         }
         if o.connSet != nil {
            o.connSet <- nil
         }
         o.conn.Close()
         break
      }
      aHeadEnd, err := strconv.ParseUint(string(aBuf[:4]), 16, 0)
      if err != nil { panic(err) }
      aHeadEnd += 4
      var aMsg tHeader
      err = json.Unmarshal(aBuf[4:aHeadEnd], &aMsg)
      if err != nil { panic(err) }
      var aData []byte
      if aLen > int(aHeadEnd) {
         aData = aBuf[aHeadEnd:]
      }
      o.HandleMsg(&aMsg, aData)
   }
}

type tHeader struct {
   Op uint8
   Uid string
   Id string
   NodeId, NewNode string
   Aliases string
   To string
   Type string
   Member string
   Alias string
   For []string
}

const (
   _ = iota
   eRegister // uid newnode aliases
   eAddNode  // uid nodeid newnode
   eLogin    // uid nodeid
   eListEdit // id to type member
   ePost     // id for
   ePing     // id alias
   eAck      // id type
)

func (o *Link) HandleMsg(iMsg *tHeader, iData []byte) {
   switch(iMsg.Op) {
   case eLogin:
      aQ := NewQueue(iMsg.Uid)
      o.conn.SetReadDeadline(time.Time{})
      o.uid = iMsg.Uid
      o.ack = aQ.ack
      o.connSet = aQ.connIn
      o.connSet <- o.conn
      fmt.Printf("%s link.handlemsg login user %s\n", o.uid, aQ.uid)
   case ePost:
      aId := sStore.MakeId()
      sStore.PutFile(aId, iData)
      aNd := GetNode(iMsg.To)
      aNd.dir.RLock()
      sStore.PutLink(aId, iMsg.To, aId)
      sStore.SyncDirs(iMsg.To)
      if aNd.queue != nil {
         aNd.queue.in <- aId
      }
      aNd.dir.RUnlock()
      sStore.RmFile(aId)
   case eAck:
      aTmr := time.NewTimer(2 * time.Second)
      select {
      case o.ack <- "bing":
         aTmr.Stop()
      case <-aTmr.C:
         fmt.Printf("%s link.handlemsg timed out waiting on ack\n", o.uid)
      }
   default:
      fmt.Printf("unknown msg type %s\n", iMsg.Type)
   }
}

type tNodes struct {
   list tNodeMap // nodes that have received msgs or loggedin
   create sync.RWMutex //todo Mutex when sync.map
}

type tNodeMap map[string]*tNode // indexed by node id

type tNode struct {
   dir sync.RWMutex // directory lock
   queue *tQueue // instantiated on login //todo free on idle
}

func GetNode(iUid string) *tNode {
   sNode.create.RLock() //todo drop for sync.map
   aNd := sNode.list[iUid]
   sNode.create.RUnlock()
   if aNd != nil {
      return aNd
   }
   sNode.create.Lock()
   aNd = sNode.list[iUid]
   if aNd == nil {
      fmt.Printf("%s getnode make node\n", iUid)
      aNd = new(tNode)
      sNode.list[iUid] = aNd
   }
   sNode.create.Unlock()
   return aNd
}

type tQueue struct {
   uid string
   connIn chan net.Conn // Link updates client ref here
   connOut chan net.Conn // runQueue receives client ref here
   ack chan string // forwards acks from client
   buf []string // elastic channel buffer
   in chan string // elastic channel input
   out chan string // elastic channel output
}

func NewQueue(iUid string) *tQueue {
   aNd := GetNode(iUid)
   if aNd.queue == nil {
      aNd.dir.Lock()
      if aNd.queue != nil { panic("attempt to recreate queue for node "+iUid) }
      fmt.Printf("%s newqueue make queue\n", iUid)
      aNd.queue = new(tQueue)
      aQ := aNd.queue
      aQ.uid = iUid
      aQ.connIn = make(chan net.Conn)
      aQ.connOut = make(chan net.Conn)
      aQ.ack = make(chan string, 10)
      aQ.in = make(chan string)
      aQ.out = make(chan string)
      var err error
      aQ.buf, err = sStore.GetDir(iUid)
      if err != nil { panic(err) }
      aNd.dir.Unlock()
      go runConnChan(aQ)
      go runElasticChan(aQ)
      go runQueue(aQ)
   }
   return aNd.queue
}

func runQueue(o *tQueue) {
   aId := <-o.out
   aConn := <-o.connOut
   for {
      aMsg, err := sStore.GetFile(o.uid, aId)
      if err != nil { panic(err) }
      _, err = aConn.Write(aMsg)
      if err == nil {
         aTimeout := time.NewTimer(5 * time.Second)
         select {
         case <-o.ack:
            aTimeout.Stop()
            sStore.RmLink(o.uid, aId)
            aId = <-o.out
            aConn = <-o.connOut
         case <-aTimeout.C:
            fmt.Printf("%s queue.runqueue timed out awaiting ack\n", o.uid)
         }
      } else if false { //todo transient
         time.Sleep(10 * time.Millisecond)
      } else {
         aConn = <-o.connOut
         fmt.Printf("%s runqueue resumed\n", o.uid)
      }
   }
}

func runConnChan(o *tQueue) {
   var aConn net.Conn
   var ok bool
   for {
      for aConn == nil {
         aConn, ok = <-o.connIn
         if !ok { goto closed }
      }
      select {
         case aConn, ok = <-o.connIn:
            if !ok { goto closed }
         case o.connOut <- aConn:
      }
   }
closed:
   close(o.connOut)
}

func runElasticChan(o *tQueue) {
   var aS string
   var ok bool
   for {
      // buf needs a value to let select multiplex consumer & producer
      if len(o.buf) == 0 {
         aS, ok = <-o.in
         if !ok { goto closed }
         o.buf = append(o.buf, aS)
      }

      select {
      case aS, ok = <-o.in:
         if !ok { goto closed }
         o.buf = append(o.buf, aS)
      case o.out <- o.buf[0]:
         o.buf = o.buf[1:]
      }
   }

closed:
   for _, aS = range o.buf {
      o.out <- aS
   }
   close(o.out)
}

type tStore struct { // queue and msg storage
   Root string // top-level directory
   temp string // msg files land here before hardlinks land in queue directories
   nextId uint64 // incrementing msg filename
   idStore chan uint64 // updates nextId on disk
}

func Init(iMain string) {
   o := &sStore
   o.Root = iMain + "/"
   o.temp = o.Root + "temp/"
   o.idStore = make(chan uint64, 1)

   err := os.MkdirAll(o.temp, 0700)
   if err != nil { panic(err) }

   var aWg sync.WaitGroup
   aWg.Add(1)
   go runIdStore(o, &aWg)
   aWg.Wait()
}

func runIdStore(o *tStore, iWg *sync.WaitGroup) {
   aBuf, err := ioutil.ReadFile(o.Root+"NEXTID")
   if err != nil {
      if !os.IsNotExist(err) { panic(err) }
      aBuf = make([]byte, 16)
   } else {
      o.nextId, err = strconv.ParseUint(string(aBuf), 16, 64)
      if err != nil { panic(err) }
   }
   o.idStore <- o.nextId

   aFd, err := os.OpenFile(o.Root+"NEXTID", os.O_WRONLY|os.O_CREATE, 0600)
   if err != nil { panic(err) }
   defer aFd.Close()

   for {
      aId := <-o.idStore + (2 * kStoreIdIncr)
      copy(aBuf, fmt.Sprintf("%016x", aId))

      _, err = aFd.Seek(0, 0)
      if err != nil { panic(err) }

      _, err = aFd.Write(aBuf)
      if err != nil { panic (err) }

      err = aFd.Sync()
      if err != nil { panic (err) }

      if iWg != nil {
         iWg.Done()
         iWg = nil
      }
   }
}

func (o *tStore) MakeId() string {
   aN := atomic.AddUint64(&o.nextId, 1)
   if aN % 1000 == 0 {
      o.idStore <- aN
   }
   return fmt.Sprintf("%016x", aN)
}

func (o *tStore) PutFile(iId string, iBuf []byte) error {
   aFd, err := os.OpenFile(o.temp+iId, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
   if err != nil { return err }
   defer aFd.Close()
   for aPos, aLen := 0,0; aPos < len(iBuf); aPos += aLen {
      aLen, err = aFd.Write(iBuf[aPos:])
      if err != nil && err != io.ErrShortWrite { return err }
   }
   err = aFd.Sync()
   return err
}

func (o *tStore) ZeroFile(iNode, iId string) error {
   aFd, err := os.OpenFile(o.nodeSub(iNode)+"/"+iId, os.O_WRONLY|os.O_TRUNC, 0600)
   if err != nil { return err }
   aFd.Close()
   return nil
}

func (o *tStore) PutLink(iSrc, iNode, iId string) error {
   aPath := o.nodeSub(iNode)
   err := os.MkdirAll(aPath, 0700)
   if err != nil { return err }
   err = os.Link(o.temp+iSrc, aPath+"/"+iId)
   return err
}

func (o *tStore) RmFile(iId string) error {
   return os.Remove(o.temp+iId)
}

func (o *tStore) RmLink(iNode, iId string) error {
   return os.Remove(o.nodeSub(iNode)+"/"+iId)
}

func (o *tStore) RmDir(iNode string) error {
   err := os.Remove(o.nodeSub(iNode))
   if os.IsNotExist(err) { return nil }
   return err
}

func (o *tStore) SyncDirs(iNode string) error {
   var aFd *os.File
   var err error
   fSync := func(aDir string) {
      aFd, err = os.Open(aDir)
      if err != nil { return }
      err = aFd.Sync()
      aFd.Close()
   }
   fSync(o.Root)
   if err != nil { return err }
   fSync(o.rootSub(iNode))
   if err != nil { return err }
   fSync(o.nodeSub(iNode))
   return err
}

func (o *tStore) GetFile(iNode, iId string) ([]byte, error) {
   return ioutil.ReadFile(o.nodeSub(iNode)+"/"+iId)
}

func (o *tStore) GetDir(iNode string) (ret []string, err error) {
   fmt.Printf("read dir %s\n", o.nodeSub(iNode))
   aFd, err := os.Open(o.nodeSub(iNode))
   if err != nil {
      if os.IsNotExist(err) { err = nil }
      return
   }
   ret, err = aFd.Readdirnames(0)
   sort.Slice(ret, func(i, j int) bool { return ret[i] < ret[j] })
   aFd.Close()
   return
}

func (o *tStore) rootSub(iNode string) string {
   return o.Root + strings.ToLower(iNode[:4])
}

func (o *tStore) nodeSub(iNode string) string {
   return o.rootSub(iNode) + "/" + strings.ToLower(iNode)
}

<commit_msg>qlib: prevent another Link from attaching to tQueue<commit_after>package qlib

import (
   "sync/atomic"
   "fmt"
   "io"
   "io/ioutil"
   "encoding/json"
   "net"
   "os"
   "sort"
   "strconv"
   "strings"
   "sync"
   "time"
)

const kLoginTimeout time.Duration =  5 * time.Second
const kQueueIdleMax time.Duration = 28 * time.Hour
const kStoreIdIncr = 1000

var sNode tNodes
var sStore tStore

func init() {
   sNode.list = make(tNodeMap)
}

type Link struct { // network client msg handler
   conn net.Conn // link to client
   queue *tQueue
   uid string
}

func NewLink(iConn net.Conn) *Link {
   aL := &Link{conn:iConn}
   go runLink(aL)
   return aL
}

func runLink(o *Link) {
   o.conn.SetReadDeadline(time.Now().Add(kLoginTimeout))
   aBuf := make([]byte, 64)
   for {
      aLen, err := o.conn.Read(aBuf)
      if err != nil {
         //todo if recoverable continue
         if err.(net.Error).Timeout() {
            o.conn.Write([]byte("login timeout"))
         }
         if o.queue != nil {
            o.queue.Unlink()
         }
         o.conn.Close()
         break
      }
      aHeadEnd, err := strconv.ParseUint(string(aBuf[:4]), 16, 0)
      if err != nil { panic(err) }
      aHeadEnd += 4
      var aMsg tHeader
      err = json.Unmarshal(aBuf[4:aHeadEnd], &aMsg)
      if err != nil { panic(err) }
      var aData []byte
      if aLen > int(aHeadEnd) {
         aData = aBuf[aHeadEnd:]
      }
      o.HandleMsg(&aMsg, aData)
   }
}

type tHeader struct {
   Op uint8
   Uid string
   Id string
   NodeId, NewNode string
   Aliases string
   To string
   Type string
   Member string
   Alias string
   For []string
}

const (
   _ = iota
   eRegister // uid newnode aliases
   eAddNode  // uid nodeid newnode
   eLogin    // uid nodeid
   eListEdit // id to type member
   ePost     // id for
   ePing     // id alias
   eAck      // id type
)

func (o *Link) HandleMsg(iMsg *tHeader, iData []byte) {
   switch(iMsg.Op) {
   case eLogin:
      //todo check credentials
      aQ := QueueLink(iMsg.Uid, o.conn)
      if aQ == nil {
         return
      }
      o.conn.SetReadDeadline(time.Time{})
      o.uid = iMsg.Uid
      o.queue = aQ
      fmt.Printf("%s link.handlemsg login user %s\n", o.uid, aQ.uid)
   case ePost:
      aId := sStore.MakeId()
      sStore.PutFile(aId, iData)
      aNd := GetNode(iMsg.To)
      aNd.dir.RLock()
      sStore.PutLink(aId, iMsg.To, aId)
      sStore.SyncDirs(iMsg.To)
      if aNd.queue != nil {
         aNd.queue.in <- aId
      }
      aNd.dir.RUnlock()
      sStore.RmFile(aId)
   case eAck:
      aTmr := time.NewTimer(2 * time.Second)
      select {
      case o.queue.ack <- "bing":
         aTmr.Stop()
      case <-aTmr.C:
         fmt.Printf("%s link.handlemsg timed out waiting on ack\n", o.uid)
      }
   default:
      fmt.Printf("unknown msg type %s\n", iMsg.Type)
   }
}

type tNodes struct {
   list tNodeMap // nodes that have received msgs or loggedin
   create sync.RWMutex //todo Mutex when sync.map
}

type tNodeMap map[string]*tNode // indexed by node id

type tNode struct {
   dir sync.RWMutex // directory lock
   queue *tQueue // instantiated on login //todo free on idle
}

func GetNode(iUid string) *tNode {
   sNode.create.RLock() //todo drop for sync.map
   aNd := sNode.list[iUid]
   sNode.create.RUnlock()
   if aNd != nil {
      return aNd
   }
   sNode.create.Lock()
   aNd = sNode.list[iUid]
   if aNd == nil {
      fmt.Printf("%s getnode make node\n", iUid)
      aNd = new(tNode)
      sNode.list[iUid] = aNd
   }
   sNode.create.Unlock()
   return aNd
}

type tQueue struct {
   uid string
   connIn chan net.Conn // Link updates client ref here
   connOut chan net.Conn // runQueue receives client ref here
   ack chan string // forwards acks from client
   buf []string // elastic channel buffer
   in chan string // elastic channel input
   out chan string // elastic channel output
   hasConn int32 // in use by Link
}

func QueueLink(iUid string, iConn net.Conn) *tQueue {
   aNd := GetNode(iUid)
   if aNd.queue == nil {
      aNd.dir.Lock()
      if aNd.queue != nil {
         aNd.dir.Unlock()
         fmt.Printf(iUid+" newqueue attempt to recreate queue\n")
      } else {
         aNd.queue = new(tQueue)
         aQ := aNd.queue
         aQ.uid = iUid
         aQ.connIn = make(chan net.Conn)
         aQ.connOut = make(chan net.Conn)
         aQ.ack = make(chan string, 10)
         aQ.in = make(chan string)
         aQ.out = make(chan string)
         var err error
         aQ.buf, err = sStore.GetDir(iUid)
         if err != nil { panic(err) }
         aNd.dir.Unlock()
         fmt.Printf(iUid+" newqueue create queue\n")
         go runConnChan(aQ)
         go runElasticChan(aQ)
         go runQueue(aQ)
      }
   }
   if atomic.CompareAndSwapInt32(&aNd.queue.hasConn, 0, 1) {
      aNd.queue.connIn <- iConn
      return aNd.queue
   }
   return nil
}

func (o *tQueue) Unlink() {
   o.connIn <- nil
   o.hasConn = 0
}

func runQueue(o *tQueue) {
   aId := <-o.out
   aConn := <-o.connOut
   for {
      aMsg, err := sStore.GetFile(o.uid, aId)
      if err != nil { panic(err) }
      _, err = aConn.Write(aMsg)
      if err == nil {
         aTimeout := time.NewTimer(5 * time.Second)
         select {
         case <-o.ack:
            aTimeout.Stop()
            sStore.RmLink(o.uid, aId)
            aId = <-o.out
            aConn = <-o.connOut
         case <-aTimeout.C:
            fmt.Printf("%s queue.runqueue timed out awaiting ack\n", o.uid)
         }
      } else if false { //todo transient
         time.Sleep(10 * time.Millisecond)
      } else {
         aConn = <-o.connOut
         fmt.Printf("%s runqueue resumed\n", o.uid)
      }
   }
}

func runConnChan(o *tQueue) {
   var aConn net.Conn
   var ok bool
   for {
      for aConn == nil {
         aConn, ok = <-o.connIn
         if !ok { goto closed }
      }
      select {
         case aConn, ok = <-o.connIn:
            if !ok { goto closed }
         case o.connOut <- aConn:
      }
   }
closed:
   close(o.connOut)
}

func runElasticChan(o *tQueue) {
   var aS string
   var ok bool
   for {
      // buf needs a value to let select multiplex consumer & producer
      if len(o.buf) == 0 {
         aS, ok = <-o.in
         if !ok { goto closed }
         o.buf = append(o.buf, aS)
      }

      select {
      case aS, ok = <-o.in:
         if !ok { goto closed }
         o.buf = append(o.buf, aS)
      case o.out <- o.buf[0]:
         o.buf = o.buf[1:]
      }
   }

closed:
   for _, aS = range o.buf {
      o.out <- aS
   }
   close(o.out)
}

type tStore struct { // queue and msg storage
   Root string // top-level directory
   temp string // msg files land here before hardlinks land in queue directories
   nextId uint64 // incrementing msg filename
   idStore chan uint64 // updates nextId on disk
}

func Init(iMain string) {
   o := &sStore
   o.Root = iMain + "/"
   o.temp = o.Root + "temp/"
   o.idStore = make(chan uint64, 1)

   err := os.MkdirAll(o.temp, 0700)
   if err != nil { panic(err) }

   var aWg sync.WaitGroup
   aWg.Add(1)
   go runIdStore(o, &aWg)
   aWg.Wait()
}

func runIdStore(o *tStore, iWg *sync.WaitGroup) {
   aBuf, err := ioutil.ReadFile(o.Root+"NEXTID")
   if err != nil {
      if !os.IsNotExist(err) { panic(err) }
      aBuf = make([]byte, 16)
   } else {
      o.nextId, err = strconv.ParseUint(string(aBuf), 16, 64)
      if err != nil { panic(err) }
   }
   o.idStore <- o.nextId

   aFd, err := os.OpenFile(o.Root+"NEXTID", os.O_WRONLY|os.O_CREATE, 0600)
   if err != nil { panic(err) }
   defer aFd.Close()

   for {
      aId := <-o.idStore + (2 * kStoreIdIncr)
      copy(aBuf, fmt.Sprintf("%016x", aId))

      _, err = aFd.Seek(0, 0)
      if err != nil { panic(err) }

      _, err = aFd.Write(aBuf)
      if err != nil { panic (err) }

      err = aFd.Sync()
      if err != nil { panic (err) }

      if iWg != nil {
         iWg.Done()
         iWg = nil
      }
   }
}

func (o *tStore) MakeId() string {
   aN := atomic.AddUint64(&o.nextId, 1)
   if aN % 1000 == 0 {
      o.idStore <- aN
   }
   return fmt.Sprintf("%016x", aN)
}

func (o *tStore) PutFile(iId string, iBuf []byte) error {
   aFd, err := os.OpenFile(o.temp+iId, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0600)
   if err != nil { return err }
   defer aFd.Close()
   for aPos, aLen := 0,0; aPos < len(iBuf); aPos += aLen {
      aLen, err = aFd.Write(iBuf[aPos:])
      if err != nil && err != io.ErrShortWrite { return err }
   }
   err = aFd.Sync()
   return err
}

func (o *tStore) ZeroFile(iNode, iId string) error {
   aFd, err := os.OpenFile(o.nodeSub(iNode)+"/"+iId, os.O_WRONLY|os.O_TRUNC, 0600)
   if err != nil { return err }
   aFd.Close()
   return nil
}

func (o *tStore) PutLink(iSrc, iNode, iId string) error {
   aPath := o.nodeSub(iNode)
   err := os.MkdirAll(aPath, 0700)
   if err != nil { return err }
   err = os.Link(o.temp+iSrc, aPath+"/"+iId)
   return err
}

func (o *tStore) RmFile(iId string) error {
   return os.Remove(o.temp+iId)
}

func (o *tStore) RmLink(iNode, iId string) error {
   return os.Remove(o.nodeSub(iNode)+"/"+iId)
}

func (o *tStore) RmDir(iNode string) error {
   err := os.Remove(o.nodeSub(iNode))
   if os.IsNotExist(err) { return nil }
   return err
}

func (o *tStore) SyncDirs(iNode string) error {
   var aFd *os.File
   var err error
   fSync := func(aDir string) {
      aFd, err = os.Open(aDir)
      if err != nil { return }
      err = aFd.Sync()
      aFd.Close()
   }
   fSync(o.Root)
   if err != nil { return err }
   fSync(o.rootSub(iNode))
   if err != nil { return err }
   fSync(o.nodeSub(iNode))
   return err
}

func (o *tStore) GetFile(iNode, iId string) ([]byte, error) {
   return ioutil.ReadFile(o.nodeSub(iNode)+"/"+iId)
}

func (o *tStore) GetDir(iNode string) (ret []string, err error) {
   fmt.Printf("read dir %s\n", o.nodeSub(iNode))
   aFd, err := os.Open(o.nodeSub(iNode))
   if err != nil {
      if os.IsNotExist(err) { err = nil }
      return
   }
   ret, err = aFd.Readdirnames(0)
   sort.Slice(ret, func(i, j int) bool { return ret[i] < ret[j] })
   aFd.Close()
   return
}

func (o *tStore) rootSub(iNode string) string {
   return o.Root + strings.ToLower(iNode[:4])
}

func (o *tStore) nodeSub(iNode string) string {
   return o.rootSub(iNode) + "/" + strings.ToLower(iNode)
}

<|endoftext|>


<commit_before>// Copyright 2021 Gravitational, Inc
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backend

import (
	"time"
)

const (
	// DefaultBufferCapacity is a default circular buffer size
	// used by backends to fan out events
	DefaultBufferCapacity = 1024
	// DefaultBacklogGracePeriod is the default amount of time
	// that the circular buffer will tolerate an event backlog
	// in one of its watchers.
	DefaultBacklogGracePeriod = time.Second * 30
	// DefaultPollStreamPeriod is a default event poll stream period
	DefaultPollStreamPeriod = time.Second
	// DefaultEventsTTL is a default events TTL period
	DefaultEventsTTL = 10 * time.Minute
	// DefaultRangeLimit is used to specify some very large limit when limit is not specified
	// explicitly to prevent OOM due to infinite loops or other issues along those lines.
	DefaultRangeLimit = 200_000
)
<commit_msg>Increase dynamo get limit (#16076)<commit_after>// Copyright 2021 Gravitational, Inc
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backend

import (
	"time"
)

const (
	// DefaultBufferCapacity is a default circular buffer size
	// used by backends to fan out events
	DefaultBufferCapacity = 1024
	// DefaultBacklogGracePeriod is the default amount of time
	// that the circular buffer will tolerate an event backlog
	// in one of its watchers.
	DefaultBacklogGracePeriod = time.Second * 30
	// DefaultPollStreamPeriod is a default event poll stream period
	DefaultPollStreamPeriod = time.Second
	// DefaultEventsTTL is a default events TTL period
	DefaultEventsTTL = 10 * time.Minute
	// DefaultRangeLimit is used to specify some very large limit when limit is not specified
	// explicitly to prevent OOM due to infinite loops or other issues along those lines.
	DefaultRangeLimit = 2_000_000
)
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
	"regexp"
	"text/template"
)

type Command func(*Changelog, []string)

type Release struct {
	Version    string
	Date       string
	Summary    string
	Added      []string
	Changed    []string
	Deprecated []string
	Removed    []string
	Fixed      []string
	Security   []string
}

type Changelog []Release

var REGEXP_FILENAME = regexp.MustCompile(`^(?i)change(-|_)?log(.yml|.yaml)?$`)
var DEFAULT_COMMAND = "release"
var COMMAND_MAPPING = map[string]Command{
	"release": release,
	"to":      to,
}
var ERROR_READING = 1
var ERROR_PARSING = 2
var ERROR_RELEASE = 3
var ERROR_TO = 4
var REGEXP_DATE = regexp.MustCompile(`^\d\d\d\d-\d\d-\d\d$`)
var REGEXP_VERSION = regexp.MustCompile(`^\d+(\.\d+)?(\.\d+)?$`)
var HTML_TEMPLATE = `<!DOCTYPE html>

<title>Change Log</title>
<meta charset="utf-8">
{{ range $stylesheet := .Stylesheets }}

{{ end }}

Change Log

{{ range $release := .Changelog }}

Release {{ .Version }} ({{ .Date }})

{{ .Summary }}

{{ if .Added }}

Added

{{ range $entry := .Added }}

{{ . }}

{{ end }}

{{ end }}
{{ if .Changed }}

Changed

{{ range $entry := .Changed }}

{{ . }}

{{ end }}

{{ end }}
{{ if .Deprecated }}

Deprecated

{{ range $entry := .Deprecated }}

{{ . }}

{{ end }}

{{ end }}
{{ if .Removed }}

Removed

{{ range $entry := .Removed }}

{{ . }}

{{ end }}

{{ end }}
{{ if .Fixed }}

Fixed

{{ range $entry := .Fixed }}

{{ . }}

{{ end }}

{{ end }}
{{ if .Security }}

Security

{{ range $entry := .Security }}

{{ . }}

{{ end }}

{{ end }}
{{ end }}

`

func Error(code int, message string) {
	fmt.Fprintln(os.Stderr, message)
	os.Exit(code)
}

func Errorf(code int, message string, args ...interface{}) {
	Error(code, fmt.Sprintf(message, args...))
}

func readChangelog() []byte {
	stat, _ := os.Stdin.Stat()
	if (stat.Mode() & os.ModeCharDevice) == 0 {
		// data is being piped to stdin
		source, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			Error(ERROR_READING, "Error reading changelog from stdin")
		}
		return source
	} else {
		// look for changelog in current directory
		files, err := ioutil.ReadDir(".")
		if err != nil {
			Error(ERROR_READING, "Could not list current directory")
		}
		for _, file := range files {
			if !file.IsDir() && REGEXP_FILENAME.MatchString(file.Name()) {
				source, err := ioutil.ReadFile(file.Name())
				if err != nil {
					Errorf(ERROR_READING, "Error reading changelog file '%s'\n", file.Name())
				}
				return source
			}
		}
		Error(ERROR_READING, "No changelog file found")
		return []byte{}
	}
}

func parseChangelog(source []byte) *Changelog {
	var changelog Changelog
	err := yaml.Unmarshal(source, &changelog)
	if err != nil {
		Errorf(ERROR_PARSING, "Error parsing changelog: %s\n", err.Error())
	}
	return &changelog
}

////////////////////////////////////////////////////////////////////////////////
//                                  COMMANDS                                  //
////////////////////////////////////////////////////////////////////////////////

func checkRelease(release Release) {
	if release.Version == "" {
		Error(ERROR_RELEASE, "Release version is empty")
	}
	if !REGEXP_VERSION.MatchString(release.Version) {
		Errorf(ERROR_RELEASE, "Release version '%s' is not a valid semantic version number", release.Version)
	}
	if release.Date == "" {
		Error(ERROR_RELEASE, "Release date is empty")
	}
	if !REGEXP_DATE.MatchString(release.Date) {
		Errorf(ERROR_RELEASE, "Release date '%s' is not valid ISO format", release.Date)
	}
	if release.Summary == "" {
		Error(ERROR_RELEASE, "Release summary is empty")
	}
}

func checkChangelog(changelog *Changelog) {
	if len(*changelog) == 0 {
		Error(ERROR_RELEASE, "Release is empy")
	}
	for _, release := range *changelog {
		checkRelease(release)
	}
}

func release(changelog *Changelog, args []string) {
	checkChangelog(changelog)
	if len(args) > 0 {
		if args[0] == "summary" {
			fmt.Println((*changelog)[0].Summary)
		} else if args[0] == "date" {
			fmt.Println((*changelog)[0].Date)
		} else if args[0] == "version" {
			fmt.Println((*changelog)[0].Version)
		} else {
			Errorf(ERROR_RELEASE, "Unknown release argument %s", args[0])
		}
	}
}

type HtmlTemplateData struct {
	Changelog   *Changelog
	Stylesheets []string
}

func toHtml(changelog *Changelog, args []string) {
	stylesheets := make([]string, 0)
	for _, file := range args {
		stylesheet, err := ioutil.ReadFile(file)
		if err != nil {
			Errorf(ERROR_TO, "Error loading stylesheet %s: %s", file, err.Error())
		}
		stylesheets = append(stylesheets, string(stylesheet))
	}
	data := HtmlTemplateData{
		Stylesheets: stylesheets,
		Changelog:   changelog,
	}
	t := template.Must(template.New("changelog").Parse(HTML_TEMPLATE))
	err := t.Execute(os.Stdout, data)
	if err != nil {
		Errorf(ERROR_TO, "Error processing template: %s", err)
	}
}

func to(changelog *Changelog, args []string) {
	checkChangelog(changelog)
	if len(args) < 1 {
		Error(ERROR_TO, "You must pass format to convert to")
	}
	format := args[0]
	if format != "html" {
		Errorf(ERROR_TO, "Unknown format %s", args[0])
	}
	toHtml(changelog, args[1:])
}

func main() {
	changelog := parseChangelog(readChangelog())
	var command string
	var args []string
	if len(os.Args) < 2 {
		command = DEFAULT_COMMAND
		args = []string(nil)
	} else {
		command = os.Args[1]
		args = os.Args[2:]
	}
	function := COMMAND_MAPPING[command]
	if function != nil {
		function(changelog, args)
	} else {
		fmt.Printf("Command %s unknown\n", command)
		os.Exit(3)
	}
}
<commit_msg>Added release date check<commit_after>package main

import (
	"fmt"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"os"
	"regexp"
	"text/template"
	"time"
)

type Command func(*Changelog, []string)

type Release struct {
	Version    string
	Date       string
	Summary    string
	Added      []string
	Changed    []string
	Deprecated []string
	Removed    []string
	Fixed      []string
	Security   []string
}

type Changelog []Release

var REGEXP_FILENAME = regexp.MustCompile(`^(?i)change(-|_)?log(.yml|.yaml)?$`)
var DEFAULT_COMMAND = "release"
var COMMAND_MAPPING = map[string]Command{
	"release": release,
	"to":      to,
}
var ERROR_READING = 1
var ERROR_PARSING = 2
var ERROR_RELEASE = 3
var ERROR_TO = 4
var REGEXP_DATE = regexp.MustCompile(`^\d\d\d\d-\d\d-\d\d$`)
var REGEXP_VERSION = regexp.MustCompile(`^\d+(\.\d+)?(\.\d+)?$`)
var HTML_TEMPLATE = `<!DOCTYPE html>

<title>Change Log</title>
<meta charset="utf-8">
{{ range $stylesheet := .Stylesheets }}

{{ end }}

Change Log

{{ range $release := .Changelog }}

Release {{ .Version }} ({{ .Date }})

{{ .Summary }}

{{ if .Added }}

Added

{{ range $entry := .Added }}

{{ . }}

{{ end }}

{{ end }}
{{ if .Changed }}

Changed

{{ range $entry := .Changed }}

{{ . }}

{{ end }}

{{ end }}
{{ if .Deprecated }}

Deprecated

{{ range $entry := .Deprecated }}

{{ . }}

{{ end }}

{{ end }}
{{ if .Removed }}

Removed

{{ range $entry := .Removed }}

{{ . }}

{{ end }}

{{ end }}
{{ if .Fixed }}

Fixed

{{ range $entry := .Fixed }}

{{ . }}

{{ end }}

{{ end }}
{{ if .Security }}

Security

{{ range $entry := .Security }}

{{ . }}

{{ end }}

{{ end }}
{{ end }}

`

func Error(code int, message string) {
	fmt.Fprintln(os.Stderr, message)
	os.Exit(code)
}

func Errorf(code int, message string, args ...interface{}) {
	Error(code, fmt.Sprintf(message, args...))
}

func readChangelog() []byte {
	stat, _ := os.Stdin.Stat()
	if (stat.Mode() & os.ModeCharDevice) == 0 {
		// data is being piped to stdin
		source, err := ioutil.ReadAll(os.Stdin)
		if err != nil {
			Error(ERROR_READING, "Error reading changelog from stdin")
		}
		return source
	} else {
		// look for changelog in current directory
		files, err := ioutil.ReadDir(".")
		if err != nil {
			Error(ERROR_READING, "Could not list current directory")
		}
		for _, file := range files {
			if !file.IsDir() && REGEXP_FILENAME.MatchString(file.Name()) {
				source, err := ioutil.ReadFile(file.Name())
				if err != nil {
					Errorf(ERROR_READING, "Error reading changelog file '%s'\n", file.Name())
				}
				return source
			}
		}
		Error(ERROR_READING, "No changelog file found")
		return []byte{}
	}
}

func parseChangelog(source []byte) *Changelog {
	var changelog Changelog
	err := yaml.Unmarshal(source, &changelog)
	if err != nil {
		Errorf(ERROR_PARSING, "Error parsing changelog: %s\n", err.Error())
	}
	return &changelog
}

////////////////////////////////////////////////////////////////////////////////
//                                  COMMANDS                                  //
////////////////////////////////////////////////////////////////////////////////

func checkRelease(release Release) {
	if release.Version == "" {
		Error(ERROR_RELEASE, "Release version is empty")
	}
	if !REGEXP_VERSION.MatchString(release.Version) {
		Errorf(ERROR_RELEASE, "Release version '%s' is not a valid semantic version number", release.Version)
	}
	if release.Date == "" {
		Error(ERROR_RELEASE, "Release date is empty")
	}
	if !REGEXP_DATE.MatchString(release.Date) {
		Errorf(ERROR_RELEASE, "Release date '%s' is not valid ISO format", release.Date)
	}
	if release.Summary == "" {
		Error(ERROR_RELEASE, "Release summary is empty")
	}
}

func checkChangelog(changelog *Changelog) {
	if len(*changelog) == 0 {
		Error(ERROR_RELEASE, "Release is empy")
	}
	for _, release := range *changelog {
		checkRelease(release)
	}
}

func release(changelog *Changelog, args []string) {
	checkChangelog(changelog)
	if len(args) > 0 {
		if args[0] == "summary" {
			fmt.Println((*changelog)[0].Summary)
		} else if args[0] == "date" {
			if len(args) > 1 {
				date := time.Now().Local().Format("2006-01-02")
				if date != (*changelog)[0].Date {
					Errorf(ERROR_RELEASE, "Release date %s is wrong (should be %s)", (*changelog)[0].Date, date)
				}
			} else {
				fmt.Println((*changelog)[0].Date)
			}
		} else if args[0] == "version" {
			fmt.Println((*changelog)[0].Version)
		} else {
			Errorf(ERROR_RELEASE, "Unknown release argument %s", args[0])
		}
	}
}

type HtmlTemplateData struct {
	Changelog   *Changelog
	Stylesheets []string
}

func toHtml(changelog *Changelog, args []string) {
	stylesheets := make([]string, 0)
	for _, file := range args {
		stylesheet, err := ioutil.ReadFile(file)
		if err != nil {
			Errorf(ERROR_TO, "Error loading stylesheet %s: %s", file, err.Error())
		}
		stylesheets = append(stylesheets, string(stylesheet))
	}
	data := HtmlTemplateData{
		Stylesheets: stylesheets,
		Changelog:   changelog,
	}
	t := template.Must(template.New("changelog").Parse(HTML_TEMPLATE))
	err := t.Execute(os.Stdout, data)
	if err != nil {
		Errorf(ERROR_TO, "Error processing template: %s", err)
	}
}

func to(changelog *Changelog, args []string) {
	checkChangelog(changelog)
	if len(args) < 1 {
		Error(ERROR_TO, "You must pass format to convert to")
	}
	format := args[0]
	if format != "html" {
		Errorf(ERROR_TO, "Unknown format %s", args[0])
	}
	toHtml(changelog, args[1:])
}

func main() {
	changelog := parseChangelog(readChangelog())
	var command string
	var args []string
	if len(os.Args) < 2 {
		command = DEFAULT_COMMAND
		args = []string(nil)
	} else {
		command = os.Args[1]
		args = os.Args[2:]
	}
	function := COMMAND_MAPPING[command]
	if function != nil {
		function(changelog, args)
	} else {
		fmt.Printf("Command %s unknown\n", command)
		os.Exit(3)
	}
}
<|endoftext|>


<commit_before>// The Harness for a Revel program.
//
// It has a couple responsibilities:
// 1. Parse the user program, generating a main.go file that registers
//    controller classes and starts the user's server.
// 2. Build and run the user program.  Show compile errors.
// 3. Monitor the user source and re-build / restart the program when necessary.
//
// Source files are generated in the app/tmp directory.

package harness

import (
	"fmt"
	"github.com/robfig/revel"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"strings"
)

var (
	watcher    *rev.Watcher
	doNotWatch = []string{"tmp", "views"}
)

// Harness reverse proxies requests to the application server.
// It builds / runs / rebuilds / restarts the server when code is changed.
type Harness struct {
	serverHost string
	port       int
	proxy      *httputil.ReverseProxy
}

func renderError(w http.ResponseWriter, r *http.Request, err error) {
	rev.RenderError(rev.NewRequest(r), rev.NewResponse(w), err)
}

// ServeHTTP handles all requests.
// It checks for changes to app, rebuilds if necessary, and forwards the request.
func (hp *Harness) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// Flush any change events and rebuild app if necessary.
	// Render an error page if the rebuild / restart failed.
	err := watcher.Notify()
	if err != nil {
		renderError(w, r, err)
		return
	}

	// Reverse proxy the request.
	// (Need special code for websockets, courtesy of bradfitz)
	if r.Header.Get("Upgrade") == "websocket" {
		proxyWebsocket(w, r, hp.serverHost)
	} else {
		hp.proxy.ServeHTTP(w, r)
	}
}

// Return a reverse proxy that forwards requests to the given port.
func NewHarness() *Harness {
	// Get a template loader to render errors.
	// Prefer the app's views/errors directory, and fall back to the stock error pages.
	rev.MainTemplateLoader = rev.NewTemplateLoader(
		rev.ViewsPath,
		rev.RevelTemplatePath)

	port := getFreePort()
	serverUrl, _ := url.ParseRequestURI(fmt.Sprintf("http://localhost:%d", port))
	harness := &Harness{
		port:       port,
		serverHost: serverUrl.String()[len("http://"):],
		proxy:      httputil.NewSingleHostReverseProxy(serverUrl),
	}
	return harness
}

func (h *Harness) Refresh() *rev.Error {
	return rebuild("", h.port)
}

func (h *Harness) WatchDir(info os.FileInfo) bool {
	return !rev.ContainsString(doNotWatch, info.Name())
}

func (h *Harness) WatchFile(filename string) bool {
	return strings.HasSuffix(filename, ".go")
}

func (h *Harness) Run() {
	// If the harness exits, be sure to kill the app server.
	defer func() {
		if cmd != nil {
			cmd.Process.Kill()
			cmd = nil
		}
	}()

	watcher = rev.NewWatcher()
	watcher.Listen(h, rev.AppPath)

	appAddr := getAppAddress()
	appPort := getAppPort()
	rev.INFO.Printf("Listening on %s:%d", appAddr, appPort)
	err := http.ListenAndServe(fmt.Sprintf("%s:%d", appAddr, appPort), h)
	if err != nil {
		rev.ERROR.Fatalln("Failed to start reverse proxy:", err)
	}
}
<commit_msg>Fix template loader panic serving go compile errors.<commit_after>// The Harness for a Revel program.
//
// It has a couple responsibilities:
// 1. Parse the user program, generating a main.go file that registers
//    controller classes and starts the user's server.
// 2. Build and run the user program.  Show compile errors.
// 3. Monitor the user source and re-build / restart the program when necessary.
//
// Source files are generated in the app/tmp directory.

package harness

import (
	"fmt"
	"github.com/robfig/revel"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"strings"
)

var (
	watcher    *rev.Watcher
	doNotWatch = []string{"tmp", "views"}
)

// Harness reverse proxies requests to the application server.
// It builds / runs / rebuilds / restarts the server when code is changed.
type Harness struct {
	serverHost string
	port       int
	proxy      *httputil.ReverseProxy
}

func renderError(w http.ResponseWriter, r *http.Request, err error) {
	rev.RenderError(rev.NewRequest(r), rev.NewResponse(w), err)
}

// ServeHTTP handles all requests.
// It checks for changes to app, rebuilds if necessary, and forwards the request.
func (hp *Harness) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// Flush any change events and rebuild app if necessary.
	// Render an error page if the rebuild / restart failed.
	err := watcher.Notify()
	if err != nil {
		renderError(w, r, err)
		return
	}

	// Reverse proxy the request.
	// (Need special code for websockets, courtesy of bradfitz)
	if r.Header.Get("Upgrade") == "websocket" {
		proxyWebsocket(w, r, hp.serverHost)
	} else {
		hp.proxy.ServeHTTP(w, r)
	}
}

// Return a reverse proxy that forwards requests to the given port.
func NewHarness() *Harness {
	// Get a template loader to render errors.
	// Prefer the app's views/errors directory, and fall back to the stock error pages.
	rev.MainTemplateLoader = rev.NewTemplateLoader(
		rev.ViewsPath,
		rev.RevelTemplatePath)
	rev.MainTemplateLoader.Refresh()

	port := getFreePort()
	serverUrl, _ := url.ParseRequestURI(fmt.Sprintf("http://localhost:%d", port))
	harness := &Harness{
		port:       port,
		serverHost: serverUrl.String()[len("http://"):],
		proxy:      httputil.NewSingleHostReverseProxy(serverUrl),
	}
	return harness
}

func (h *Harness) Refresh() *rev.Error {
	return rebuild("", h.port)
}

func (h *Harness) WatchDir(info os.FileInfo) bool {
	return !rev.ContainsString(doNotWatch, info.Name())
}

func (h *Harness) WatchFile(filename string) bool {
	return strings.HasSuffix(filename, ".go")
}

func (h *Harness) Run() {
	// If the harness exits, be sure to kill the app server.
	defer func() {
		if cmd != nil {
			cmd.Process.Kill()
			cmd = nil
		}
	}()

	watcher = rev.NewWatcher()
	watcher.Listen(h, rev.AppPath)

	appAddr := getAppAddress()
	appPort := getAppPort()
	rev.INFO.Printf("Listening on %s:%d", appAddr, appPort)
	err := http.ListenAndServe(fmt.Sprintf("%s:%d", appAddr, appPort), h)
	if err != nil {
		rev.ERROR.Fatalln("Failed to start reverse proxy:", err)
	}
}
<|endoftext|>


<commit_before>package diff

import (
	"sort"

	. "github.com/zrepl/zrepl/replication/logic/pdu"
)

type ConflictNoCommonAncestor struct {
	SortedSenderVersions, SortedReceiverVersions []*FilesystemVersion
}

func (c *ConflictNoCommonAncestor) Error() string {
	return "no common snapshot or suitable bookmark between sender and receiver"
}

type ConflictDiverged struct {
	SortedSenderVersions, SortedReceiverVersions []*FilesystemVersion
	CommonAncestor                               *FilesystemVersion
	SenderOnly, ReceiverOnly                     []*FilesystemVersion
}

func (c *ConflictDiverged) Error() string {
	return "the receiver's latest snapshot is not present on sender"
}

func SortVersionListByCreateTXGThenBookmarkLTSnapshot(fsvslice []*FilesystemVersion) []*FilesystemVersion {
	lesser := func(s []*FilesystemVersion) func(i, j int) bool {
		return func(i, j int) bool {
			if s[i].CreateTXG < s[j].CreateTXG {
				return true
			}
			if s[i].CreateTXG == s[j].CreateTXG {
				//  Bookmark < Snapshot
				return s[i].Type == FilesystemVersion_Bookmark && s[j].Type == FilesystemVersion_Snapshot
			}
			return false
		}
	}
	if sort.SliceIsSorted(fsvslice, lesser(fsvslice)) {
		return fsvslice
	}
	sorted := make([]*FilesystemVersion, len(fsvslice))
	copy(sorted, fsvslice)
	sort.Slice(sorted, lesser(sorted))
	return sorted
}

// conflict may be a *ConflictDiverged or a *ConflictNoCommonAncestor
func IncrementalPath(receiver, sender []*FilesystemVersion) (incPath []*FilesystemVersion, conflict error) {

	receiver = SortVersionListByCreateTXGThenBookmarkLTSnapshot(receiver)
	sender = SortVersionListByCreateTXGThenBookmarkLTSnapshot(sender)

	// Find most recent common ancestor by name, preferring snapshots over bookmarks

	mrcaRcv := len(receiver) - 1
	mrcaSnd := len(sender) - 1

	for mrcaRcv >= 0 && mrcaSnd >= 0 {
		if receiver[mrcaRcv].Guid == sender[mrcaSnd].Guid {
			// Since we arrive from the end of the array, and because we defined bookmark < snapshot,
			// this condition will match snapshot first, which is what we want because it gives us
			// size estimation
			break
		}
		receiverCreation, err := receiver[mrcaRcv].CreationAsTime()
		if err != nil {
			panic(err) // FIXME move this to a sorting phase before
		}
		senderCreation, err := sender[mrcaSnd].CreationAsTime()
		if err != nil {
			panic(err) // FIXME move this to the sorting phase before
		}

		if receiverCreation.Before(senderCreation) {
			mrcaSnd--
		} else {
			mrcaRcv--
		}
	}

	if mrcaRcv == -1 || mrcaSnd == -1 {
		return nil, &ConflictNoCommonAncestor{
			SortedSenderVersions:   sender,
			SortedReceiverVersions: receiver,
		}
	}

	if mrcaRcv != len(receiver)-1 {
		return nil, &ConflictDiverged{
			SortedSenderVersions:   sender,
			SortedReceiverVersions: receiver,
			CommonAncestor:         sender[mrcaSnd],
			SenderOnly:             sender[mrcaSnd+1:],
			ReceiverOnly:           receiver[mrcaRcv+1:],
		}
	}

	// incPath must not contain bookmarks except initial one,
	incPath = make([]*FilesystemVersion, 0, len(sender))
	incPath = append(incPath, sender[mrcaSnd])
	// it's ok if incPath[0] is a bookmark, but not the subsequent ones in the incPath
	for i := mrcaSnd + 1; i < len(sender); i++ {
		if sender[i].Type == FilesystemVersion_Snapshot && incPath[len(incPath)-1].Guid != sender[i].Guid {
			incPath = append(incPath, sender[i])
		}
	}
	if len(incPath) == 1 {
		// nothing to do
		incPath = incPath[1:]
	}
	return incPath, nil
}
<commit_msg>replication: more elaborate messages for Conflict errors<commit_after>package diff

import (
	"fmt"
	"sort"
	"strings"

	. "github.com/zrepl/zrepl/replication/logic/pdu"
)

type ConflictNoCommonAncestor struct {
	SortedSenderVersions, SortedReceiverVersions []*FilesystemVersion
}

func (c *ConflictNoCommonAncestor) Error() string {
	var buf strings.Builder
	buf.WriteString("no common snapshot or suitable bookmark between sender and receiver")
	if len(c.SortedReceiverVersions) > 0 || len(c.SortedSenderVersions) > 0 {
		buf.WriteString(":\n  sorted sender versions:\n")
		for _, v := range c.SortedSenderVersions {
			fmt.Fprintf(&buf, "    %s\n", v.RelName())
		}
		buf.WriteString("  sorted receiver versions:\n")
		for _, v := range c.SortedReceiverVersions {
			fmt.Fprintf(&buf, "    %s\n", v.RelName())
		}
	}
	return buf.String()
}

type ConflictDiverged struct {
	SortedSenderVersions, SortedReceiverVersions []*FilesystemVersion
	CommonAncestor                               *FilesystemVersion
	SenderOnly, ReceiverOnly                     []*FilesystemVersion
}

func (c *ConflictDiverged) Error() string {
	var buf strings.Builder
	buf.WriteString("the receiver's latest snapshot is not present on sender:\n")
	fmt.Fprintf(&buf, "  last common: %s\n", c.CommonAncestor.RelName())
	fmt.Fprintf(&buf, "  sender-only:\n")
	for _, v := range c.SenderOnly {
		fmt.Fprintf(&buf, "    %s\n", v.RelName())
	}
	fmt.Fprintf(&buf, "  receiver-only:\n")
	for _, v := range c.ReceiverOnly {
		fmt.Fprintf(&buf, "    %s\n", v.RelName())
	}
	return buf.String()
}

func SortVersionListByCreateTXGThenBookmarkLTSnapshot(fsvslice []*FilesystemVersion) []*FilesystemVersion {
	lesser := func(s []*FilesystemVersion) func(i, j int) bool {
		return func(i, j int) bool {
			if s[i].CreateTXG < s[j].CreateTXG {
				return true
			}
			if s[i].CreateTXG == s[j].CreateTXG {
				//  Bookmark < Snapshot
				return s[i].Type == FilesystemVersion_Bookmark && s[j].Type == FilesystemVersion_Snapshot
			}
			return false
		}
	}
	if sort.SliceIsSorted(fsvslice, lesser(fsvslice)) {
		return fsvslice
	}
	sorted := make([]*FilesystemVersion, len(fsvslice))
	copy(sorted, fsvslice)
	sort.Slice(sorted, lesser(sorted))
	return sorted
}

// conflict may be a *ConflictDiverged or a *ConflictNoCommonAncestor
func IncrementalPath(receiver, sender []*FilesystemVersion) (incPath []*FilesystemVersion, conflict error) {

	receiver = SortVersionListByCreateTXGThenBookmarkLTSnapshot(receiver)
	sender = SortVersionListByCreateTXGThenBookmarkLTSnapshot(sender)

	// Find most recent common ancestor by name, preferring snapshots over bookmarks

	mrcaRcv := len(receiver) - 1
	mrcaSnd := len(sender) - 1

	for mrcaRcv >= 0 && mrcaSnd >= 0 {
		if receiver[mrcaRcv].Guid == sender[mrcaSnd].Guid {
			// Since we arrive from the end of the array, and because we defined bookmark < snapshot,
			// this condition will match snapshot first, which is what we want because it gives us
			// size estimation
			break
		}
		receiverCreation, err := receiver[mrcaRcv].CreationAsTime()
		if err != nil {
			panic(err) // FIXME move this to a sorting phase before
		}
		senderCreation, err := sender[mrcaSnd].CreationAsTime()
		if err != nil {
			panic(err) // FIXME move this to the sorting phase before
		}

		if receiverCreation.Before(senderCreation) {
			mrcaSnd--
		} else {
			mrcaRcv--
		}
	}

	if mrcaRcv == -1 || mrcaSnd == -1 {
		return nil, &ConflictNoCommonAncestor{
			SortedSenderVersions:   sender,
			SortedReceiverVersions: receiver,
		}
	}

	if mrcaRcv != len(receiver)-1 {
		return nil, &ConflictDiverged{
			SortedSenderVersions:   sender,
			SortedReceiverVersions: receiver,
			CommonAncestor:         sender[mrcaSnd],
			SenderOnly:             sender[mrcaSnd+1:],
			ReceiverOnly:           receiver[mrcaRcv+1:],
		}
	}

	// incPath must not contain bookmarks except initial one,
	incPath = make([]*FilesystemVersion, 0, len(sender))
	incPath = append(incPath, sender[mrcaSnd])
	// it's ok if incPath[0] is a bookmark, but not the subsequent ones in the incPath
	for i := mrcaSnd + 1; i < len(sender); i++ {
		if sender[i].Type == FilesystemVersion_Snapshot && incPath[len(incPath)-1].Guid != sender[i].Guid {
			incPath = append(incPath, sender[i])
		}
	}
	if len(incPath) == 1 {
		// nothing to do
		incPath = incPath[1:]
	}
	return incPath, nil
}
<|endoftext|>


<commit_before>// +build linux

package raw

import (
	"net"
	"os"
	"sync/atomic"
	"syscall"
	"time"
	"unsafe"

	"golang.org/x/net/bpf"
	"golang.org/x/sys/unix"
)

// Must implement net.PacketConn at compile-time.
var _ net.PacketConn = &packetConn{}

// packetConn is the Linux-specific implementation of net.PacketConn for this
// package.
type packetConn struct {
	ifi *net.Interface
	s   socket
	pbe uint16

	// Should stats be accumulated instead of reset on each call?
	noCumulativeStats bool

	// Internal storage for cumulative stats.
	stats Stats
}

// socket is an interface which enables swapping out socket syscalls for
// testing.
type socket interface {
	Bind(unix.Sockaddr) error
	Close() error
	GetSockoptTpacketStats(level, name int) (*unix.TpacketStats, error)
	Recvfrom([]byte, int) (int, unix.Sockaddr, error)
	Sendto([]byte, int, unix.Sockaddr) error
	SetSockoptPacketMreq(level, name int, mreq *unix.PacketMreq) error
	SetSockoptSockFprog(level, name int, fprog *unix.SockFprog) error
	SetDeadline(time.Time) error
	SetReadDeadline(time.Time) error
	SetWriteDeadline(time.Time) error
}

// htons converts a short (uint16) from host-to-network byte order.
// Thanks to mikioh for this neat trick:
// https://github.com/mikioh/-stdyng/blob/master/afpacket.go
func htons(i uint16) uint16 {
	return (i<<8)&0xff00 | i>>8
}

// listenPacket creates a net.PacketConn which can be used to send and receive
// data at the device driver level.
func listenPacket(ifi *net.Interface, proto uint16, cfg Config) (*packetConn, error) {
	filename := "eth-packet-socket"
	// Enabling overriding the socket type via config.
	typ := unix.SOCK_RAW
	if cfg.LinuxSockDGRAM {
		filename = "packet-socket"
		typ = unix.SOCK_DGRAM
	}

	// Open a packet socket using specified socket type. Do not specify
	// a protocol to avoid capturing packets which to not match cfg.Filter.
	// The later call to bind() will set up the correct protocol for us.
	sock, err := unix.Socket(unix.AF_PACKET, typ, 0)
	if err != nil {
		return nil, err
	}

	if err := unix.SetNonblock(sock, true); err != nil {
		unix.Close(sock)
		return nil, err
	}

	// When using Go 1.12+, the SetNonblock call we just did puts the file
	// descriptor into non-blocking mode. In that case, os.NewFile
	// registers the file descriptor with the runtime poller, which is then
	// used for all subsequent operations.
	//
	// See also: https://golang.org/pkg/os/#NewFile
	f := os.NewFile(uintptr(sock), filename)
	sc, err := f.SyscallConn()
	if err != nil {
		return nil, err
	}

	// Wrap raw socket in socket interface.
	pc, err := newPacketConn(ifi, &sysSocket{f: f, rc: sc}, htons(proto), cfg.Filter)
	if err != nil {
		return nil, err
	}

	pc.noCumulativeStats = cfg.NoCumulativeStats
	return pc, nil
}

// newPacketConn creates a net.PacketConn using the specified network
// interface, wrapped socket and big endian protocol number.
//
// It is the entry point for tests in this package.
func newPacketConn(ifi *net.Interface, s socket, pbe uint16, filter []bpf.RawInstruction) (*packetConn, error) {
	pc := &packetConn{
		ifi: ifi,
		s:   s,
		pbe: pbe,
	}

	if len(filter) > 0 {
		if err := pc.SetBPF(filter); err != nil {
			return nil, err
		}
	}

	// Bind the packet socket to the interface specified by ifi
	// packet(7):
	//   Only the sll_protocol and the sll_ifindex address fields are used for
	//   purposes of binding.
	// This overrides the protocol given to socket(AF_PACKET).
	err := s.Bind(&unix.SockaddrLinklayer{
		Protocol: pc.pbe,
		Ifindex:  ifi.Index,
	})
	if err != nil {
		return nil, err
	}

	return pc, nil
}

// ReadFrom implements the net.PacketConn.ReadFrom method.
func (p *packetConn) ReadFrom(b []byte) (int, net.Addr, error) {
	// Attempt to receive on socket
	n, addr, err := p.s.Recvfrom(b, 0)
	if err != nil {
		return n, nil, err
	}

	// Retrieve hardware address and other information from addr.
	sa, ok := addr.(*unix.SockaddrLinklayer)
	if !ok {
		return n, nil, unix.EINVAL
	}

	// Use length specified to convert byte array into a hardware address slice.
	mac := make(net.HardwareAddr, sa.Halen)
	copy(mac, sa.Addr[:])

	// packet(7):
	//   sll_hatype and sll_pkttype are set on received packets for your
	//   information.
	// TODO(mdlayher): determine if similar fields exist and are useful on
	// non-Linux platforms
	return n, &Addr{
		HardwareAddr: mac,
	}, nil
}

// WriteTo implements the net.PacketConn.WriteTo method.
func (p *packetConn) WriteTo(b []byte, addr net.Addr) (int, error) {
	// Ensure correct Addr type.
	a, ok := addr.(*Addr)
	if !ok || a.HardwareAddr == nil {
		return 0, unix.EINVAL
	}

	// Convert hardware address back to byte array form.
	var baddr [8]byte
	copy(baddr[:], a.HardwareAddr)

	// Send message on socket to the specified hardware address from addr
	// packet(7):
	//   When you send packets it is enough to specify sll_family, sll_addr,
	//   sll_halen, sll_ifindex, and sll_protocol. The other fields should
	//   be 0.
	// In this case, sll_family is taken care of automatically by unix.
	err := p.s.Sendto(b, 0, &unix.SockaddrLinklayer{
		Ifindex:  p.ifi.Index,
		Halen:    uint8(len(a.HardwareAddr)),
		Addr:     baddr,
		Protocol: p.pbe,
	})
	return len(b), err
}

// Close closes the connection.
func (p *packetConn) Close() error {
	return p.s.Close()
}

// LocalAddr returns the local network address.
func (p *packetConn) LocalAddr() net.Addr {
	return &Addr{
		HardwareAddr: p.ifi.HardwareAddr,
	}
}

// SetDeadline implements the net.PacketConn.SetDeadline method.
func (p *packetConn) SetDeadline(t time.Time) error {
	return p.s.SetDeadline(t)
}

// SetReadDeadline implements the net.PacketConn.SetReadDeadline method.
func (p *packetConn) SetReadDeadline(t time.Time) error {
	return p.s.SetReadDeadline(t)
}

// SetWriteDeadline implements the net.PacketConn.SetWriteDeadline method.
func (p *packetConn) SetWriteDeadline(t time.Time) error {
	return p.s.SetWriteDeadline(t)
}

// SetBPF attaches an assembled BPF program to a raw net.PacketConn.
func (p *packetConn) SetBPF(filter []bpf.RawInstruction) error {
	prog := unix.SockFprog{
		Len:    uint16(len(filter)),
		Filter: (*unix.SockFilter)(unsafe.Pointer(&filter[0])),
	}

	err := p.s.SetSockoptSockFprog(
		unix.SOL_SOCKET,
		unix.SO_ATTACH_FILTER,
		&prog,
	)
	if err != nil {
		return os.NewSyscallError("setsockopt", err)
	}
	return nil
}

// SetPromiscuous enables or disables promiscuous mode on the interface, allowing it
// to receive traffic that is not addressed to the interface.
func (p *packetConn) SetPromiscuous(b bool) error {
	mreq := unix.PacketMreq{
		Ifindex: int32(p.ifi.Index),
		Type:    unix.PACKET_MR_PROMISC,
	}

	membership := unix.PACKET_ADD_MEMBERSHIP
	if !b {
		membership = unix.PACKET_DROP_MEMBERSHIP
	}

	return p.s.SetSockoptPacketMreq(unix.SOL_PACKET, membership, &mreq)
}

// Stats retrieves statistics from the Conn.
func (p *packetConn) Stats() (*Stats, error) {
	stats, err := p.s.GetSockoptTpacketStats(unix.SOL_PACKET, unix.PACKET_STATISTICS)
	if err != nil {
		return nil, err
	}

	return p.handleStats(stats), nil
}

// handleStats handles creation of Stats structures from raw packet socket stats.
func (p *packetConn) handleStats(s *unix.TpacketStats) *Stats {
	// Does the caller want instantaneous stats as provided by Linux?  If so,
	// return the structure directly.
	if p.noCumulativeStats {
		return &Stats{
			Packets: uint64(s.Packets),
			Drops:   uint64(s.Drops),
		}
	}

	// The caller wants cumulative stats.  Add stats with the internal stats
	// structure and return a copy of the resulting stats.
	packets := atomic.AddUint64(&p.stats.Packets, uint64(s.Packets))
	drops := atomic.AddUint64(&p.stats.Drops, uint64(s.Drops))

	return &Stats{
		Packets: packets,
		Drops:   drops,
	}
}

// sysSocket is the default socket implementation.  It makes use of
// Linux-specific system calls to handle raw socket functionality.
type sysSocket struct {
	f  *os.File
	rc syscall.RawConn
}

func (s *sysSocket) SetDeadline(t time.Time) error {
	return s.f.SetDeadline(t)
}

func (s *sysSocket) SetReadDeadline(t time.Time) error {
	return s.f.SetReadDeadline(t)
}

func (s *sysSocket) SetWriteDeadline(t time.Time) error {
	return s.f.SetWriteDeadline(t)
}

func (s *sysSocket) Bind(sa unix.Sockaddr) error {
	var err error
	cerr := s.rc.Control(func(fd uintptr) {
		err = unix.Bind(int(fd), sa)
	})
	if err != nil {
		return err
	}
	return cerr
}

func (s *sysSocket) Close() error {
	return s.f.Close()
}

func (s *sysSocket) GetSockoptTpacketStats(level, name int) (*unix.TpacketStats, error) {
	var stats *unix.TpacketStats
	var err error
	cerr := s.rc.Control(func(fd uintptr) {
		s, errno := unix.GetsockoptTpacketStats(int(fd), level, name)
		stats = s
		if errno != nil {
			err = os.NewSyscallError("getsockopt", errno)
		}
	})
	if err != nil {
		return stats, err
	}
	return stats, cerr
}

func (s *sysSocket) Recvfrom(p []byte, flags int) (n int, addr unix.Sockaddr, err error) {
	cerr := s.rc.Read(func(fd uintptr) bool {
		n, addr, err = unix.Recvfrom(int(fd), p, flags)
		// When the socket is in non-blocking mode, we might see EAGAIN
		// and end up here. In that case, return false to let the
		// poller wait for readiness. See the source code for
		// internal/poll.FD.RawRead for more details.
		//
		// If the socket is in blocking mode, EAGAIN should never occur.
		return err != unix.EAGAIN
	})
	if err != nil {
		return n, addr, err
	}
	return n, addr, cerr
}

func (s *sysSocket) Sendto(p []byte, flags int, to unix.Sockaddr) error {
	var err error
	cerr := s.rc.Write(func(fd uintptr) bool {
		err = unix.Sendto(int(fd), p, flags, to)
		// See comment in Recvfrom.
		return err != unix.EAGAIN
	})
	if err != nil {
		return err
	}
	return cerr
}

func (s *sysSocket) SetSockoptSockFprog(level, name int, fprog *unix.SockFprog) error {
	var err error
	cerr := s.rc.Control(func(fd uintptr) {
		errno := unix.SetsockoptSockFprog(int(fd), level, name, fprog)
		if errno != nil {
			err = os.NewSyscallError("setsockopt", errno)
		}
	})
	if err != nil {
		return err
	}
	return cerr
}

func (s *sysSocket) SetSockoptPacketMreq(level, name int, mreq *unix.PacketMreq) error {
	var err error
	cerr := s.rc.Control(func(fd uintptr) {
		errno := unix.SetsockoptPacketMreq(int(fd), level, name, mreq)
		if errno != nil {
			err = os.NewSyscallError("setsockopt", errno)
		}
	})
	if err != nil {
		return err
	}
	return cerr
}
<commit_msg>raw: return poller error from Recvfrom on i/o timeout<commit_after>// +build linux

package raw

import (
	"net"
	"os"
	"sync/atomic"
	"syscall"
	"time"
	"unsafe"

	"golang.org/x/net/bpf"
	"golang.org/x/sys/unix"
)

// Must implement net.PacketConn at compile-time.
var _ net.PacketConn = &packetConn{}

// packetConn is the Linux-specific implementation of net.PacketConn for this
// package.
type packetConn struct {
	ifi *net.Interface
	s   socket
	pbe uint16

	// Should stats be accumulated instead of reset on each call?
	noCumulativeStats bool

	// Internal storage for cumulative stats.
	stats Stats
}

// socket is an interface which enables swapping out socket syscalls for
// testing.
type socket interface {
	Bind(unix.Sockaddr) error
	Close() error
	GetSockoptTpacketStats(level, name int) (*unix.TpacketStats, error)
	Recvfrom([]byte, int) (int, unix.Sockaddr, error)
	Sendto([]byte, int, unix.Sockaddr) error
	SetSockoptPacketMreq(level, name int, mreq *unix.PacketMreq) error
	SetSockoptSockFprog(level, name int, fprog *unix.SockFprog) error
	SetDeadline(time.Time) error
	SetReadDeadline(time.Time) error
	SetWriteDeadline(time.Time) error
}

// htons converts a short (uint16) from host-to-network byte order.
// Thanks to mikioh for this neat trick:
// https://github.com/mikioh/-stdyng/blob/master/afpacket.go
func htons(i uint16) uint16 {
	return (i<<8)&0xff00 | i>>8
}

// listenPacket creates a net.PacketConn which can be used to send and receive
// data at the device driver level.
func listenPacket(ifi *net.Interface, proto uint16, cfg Config) (*packetConn, error) {
	filename := "eth-packet-socket"
	// Enabling overriding the socket type via config.
	typ := unix.SOCK_RAW
	if cfg.LinuxSockDGRAM {
		filename = "packet-socket"
		typ = unix.SOCK_DGRAM
	}

	// Open a packet socket using specified socket type. Do not specify
	// a protocol to avoid capturing packets which to not match cfg.Filter.
	// The later call to bind() will set up the correct protocol for us.
	sock, err := unix.Socket(unix.AF_PACKET, typ, 0)
	if err != nil {
		return nil, err
	}

	if err := unix.SetNonblock(sock, true); err != nil {
		unix.Close(sock)
		return nil, err
	}

	// When using Go 1.12+, the SetNonblock call we just did puts the file
	// descriptor into non-blocking mode. In that case, os.NewFile
	// registers the file descriptor with the runtime poller, which is then
	// used for all subsequent operations.
	//
	// See also: https://golang.org/pkg/os/#NewFile
	f := os.NewFile(uintptr(sock), filename)
	sc, err := f.SyscallConn()
	if err != nil {
		return nil, err
	}

	// Wrap raw socket in socket interface.
	pc, err := newPacketConn(ifi, &sysSocket{f: f, rc: sc}, htons(proto), cfg.Filter)
	if err != nil {
		return nil, err
	}

	pc.noCumulativeStats = cfg.NoCumulativeStats
	return pc, nil
}

// newPacketConn creates a net.PacketConn using the specified network
// interface, wrapped socket and big endian protocol number.
//
// It is the entry point for tests in this package.
func newPacketConn(ifi *net.Interface, s socket, pbe uint16, filter []bpf.RawInstruction) (*packetConn, error) {
	pc := &packetConn{
		ifi: ifi,
		s:   s,
		pbe: pbe,
	}

	if len(filter) > 0 {
		if err := pc.SetBPF(filter); err != nil {
			return nil, err
		}
	}

	// Bind the packet socket to the interface specified by ifi
	// packet(7):
	//   Only the sll_protocol and the sll_ifindex address fields are used for
	//   purposes of binding.
	// This overrides the protocol given to socket(AF_PACKET).
	err := s.Bind(&unix.SockaddrLinklayer{
		Protocol: pc.pbe,
		Ifindex:  ifi.Index,
	})
	if err != nil {
		return nil, err
	}

	return pc, nil
}

// ReadFrom implements the net.PacketConn.ReadFrom method.
func (p *packetConn) ReadFrom(b []byte) (int, net.Addr, error) {
	// Attempt to receive on socket
	n, addr, err := p.s.Recvfrom(b, 0)
	if err != nil {
		return n, nil, err
	}

	// Retrieve hardware address and other information from addr.
	sa, ok := addr.(*unix.SockaddrLinklayer)
	if !ok {
		return n, nil, unix.EINVAL
	}

	// Use length specified to convert byte array into a hardware address slice.
	mac := make(net.HardwareAddr, sa.Halen)
	copy(mac, sa.Addr[:])

	// packet(7):
	//   sll_hatype and sll_pkttype are set on received packets for your
	//   information.
	// TODO(mdlayher): determine if similar fields exist and are useful on
	// non-Linux platforms
	return n, &Addr{
		HardwareAddr: mac,
	}, nil
}

// WriteTo implements the net.PacketConn.WriteTo method.
func (p *packetConn) WriteTo(b []byte, addr net.Addr) (int, error) {
	// Ensure correct Addr type.
	a, ok := addr.(*Addr)
	if !ok || a.HardwareAddr == nil {
		return 0, unix.EINVAL
	}

	// Convert hardware address back to byte array form.
	var baddr [8]byte
	copy(baddr[:], a.HardwareAddr)

	// Send message on socket to the specified hardware address from addr
	// packet(7):
	//   When you send packets it is enough to specify sll_family, sll_addr,
	//   sll_halen, sll_ifindex, and sll_protocol. The other fields should
	//   be 0.
	// In this case, sll_family is taken care of automatically by unix.
	err := p.s.Sendto(b, 0, &unix.SockaddrLinklayer{
		Ifindex:  p.ifi.Index,
		Halen:    uint8(len(a.HardwareAddr)),
		Addr:     baddr,
		Protocol: p.pbe,
	})
	return len(b), err
}

// Close closes the connection.
func (p *packetConn) Close() error {
	return p.s.Close()
}

// LocalAddr returns the local network address.
func (p *packetConn) LocalAddr() net.Addr {
	return &Addr{
		HardwareAddr: p.ifi.HardwareAddr,
	}
}

// SetDeadline implements the net.PacketConn.SetDeadline method.
func (p *packetConn) SetDeadline(t time.Time) error {
	return p.s.SetDeadline(t)
}

// SetReadDeadline implements the net.PacketConn.SetReadDeadline method.
func (p *packetConn) SetReadDeadline(t time.Time) error {
	return p.s.SetReadDeadline(t)
}

// SetWriteDeadline implements the net.PacketConn.SetWriteDeadline method.
func (p *packetConn) SetWriteDeadline(t time.Time) error {
	return p.s.SetWriteDeadline(t)
}

// SetBPF attaches an assembled BPF program to a raw net.PacketConn.
func (p *packetConn) SetBPF(filter []bpf.RawInstruction) error {
	prog := unix.SockFprog{
		Len:    uint16(len(filter)),
		Filter: (*unix.SockFilter)(unsafe.Pointer(&filter[0])),
	}

	err := p.s.SetSockoptSockFprog(
		unix.SOL_SOCKET,
		unix.SO_ATTACH_FILTER,
		&prog,
	)
	if err != nil {
		return os.NewSyscallError("setsockopt", err)
	}
	return nil
}

// SetPromiscuous enables or disables promiscuous mode on the interface, allowing it
// to receive traffic that is not addressed to the interface.
func (p *packetConn) SetPromiscuous(b bool) error {
	mreq := unix.PacketMreq{
		Ifindex: int32(p.ifi.Index),
		Type:    unix.PACKET_MR_PROMISC,
	}

	membership := unix.PACKET_ADD_MEMBERSHIP
	if !b {
		membership = unix.PACKET_DROP_MEMBERSHIP
	}

	return p.s.SetSockoptPacketMreq(unix.SOL_PACKET, membership, &mreq)
}

// Stats retrieves statistics from the Conn.
func (p *packetConn) Stats() (*Stats, error) {
	stats, err := p.s.GetSockoptTpacketStats(unix.SOL_PACKET, unix.PACKET_STATISTICS)
	if err != nil {
		return nil, err
	}

	return p.handleStats(stats), nil
}

// handleStats handles creation of Stats structures from raw packet socket stats.
func (p *packetConn) handleStats(s *unix.TpacketStats) *Stats {
	// Does the caller want instantaneous stats as provided by Linux?  If so,
	// return the structure directly.
	if p.noCumulativeStats {
		return &Stats{
			Packets: uint64(s.Packets),
			Drops:   uint64(s.Drops),
		}
	}

	// The caller wants cumulative stats.  Add stats with the internal stats
	// structure and return a copy of the resulting stats.
	packets := atomic.AddUint64(&p.stats.Packets, uint64(s.Packets))
	drops := atomic.AddUint64(&p.stats.Drops, uint64(s.Drops))

	return &Stats{
		Packets: packets,
		Drops:   drops,
	}
}

// sysSocket is the default socket implementation.  It makes use of
// Linux-specific system calls to handle raw socket functionality.
type sysSocket struct {
	f  *os.File
	rc syscall.RawConn
}

func (s *sysSocket) SetDeadline(t time.Time) error {
	return s.f.SetDeadline(t)
}

func (s *sysSocket) SetReadDeadline(t time.Time) error {
	return s.f.SetReadDeadline(t)
}

func (s *sysSocket) SetWriteDeadline(t time.Time) error {
	return s.f.SetWriteDeadline(t)
}

func (s *sysSocket) Bind(sa unix.Sockaddr) error {
	var err error
	cerr := s.rc.Control(func(fd uintptr) {
		err = unix.Bind(int(fd), sa)
	})
	if err != nil {
		return err
	}
	return cerr
}

func (s *sysSocket) Close() error {
	return s.f.Close()
}

func (s *sysSocket) GetSockoptTpacketStats(level, name int) (*unix.TpacketStats, error) {
	var stats *unix.TpacketStats
	var err error
	cerr := s.rc.Control(func(fd uintptr) {
		s, errno := unix.GetsockoptTpacketStats(int(fd), level, name)
		stats = s
		if errno != nil {
			err = os.NewSyscallError("getsockopt", errno)
		}
	})
	if err != nil {
		return stats, err
	}
	return stats, cerr
}

func (s *sysSocket) Recvfrom(p []byte, flags int) (n int, addr unix.Sockaddr, err error) {
	cerr := s.rc.Read(func(fd uintptr) bool {
		n, addr, err = unix.Recvfrom(int(fd), p, flags)
		// When the socket is in non-blocking mode, we might see EAGAIN
		// and end up here. In that case, return false to let the
		// poller wait for readiness. See the source code for
		// internal/poll.FD.RawRead for more details.
		//
		// If the socket is in blocking mode, EAGAIN should never occur.
		return err != unix.EAGAIN
	})
	if cerr != nil {
		return n, addr, cerr
	}
	return n, addr, err
}

func (s *sysSocket) Sendto(p []byte, flags int, to unix.Sockaddr) error {
	var err error
	cerr := s.rc.Write(func(fd uintptr) bool {
		err = unix.Sendto(int(fd), p, flags, to)
		// See comment in Recvfrom.
		return err != unix.EAGAIN
	})
	if err != nil {
		return err
	}
	return cerr
}

func (s *sysSocket) SetSockoptSockFprog(level, name int, fprog *unix.SockFprog) error {
	var err error
	cerr := s.rc.Control(func(fd uintptr) {
		errno := unix.SetsockoptSockFprog(int(fd), level, name, fprog)
		if errno != nil {
			err = os.NewSyscallError("setsockopt", errno)
		}
	})
	if err != nil {
		return err
	}
	return cerr
}

func (s *sysSocket) SetSockoptPacketMreq(level, name int, mreq *unix.PacketMreq) error {
	var err error
	cerr := s.rc.Control(func(fd uintptr) {
		errno := unix.SetsockoptPacketMreq(int(fd), level, name, mreq)
		if errno != nil {
			err = os.NewSyscallError("setsockopt", errno)
		}
	})
	if err != nil {
		return err
	}
	return cerr
}
<|endoftext|>