<commit_before>package obs

import (
	"context"
	"strings"

	rcmgr "github.com/libp2p/go-libp2p-resource-manager"
	"go.opencensus.io/stats"
	"go.opencensus.io/stats/view"
	"go.opencensus.io/tag"
)

var (
	conns = stats.Int64("connections", "Number of Connections", stats.UnitDimensionless)

	peerConns         = stats.Int64("peer/connections", "Number of connections this peer has", stats.UnitDimensionless)
	peerConnsNegative = stats.Int64("peer/connections_negative", "Number of connections this peer had. This is used to get the current connection number per peer histogram by subtracting this from the peer/connections histogram", stats.UnitDimensionless)

	streams = stats.Int64("streams", "Number of Streams", stats.UnitDimensionless)

	peerStreams         = stats.Int64("peer/streams", "Number of streams this peer has", stats.UnitDimensionless)
	peerStreamsNegative = stats.Int64("peer/streams_negative", "Number of streams this peer had. This is used to get the current streams number per peer histogram by subtracting this from the peer/streams histogram", stats.UnitDimensionless)

	memory             = stats.Int64("memory", "Amount of memory reserved as reported to the Resource Manager", stats.UnitDimensionless)
	peerMemory         = stats.Int64("peer/memory", "Amount of memory currently reseved for peer", stats.UnitDimensionless)
	peerMemoryNegative = stats.Int64("peer/memory_negative", "Amount of memory previously reseved for peer. This is used to get the current memory per peer histogram by subtracting this from the peer/memory histogram", stats.UnitDimensionless)

	connMemory         = stats.Int64("conn/memory", "Amount of memory currently reseved for the connection", stats.UnitDimensionless)
	connMemoryNegative = stats.Int64("conn/memory_negative", "Amount of memory previously reseved for the connection.  This is used to get the current memory per connection histogram by subtracting this from the conn/memory histogram", stats.UnitDimensionless)

	fds = stats.Int64("fds", "Number of fds as reported to the Resource Manager", stats.UnitDimensionless)

	blockedResources = stats.Int64("blocked_resources", "Number of resource requests blocked", stats.UnitDimensionless)
)

var (
	directionTag, _ = tag.NewKey("dir")
	scopeTag, _     = tag.NewKey("scope")
	serviceTag, _   = tag.NewKey("service")
	protocolTag, _  = tag.NewKey("protocol")
	resourceTag, _  = tag.NewKey("resource")
)

var (
	ConnView = &view.View{Measure: conns, Aggregation: view.Sum(), TagKeys: []tag.Key{directionTag, scopeTag}}

	oneTenThenExpDistribution = []float64{
		1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1, 8.1, 9.1, 10.1, 16.1, 32.1, 64.1, 128.1, 256.1,
	}

	PeerConnsView = &view.View{
		Measure:     peerConns,
		Aggregation: view.Distribution(oneTenThenExpDistribution...),
		TagKeys:     []tag.Key{directionTag},
	}
	PeerConnsNegativeView = &view.View{
		Measure:     peerConnsNegative,
		Aggregation: view.Distribution(oneTenThenExpDistribution...),
		TagKeys:     []tag.Key{directionTag},
	}

	StreamView             = &view.View{Measure: streams, Aggregation: view.Sum(), TagKeys: []tag.Key{directionTag, scopeTag, serviceTag, protocolTag}}
	PeerStreamsView        = &view.View{Measure: peerStreams, Aggregation: view.Distribution(oneTenThenExpDistribution...), TagKeys: []tag.Key{directionTag}}
	PeerStreamNegativeView = &view.View{Measure: peerStreamsNegative, Aggregation: view.Distribution(oneTenThenExpDistribution...), TagKeys: []tag.Key{directionTag}}

	MemoryView = &view.View{Measure: memory, Aggregation: view.Sum(), TagKeys: []tag.Key{scopeTag, serviceTag, protocolTag}}

	memDistribution = []float64{
		1 << 10,   // 1KB
		4 << 10,   // 4KB
		32 << 10,  // 32KB
		1 << 20,   // 1MB
		32 << 20,  // 32MB
		256 << 20, // 256MB
		512 << 20, // 512MB
		1 << 30,   // 1GB
		2 << 30,   // 2GB
		4 << 30,   // 4GB
	}
	PeerMemoryView = &view.View{
		Measure:     peerMemory,
		Aggregation: view.Distribution(memDistribution...),
	}
	PeerMemoryNegativeView = &view.View{
		Measure:     peerMemoryNegative,
		Aggregation: view.Distribution(memDistribution...),
	}

	// Not setup yet. Memory isn't attached to a given connection.
	ConnMemoryView = &view.View{
		Measure:     connMemory,
		Aggregation: view.Distribution(memDistribution...),
	}
	ConnMemoryNegativeView = &view.View{
		Measure:     connMemoryNegative,
		Aggregation: view.Distribution(memDistribution...),
	}

	FDsView = &view.View{Measure: fds, Aggregation: view.Sum(), TagKeys: []tag.Key{scopeTag}}

	BlockedResourcesView = &view.View{
		Measure:     blockedResources,
		Aggregation: view.Sum(),
		TagKeys:     []tag.Key{scopeTag, resourceTag},
	}
)

var DefaultViews []*view.View = []*view.View{
	ConnView,
	PeerConnsView,
	PeerConnsNegativeView,
	FDsView,

	StreamView,
	PeerStreamsView,
	PeerStreamNegativeView,

	MemoryView,
	PeerMemoryView,
	PeerMemoryNegativeView,

	BlockedResourcesView,
}

// StatsTraceReporter reports stats on the resource manager using its traces.
type StatsTraceReporter struct{}

func NewStatsTraceReporter() (StatsTraceReporter, error) {
	// TODO tell prometheus the system limits
	return StatsTraceReporter{}, nil
}

func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) {
	ctx := context.Background()

	switch evt.Type {
	case rcmgr.TraceAddStreamEvt, rcmgr.TraceRemoveStreamEvt:
		if p := rcmgr.ParsePeerScopeName(evt.Name); p.Validate() == nil {
			// Aggregated peer stats. Counts how many peers have N number of streams open.
			// Uses two buckets aggregations. One to count how many streams the
			// peer has now. The other to count the negative value, or how many
			// streams did the peer use to have. When looking at the data you
			// take the difference from the two.

			oldStreamsOut := int64(evt.StreamsOut - evt.DeltaOut)
			peerStreamsOut := int64(evt.StreamsOut)
			if oldStreamsOut != peerStreamsOut {
				if oldStreamsOut != 0 {
					stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "outbound")}, peerStreamsNegative.M(oldStreamsOut))
				}
				if peerStreamsOut != 0 {
					stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "outbound")}, peerStreams.M(peerStreamsOut))
				}
			}

			oldStreamsIn := int64(evt.StreamsIn - evt.DeltaIn)
			peerStreamsIn := int64(evt.StreamsIn)
			if oldStreamsIn != peerStreamsIn {
				if oldStreamsIn != 0 {
					stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "inbound")}, peerStreamsNegative.M(oldStreamsIn))
				}
				if peerStreamsIn != 0 {
					stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "inbound")}, peerStreams.M(peerStreamsIn))
				}
			}
		} else {
			var tags []tag.Mutator
			if rcmgr.IsSystemScope(evt.Name) || rcmgr.IsTransientScope(evt.Name) {
				tags = append(tags, tag.Upsert(scopeTag, evt.Name))
			} else if svc := rcmgr.ParseServiceScopeName(evt.Name); svc != "" {
				tags = append(tags, tag.Upsert(scopeTag, "service"), tag.Upsert(serviceTag, svc))
			} else if proto := rcmgr.ParseProtocolScopeName(evt.Name); proto != "" {
				tags = append(tags, tag.Upsert(scopeTag, "protocol"), tag.Upsert(protocolTag, proto))
			} else {
				// Not measuring connscope, servicepeer and protocolpeer. Lots of data, and
				// you can use aggregated peer stats + service stats to infer
				// this.
				break
			}

			if evt.DeltaOut != 0 {
				stats.RecordWithTags(
					ctx,
					append([]tag.Mutator{tag.Upsert(directionTag, "outbound")}, tags...),
					streams.M(int64(evt.DeltaOut)),
				)
			}

			if evt.DeltaIn != 0 {
				stats.RecordWithTags(
					ctx,
					append([]tag.Mutator{tag.Upsert(directionTag, "inbound")}, tags...),
					streams.M(int64(evt.DeltaIn)),
				)
			}
		}

	case rcmgr.TraceAddConnEvt, rcmgr.TraceRemoveConnEvt:
		if p := rcmgr.ParsePeerScopeName(evt.Name); p.Validate() == nil {
			// Aggregated peer stats. Counts how many peers have N number of connections.
			// Uses two buckets aggregations. One to count how many streams the
			// peer has now. The other to count the negative value, or how many
			// conns did the peer use to have. When looking at the data you
			// take the difference from the two.

			oldConnsOut := int64(evt.ConnsOut - evt.DeltaOut)
			connsOut := int64(evt.ConnsOut)
			if oldConnsOut != connsOut {
				if oldConnsOut != 0 {
					stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "outbound")}, peerConnsNegative.M(oldConnsOut))
				}
				if connsOut != 0 {
					stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "outbound")}, peerConns.M(connsOut))
				}
			}

			oldConnsIn := int64(evt.ConnsIn - evt.DeltaIn)
			connsIn := int64(evt.ConnsIn)
			if oldConnsIn != connsIn {
				if oldConnsIn != 0 {
					stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "inbound")}, peerConnsNegative.M(oldConnsIn))
				}
				if connsIn != 0 {
					stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "inbound")}, peerConns.M(connsIn))
				}
			}
		} else {
			var tags []tag.Mutator
			if rcmgr.IsSystemScope(evt.Name) || rcmgr.IsTransientScope(evt.Name) {
				tags = append(tags, tag.Upsert(scopeTag, evt.Name))
			} else if rcmgr.IsConnScope(evt.Name) {
				// Not measuring this. I don't think it's useful.
				break
			} else {
				// This could be a span
				break
			}

			if evt.DeltaOut != 0 {
				stats.RecordWithTags(
					ctx,
					append([]tag.Mutator{tag.Upsert(directionTag, "outbound")}, tags...),
					conns.M(int64(evt.DeltaOut)),
				)
			}

			if evt.DeltaIn != 0 {
				stats.RecordWithTags(
					ctx,
					append([]tag.Mutator{tag.Upsert(directionTag, "inbound")}, tags...),
					conns.M(int64(evt.DeltaIn)),
				)
			}

			// Represents the delta in fds
			if evt.Delta != 0 {
				stats.RecordWithTags(
					ctx,
					tags,
					fds.M(int64(evt.Delta)),
				)
			}
		}
	case rcmgr.TraceReserveMemoryEvt, rcmgr.TraceReleaseMemoryEvt:
		if p := rcmgr.ParsePeerScopeName(evt.Name); p.Validate() == nil {
			oldMem := evt.Memory - evt.Delta
			if oldMem != evt.Memory {
				if oldMem != 0 {
					stats.Record(ctx, peerMemoryNegative.M(oldMem))
				}
				if evt.Memory != 0 {
					stats.Record(ctx, peerMemory.M(evt.Memory))
				}
			}
		} else if rcmgr.IsConnScope(evt.Name) {
			oldMem := evt.Memory - evt.Delta
			if oldMem != evt.Memory {
				if oldMem != 0 {
					stats.Record(ctx, connMemoryNegative.M(oldMem))
				}
				if evt.Memory != 0 {
					stats.Record(ctx, connMemory.M(evt.Memory))
				}
			}
		} else {
			var tags []tag.Mutator
			if rcmgr.IsSystemScope(evt.Name) || rcmgr.IsTransientScope(evt.Name) {
				tags = append(tags, tag.Upsert(scopeTag, evt.Name))
			} else if svc := rcmgr.ParseServiceScopeName(evt.Name); svc != "" {
				tags = append(tags, tag.Upsert(scopeTag, "service"), tag.Upsert(serviceTag, svc))
			} else if proto := rcmgr.ParseProtocolScopeName(evt.Name); proto != "" {
				tags = append(tags, tag.Upsert(scopeTag, "protocol"), tag.Upsert(protocolTag, proto))
			} else {
				// Not measuring connscope, servicepeer and protocolpeer. Lots of data, and
				// you can use aggregated peer stats + service stats to infer
				// this.
				break
			}

			if evt.Delta != 0 {
				stats.RecordWithTags(ctx, tags, memory.M(int64(evt.Delta)))
			}
		}

	case rcmgr.TraceBlockAddConnEvt, rcmgr.TraceBlockAddStreamEvt, rcmgr.TraceBlockReserveMemoryEvt:
		var resource string
		if evt.Type == rcmgr.TraceBlockAddConnEvt {
			resource = "connection"
		} else if evt.Type == rcmgr.TraceBlockAddStreamEvt {
			resource = "stream"
		} else {
			resource = "memory"
		}

		// Only the top scopeName. We don't want to get the peerid here.
		scopeName := strings.SplitN(evt.Name, ":", 2)[0]
		// Drop the connection or stream id
		scopeName = strings.SplitN(scopeName, "-", 2)[0]

		tags := []tag.Mutator{tag.Upsert(scopeTag, scopeName), tag.Upsert(resourceTag, resource)}

		if evt.DeltaIn != 0 {
			stats.RecordWithTags(ctx, tags, blockedResources.M(int64(1)))
		}

		if evt.DeltaOut != 0 {
			stats.RecordWithTags(ctx, tags, blockedResources.M(int64(1)))
		}

		if evt.Delta != 0 {
			stats.RecordWithTags(ctx, tags, blockedResources.M(1))
		}
	}
}
<commit_msg>Record which direction the resource was blocked<commit_after>package obs

import (
	"context"
	"strings"

	rcmgr "github.com/libp2p/go-libp2p-resource-manager"
	"go.opencensus.io/stats"
	"go.opencensus.io/stats/view"
	"go.opencensus.io/tag"
)

var (
	conns = stats.Int64("connections", "Number of Connections", stats.UnitDimensionless)

	peerConns         = stats.Int64("peer/connections", "Number of connections this peer has", stats.UnitDimensionless)
	peerConnsNegative = stats.Int64("peer/connections_negative", "Number of connections this peer had. This is used to get the current connection number per peer histogram by subtracting this from the peer/connections histogram", stats.UnitDimensionless)

	streams = stats.Int64("streams", "Number of Streams", stats.UnitDimensionless)

	peerStreams         = stats.Int64("peer/streams", "Number of streams this peer has", stats.UnitDimensionless)
	peerStreamsNegative = stats.Int64("peer/streams_negative", "Number of streams this peer had. This is used to get the current streams number per peer histogram by subtracting this from the peer/streams histogram", stats.UnitDimensionless)

	memory             = stats.Int64("memory", "Amount of memory reserved as reported to the Resource Manager", stats.UnitDimensionless)
	peerMemory         = stats.Int64("peer/memory", "Amount of memory currently reseved for peer", stats.UnitDimensionless)
	peerMemoryNegative = stats.Int64("peer/memory_negative", "Amount of memory previously reseved for peer. This is used to get the current memory per peer histogram by subtracting this from the peer/memory histogram", stats.UnitDimensionless)

	connMemory         = stats.Int64("conn/memory", "Amount of memory currently reseved for the connection", stats.UnitDimensionless)
	connMemoryNegative = stats.Int64("conn/memory_negative", "Amount of memory previously reseved for the connection.  This is used to get the current memory per connection histogram by subtracting this from the conn/memory histogram", stats.UnitDimensionless)

	fds = stats.Int64("fds", "Number of fds as reported to the Resource Manager", stats.UnitDimensionless)

	blockedResources = stats.Int64("blocked_resources", "Number of resource requests blocked", stats.UnitDimensionless)
)

var (
	directionTag, _ = tag.NewKey("dir")
	scopeTag, _     = tag.NewKey("scope")
	serviceTag, _   = tag.NewKey("service")
	protocolTag, _  = tag.NewKey("protocol")
	resourceTag, _  = tag.NewKey("resource")
)

var (
	ConnView = &view.View{Measure: conns, Aggregation: view.Sum(), TagKeys: []tag.Key{directionTag, scopeTag}}

	oneTenThenExpDistribution = []float64{
		1.1, 2.1, 3.1, 4.1, 5.1, 6.1, 7.1, 8.1, 9.1, 10.1, 16.1, 32.1, 64.1, 128.1, 256.1,
	}

	PeerConnsView = &view.View{
		Measure:     peerConns,
		Aggregation: view.Distribution(oneTenThenExpDistribution...),
		TagKeys:     []tag.Key{directionTag},
	}
	PeerConnsNegativeView = &view.View{
		Measure:     peerConnsNegative,
		Aggregation: view.Distribution(oneTenThenExpDistribution...),
		TagKeys:     []tag.Key{directionTag},
	}

	StreamView             = &view.View{Measure: streams, Aggregation: view.Sum(), TagKeys: []tag.Key{directionTag, scopeTag, serviceTag, protocolTag}}
	PeerStreamsView        = &view.View{Measure: peerStreams, Aggregation: view.Distribution(oneTenThenExpDistribution...), TagKeys: []tag.Key{directionTag}}
	PeerStreamNegativeView = &view.View{Measure: peerStreamsNegative, Aggregation: view.Distribution(oneTenThenExpDistribution...), TagKeys: []tag.Key{directionTag}}

	MemoryView = &view.View{Measure: memory, Aggregation: view.Sum(), TagKeys: []tag.Key{scopeTag, serviceTag, protocolTag}}

	memDistribution = []float64{
		1 << 10,   // 1KB
		4 << 10,   // 4KB
		32 << 10,  // 32KB
		1 << 20,   // 1MB
		32 << 20,  // 32MB
		256 << 20, // 256MB
		512 << 20, // 512MB
		1 << 30,   // 1GB
		2 << 30,   // 2GB
		4 << 30,   // 4GB
	}
	PeerMemoryView = &view.View{
		Measure:     peerMemory,
		Aggregation: view.Distribution(memDistribution...),
	}
	PeerMemoryNegativeView = &view.View{
		Measure:     peerMemoryNegative,
		Aggregation: view.Distribution(memDistribution...),
	}

	// Not setup yet. Memory isn't attached to a given connection.
	ConnMemoryView = &view.View{
		Measure:     connMemory,
		Aggregation: view.Distribution(memDistribution...),
	}
	ConnMemoryNegativeView = &view.View{
		Measure:     connMemoryNegative,
		Aggregation: view.Distribution(memDistribution...),
	}

	FDsView = &view.View{Measure: fds, Aggregation: view.Sum(), TagKeys: []tag.Key{scopeTag}}

	BlockedResourcesView = &view.View{
		Measure:     blockedResources,
		Aggregation: view.Sum(),
		TagKeys:     []tag.Key{scopeTag, resourceTag},
	}
)

var DefaultViews []*view.View = []*view.View{
	ConnView,
	PeerConnsView,
	PeerConnsNegativeView,
	FDsView,

	StreamView,
	PeerStreamsView,
	PeerStreamNegativeView,

	MemoryView,
	PeerMemoryView,
	PeerMemoryNegativeView,

	BlockedResourcesView,
}

// StatsTraceReporter reports stats on the resource manager using its traces.
type StatsTraceReporter struct{}

func NewStatsTraceReporter() (StatsTraceReporter, error) {
	// TODO tell prometheus the system limits
	return StatsTraceReporter{}, nil
}

func (r StatsTraceReporter) ConsumeEvent(evt rcmgr.TraceEvt) {
	ctx := context.Background()

	switch evt.Type {
	case rcmgr.TraceAddStreamEvt, rcmgr.TraceRemoveStreamEvt:
		if p := rcmgr.ParsePeerScopeName(evt.Name); p.Validate() == nil {
			// Aggregated peer stats. Counts how many peers have N number of streams open.
			// Uses two buckets aggregations. One to count how many streams the
			// peer has now. The other to count the negative value, or how many
			// streams did the peer use to have. When looking at the data you
			// take the difference from the two.

			oldStreamsOut := int64(evt.StreamsOut - evt.DeltaOut)
			peerStreamsOut := int64(evt.StreamsOut)
			if oldStreamsOut != peerStreamsOut {
				if oldStreamsOut != 0 {
					stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "outbound")}, peerStreamsNegative.M(oldStreamsOut))
				}
				if peerStreamsOut != 0 {
					stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "outbound")}, peerStreams.M(peerStreamsOut))
				}
			}

			oldStreamsIn := int64(evt.StreamsIn - evt.DeltaIn)
			peerStreamsIn := int64(evt.StreamsIn)
			if oldStreamsIn != peerStreamsIn {
				if oldStreamsIn != 0 {
					stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "inbound")}, peerStreamsNegative.M(oldStreamsIn))
				}
				if peerStreamsIn != 0 {
					stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "inbound")}, peerStreams.M(peerStreamsIn))
				}
			}
		} else {
			var tags []tag.Mutator
			if rcmgr.IsSystemScope(evt.Name) || rcmgr.IsTransientScope(evt.Name) {
				tags = append(tags, tag.Upsert(scopeTag, evt.Name))
			} else if svc := rcmgr.ParseServiceScopeName(evt.Name); svc != "" {
				tags = append(tags, tag.Upsert(scopeTag, "service"), tag.Upsert(serviceTag, svc))
			} else if proto := rcmgr.ParseProtocolScopeName(evt.Name); proto != "" {
				tags = append(tags, tag.Upsert(scopeTag, "protocol"), tag.Upsert(protocolTag, proto))
			} else {
				// Not measuring connscope, servicepeer and protocolpeer. Lots of data, and
				// you can use aggregated peer stats + service stats to infer
				// this.
				break
			}

			if evt.DeltaOut != 0 {
				stats.RecordWithTags(
					ctx,
					append([]tag.Mutator{tag.Upsert(directionTag, "outbound")}, tags...),
					streams.M(int64(evt.DeltaOut)),
				)
			}

			if evt.DeltaIn != 0 {
				stats.RecordWithTags(
					ctx,
					append([]tag.Mutator{tag.Upsert(directionTag, "inbound")}, tags...),
					streams.M(int64(evt.DeltaIn)),
				)
			}
		}

	case rcmgr.TraceAddConnEvt, rcmgr.TraceRemoveConnEvt:
		if p := rcmgr.ParsePeerScopeName(evt.Name); p.Validate() == nil {
			// Aggregated peer stats. Counts how many peers have N number of connections.
			// Uses two buckets aggregations. One to count how many streams the
			// peer has now. The other to count the negative value, or how many
			// conns did the peer use to have. When looking at the data you
			// take the difference from the two.

			oldConnsOut := int64(evt.ConnsOut - evt.DeltaOut)
			connsOut := int64(evt.ConnsOut)
			if oldConnsOut != connsOut {
				if oldConnsOut != 0 {
					stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "outbound")}, peerConnsNegative.M(oldConnsOut))
				}
				if connsOut != 0 {
					stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "outbound")}, peerConns.M(connsOut))
				}
			}

			oldConnsIn := int64(evt.ConnsIn - evt.DeltaIn)
			connsIn := int64(evt.ConnsIn)
			if oldConnsIn != connsIn {
				if oldConnsIn != 0 {
					stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "inbound")}, peerConnsNegative.M(oldConnsIn))
				}
				if connsIn != 0 {
					stats.RecordWithTags(ctx, []tag.Mutator{tag.Upsert(directionTag, "inbound")}, peerConns.M(connsIn))
				}
			}
		} else {
			var tags []tag.Mutator
			if rcmgr.IsSystemScope(evt.Name) || rcmgr.IsTransientScope(evt.Name) {
				tags = append(tags, tag.Upsert(scopeTag, evt.Name))
			} else if rcmgr.IsConnScope(evt.Name) {
				// Not measuring this. I don't think it's useful.
				break
			} else {
				// This could be a span
				break
			}

			if evt.DeltaOut != 0 {
				stats.RecordWithTags(
					ctx,
					append([]tag.Mutator{tag.Upsert(directionTag, "outbound")}, tags...),
					conns.M(int64(evt.DeltaOut)),
				)
			}

			if evt.DeltaIn != 0 {
				stats.RecordWithTags(
					ctx,
					append([]tag.Mutator{tag.Upsert(directionTag, "inbound")}, tags...),
					conns.M(int64(evt.DeltaIn)),
				)
			}

			// Represents the delta in fds
			if evt.Delta != 0 {
				stats.RecordWithTags(
					ctx,
					tags,
					fds.M(int64(evt.Delta)),
				)
			}
		}
	case rcmgr.TraceReserveMemoryEvt, rcmgr.TraceReleaseMemoryEvt:
		if p := rcmgr.ParsePeerScopeName(evt.Name); p.Validate() == nil {
			oldMem := evt.Memory - evt.Delta
			if oldMem != evt.Memory {
				if oldMem != 0 {
					stats.Record(ctx, peerMemoryNegative.M(oldMem))
				}
				if evt.Memory != 0 {
					stats.Record(ctx, peerMemory.M(evt.Memory))
				}
			}
		} else if rcmgr.IsConnScope(evt.Name) {
			oldMem := evt.Memory - evt.Delta
			if oldMem != evt.Memory {
				if oldMem != 0 {
					stats.Record(ctx, connMemoryNegative.M(oldMem))
				}
				if evt.Memory != 0 {
					stats.Record(ctx, connMemory.M(evt.Memory))
				}
			}
		} else {
			var tags []tag.Mutator
			if rcmgr.IsSystemScope(evt.Name) || rcmgr.IsTransientScope(evt.Name) {
				tags = append(tags, tag.Upsert(scopeTag, evt.Name))
			} else if svc := rcmgr.ParseServiceScopeName(evt.Name); svc != "" {
				tags = append(tags, tag.Upsert(scopeTag, "service"), tag.Upsert(serviceTag, svc))
			} else if proto := rcmgr.ParseProtocolScopeName(evt.Name); proto != "" {
				tags = append(tags, tag.Upsert(scopeTag, "protocol"), tag.Upsert(protocolTag, proto))
			} else {
				// Not measuring connscope, servicepeer and protocolpeer. Lots of data, and
				// you can use aggregated peer stats + service stats to infer
				// this.
				break
			}

			if evt.Delta != 0 {
				stats.RecordWithTags(ctx, tags, memory.M(int64(evt.Delta)))
			}
		}

	case rcmgr.TraceBlockAddConnEvt, rcmgr.TraceBlockAddStreamEvt, rcmgr.TraceBlockReserveMemoryEvt:
		var resource string
		if evt.Type == rcmgr.TraceBlockAddConnEvt {
			resource = "connection"
		} else if evt.Type == rcmgr.TraceBlockAddStreamEvt {
			resource = "stream"
		} else {
			resource = "memory"
		}

		// Only the top scopeName. We don't want to get the peerid here.
		scopeName := strings.SplitN(evt.Name, ":", 2)[0]
		// Drop the connection or stream id
		scopeName = strings.SplitN(scopeName, "-", 2)[0]

		tags := []tag.Mutator{tag.Upsert(scopeTag, scopeName), tag.Upsert(resourceTag, resource)}

		if evt.DeltaIn != 0 {
			tagsWithDir := append([]tag.Mutator{tag.Insert(directionTag, "inbound")}, tags...)
			stats.RecordWithTags(ctx, tagsWithDir, blockedResources.M(int64(1)))
		}

		if evt.DeltaOut != 0 {
			tagsWithDir := append([]tag.Mutator{tag.Insert(directionTag, "outbound")}, tags...)
			stats.RecordWithTags(ctx, tagsWithDir, blockedResources.M(int64(1)))
		}

		if evt.Delta != 0 {
			stats.RecordWithTags(ctx, tags, blockedResources.M(1))
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"encoding/hex"
	"fmt"
	"net"
	"os"
	"sync"

	"github.com/containers/virtcontainers/hyperstart"
	"github.com/golang/glog"
)

// Represents a single qemu/hyperstart instance on the system
type vm struct {
	sync.Mutex

	containerID string

	hyperHandler *hyperstart.Hyperstart

	// Socket to the VM console
	console struct {
		socketPath string
		conn       net.Conn
	}

	// Used to allocate globally unique IO sequence numbers
	nextIoBase uint64

	// ios are hashed by their sequence numbers. If 2 sequence numbers are
	// allocated for one process (stdin/stdout and stderr) both sequence
	// numbers appear in this map.
	ioSessions map[uint64]*ioSession

	// Used to wait for all VM-global goroutines to finish on Close()
	wg sync.WaitGroup

	// Channel to signal qemu has terminated.
	vmLost chan interface{}
}

// A set of I/O streams between a client and a process running inside the VM
type ioSession struct {
	nStreams int
	ioBase   uint64

	// id  of the client owning that ioSession
	clientID uint64

	// socket connected to the fd sent over to the client
	client net.Conn

	// Used to wait for per-ioSession goroutines. Currently there's only
	// one such goroutine, the one reading stdin data from client socket.
	wg sync.WaitGroup
}

func newVM(id, ctlSerial, ioSerial string) *vm {
	h := hyperstart.NewHyperstart(ctlSerial, ioSerial, "unix")

	return &vm{
		containerID:  id,
		hyperHandler: h,
		nextIoBase:   1,
		ioSessions:   make(map[uint64]*ioSession),
		vmLost:       make(chan interface{}),
	}
}

// setConsole() will make the proxy output the console data on stderr
func (vm *vm) setConsole(path string) {
	vm.console.socketPath = path
}

func (vm *vm) shortName() string {
	length := 8
	if len(vm.containerID) < 8 {
		length = len(vm.containerID)
	}
	return vm.containerID[0:length]
}

func (vm *vm) info(lvl glog.Level, channel string, msg string) {
	if !glog.V(lvl) {
		return
	}
	glog.Infof("[vm %s %s] %s", vm.shortName(), channel, msg)
}

func (vm *vm) infof(lvl glog.Level, channel string, fmt string, a ...interface{}) {
	if !glog.V(lvl) {
		return
	}
	a = append(a, 0, 0)
	copy(a[2:], a[0:])
	a[0] = vm.shortName()
	a[1] = channel
	glog.Infof("[vm %s %s] "+fmt, a...)
}

func (vm *vm) dump(lvl glog.Level, data []byte) {
	if !glog.V(lvl) {
		return
	}
	glog.Infof("\n%s", hex.Dump(data))
}

func (vm *vm) findSession(seq uint64) *ioSession {
	vm.Lock()
	defer vm.Unlock()

	return vm.ioSessions[seq]
}

// This function runs in a goroutine, reading data from the io channel and
// dispatching it to the right client (the one with matching seq number)
// There's only one instance of this goroutine per-VM
func (vm *vm) ioHyperToClients() {
	for {
		msg, err := vm.hyperHandler.ReadIoMessage()
		if err != nil {
			// VM process is gone
			vm.signalVMLost()
			break
		}

		session := vm.findSession(msg.Session)
		if session == nil {
			fmt.Fprintf(os.Stderr,
				"couldn't find client with seq number %d\n", msg.Session)
			continue
		}

		vm.infof(1, "io", "<- writing to client #%d", session.clientID)
		vm.dump(2, msg.Message)

		err = hyperstart.SendIoMessageWithConn(session.client, msg)
		if err != nil {
			fmt.Fprintf(os.Stderr,
				"error writing I/O data to client: %v\n", err)
			break
		}
	}

	vm.wg.Done()
}

// Stream the VM console to stderr
func (vm *vm) consoleToLog() {
	reader := bufio.NewReader(vm.console.conn)
	for {
		line, err := reader.ReadString('\n')
		if err != nil {
			break
		}

		vm.infof(3, "hyperstart", line)
	}

	vm.wg.Done()
}

func (vm *vm) Connect() error {
	if vm.console.socketPath != "" {
		var err error

		vm.console.conn, err = net.Dial("unix", vm.console.socketPath)
		if err != nil {
			return err
		}

		vm.wg.Add(1)
		go vm.consoleToLog()
	}

	if err := vm.hyperHandler.OpenSockets(); err != nil {
		return err
	}

	if err := vm.hyperHandler.WaitForReady(); err != nil {
		vm.hyperHandler.CloseSockets()
		return err
	}

	vm.wg.Add(1)
	go vm.ioHyperToClients()

	return nil
}

func (vm *vm) SendMessage(cmd string, data []byte) error {
	_, err := vm.hyperHandler.SendCtlMessage(cmd, data)
	return err
}

// This function runs in a goroutine, reading data from the client socket and
// writing data to the hyperstart I/O chanel.
// There's one instance of this goroutine per client having done an allocateIO.
func (vm *vm) ioClientToHyper(session *ioSession) {
	for {
		msg, err := hyperstart.ReadIoMessageWithConn(session.client)
		if err != nil {
			// client process is gone
			break
		}

		if msg.Session != session.ioBase {
			fmt.Fprintf(os.Stderr, "stdin seq %d not matching ioBase %d\n", msg.Session, session.ioBase)
			session.client.Close()
			break
		}

		vm.infof(1, "io", "-> writing to hyper from #%d", session.clientID)
		vm.dump(2, msg.Message)

		err = vm.hyperHandler.SendIoMessage(msg)
		if err != nil {
			fmt.Fprintf(os.Stderr,
				"error writing I/O data to hyperstart: %v\n", err)
			break
		}
	}

	session.wg.Done()
}

func (vm *vm) AllocateIo(n int, clientID uint64, c net.Conn) uint64 {
	// Allocate ioBase
	vm.Lock()
	ioBase := vm.nextIoBase
	vm.nextIoBase += uint64(n)

	session := &ioSession{
		nStreams: n,
		ioBase:   ioBase,
		clientID: clientID,
		client:   c,
	}

	for i := 0; i < n; i++ {
		vm.ioSessions[ioBase+uint64(i)] = session
	}
	vm.Unlock()

	// Starts stdin forwarding between client and hyper
	session.wg.Add(1)
	go vm.ioClientToHyper(session)

	return ioBase
}

func (session *ioSession) Close() {
	session.client.Close()
	session.wg.Wait()
}

func (vm *vm) CloseIo(seq uint64) {
	vm.Lock()
	session := vm.ioSessions[seq]
	if session == nil {
		vm.Unlock()
		return
	}
	for i := 0; i < session.nStreams; i++ {
		delete(vm.ioSessions, seq+uint64(i))
	}
	vm.Unlock()

	session.Close()
}

func (vm *vm) Close() {
	vm.hyperHandler.CloseSockets()
	if vm.console.conn != nil {
		vm.console.conn.Close()
	}

	// Wait for per-client goroutines
	vm.Lock()
	for seq, session := range vm.ioSessions {
		delete(vm.ioSessions, seq)
		if seq != session.ioBase {
			continue
		}

		session.Close()
	}
	vm.Unlock()

	// Wait for VM global goroutines
	vm.wg.Wait()
}

// OnVmLost returns a channel can be waited on to signal the end of the qemu
// process.
func (vm *vm) OnVMLost() <-chan interface{} {
	return vm.vmLost
}

func (vm *vm) signalVMLost() {
	close(vm.vmLost)
}
<commit_msg>proxy: Fix the VM lost detection site<commit_after>// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"encoding/hex"
	"fmt"
	"net"
	"os"
	"sync"

	"github.com/containers/virtcontainers/hyperstart"
	"github.com/golang/glog"
)

// Represents a single qemu/hyperstart instance on the system
type vm struct {
	sync.Mutex

	containerID string

	hyperHandler *hyperstart.Hyperstart

	// Socket to the VM console
	console struct {
		socketPath string
		conn       net.Conn
	}

	// Used to allocate globally unique IO sequence numbers
	nextIoBase uint64

	// ios are hashed by their sequence numbers. If 2 sequence numbers are
	// allocated for one process (stdin/stdout and stderr) both sequence
	// numbers appear in this map.
	ioSessions map[uint64]*ioSession

	// Used to wait for all VM-global goroutines to finish on Close()
	wg sync.WaitGroup

	// Channel to signal qemu has terminated.
	vmLost chan interface{}
}

// A set of I/O streams between a client and a process running inside the VM
type ioSession struct {
	nStreams int
	ioBase   uint64

	// id  of the client owning that ioSession
	clientID uint64

	// socket connected to the fd sent over to the client
	client net.Conn

	// Used to wait for per-ioSession goroutines. Currently there's only
	// one such goroutine, the one reading stdin data from client socket.
	wg sync.WaitGroup
}

func newVM(id, ctlSerial, ioSerial string) *vm {
	h := hyperstart.NewHyperstart(ctlSerial, ioSerial, "unix")

	return &vm{
		containerID:  id,
		hyperHandler: h,
		nextIoBase:   1,
		ioSessions:   make(map[uint64]*ioSession),
		vmLost:       make(chan interface{}),
	}
}

// setConsole() will make the proxy output the console data on stderr
func (vm *vm) setConsole(path string) {
	vm.console.socketPath = path
}

func (vm *vm) shortName() string {
	length := 8
	if len(vm.containerID) < 8 {
		length = len(vm.containerID)
	}
	return vm.containerID[0:length]
}

func (vm *vm) info(lvl glog.Level, channel string, msg string) {
	if !glog.V(lvl) {
		return
	}
	glog.Infof("[vm %s %s] %s", vm.shortName(), channel, msg)
}

func (vm *vm) infof(lvl glog.Level, channel string, fmt string, a ...interface{}) {
	if !glog.V(lvl) {
		return
	}
	a = append(a, 0, 0)
	copy(a[2:], a[0:])
	a[0] = vm.shortName()
	a[1] = channel
	glog.Infof("[vm %s %s] "+fmt, a...)
}

func (vm *vm) dump(lvl glog.Level, data []byte) {
	if !glog.V(lvl) {
		return
	}
	glog.Infof("\n%s", hex.Dump(data))
}

func (vm *vm) findSession(seq uint64) *ioSession {
	vm.Lock()
	defer vm.Unlock()

	return vm.ioSessions[seq]
}

// This function runs in a goroutine, reading data from the io channel and
// dispatching it to the right client (the one with matching seq number)
// There's only one instance of this goroutine per-VM
func (vm *vm) ioHyperToClients() {
	for {
		msg, err := vm.hyperHandler.ReadIoMessage()
		if err != nil {
			break
		}

		session := vm.findSession(msg.Session)
		if session == nil {
			fmt.Fprintf(os.Stderr,
				"couldn't find client with seq number %d\n", msg.Session)
			continue
		}

		vm.infof(1, "io", "<- writing to client #%d", session.clientID)
		vm.dump(2, msg.Message)

		err = hyperstart.SendIoMessageWithConn(session.client, msg)
		if err != nil {
			fmt.Fprintf(os.Stderr,
				"error writing I/O data to client: %v\n", err)
			break
		}
	}

	// Having an error on read/write is interpreted as having lost the VM.
	vm.signalVMLost()
	vm.wg.Done()
}

// Stream the VM console to stderr
func (vm *vm) consoleToLog() {
	reader := bufio.NewReader(vm.console.conn)
	for {
		line, err := reader.ReadString('\n')
		if err != nil {
			break
		}

		vm.infof(3, "hyperstart", line)
	}

	vm.wg.Done()
}

func (vm *vm) Connect() error {
	if vm.console.socketPath != "" {
		var err error

		vm.console.conn, err = net.Dial("unix", vm.console.socketPath)
		if err != nil {
			return err
		}

		vm.wg.Add(1)
		go vm.consoleToLog()
	}

	if err := vm.hyperHandler.OpenSockets(); err != nil {
		return err
	}

	if err := vm.hyperHandler.WaitForReady(); err != nil {
		vm.hyperHandler.CloseSockets()
		return err
	}

	vm.wg.Add(1)
	go vm.ioHyperToClients()

	return nil
}

func (vm *vm) SendMessage(cmd string, data []byte) error {
	_, err := vm.hyperHandler.SendCtlMessage(cmd, data)
	return err
}

// This function runs in a goroutine, reading data from the client socket and
// writing data to the hyperstart I/O chanel.
// There's one instance of this goroutine per client having done an allocateIO.
func (vm *vm) ioClientToHyper(session *ioSession) {
	for {
		msg, err := hyperstart.ReadIoMessageWithConn(session.client)
		if err != nil {
			// client process is gone
			break
		}

		if msg.Session != session.ioBase {
			fmt.Fprintf(os.Stderr, "stdin seq %d not matching ioBase %d\n", msg.Session, session.ioBase)
			session.client.Close()
			break
		}

		vm.infof(1, "io", "-> writing to hyper from #%d", session.clientID)
		vm.dump(2, msg.Message)

		err = vm.hyperHandler.SendIoMessage(msg)
		if err != nil {
			fmt.Fprintf(os.Stderr,
				"error writing I/O data to hyperstart: %v\n", err)
			break
		}
	}

	session.wg.Done()
}

func (vm *vm) AllocateIo(n int, clientID uint64, c net.Conn) uint64 {
	// Allocate ioBase
	vm.Lock()
	ioBase := vm.nextIoBase
	vm.nextIoBase += uint64(n)

	session := &ioSession{
		nStreams: n,
		ioBase:   ioBase,
		clientID: clientID,
		client:   c,
	}

	for i := 0; i < n; i++ {
		vm.ioSessions[ioBase+uint64(i)] = session
	}
	vm.Unlock()

	// Starts stdin forwarding between client and hyper
	session.wg.Add(1)
	go vm.ioClientToHyper(session)

	return ioBase
}

func (session *ioSession) Close() {
	session.client.Close()
	session.wg.Wait()
}

func (vm *vm) CloseIo(seq uint64) {
	vm.Lock()
	session := vm.ioSessions[seq]
	if session == nil {
		vm.Unlock()
		return
	}
	for i := 0; i < session.nStreams; i++ {
		delete(vm.ioSessions, seq+uint64(i))
	}
	vm.Unlock()

	session.Close()
}

func (vm *vm) Close() {
	vm.hyperHandler.CloseSockets()
	if vm.console.conn != nil {
		vm.console.conn.Close()
	}

	// Wait for per-client goroutines
	vm.Lock()
	for seq, session := range vm.ioSessions {
		delete(vm.ioSessions, seq)
		if seq != session.ioBase {
			continue
		}

		session.Close()
	}
	vm.Unlock()

	// Wait for VM global goroutines
	vm.wg.Wait()
}

// OnVmLost returns a channel can be waited on to signal the end of the qemu
// process.
func (vm *vm) OnVMLost() <-chan interface{} {
	return vm.vmLost
}

func (vm *vm) signalVMLost() {
	close(vm.vmLost)
}
<|endoftext|>


<commit_before>package render_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/flant/werf/pkg/testing/utils"
)

var _ = Describe("helm render with secrets", func() {
	BeforeEach(func() {
		utils.CopyIn(utils.FixturePath("secret"), testDirPath)
	})

	It("should be rendered", func() {
		output := utils.SucceedCommandOutputString(
			testDirPath,
			werfBinPath,
			"helm", "render",
		)

		for _, substr := range []string{
			"int: MTc=",
			`json:
    web:
      id: "220821453116"
      redirect_uris:
      - https://test.ru`,
			"quoted: '''password'''",
			"secret_file: TUlJSktBSUJBQUtDQWdFQXFQMEFDcHZKNmUrSFJjdndDMUlFM201TzJNQXo4ci90QU1RZldwK2w3L0g1TE1zTgpGTzJKSVJPUis1VE95R2VWc1dzekRZMTEzcFhhRTJoaDZjam16Qkhvc1h6azZSenpVd1B4TlZzUy9MN3ZkRy9iClZxTnB1UGhyRllMdVNWcFgzZC9Cc0wzUmZXVnpxS1JlYVpnb05SQnpUN044UWNFVHdCMEEzbExvYy9TcGxNY0UKOTJ5QUJJMUNwcEt4b1RWVnlGZ09vMVNhSlFBRlRwOUtLbDY4K2tkTTFhcUlhV1Y0eWxKbXB2Umo3bVR2eGlmYwp3SVdLc1FLQ0FRQmo4c05RTHl3bnJIY2pJSnpwemRKZVArTUlCN1BaUFFIVDQzWDBpZzRFaC9kS3BIVHlWb21ECituWjZLWktqRUcySmY1NnNKUysxa202cWNhYjU1MnIxTGtNcXNoRVZWb2s2K2ovanNidHROOHVCVGliMkRyUksKbFlWMmI5TTh2RnF6LzNxQmM2VkxISWNkZ29pRGlOdWl3QlhlZFdsRHBnNUN2V3BIRWx1anRKVjNMNmNpQjFRNwpqeG9lMGNtUXY5TGJYWUNCRjBBNVZVNWZqeHdJN0tQVmRCSm1rWE9RS3EvZ1ZuVGFQOG9DSlN0dk95Lzd6bm05CjgwemhRQUtJMkgzVjZMOG9ONG5vTm16TW50bkNUWUl5UWFpMlRlUVgyckswbGlpeXNjVWQxaGJGaWE5bUZMNWEKd2g2Y3EyVElNcDNpK1h2MWw4VU5pTXZOM0JGWEhKR3hBb0lCQUMrajZZY2wxRVZuMjJCOFk3bHd1Mmp1SFFuYQpiZEpTR2QrMk5mUnp3TGcxaGxDc3pmL3JPNzZkUlYyVkFnR0NTM1NUWE9iQ2s2UHR5RTJHLy9TeG5pTG1nNCtyCklkLzF6SVNFZERUdkg1TG1OejNqNmJKNXFBTUttdkYyVjRoeHhtMmYwQ2xZSjJBWFFZdGNrL2N1bzQrWUxjVkQKOTRqNXowZkd4Y0xhSVkxVVpDUXZpbHZzVmdFaEV0cEQ0enpaTGFuOXNqRVVLLzAwV2E3NG8vVzBKWmpUS3Q3RAoxaDAwbUJxcTJDN1J0OElnNitreE9QSVdCUGxTNk1CQkJuS2Q1L1VqeS9hanJERnlHVjZjNjRtRFJHR1FFKy8rCk5JcDcwZjcxaWFBQVFQSVBkaFJUdk9zaEhna2RNQTFqRlBOWUNUVlVmWk56NllCSVpzajhBTktqNnNFPQo=",
		} {
			Ω(output).Should(ContainSubstring(substr))
		}
	})
})
<commit_msg>[tests] Update Helm/Secret suite<commit_after>package render_test

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/flant/werf/pkg/testing/utils"
)

var _ = Describe("helm render with secrets", func() {
	BeforeEach(func() {
		utils.CopyIn(utils.FixturePath("secret"), testDirPath)
	})

	It("should be rendered", func() {
		output := utils.SucceedCommandOutputString(
			testDirPath,
			werfBinPath,
			"helm", "render",
		)

		for _, substr := range []string{
			"int: MTc=",
			`json: {"web":{"id":"220821453116","redirect_uris":["https://test.ru"]}}`,
			`quoted: "'password'"`,
			"secret_file: TUlJSktBSUJBQUtDQWdFQXFQMEFDcHZKNmUrSFJjdndDMUlFM201TzJNQXo4ci90QU1RZldwK2w3L0g1TE1zTgpGTzJKSVJPUis1VE95R2VWc1dzekRZMTEzcFhhRTJoaDZjam16Qkhvc1h6azZSenpVd1B4TlZzUy9MN3ZkRy9iClZxTnB1UGhyRllMdVNWcFgzZC9Cc0wzUmZXVnpxS1JlYVpnb05SQnpUN044UWNFVHdCMEEzbExvYy9TcGxNY0UKOTJ5QUJJMUNwcEt4b1RWVnlGZ09vMVNhSlFBRlRwOUtLbDY4K2tkTTFhcUlhV1Y0eWxKbXB2Umo3bVR2eGlmYwp3SVdLc1FLQ0FRQmo4c05RTHl3bnJIY2pJSnpwemRKZVArTUlCN1BaUFFIVDQzWDBpZzRFaC9kS3BIVHlWb21ECituWjZLWktqRUcySmY1NnNKUysxa202cWNhYjU1MnIxTGtNcXNoRVZWb2s2K2ovanNidHROOHVCVGliMkRyUksKbFlWMmI5TTh2RnF6LzNxQmM2VkxISWNkZ29pRGlOdWl3QlhlZFdsRHBnNUN2V3BIRWx1anRKVjNMNmNpQjFRNwpqeG9lMGNtUXY5TGJYWUNCRjBBNVZVNWZqeHdJN0tQVmRCSm1rWE9RS3EvZ1ZuVGFQOG9DSlN0dk95Lzd6bm05CjgwemhRQUtJMkgzVjZMOG9ONG5vTm16TW50bkNUWUl5UWFpMlRlUVgyckswbGlpeXNjVWQxaGJGaWE5bUZMNWEKd2g2Y3EyVElNcDNpK1h2MWw4VU5pTXZOM0JGWEhKR3hBb0lCQUMrajZZY2wxRVZuMjJCOFk3bHd1Mmp1SFFuYQpiZEpTR2QrMk5mUnp3TGcxaGxDc3pmL3JPNzZkUlYyVkFnR0NTM1NUWE9iQ2s2UHR5RTJHLy9TeG5pTG1nNCtyCklkLzF6SVNFZERUdkg1TG1OejNqNmJKNXFBTUttdkYyVjRoeHhtMmYwQ2xZSjJBWFFZdGNrL2N1bzQrWUxjVkQKOTRqNXowZkd4Y0xhSVkxVVpDUXZpbHZzVmdFaEV0cEQ0enpaTGFuOXNqRVVLLzAwV2E3NG8vVzBKWmpUS3Q3RAoxaDAwbUJxcTJDN1J0OElnNitreE9QSVdCUGxTNk1CQkJuS2Q1L1VqeS9hanJERnlHVjZjNjRtRFJHR1FFKy8rCk5JcDcwZjcxaWFBQVFQSVBkaFJUdk9zaEhna2RNQTFqRlBOWUNUVlVmWk56NllCSVpzajhBTktqNnNFPQo=",
		} {
			Ω(output).Should(ContainSubstring(substr))
		}
	})
})
<|endoftext|>


<commit_before>/*
   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package api

import (
	"fmt"
	"net/http"
	"unicode"

	"github.com/vmware/harbor/dao"
	"github.com/vmware/harbor/models"
	"github.com/vmware/harbor/utils/log"

	"strconv"
	"time"
)

// ProjectAPI handles request to /api/projects/{} /api/projects/{}/logs
type ProjectAPI struct {
	BaseAPI
	userID    int
	projectID int64
}

type projectReq struct {
	ProjectName string `json:"project_name"`
	Public      bool   `json:"public"`
}

const projectNameMaxLen int = 30
const projectNameMinLen int = 4

// Prepare validates the URL and the user
func (p *ProjectAPI) Prepare() {
	idStr := p.Ctx.Input.Param(":id")
	if len(idStr) > 0 {
		var err error
		p.projectID, err = strconv.ParseInt(idStr, 10, 64)
		if err != nil {
			log.Errorf("Error parsing project id: %s, error: %v", idStr, err)
			p.CustomAbort(http.StatusBadRequest, "invalid project id")
		}
		exist, err := dao.ProjectExists(p.projectID)
		if err != nil {
			log.Errorf("Error occurred in ProjectExists, error: %v", err)
			p.CustomAbort(http.StatusInternalServerError, "Internal error.")
		}
		if !exist {
			p.CustomAbort(http.StatusNotFound, fmt.Sprintf("project does not exist, id: %v", p.projectID))
		}
	}
}

// Post ...
func (p *ProjectAPI) Post() {
	p.userID = p.ValidateUser()

	var req projectReq
	var public int
	p.DecodeJSONReq(&req)
	if req.Public {
		public = 1
	}
	err := validateProjectReq(req)
	if err != nil {
		log.Errorf("Invalid project request, error: %v", err)
		p.RenderError(http.StatusBadRequest, "Invalid request for creating project")
		return
	}
	projectName := req.ProjectName
	exist, err := dao.ProjectExists(projectName)
	if err != nil {
		log.Errorf("Error happened checking project existence in db, error: %v, project name: %s", err, projectName)
	}
	if exist {
		p.RenderError(http.StatusConflict, "")
		return
	}
	project := models.Project{OwnerID: p.userID, Name: projectName, CreationTime: time.Now(), Public: public}
	projectID, err := dao.AddProject(project)
	if err != nil {
		log.Errorf("Failed to add project, error: %v", err)
		p.RenderError(http.StatusInternalServerError, "Failed to add project")
	}

	p.Redirect(http.StatusCreated, strconv.FormatInt(projectID, 10))
}

// Head ...
func (p *ProjectAPI) Head() {
	projectName := p.GetString("project_name")
	if len(projectName) == 0 {
		p.CustomAbort(http.StatusBadRequest, "project_name is needed")
	}

	project, err := dao.GetProjectByName(projectName)
	if err != nil {
		log.Errorf("error occurred in GetProjectByName: %v", err)
		p.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
	}

	// only public project can be Headed by user without login
	if project != nil && project.Public == 1 {
		return
	}

	userID := p.ValidateUser()
	if project == nil {
		p.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound))
	}

	if !checkProjectPermission(userID, project.ProjectID) {
		p.CustomAbort(http.StatusForbidden, http.StatusText(http.StatusForbidden))
	}
}

// Get ...
func (p *ProjectAPI) Get() {
	project, err := dao.GetProjectByID(p.projectID)
	if err != nil {
		log.Errorf("failed to get project %d: %v", p.projectID, err)
		p.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
	}

	if project.Public == 0 {
		userID := p.ValidateUser()
		if !checkProjectPermission(userID, p.projectID) {
			p.CustomAbort(http.StatusUnauthorized, http.StatusText(http.StatusUnauthorized))
		}
	}

	p.Data["json"] = project
	p.ServeJSON()
}

// List ...
func (p *ProjectAPI) List() {
	var projectList []models.Project
	projectName := p.GetString("project_name")
	if len(projectName) > 0 {
		projectName = "%" + projectName + "%"
	}
	var public int
	var err error
	isPublic := p.GetString("is_public")
	if len(isPublic) > 0 {
		public, err = strconv.Atoi(isPublic)
		if err != nil {
			log.Errorf("Error parsing public property: %d, error: %v", isPublic, err)
			p.CustomAbort(http.StatusBadRequest, "invalid project Id")
		}
	}
	isAdmin := false
	if public == 1 {
		projectList, err = dao.GetPublicProjects(projectName)
	} else {
		//if the request is not for public projects, user must login or provide credential
		p.userID = p.ValidateUser()
		isAdmin, err = dao.IsAdminRole(p.userID)
		if err != nil {
			log.Errorf("Error occured in check admin, error: %v", err)
			p.CustomAbort(http.StatusInternalServerError, "Internal error.")
		}
		if isAdmin {
			projectList, err = dao.GetAllProjects(projectName)
		} else {
			projectList, err = dao.GetUserRelevantProjects(p.userID, projectName)
		}
	}
	if err != nil {
		log.Errorf("Error occured in get projects info, error: %v", err)
		p.CustomAbort(http.StatusInternalServerError, "Internal error.")
	}
	for i := 0; i < len(projectList); i++ {
		if public != 1 {
			if isAdmin {
				projectList[i].Role = models.PROJECTADMIN
			}
			if projectList[i].Role == models.PROJECTADMIN {
				projectList[i].Togglable = true
			}
		}
		projectList[i].RepoCount = getRepoCountByProject(projectList[i].Name)
	}
	p.Data["json"] = projectList
	p.ServeJSON()
}

// ToggleProjectPulic handles request PUT /api/projects/:id/publicity
func (p *ProjectAPI) ToggleProjectPublic() {
	p.userID = p.ValidateUser()
	var req projectReq
	var public int

	projectID, err := strconv.ParseInt(p.Ctx.Input.Param(":id"), 10, 64)
	if err != nil {
		log.Errorf("Error parsing project id: %d, error: %v", projectID, err)
		p.RenderError(http.StatusBadRequest, "invalid project id")
		return
	}

	p.DecodeJSONReq(&req)
	if req.Public {
		public = 1
	}
	if !isProjectAdmin(p.userID, projectID) {
		log.Warningf("Current user, id: %d does not have project admin role for project, id: %d", p.userID, projectID)
		p.RenderError(http.StatusForbidden, "")
		return
	}
	err = dao.ToggleProjectPublicity(p.projectID, public)
	if err != nil {
		log.Errorf("Error while updating project, project id: %d, error: %v", projectID, err)
		p.RenderError(http.StatusInternalServerError, "Failed to update project")
	}
}

// FilterAccessLog handles GET to /api/projects/{}/logs
func (p *ProjectAPI) FilterAccessLog() {
	p.userID = p.ValidateUser()

	var filter models.AccessLog
	p.DecodeJSONReq(&filter)

	username := filter.Username
	keywords := filter.Keywords

	beginTime := time.Unix(filter.BeginTimestamp, 0)
	endTime := time.Unix(filter.EndTimestamp, 0)

	query := models.AccessLog{ProjectID: p.projectID, Username: "%" + username + "%", Keywords: keywords, BeginTime: beginTime, BeginTimestamp: filter.BeginTimestamp, EndTime: endTime, EndTimestamp: filter.EndTimestamp}

	log.Infof("Query AccessLog: begin: %v, end: %v, keywords: %s", query.BeginTime, query.EndTime, query.Keywords)

	accessLogList, err := dao.GetAccessLogs(query)
	if err != nil {
		log.Errorf("Error occurred in GetAccessLogs, error: %v", err)
		p.CustomAbort(http.StatusInternalServerError, "Internal error.")
	}
	p.Data["json"] = accessLogList

	p.ServeJSON()
}

func isProjectAdmin(userID int, pid int64) bool {
	isSysAdmin, err := dao.IsAdminRole(userID)
	if err != nil {
		log.Errorf("Error occurred in IsAdminRole, returning false, error: %v", err)
		return false
	}

	if isSysAdmin {
		return true
	}

	rolelist, err := dao.GetUserProjectRoles(userID, pid)
	if err != nil {
		log.Errorf("Error occurred in GetUserProjectRoles, returning false, error: %v", err)
		return false
	}

	hasProjectAdminRole := false
	for _, role := range rolelist {
		if role.RoleID == models.PROJECTADMIN {
			hasProjectAdminRole = true
			break
		}
	}

	return hasProjectAdminRole
}

func validateProjectReq(req projectReq) error {
	pn := req.ProjectName
	if len(pn) == 0 {
		return fmt.Errorf("Project name can not be empty")
	}
	if isIllegalLength(req.ProjectName, projectNameMinLen, projectNameMaxLen) {
		return fmt.Errorf("project name is illegal in length. (greater than 4 or less than 30)")
	}
	if isContainIllegalChar(req.ProjectName, []string{"~", "-", "$", "\\", "[", "]", "{", "}", "(", ")", "&", "^", "%", "*", "<", ">", "\"", "'", "/", "?", "@"}) {
		return fmt.Errorf("project name contains illegal characters")
	}
	for _, v := range pn {
		if !unicode.IsLower(v) {
			return fmt.Errorf("project name must be in lower case")
		}
	}
	return nil
}
<commit_msg>modify as travis CI required<commit_after>/*
   Copyright (c) 2016 VMware, Inc. All Rights Reserved.
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package api

import (
	"fmt"
	"net/http"
	"unicode"

	"github.com/vmware/harbor/dao"
	"github.com/vmware/harbor/models"
	"github.com/vmware/harbor/utils/log"

	"strconv"
	"time"
)

// ProjectAPI handles request to /api/projects/{} /api/projects/{}/logs
type ProjectAPI struct {
	BaseAPI
	userID    int
	projectID int64
}

type projectReq struct {
	ProjectName string `json:"project_name"`
	Public      bool   `json:"public"`
}

const projectNameMaxLen int = 30
const projectNameMinLen int = 4

// Prepare validates the URL and the user
func (p *ProjectAPI) Prepare() {
	idStr := p.Ctx.Input.Param(":id")
	if len(idStr) > 0 {
		var err error
		p.projectID, err = strconv.ParseInt(idStr, 10, 64)
		if err != nil {
			log.Errorf("Error parsing project id: %s, error: %v", idStr, err)
			p.CustomAbort(http.StatusBadRequest, "invalid project id")
		}
		exist, err := dao.ProjectExists(p.projectID)
		if err != nil {
			log.Errorf("Error occurred in ProjectExists, error: %v", err)
			p.CustomAbort(http.StatusInternalServerError, "Internal error.")
		}
		if !exist {
			p.CustomAbort(http.StatusNotFound, fmt.Sprintf("project does not exist, id: %v", p.projectID))
		}
	}
}

// Post ...
func (p *ProjectAPI) Post() {
	p.userID = p.ValidateUser()

	var req projectReq
	var public int
	p.DecodeJSONReq(&req)
	if req.Public {
		public = 1
	}
	err := validateProjectReq(req)
	if err != nil {
		log.Errorf("Invalid project request, error: %v", err)
		p.RenderError(http.StatusBadRequest, "Invalid request for creating project")
		return
	}
	projectName := req.ProjectName
	exist, err := dao.ProjectExists(projectName)
	if err != nil {
		log.Errorf("Error happened checking project existence in db, error: %v, project name: %s", err, projectName)
	}
	if exist {
		p.RenderError(http.StatusConflict, "")
		return
	}
	project := models.Project{OwnerID: p.userID, Name: projectName, CreationTime: time.Now(), Public: public}
	projectID, err := dao.AddProject(project)
	if err != nil {
		log.Errorf("Failed to add project, error: %v", err)
		p.RenderError(http.StatusInternalServerError, "Failed to add project")
	}

	p.Redirect(http.StatusCreated, strconv.FormatInt(projectID, 10))
}

// Head ...
func (p *ProjectAPI) Head() {
	projectName := p.GetString("project_name")
	if len(projectName) == 0 {
		p.CustomAbort(http.StatusBadRequest, "project_name is needed")
	}

	project, err := dao.GetProjectByName(projectName)
	if err != nil {
		log.Errorf("error occurred in GetProjectByName: %v", err)
		p.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
	}

	// only public project can be Headed by user without login
	if project != nil && project.Public == 1 {
		return
	}

	userID := p.ValidateUser()
	if project == nil {
		p.CustomAbort(http.StatusNotFound, http.StatusText(http.StatusNotFound))
	}

	if !checkProjectPermission(userID, project.ProjectID) {
		p.CustomAbort(http.StatusForbidden, http.StatusText(http.StatusForbidden))
	}
}

// Get ...
func (p *ProjectAPI) Get() {
	project, err := dao.GetProjectByID(p.projectID)
	if err != nil {
		log.Errorf("failed to get project %d: %v", p.projectID, err)
		p.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
	}

	if project.Public == 0 {
		userID := p.ValidateUser()
		if !checkProjectPermission(userID, p.projectID) {
			p.CustomAbort(http.StatusUnauthorized, http.StatusText(http.StatusUnauthorized))
		}
	}

	p.Data["json"] = project
	p.ServeJSON()
}

// List ...
func (p *ProjectAPI) List() {
	var projectList []models.Project
	projectName := p.GetString("project_name")
	if len(projectName) > 0 {
		projectName = "%" + projectName + "%"
	}
	var public int
	var err error
	isPublic := p.GetString("is_public")
	if len(isPublic) > 0 {
		public, err = strconv.Atoi(isPublic)
		if err != nil {
			log.Errorf("Error parsing public property: %d, error: %v", isPublic, err)
			p.CustomAbort(http.StatusBadRequest, "invalid project Id")
		}
	}
	isAdmin := false
	if public == 1 {
		projectList, err = dao.GetPublicProjects(projectName)
	} else {
		//if the request is not for public projects, user must login or provide credential
		p.userID = p.ValidateUser()
		isAdmin, err = dao.IsAdminRole(p.userID)
		if err != nil {
			log.Errorf("Error occured in check admin, error: %v", err)
			p.CustomAbort(http.StatusInternalServerError, "Internal error.")
		}
		if isAdmin {
			projectList, err = dao.GetAllProjects(projectName)
		} else {
			projectList, err = dao.GetUserRelevantProjects(p.userID, projectName)
		}
	}
	if err != nil {
		log.Errorf("Error occured in get projects info, error: %v", err)
		p.CustomAbort(http.StatusInternalServerError, "Internal error.")
	}
	for i := 0; i < len(projectList); i++ {
		if public != 1 {
			if isAdmin {
				projectList[i].Role = models.PROJECTADMIN
			}
			if projectList[i].Role == models.PROJECTADMIN {
				projectList[i].Togglable = true
			}
		}
		projectList[i].RepoCount = getRepoCountByProject(projectList[i].Name)
	}
	p.Data["json"] = projectList
	p.ServeJSON()
}

// ToggleProjectPublic handles request PUT /api/projects/:id/publicity
func (p *ProjectAPI) ToggleProjectPublic() {
	p.userID = p.ValidateUser()
	var req projectReq
	var public int

	projectID, err := strconv.ParseInt(p.Ctx.Input.Param(":id"), 10, 64)
	if err != nil {
		log.Errorf("Error parsing project id: %d, error: %v", projectID, err)
		p.RenderError(http.StatusBadRequest, "invalid project id")
		return
	}

	p.DecodeJSONReq(&req)
	if req.Public {
		public = 1
	}
	if !isProjectAdmin(p.userID, projectID) {
		log.Warningf("Current user, id: %d does not have project admin role for project, id: %d", p.userID, projectID)
		p.RenderError(http.StatusForbidden, "")
		return
	}
	err = dao.ToggleProjectPublicity(p.projectID, public)
	if err != nil {
		log.Errorf("Error while updating project, project id: %d, error: %v", projectID, err)
		p.RenderError(http.StatusInternalServerError, "Failed to update project")
	}
}

// FilterAccessLog handles GET to /api/projects/{}/logs
func (p *ProjectAPI) FilterAccessLog() {
	p.userID = p.ValidateUser()

	var filter models.AccessLog
	p.DecodeJSONReq(&filter)

	username := filter.Username
	keywords := filter.Keywords

	beginTime := time.Unix(filter.BeginTimestamp, 0)
	endTime := time.Unix(filter.EndTimestamp, 0)

	query := models.AccessLog{ProjectID: p.projectID, Username: "%" + username + "%", Keywords: keywords, BeginTime: beginTime, BeginTimestamp: filter.BeginTimestamp, EndTime: endTime, EndTimestamp: filter.EndTimestamp}

	log.Infof("Query AccessLog: begin: %v, end: %v, keywords: %s", query.BeginTime, query.EndTime, query.Keywords)

	accessLogList, err := dao.GetAccessLogs(query)
	if err != nil {
		log.Errorf("Error occurred in GetAccessLogs, error: %v", err)
		p.CustomAbort(http.StatusInternalServerError, "Internal error.")
	}
	p.Data["json"] = accessLogList

	p.ServeJSON()
}

func isProjectAdmin(userID int, pid int64) bool {
	isSysAdmin, err := dao.IsAdminRole(userID)
	if err != nil {
		log.Errorf("Error occurred in IsAdminRole, returning false, error: %v", err)
		return false
	}

	if isSysAdmin {
		return true
	}

	rolelist, err := dao.GetUserProjectRoles(userID, pid)
	if err != nil {
		log.Errorf("Error occurred in GetUserProjectRoles, returning false, error: %v", err)
		return false
	}

	hasProjectAdminRole := false
	for _, role := range rolelist {
		if role.RoleID == models.PROJECTADMIN {
			hasProjectAdminRole = true
			break
		}
	}

	return hasProjectAdminRole
}

func validateProjectReq(req projectReq) error {
	pn := req.ProjectName
	if len(pn) == 0 {
		return fmt.Errorf("Project name can not be empty")
	}
	if isIllegalLength(req.ProjectName, projectNameMinLen, projectNameMaxLen) {
		return fmt.Errorf("project name is illegal in length. (greater than 4 or less than 30)")
	}
	if isContainIllegalChar(req.ProjectName, []string{"~", "-", "$", "\\", "[", "]", "{", "}", "(", ")", "&", "^", "%", "*", "<", ">", "\"", "'", "/", "?", "@"}) {
		return fmt.Errorf("project name contains illegal characters")
	}
	for _, v := range pn {
		if !unicode.IsLower(v) {
			return fmt.Errorf("project name must be in lower case")
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package ipfix

import (
	"errors"
	"io"
	"sync"
)

type IPFIXDecoder struct {
	reader *Reader
}

type MessageHeader struct {
	Version    uint16 // Version of IPFIX to which this Message conforms
	Length     uint16 // Total length of the IPFIX Message, measured in octets
	ExportTime uint32 // Time at which the IPFIX Message Header leaves the Exporter
	SequenceNo uint32 // Incremental sequence counter modulo 2^32
	DomainID   uint32 // A 32-bit id that is locally unique to the Exporting Process
}

type TemplateHeader struct {
	TemplateID uint16
	FieldCount uint16
}

type Message struct {
	Header       MessageHeader
	TemplateSets []TemplateSet
	DataSets     []DataSet
}

type TemplateSet struct {
}

type DataSet struct {
}

type Session struct {
	buff *sync.Pool
}

type SetHeader struct {
	SetID  uint16
	Length uint16
}

var (
	errInvalidVersion = errors.New("invalid ipfix version")
)

func NewDecoder(r io.Reader) (*IPFIXDecoder, error) {
	data := make([]byte, 1500)
	n, err := r.Read(data)
	if err != nil {
		return nil, err
	}
	return &IPFIXDecoder{NewReader(data[:n])}, nil
}

func (d *IPFIXDecoder) Decode() error {
	var (
		msg Message
		err error
	)

	// IPFIX Message Header decoding
	if err = msg.Header.unmarshal(d.reader); err != nil {
		return err
	}
	// IPFIX Message Header validation
	if err = msg.Header.validate(); err != nil {
		return err
	}

	for d.reader.Len() > 0 {

		setHeader := new(SetHeader)
		setHeader.unmarshal(d.reader)

		if setHeader.Length < 4 {
			return io.ErrUnexpectedEOF
		}

		switch {
		case setHeader.SetID == 2:
			// Template set
			ts := new(TemplateSet)
			ts.unmarshal(d.reader)
		case setHeader.SetID == 3:
			// Option set
		case setHeader.SetID >= 4 && setHeader.SetID <= 255:
			// Reserved
		default:
			// data
		}

		break

	}

	return nil
}

// RFC 7011 - part 3.1. Message Header Format
// 0                   1                   2                   3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |       Version Number          |            Length             |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |                           Export Time                         |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |                       Sequence Number                         |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |                    Observation Domain ID                      |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (h *MessageHeader) unmarshal(r *Reader) error {
	var err error

	if h.Version, err = r.Uint16(); err != nil {
		return err
	}

	if h.Length, err = r.Uint16(); err != nil {
		return err
	}

	if h.ExportTime, err = r.Uint32(); err != nil {
		return err
	}

	if h.SequenceNo, err = r.Uint32(); err != nil {
		return err
	}

	if h.DomainID, err = r.Uint32(); err != nil {
		return err
	}

	return nil
}

func (h *MessageHeader) validate() error {
	if h.Version != 0x000a {
		return errInvalidVersion
	}

	// TODO: needs more validation

	return nil
}

// RFC 7011 - part 3.3.2 Set Header Format
// 0                   1                   2                   3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |          Set ID               |          Length               |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (h *SetHeader) unmarshal(r *Reader) error {
	var err error

	if h.SetID, err = r.Uint16(); err != nil {
		return err
	}

	if h.Length, err = r.Uint16(); err != nil {
		return err
	}

	return nil
}

// RFC 7011
// 0                   1                   2                   3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |         Template ID           |         Field Count           |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (t *TemplateHeader) unmarshal(r *Reader) error {
	var err error

	if t.TemplateID, err = r.Uint16(); err != nil {
		return err
	}

	if t.FieldCount, err = r.Uint16(); err != nil {
		return err
	}

	return nil

}

func (t *TemplateSet) unmarshal(r *Reader) error {
	th := new(TemplateHeader)
	th.unmarshal(r)

	println(th.TemplateID, th.FieldCount)

	return nil
}
<commit_msg>add TemplateFieldSpecifier decoder<commit_after>package ipfix

import (
	"errors"
	"io"
	"sync"
)

type IPFIXDecoder struct {
	reader *Reader
}

type MessageHeader struct {
	Version    uint16 // Version of IPFIX to which this Message conforms
	Length     uint16 // Total length of the IPFIX Message, measured in octets
	ExportTime uint32 // Time at which the IPFIX Message Header leaves the Exporter
	SequenceNo uint32 // Incremental sequence counter modulo 2^32
	DomainID   uint32 // A 32-bit id that is locally unique to the Exporting Process
}

type TemplateHeader struct {
	TemplateID uint16
	FieldCount uint16
}

type TemplateRecord struct {
	TemplateID              uint16
	TemplateFieldSpecifiers []TemplateFieldSpecifier
}

type TemplateFieldSpecifier struct {
	ElementID    uint16
	Length       uint16
	EnterpriseNo uint32
}

type Message struct {
	Header       MessageHeader
	TemplateSets []TemplateSet
	DataSets     []DataSet
}

type TemplateSet struct {
}

type DataSet struct {
}

type Session struct {
	buff *sync.Pool
}

type SetHeader struct {
	SetID  uint16
	Length uint16
}

var (
	errInvalidVersion = errors.New("invalid ipfix version")
)

func NewDecoder(r io.Reader) (*IPFIXDecoder, error) {
	data := make([]byte, 1500)
	n, err := r.Read(data)
	if err != nil {
		return nil, err
	}
	return &IPFIXDecoder{NewReader(data[:n])}, nil
}

func (d *IPFIXDecoder) Decode() error {
	var (
		msg Message
		err error
	)

	// IPFIX Message Header decoding
	if err = msg.Header.unmarshal(d.reader); err != nil {
		return err
	}
	// IPFIX Message Header validation
	if err = msg.Header.validate(); err != nil {
		return err
	}

	for d.reader.Len() > 0 {

		setHeader := new(SetHeader)
		setHeader.unmarshal(d.reader)

		if setHeader.Length < 4 {
			return io.ErrUnexpectedEOF
		}

		switch {
		case setHeader.SetID == 2:
			// Template set
			ts := new(TemplateSet)
			ts.unmarshal(d.reader)
		case setHeader.SetID == 3:
			// Option set
		case setHeader.SetID >= 4 && setHeader.SetID <= 255:
			// Reserved
		default:
			// data
		}

		break

	}

	return nil
}

// RFC 7011 - part 3.1. Message Header Format
// 0                   1                   2                   3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |       Version Number          |            Length             |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |                           Export Time                         |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |                       Sequence Number                         |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |                    Observation Domain ID                      |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (h *MessageHeader) unmarshal(r *Reader) error {
	var err error

	if h.Version, err = r.Uint16(); err != nil {
		return err
	}

	if h.Length, err = r.Uint16(); err != nil {
		return err
	}

	if h.ExportTime, err = r.Uint32(); err != nil {
		return err
	}

	if h.SequenceNo, err = r.Uint32(); err != nil {
		return err
	}

	if h.DomainID, err = r.Uint32(); err != nil {
		return err
	}

	return nil
}

func (h *MessageHeader) validate() error {
	if h.Version != 0x000a {
		return errInvalidVersion
	}

	// TODO: needs more validation

	return nil
}

// RFC 7011 - part 3.3.2 Set Header Format
// 0                   1                   2                   3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |          Set ID               |          Length               |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (h *SetHeader) unmarshal(r *Reader) error {
	var err error

	if h.SetID, err = r.Uint16(); err != nil {
		return err
	}

	if h.Length, err = r.Uint16(); err != nil {
		return err
	}

	return nil
}

// RFC 7011
// 0                   1                   2                   3
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |       Set ID = (2 or 3)       |          Length               |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |         Template ID           |         Field Count           |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (t *TemplateHeader) unmarshal(r *Reader) error {
	var err error

	if t.TemplateID, err = r.Uint16(); err != nil {
		return err
	}

	if t.FieldCount, err = r.Uint16(); err != nil {
		return err
	}

	return nil

}

// RFC 7011
// 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |E|  Information Element ident. |        Field Length           |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |                      Enterprise Number                        |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

func (f *TemplateFieldSpecifier) unmarshal(r *Reader) error {
	var err error

	if f.ElementID, err = r.Uint16(); err != nil {
		return err
	}
	if f.Length, err = r.Uint16(); err != nil {
		return err
	}
	println(f.ElementID)
	if f.ElementID > 0x8000 {
		f.ElementID = f.ElementID & 0x7fff
		if f.EnterpriseNo, err = r.Uint32(); err != nil {
			return err
		}
	}

	return nil
}

func (t *TemplateSet) unmarshal(r *Reader) error {
	th := new(TemplateHeader)
	tr := new(TemplateRecord)
	tf := new(TemplateFieldSpecifier)

	th.unmarshal(r)
	tf.unmarshal(r)
	_ = tr

	return nil
}
<|endoftext|>


<commit_before>// Package inject 提供类似Guice Injector的依赖注入功能,可以用于简化服务的组装
package inject

import (
	"fmt"
	"reflect"
	"strings"

	c "github.com/d0ngw/go/common"
)

// Injector 类似Guice的注入器
// 绑定约束:相同类型的实例和名称在一个Injector中必须唯一
// Injector对于struct中可导出的字段进行注入,需要注入的字段应该使用inject tag进行说明
//  type InjectStruct struct{
//   s ServiceInterface{} `inject:"<name>,[optional]"`
//  }
// inject格式:
//  name: 绑定的名称,必须的,如果没有名称,使用`_`代替
//  optional: 标识可选的注入,即不是必须注入
type Injector struct {
	ununamed []*internalBind            //未命名的绑定
	named    map[string][]*internalBind //命名的绑定
	all      []*internalBind            //所有的绑定
}

// Inject tag常量
const (
	injectTagNoname = "_" //无名称注入
)

func (p *Injector) String() string {
	return fmt.Sprintf("Injector,unnamed binds:%v,named binds:%v", len(p.ununamed), len(p.named))
}

// NewInjector 创建一个Injector
func NewInjector(modules []*Module) *Injector {
	injector := &Injector{}
	injectorModule := NewModule()
	injectorModule.Bind(injector)
	modules = append(modules, injectorModule)
	unnamed, named, all := mergeBinds(modules)
	injector.ununamed = unnamed
	injector.named = named
	injector.all = all
	injector.injectModules()
	return injector
}

// Initialize 初始化injector
func (p *Injector) Initialize() error {
	// 执行初始化操作
	inits := p.GetInstancesByPrototype(struct{ s c.Initable }{})
	for _, init := range inits {
		if service, ok := init.(c.Service); ok {
			if !c.ServiceInit(service) {
				return fmt.Errorf("Init %s fail", service.Name())
			}
		} else {
			if err := init.(c.Initable).Init(); err != nil {
				return fmt.Errorf("Init %T fail,err:%s", init, err)
			}

		}
	}
	return nil
}

// Start 启动服务
func (p *Injector) Start() error {
	return p.startOrStop(true)
}

// Stop 停止服务
func (p *Injector) Stop() error {
	return p.startOrStop(false)
}

func (p *Injector) startOrStop(start bool) error {
	var services []c.Service
	for _, service := range p.GetInstancesByPrototype(struct{ s c.Service }{}) {
		services = append(services, service.(c.Service))
	}
	if len(services) == 0 {
		c.Infof("No service found,skip")
		return nil
	}

	sortedServices := c.NewServices(services, start)
	if start {
		if !sortedServices.Start() {
			return fmt.Errorf("Start servcie fail")
		}
	} else {
		if !sortedServices.Stop() {
			return fmt.Errorf("Stop servcie fail")
		}
	}
	return nil
}

// RequireInject 由Injector向targets注入服务
func (p *Injector) RequireInject(targets ...interface{}) {
	for _, target := range targets {
		p.injectInstance(target)
	}
}

// RequireInjectWithOverrideTags 由Injector向target注入服务,injectTags用于覆盖target struct字段中定义的inject tag
func (p *Injector) RequireInjectWithOverrideTags(target interface{}, injectTags map[string]string) {
	p.injectInstanceWithOverrideTags(target, injectTags)
}

// GetInstance 从Injector中查找与name和expectedType匹配的实例,如果没有找到返回nil
func (p *Injector) GetInstance(name string, expectedType reflect.Type) interface{} {
	bind := p.findBind(name, expectedType)
	if bind == nil {
		return nil
	}
	return bind.instance
}

// GetInstanceByPrototype 从Injector重查找与name和prototype匹配的实例,如果没有找到返回nil
func (p *Injector) GetInstanceByPrototype(name string, prototype interface{}) interface{} {
	return p.GetInstance(name, getFieldType(prototype, 0))
}

// GetInstancesByPrototype 从Injector里查找与prototype匹配的所有实例
func (p *Injector) GetInstancesByPrototype(prototype interface{}) []interface{} {
	expectedType := getFieldType(prototype, 0)
	var ret = p.findAllBind(p.all, expectedType)
	var retInstances []interface{}
	for _, bind := range ret {
		retInstances = append(retInstances, bind.instance)
	}
	return retInstances
}

func (p *Injector) findAllBind(binds []*internalBind, expectedType reflect.Type) []*internalBind {
	var ret []*internalBind
	for _, bind := range binds {
		canAssign := bind.injectType.AssignableTo(expectedType)
		if canAssign {
			ret = append(ret, bind)
		}
	}
	return ret
}

// findBind 寻找一个可用的绑定
func (p *Injector) findBind(name string, expectedType reflect.Type) *internalBind {
	var toFind []*internalBind
	if len(name) != 0 {
		//按名称查找
		if binds, ok := p.named[name]; ok {
			toFind = binds
		}
	} else {
		toFind = p.ununamed
	}

	var ret []*internalBind
	for _, bind := range toFind {
		canAssign := bind.injectType.AssignableTo(expectedType)
		if canAssign {
			ret = append(ret, bind)
		}
	}

	if len(ret) == 0 {
		return nil
	} else if len(ret) > 1 {
		bindsDesc := make([]string, 0, len(ret))
		for _, bind := range ret {
			bindsDesc = append(bindsDesc, bind.String())
		}
		panic(fmt.Errorf("Duplicate for %v#%s,available binds:[%v]", expectedType, name, strings.Join(bindsDesc, ",")))
	} else {
		return ret[0]
	}
}

// injectInstance 向target注入实例
func (p *Injector) injectInstance(target interface{}) {
	p.injectInstanceWithOverrideTags(target, map[string]string{})
}

// injectInstanceWithOverrideTAgs 向target注入实例,injectTags用于覆盖target struct中定义的inject tag定义
func (p *Injector) injectInstanceWithOverrideTags(target interface{}, injectTags map[string]string) {
	if len(injectTags) == 0 {
		if ok, injected := IsInjected(target); ok && injected {
			//已经完成注入,直接返回
			c.Debugf("%s has been injected,skip", target)
			return
		}
	}
	val := reflect.ValueOf(target)
	ind := reflect.Indirect(val)
	typ := ind.Type()

	//仅struct的指针才能被注入
	if val.Kind() != reflect.Ptr || ind.Kind() != reflect.Struct {
		c.Debugf("Skip inject:%#v", target)
		return
	}

	fieldNum := ind.NumField()
	for i := 0; i < fieldNum; i++ {
		field := typ.Field(i)
		fieldVal := ind.Field(i)
		if !fieldVal.CanSet() {
			c.Debugf("skip inject filed %s", field.Name)
			continue
		}

		if reflect.Indirect(fieldVal).Kind() == reflect.Struct {
			if fieldVal.Kind() == reflect.Ptr {
				p.injectInstanceWithOverrideTags(fieldVal.Interface(), injectTags)
			} else if fieldVal.CanAddr() && fieldVal.CanInterface() {
				p.injectInstanceWithOverrideTags(fieldVal.Addr().Interface(), injectTags)
			}
			continue
		}
		structField := typ.Field(i)
		sfTag := structField.Tag

		tag, ok := injectTags[structField.Name]
		if !ok {
			tag = sfTag.Get("inject")
		}
		found, name, optional := parseInjectTag(tag)
		if !found {
			continue
		}

		if name == injectTagNoname {
			name = ""
		}

		foundBind := p.findBind(name, structField.Type)
		if foundBind == nil {
			if !optional {
				panic(fmt.Errorf("Can't find bind instance for %v.%s", typ, structField.Name))
			} else {
				c.Infof("skip optional %s %s ", name, structField.Type)
				continue
			}
		}

		if !fieldVal.CanSet() {
			panic(fmt.Errorf("Can't inject unexported  %v.%s", typ, structField.Name))
		}

		fieldInjectDesc := fmt.Sprintf("%s.%s", typ.Name(), structField.Name)
		c.Debugf("Inject %T#%s to %s", foundBind.instance, foundBind.name, fieldInjectDesc)
		if foundBind.injectValue.Kind() == reflect.Ptr && foundBind.injectValue.Pointer() == val.Pointer() {
			injectSrcDesc := fmt.Sprintf("%#v(addr:%p)", foundBind.instance, foundBind.instance)
			injectTargetDesc := fmt.Sprintf("%s of %#v(addr:%p)", fieldInjectDesc, target, target)
			panic(fmt.Errorf("Found circular inject,src:%s,target:%s", injectSrcDesc, injectTargetDesc))
		}
		fieldVal.Set(foundBind.injectValue)
	}
}

// injectModules 注入Injector中各个Module中的绑定
func (p *Injector) injectModules() {
	c.Debugf("Inject internal")
	for _, bind := range p.ununamed {
		p.injectInstanceWithOverrideTags(bind.instance, bind.injectTags)
	}

	for _, binds := range p.named {
		for _, bind := range binds {
			p.injectInstanceWithOverrideTags(bind.instance, bind.injectTags)
		}
	}
}

// getFieldType 取得struct中field的类型
func getFieldType(structObj interface{}, fieldIndex int) reflect.Type {
	val := reflect.Indirect(reflect.ValueOf(structObj))
	return val.Field(fieldIndex).Type()
}

// parseInjectTag 解析inject tag,inject tag的格式
func parseInjectTag(tag string) (found bool, name string, optional bool) {
	tag = strings.TrimSpace(tag)
	if len(tag) == 0 {
		return
	}

	vals := strings.Split(tag, ",")
	if len(vals) > 0 {
		found = true
		name = strings.TrimSpace(vals[0])
	}

	others := vals[1:]
	for _, o := range others {
		o = strings.ToLower(strings.TrimSpace(o))
		if o == "optional" {
			optional = true
		}
	}
	return
}
<commit_msg>change log message<commit_after>// Package inject 提供类似Guice Injector的依赖注入功能,可以用于简化服务的组装
package inject

import (
	"fmt"
	"reflect"
	"strings"

	c "github.com/d0ngw/go/common"
)

// Injector 类似Guice的注入器
// 绑定约束:相同类型的实例和名称在一个Injector中必须唯一
// Injector对于struct中可导出的字段进行注入,需要注入的字段应该使用inject tag进行说明
//  type InjectStruct struct{
//   s ServiceInterface{} `inject:"<name>,[optional]"`
//  }
// inject格式:
//  name: 绑定的名称,必须的,如果没有名称,使用`_`代替
//  optional: 标识可选的注入,即不是必须注入
type Injector struct {
	ununamed []*internalBind            //未命名的绑定
	named    map[string][]*internalBind //命名的绑定
	all      []*internalBind            //所有的绑定
}

// Inject tag常量
const (
	injectTagNoname = "_" //无名称注入
)

func (p *Injector) String() string {
	return fmt.Sprintf("Injector,unnamed binds:%v,named binds:%v", len(p.ununamed), len(p.named))
}

// NewInjector 创建一个Injector
func NewInjector(modules []*Module) *Injector {
	injector := &Injector{}
	injectorModule := NewModule()
	injectorModule.Bind(injector)
	modules = append(modules, injectorModule)
	unnamed, named, all := mergeBinds(modules)
	injector.ununamed = unnamed
	injector.named = named
	injector.all = all
	injector.injectModules()
	return injector
}

// Initialize 初始化injector
func (p *Injector) Initialize() error {
	// 执行初始化操作
	inits := p.GetInstancesByPrototype(struct{ s c.Initable }{})
	for _, init := range inits {
		if service, ok := init.(c.Service); ok {
			if !c.ServiceInit(service) {
				return fmt.Errorf("Init %s fail", service.Name())
			}
		} else {
			if err := init.(c.Initable).Init(); err != nil {
				return fmt.Errorf("Init %T fail,err:%s", init, err)
			}

		}
	}
	return nil
}

// Start 启动服务
func (p *Injector) Start() error {
	return p.startOrStop(true)
}

// Stop 停止服务
func (p *Injector) Stop() error {
	return p.startOrStop(false)
}

func (p *Injector) startOrStop(start bool) error {
	var services []c.Service
	for _, service := range p.GetInstancesByPrototype(struct{ s c.Service }{}) {
		services = append(services, service.(c.Service))
	}
	if len(services) == 0 {
		c.Infof("No service found,skip")
		return nil
	}

	sortedServices := c.NewServices(services, start)
	if start {
		if !sortedServices.Start() {
			return fmt.Errorf("Start servcie fail")
		}
	} else {
		if !sortedServices.Stop() {
			return fmt.Errorf("Stop servcie fail")
		}
	}
	return nil
}

// RequireInject 由Injector向targets注入服务
func (p *Injector) RequireInject(targets ...interface{}) {
	for _, target := range targets {
		p.injectInstance(target)
	}
}

// RequireInjectWithOverrideTags 由Injector向target注入服务,injectTags用于覆盖target struct字段中定义的inject tag
func (p *Injector) RequireInjectWithOverrideTags(target interface{}, injectTags map[string]string) {
	p.injectInstanceWithOverrideTags(target, injectTags)
}

// GetInstance 从Injector中查找与name和expectedType匹配的实例,如果没有找到返回nil
func (p *Injector) GetInstance(name string, expectedType reflect.Type) interface{} {
	bind := p.findBind(name, expectedType)
	if bind == nil {
		return nil
	}
	return bind.instance
}

// GetInstanceByPrototype 从Injector重查找与name和prototype匹配的实例,如果没有找到返回nil
func (p *Injector) GetInstanceByPrototype(name string, prototype interface{}) interface{} {
	return p.GetInstance(name, getFieldType(prototype, 0))
}

// GetInstancesByPrototype 从Injector里查找与prototype匹配的所有实例
func (p *Injector) GetInstancesByPrototype(prototype interface{}) []interface{} {
	expectedType := getFieldType(prototype, 0)
	var ret = p.findAllBind(p.all, expectedType)
	var retInstances []interface{}
	for _, bind := range ret {
		retInstances = append(retInstances, bind.instance)
	}
	return retInstances
}

func (p *Injector) findAllBind(binds []*internalBind, expectedType reflect.Type) []*internalBind {
	var ret []*internalBind
	for _, bind := range binds {
		canAssign := bind.injectType.AssignableTo(expectedType)
		if canAssign {
			ret = append(ret, bind)
		}
	}
	return ret
}

// findBind 寻找一个可用的绑定
func (p *Injector) findBind(name string, expectedType reflect.Type) *internalBind {
	var toFind []*internalBind
	if len(name) != 0 {
		//按名称查找
		if binds, ok := p.named[name]; ok {
			toFind = binds
		}
	} else {
		toFind = p.ununamed
	}

	var ret []*internalBind
	for _, bind := range toFind {
		canAssign := bind.injectType.AssignableTo(expectedType)
		if canAssign {
			ret = append(ret, bind)
		}
	}

	if len(ret) == 0 {
		return nil
	} else if len(ret) > 1 {
		bindsDesc := make([]string, 0, len(ret))
		for _, bind := range ret {
			bindsDesc = append(bindsDesc, bind.String())
		}
		panic(fmt.Errorf("Duplicate for %v#%s,available binds:[%v]", expectedType, name, strings.Join(bindsDesc, ",")))
	} else {
		return ret[0]
	}
}

// injectInstance 向target注入实例
func (p *Injector) injectInstance(target interface{}) {
	p.injectInstanceWithOverrideTags(target, map[string]string{})
}

// injectInstanceWithOverrideTAgs 向target注入实例,injectTags用于覆盖target struct中定义的inject tag定义
func (p *Injector) injectInstanceWithOverrideTags(target interface{}, injectTags map[string]string) {
	if len(injectTags) == 0 {
		if ok, injected := IsInjected(target); ok && injected {
			//已经完成注入,直接返回
			c.Debugf("%T has been injected,skip", target)
			return
		}
	}
	val := reflect.ValueOf(target)
	ind := reflect.Indirect(val)
	typ := ind.Type()

	//仅struct的指针才能被注入
	if val.Kind() != reflect.Ptr || ind.Kind() != reflect.Struct {
		c.Debugf("Skip inject:%#v", target)
		return
	}

	fieldNum := ind.NumField()
	for i := 0; i < fieldNum; i++ {
		field := typ.Field(i)
		fieldVal := ind.Field(i)
		if !fieldVal.CanSet() {
			c.Debugf("skip inject filed %s", field.Name)
			continue
		}

		if reflect.Indirect(fieldVal).Kind() == reflect.Struct {
			if fieldVal.Kind() == reflect.Ptr {
				p.injectInstanceWithOverrideTags(fieldVal.Interface(), injectTags)
			} else if fieldVal.CanAddr() && fieldVal.CanInterface() {
				p.injectInstanceWithOverrideTags(fieldVal.Addr().Interface(), injectTags)
			}
			continue
		}
		structField := typ.Field(i)
		sfTag := structField.Tag

		tag, ok := injectTags[structField.Name]
		if !ok {
			tag = sfTag.Get("inject")
		}
		found, name, optional := parseInjectTag(tag)
		if !found {
			continue
		}

		if name == injectTagNoname {
			name = ""
		}

		foundBind := p.findBind(name, structField.Type)
		if foundBind == nil {
			if !optional {
				panic(fmt.Errorf("Can't find bind instance for %v.%s", typ, structField.Name))
			} else {
				c.Infof("skip optional %s %s ", name, structField.Type)
				continue
			}
		}

		if !fieldVal.CanSet() {
			panic(fmt.Errorf("Can't inject unexported  %v.%s", typ, structField.Name))
		}

		fieldInjectDesc := fmt.Sprintf("%s.%s", typ.Name(), structField.Name)
		c.Debugf("Inject %T#%s to %s", foundBind.instance, foundBind.name, fieldInjectDesc)
		if foundBind.injectValue.Kind() == reflect.Ptr && foundBind.injectValue.Pointer() == val.Pointer() {
			injectSrcDesc := fmt.Sprintf("%#v(addr:%p)", foundBind.instance, foundBind.instance)
			injectTargetDesc := fmt.Sprintf("%s of %#v(addr:%p)", fieldInjectDesc, target, target)
			panic(fmt.Errorf("Found circular inject,src:%s,target:%s", injectSrcDesc, injectTargetDesc))
		}
		fieldVal.Set(foundBind.injectValue)
	}
}

// injectModules 注入Injector中各个Module中的绑定
func (p *Injector) injectModules() {
	c.Debugf("Inject internal")
	for _, bind := range p.ununamed {
		p.injectInstanceWithOverrideTags(bind.instance, bind.injectTags)
	}

	for _, binds := range p.named {
		for _, bind := range binds {
			p.injectInstanceWithOverrideTags(bind.instance, bind.injectTags)
		}
	}
}

// getFieldType 取得struct中field的类型
func getFieldType(structObj interface{}, fieldIndex int) reflect.Type {
	val := reflect.Indirect(reflect.ValueOf(structObj))
	return val.Field(fieldIndex).Type()
}

// parseInjectTag 解析inject tag,inject tag的格式
func parseInjectTag(tag string) (found bool, name string, optional bool) {
	tag = strings.TrimSpace(tag)
	if len(tag) == 0 {
		return
	}

	vals := strings.Split(tag, ",")
	if len(vals) > 0 {
		found = true
		name = strings.TrimSpace(vals[0])
	}

	others := vals[1:]
	for _, o := range others {
		o = strings.ToLower(strings.TrimSpace(o))
		if o == "optional" {
			optional = true
		}
	}
	return
}
<|endoftext|>


<commit_before>package client

import (
	"os"
	"os/exec"
	"runtime"

	"golang.org/x/net/context"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/client"
	"github.com/maliceio/go-plugin-utils/utils"
	"github.com/maliceio/malice/config"
)

// NOTE: https://github.com/eris-ltd/eris-cli/blob/master/perform/docker_run.go

// Docker is the Malice docker client
type Docker struct {
	Client *client.Client
	ip     string
	port   string
}

// NewDockerClient creates a new Docker Client
func NewDockerClient() *Docker {
	var docker *client.Client
	var ip, port string
	var err error

	if _, found := os.LookupEnv("MALICE_IN_DOCKER"); found {
		defaultHeaders := map[string]string{"User-Agent": "engine-api-cli-1.0"}
		docker, err = client.NewClient("unix:///var/run/docker.sock", "v1.22", nil, defaultHeaders)
		if err != nil {
			log.Fatal(err)
		}
		// Check if client can connect
		if _, err = docker.Info(context.Background()); err != nil {
			handleClientError(err)
		} else {
			ip = "localhost"
			port = "2375"
			log.WithFields(log.Fields{"ip": ip, "port": port}).Debug("Connected to docker daemon native client")
		}
	} else {
		docker, err = client.NewEnvClient()

		// Check if client can connect
		if _, err = docker.Info(context.Background()); err != nil {
			// If failed to connect try to create docker client via socket

		} else {
			_, _, _, err := client.ParseHost(utils.Getopt("DOCKER_HOST", client.DefaultDockerHost))
			if err != nil {
				log.Error(err)
			}
			ip, port, err = parseDockerEndoint(utils.Getopt("DOCKER_HOST", config.Conf.Docker.EndPoint))
			if err != nil {
				log.Error(err)
			}
			log.WithFields(log.Fields{"ip": ip, "port": port}).Debug("Connected to docker daemon with docker-machine")
		}
	}

	return &Docker{
		Client: docker,
		ip:     ip,
		port:   port,
	}
}

// GetIP returns IP of docker client
func (docker *Docker) GetIP() string {
	return docker.ip
}

// TODO: Make this betta MUCHO betta
func handleClientError(dockerError error) {
	if dockerError != nil {
		log.WithFields(log.Fields{"env": config.Conf.Environment.Run}).Error("Unable to connect to docker client")
		switch runtime.GOOS {
		case "darwin":
			if _, err := exec.LookPath("docker-machine"); err != nil {
				log.Info("Please install docker-machine by running: ")
				log.Info(" - brew install docker-machine")
				log.Infof(" - brew install docker-machine\n\tdocker-machine create -d virtualbox %s", config.Conf.Docker.Name)
				log.Infof(" - eval $(docker-machine env %s)", config.Conf.Docker.Name)
			} else {
				log.Info("Please start and source the docker-machine env by running: ")
				log.Infof(" - docker-machine start %s", config.Conf.Docker.Name)
				log.Infof(" - eval $(docker-machine env %s)", config.Conf.Docker.Name)
			}
		case "linux":
			log.Info("Please start the docker daemon.")
		case "windows":
			if _, err := exec.LookPath("docker-machine.exe"); err != nil {
				log.Info("Please install docker-machine - https://www.docker.com/docker-toolbox")
			} else {
				log.Info("Please start and source the docker-machine env by running: ")
				log.Infof(" - docker-machine start %", config.Conf.Docker.Name)
				log.Infof(" - eval $(docker-machine env %s)", config.Conf.Docker.Name)
			}
		}
		// TODO Decide if I want to make docker machines or rely on user to create their own.
		// log.Info("Trying to create new docker-machine: ", "test")
		// MakeDockerMachine("test")
		os.Exit(2)
	}
}
<commit_msg>add more debugging<commit_after>package client

import (
	"os"
	"os/exec"
	"runtime"

	"golang.org/x/net/context"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/docker/client"
	"github.com/maliceio/go-plugin-utils/utils"
	"github.com/maliceio/malice/config"
)

// NOTE: https://github.com/eris-ltd/eris-cli/blob/master/perform/docker_run.go

// Docker is the Malice docker client
type Docker struct {
	Client *client.Client
	ip     string
	port   string
}

// NewDockerClient creates a new Docker Client
func NewDockerClient() *Docker {
	var docker *client.Client
	var ip, port string
	var err error

	if _, found := os.LookupEnv("MALICE_IN_DOCKER"); found {
		log.Debug("Running inside Docker...")
		log.Debug("Creating NewClient...")
		defaultHeaders := map[string]string{"User-Agent": "engine-api-cli-1.0"}
		docker, err = client.NewClient("unix:///var/run/docker.sock", "v1.23", nil, defaultHeaders)
		if err != nil {
			log.Fatal(err)
		}
		// Check if client can connect
		log.Debug("Docker Info...")
		if _, err = docker.Info(context.Background()); err != nil {
			log.Debug("Docker Info FAILED...")
			handleClientError(err)
		} else {
			ip = "localhost"
			port = "2375"
			log.WithFields(log.Fields{"ip": ip, "port": port}).Debug("Connected to docker daemon native client")
		}
	} else {
		log.Debug("Running outside Docker...")
		log.Debug("Creating NewEnvClient...")
		docker, err = client.NewEnvClient()

		// Check if client can connect
		log.Debug("Docker Info2...")
		if _, err = docker.Info(context.Background()); err != nil {
			// If failed to connect try to create docker client via socket
			log.Debug("Docker Info2 FAILED...")
		} else {
			_, _, _, err := client.ParseHost(utils.Getopt("DOCKER_HOST", client.DefaultDockerHost))
			if err != nil {
				log.Error(err)
			}
			ip, port, err = parseDockerEndoint(utils.Getopt("DOCKER_HOST", config.Conf.Docker.EndPoint))
			if err != nil {
				log.Error(err)
			}
			log.WithFields(log.Fields{"ip": ip, "port": port}).Debug("Connected to docker daemon with docker-machine")
		}
	}

	return &Docker{
		Client: docker,
		ip:     ip,
		port:   port,
	}
}

// GetIP returns IP of docker client
func (docker *Docker) GetIP() string {
	return docker.ip
}

// TODO: Make this betta MUCHO betta
func handleClientError(dockerError error) {
	if dockerError != nil {
		log.WithFields(log.Fields{"env": config.Conf.Environment.Run}).Error("Unable to connect to docker client")
		switch runtime.GOOS {
		case "darwin":
			if _, err := exec.LookPath("docker-machine"); err != nil {
				log.Info("Please install docker-machine by running: ")
				log.Info(" - brew install docker-machine")
				log.Infof(" - brew install docker-machine\n\tdocker-machine create -d virtualbox %s", config.Conf.Docker.Name)
				log.Infof(" - eval $(docker-machine env %s)", config.Conf.Docker.Name)
			} else {
				log.Info("Please start and source the docker-machine env by running: ")
				log.Infof(" - docker-machine start %s", config.Conf.Docker.Name)
				log.Infof(" - eval $(docker-machine env %s)", config.Conf.Docker.Name)
			}
		case "linux":
			log.Info("Please start the docker daemon.")
		case "windows":
			if _, err := exec.LookPath("docker-machine.exe"); err != nil {
				log.Info("Please install docker-machine - https://www.docker.com/docker-toolbox")
			} else {
				log.Info("Please start and source the docker-machine env by running: ")
				log.Infof(" - docker-machine start %", config.Conf.Docker.Name)
				log.Infof(" - eval $(docker-machine env %s)", config.Conf.Docker.Name)
			}
		}
		// TODO Decide if I want to make docker machines or rely on user to create their own.
		// log.Info("Trying to create new docker-machine: ", "test")
		// MakeDockerMachine("test")
		os.Exit(2)
	}
}
<|endoftext|>