<commit_before>package llbsolver

import (
	"strings"

	"github.com/containerd/containerd/platforms"
	"github.com/moby/buildkit/solver"
	"github.com/moby/buildkit/solver/pb"
	"github.com/moby/buildkit/source"
	"github.com/moby/buildkit/util/entitlements"
	digest "github.com/opencontainers/go-digest"
	specs "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

type vertex struct {
	sys     interface{}
	options solver.VertexOptions
	inputs  []solver.Edge
	digest  digest.Digest
	name    string
}

func (v *vertex) Digest() digest.Digest {
	return v.digest
}

func (v *vertex) Sys() interface{} {
	return v.sys
}

func (v *vertex) Options() solver.VertexOptions {
	return v.options
}

func (v *vertex) Inputs() []solver.Edge {
	return v.inputs
}

func (v *vertex) Name() string {
	if name, ok := v.options.Description["llb.customname"]; ok {
		return name
	}
	return v.name
}

type LoadOpt func(*pb.Op, *pb.OpMetadata, *solver.VertexOptions) error

func WithValidateCaps() LoadOpt {
	cs := pb.Caps.CapSet(pb.Caps.All())
	return func(_ *pb.Op, md *pb.OpMetadata, opt *solver.VertexOptions) error {
		if md != nil {
			for c := range md.Caps {
				if err := cs.Supports(c); err != nil {
					return err
				}
			}
		}
		return nil
	}
}

func WithCacheSources(cms []solver.CacheManager) LoadOpt {
	return func(_ *pb.Op, _ *pb.OpMetadata, opt *solver.VertexOptions) error {
		opt.CacheSources = cms
		return nil
	}
}

func RuntimePlatforms(p []specs.Platform) LoadOpt {
	var defaultPlatform *pb.Platform
	pp := make([]specs.Platform, len(p))
	for i := range p {
		pp[i] = platforms.Normalize(p[i])
	}
	return func(op *pb.Op, _ *pb.OpMetadata, opt *solver.VertexOptions) error {
		if op.Platform == nil {
			if defaultPlatform == nil {
				p := platforms.DefaultSpec()
				defaultPlatform = &pb.Platform{
					OS:           p.OS,
					Architecture: p.Architecture,
				}
			}
			op.Platform = defaultPlatform
		}
		if _, ok := op.Op.(*pb.Op_Exec); ok {
			var found bool
			for _, pp := range pp {
				if pp.OS == op.Platform.OS && pp.Architecture == op.Platform.Architecture && pp.Variant == op.Platform.Variant {
					found = true
					break
				}
			}
			if !found {
				return errors.Errorf("runtime execution on platform %s not supported", platforms.Format(specs.Platform{OS: op.Platform.OS, Architecture: op.Platform.Architecture, Variant: op.Platform.Variant}))
			}
		}
		return nil
	}
}

func ValidateEntitlements(ent entitlements.Set) LoadOpt {
	return func(op *pb.Op, _ *pb.OpMetadata, opt *solver.VertexOptions) error {
		switch op := op.Op.(type) {
		case *pb.Op_Exec:
			if op.Exec.Network == pb.NetMode_HOST {
				if !ent.Allowed(entitlements.EntitlementNetworkHost) {
					return errors.Errorf("%s is not allowed", entitlements.EntitlementNetworkHost)
				}
			}
			if op.Exec.Network == pb.NetMode_NONE {
				if !ent.Allowed(entitlements.EntitlementNetworkNone) {
					return errors.Errorf("%s is not allowed", entitlements.EntitlementNetworkNone)
				}
			}
		}
		return nil
	}
}

func Load(def *pb.Definition, opts ...LoadOpt) (solver.Edge, error) {
	return loadLLB(def, func(dgst digest.Digest, pbOp *pb.Op, load func(digest.Digest) (solver.Vertex, error)) (solver.Vertex, error) {
		opMetadata := def.Metadata[dgst]
		vtx, err := newVertex(dgst, pbOp, &opMetadata, load, opts...)
		if err != nil {
			return nil, err
		}
		return vtx, nil
	})
}

func newVertex(dgst digest.Digest, op *pb.Op, opMeta *pb.OpMetadata, load func(digest.Digest) (solver.Vertex, error), opts ...LoadOpt) (*vertex, error) {
	opt := solver.VertexOptions{}
	if opMeta != nil {
		opt.IgnoreCache = opMeta.IgnoreCache
		opt.Description = opMeta.Description
		if opMeta.ExportCache != nil {
			opt.ExportCache = &opMeta.ExportCache.Value
		}
	}
	for _, fn := range opts {
		if err := fn(op, opMeta, &opt); err != nil {
			return nil, err
		}
	}
	vtx := &vertex{sys: op, options: opt, digest: dgst, name: llbOpName(op)}
	for _, in := range op.Inputs {
		sub, err := load(in.Digest)
		if err != nil {
			return nil, err
		}
		vtx.inputs = append(vtx.inputs, solver.Edge{Index: solver.Index(in.Index), Vertex: sub})
	}
	return vtx, nil
}

// loadLLB loads LLB.
// fn is executed sequentially.
func loadLLB(def *pb.Definition, fn func(digest.Digest, *pb.Op, func(digest.Digest) (solver.Vertex, error)) (solver.Vertex, error)) (solver.Edge, error) {
	if len(def.Def) == 0 {
		return solver.Edge{}, errors.New("invalid empty definition")
	}

	allOps := make(map[digest.Digest]*pb.Op)

	var dgst digest.Digest

	for _, dt := range def.Def {
		var op pb.Op
		if err := (&op).Unmarshal(dt); err != nil {
			return solver.Edge{}, errors.Wrap(err, "failed to parse llb proto op")
		}
		dgst = digest.FromBytes(dt)
		allOps[dgst] = &op
	}

	lastOp := allOps[dgst]
	delete(allOps, dgst)
	dgst = lastOp.Inputs[0].Digest

	cache := make(map[digest.Digest]solver.Vertex)

	var rec func(dgst digest.Digest) (solver.Vertex, error)
	rec = func(dgst digest.Digest) (solver.Vertex, error) {
		if v, ok := cache[dgst]; ok {
			return v, nil
		}
		op, ok := allOps[dgst]
		if !ok {
			return nil, errors.Errorf("invalid missing input digest %s", dgst)
		}
		v, err := fn(dgst, op, rec)
		if err != nil {
			return nil, err
		}
		cache[dgst] = v
		return v, nil
	}

	v, err := rec(dgst)
	if err != nil {
		return solver.Edge{}, err
	}
	return solver.Edge{Vertex: v, Index: solver.Index(lastOp.Inputs[0].Index)}, nil
}

func llbOpName(op *pb.Op) string {
	switch op := op.Op.(type) {
	case *pb.Op_Source:
		if id, err := source.FromLLB(op, nil); err == nil {
			if id, ok := id.(*source.LocalIdentifier); ok {
				if len(id.IncludePatterns) == 1 {
					return op.Source.Identifier + " (" + id.IncludePatterns[0] + ")"
				}
			}
		}
		return op.Source.Identifier
	case *pb.Op_Exec:
		return strings.Join(op.Exec.Meta.Args, " ")
	case *pb.Op_Build:
		return "build"
	default:
		return "unknown"
	}
}
<commit_msg>Don't miss the Variant for arm2 platform<commit_after>package llbsolver

import (
	"strings"

	"github.com/containerd/containerd/platforms"
	"github.com/moby/buildkit/solver"
	"github.com/moby/buildkit/solver/pb"
	"github.com/moby/buildkit/source"
	"github.com/moby/buildkit/util/entitlements"
	digest "github.com/opencontainers/go-digest"
	specs "github.com/opencontainers/image-spec/specs-go/v1"
	"github.com/pkg/errors"
)

type vertex struct {
	sys     interface{}
	options solver.VertexOptions
	inputs  []solver.Edge
	digest  digest.Digest
	name    string
}

func (v *vertex) Digest() digest.Digest {
	return v.digest
}

func (v *vertex) Sys() interface{} {
	return v.sys
}

func (v *vertex) Options() solver.VertexOptions {
	return v.options
}

func (v *vertex) Inputs() []solver.Edge {
	return v.inputs
}

func (v *vertex) Name() string {
	if name, ok := v.options.Description["llb.customname"]; ok {
		return name
	}
	return v.name
}

type LoadOpt func(*pb.Op, *pb.OpMetadata, *solver.VertexOptions) error

func WithValidateCaps() LoadOpt {
	cs := pb.Caps.CapSet(pb.Caps.All())
	return func(_ *pb.Op, md *pb.OpMetadata, opt *solver.VertexOptions) error {
		if md != nil {
			for c := range md.Caps {
				if err := cs.Supports(c); err != nil {
					return err
				}
			}
		}
		return nil
	}
}

func WithCacheSources(cms []solver.CacheManager) LoadOpt {
	return func(_ *pb.Op, _ *pb.OpMetadata, opt *solver.VertexOptions) error {
		opt.CacheSources = cms
		return nil
	}
}

func RuntimePlatforms(p []specs.Platform) LoadOpt {
	var defaultPlatform *pb.Platform
	pp := make([]specs.Platform, len(p))
	for i := range p {
		pp[i] = platforms.Normalize(p[i])
	}
	return func(op *pb.Op, _ *pb.OpMetadata, opt *solver.VertexOptions) error {
		if op.Platform == nil {
			if defaultPlatform == nil {
				p := platforms.DefaultSpec()
				defaultPlatform = &pb.Platform{
					OS:           p.OS,
					Architecture: p.Architecture,
					Variant:      p.Variant,
				}
			}
			op.Platform = defaultPlatform
		}
		platform := specs.Platform{OS: op.Platform.OS, Architecture: op.Platform.Architecture, Variant: op.Platform.Variant}
		normalizedPlatform := platforms.Normalize(platform)

		op.Platform = &pb.Platform{
			OS:           normalizedPlatform.OS,
			Architecture: normalizedPlatform.Architecture,
			Variant:      normalizedPlatform.Variant,
		}

		if _, ok := op.Op.(*pb.Op_Exec); ok {
			var found bool
			for _, pp := range pp {
				if pp.OS == op.Platform.OS && pp.Architecture == op.Platform.Architecture && pp.Variant == op.Platform.Variant {
					found = true
					break
				}
			}
			if !found {
				return errors.Errorf("runtime execution on platform %s not supported", platforms.Format(specs.Platform{OS: op.Platform.OS, Architecture: op.Platform.Architecture, Variant: op.Platform.Variant}))
			}
		}
		return nil
	}
}

func ValidateEntitlements(ent entitlements.Set) LoadOpt {
	return func(op *pb.Op, _ *pb.OpMetadata, opt *solver.VertexOptions) error {
		switch op := op.Op.(type) {
		case *pb.Op_Exec:
			if op.Exec.Network == pb.NetMode_HOST {
				if !ent.Allowed(entitlements.EntitlementNetworkHost) {
					return errors.Errorf("%s is not allowed", entitlements.EntitlementNetworkHost)
				}
			}
			if op.Exec.Network == pb.NetMode_NONE {
				if !ent.Allowed(entitlements.EntitlementNetworkNone) {
					return errors.Errorf("%s is not allowed", entitlements.EntitlementNetworkNone)
				}
			}
		}
		return nil
	}
}

func Load(def *pb.Definition, opts ...LoadOpt) (solver.Edge, error) {
	return loadLLB(def, func(dgst digest.Digest, pbOp *pb.Op, load func(digest.Digest) (solver.Vertex, error)) (solver.Vertex, error) {
		opMetadata := def.Metadata[dgst]
		vtx, err := newVertex(dgst, pbOp, &opMetadata, load, opts...)
		if err != nil {
			return nil, err
		}
		return vtx, nil
	})
}

func newVertex(dgst digest.Digest, op *pb.Op, opMeta *pb.OpMetadata, load func(digest.Digest) (solver.Vertex, error), opts ...LoadOpt) (*vertex, error) {
	opt := solver.VertexOptions{}
	if opMeta != nil {
		opt.IgnoreCache = opMeta.IgnoreCache
		opt.Description = opMeta.Description
		if opMeta.ExportCache != nil {
			opt.ExportCache = &opMeta.ExportCache.Value
		}
	}
	for _, fn := range opts {
		if err := fn(op, opMeta, &opt); err != nil {
			return nil, err
		}
	}
	vtx := &vertex{sys: op, options: opt, digest: dgst, name: llbOpName(op)}
	for _, in := range op.Inputs {
		sub, err := load(in.Digest)
		if err != nil {
			return nil, err
		}
		vtx.inputs = append(vtx.inputs, solver.Edge{Index: solver.Index(in.Index), Vertex: sub})
	}
	return vtx, nil
}

// loadLLB loads LLB.
// fn is executed sequentially.
func loadLLB(def *pb.Definition, fn func(digest.Digest, *pb.Op, func(digest.Digest) (solver.Vertex, error)) (solver.Vertex, error)) (solver.Edge, error) {
	if len(def.Def) == 0 {
		return solver.Edge{}, errors.New("invalid empty definition")
	}

	allOps := make(map[digest.Digest]*pb.Op)

	var dgst digest.Digest

	for _, dt := range def.Def {
		var op pb.Op
		if err := (&op).Unmarshal(dt); err != nil {
			return solver.Edge{}, errors.Wrap(err, "failed to parse llb proto op")
		}
		dgst = digest.FromBytes(dt)
		allOps[dgst] = &op
	}

	lastOp := allOps[dgst]
	delete(allOps, dgst)
	dgst = lastOp.Inputs[0].Digest

	cache := make(map[digest.Digest]solver.Vertex)

	var rec func(dgst digest.Digest) (solver.Vertex, error)
	rec = func(dgst digest.Digest) (solver.Vertex, error) {
		if v, ok := cache[dgst]; ok {
			return v, nil
		}
		op, ok := allOps[dgst]
		if !ok {
			return nil, errors.Errorf("invalid missing input digest %s", dgst)
		}
		v, err := fn(dgst, op, rec)
		if err != nil {
			return nil, err
		}
		cache[dgst] = v
		return v, nil
	}

	v, err := rec(dgst)
	if err != nil {
		return solver.Edge{}, err
	}
	return solver.Edge{Vertex: v, Index: solver.Index(lastOp.Inputs[0].Index)}, nil
}

func llbOpName(op *pb.Op) string {
	switch op := op.Op.(type) {
	case *pb.Op_Source:
		if id, err := source.FromLLB(op, nil); err == nil {
			if id, ok := id.(*source.LocalIdentifier); ok {
				if len(id.IncludePatterns) == 1 {
					return op.Source.Identifier + " (" + id.IncludePatterns[0] + ")"
				}
			}
		}
		return op.Source.Identifier
	case *pb.Op_Exec:
		return strings.Join(op.Exec.Meta.Args, " ")
	case *pb.Op_Build:
		return "build"
	default:
		return "unknown"
	}
}
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"

	"github.com/hashicorp/terraform/config"
)

// ResourceProvisionerConfig is used to pair a provisioner
// with its provided configuration. This allows us to use singleton
// instances of each ResourceProvisioner and to keep the relevant
// configuration instead of instantiating a new Provisioner for each
// resource.
type ResourceProvisionerConfig struct {
	Type        string
	Provisioner ResourceProvisioner
	Config      *ResourceConfig
	RawConfig   *config.RawConfig
	ConnInfo    *config.RawConfig
}

// Resource encapsulates a resource, its configuration, its provider,
// its current state, and potentially a desired diff from the state it
// wants to reach.
type Resource struct {
	// These are all used by the new EvalNode stuff.
	Name       string
	Type       string
	CountIndex int

	// These aren't really used anymore anywhere, but we keep them around
	// since we haven't done a proper cleanup yet.
	Id           string
	Info         *InstanceInfo
	Config       *ResourceConfig
	Dependencies []string
	Diff         *InstanceDiff
	Provider     ResourceProvider
	State        *InstanceState
	Provisioners []*ResourceProvisionerConfig
	Flags        ResourceFlag
}

// ResourceKind specifies what kind of instance we're working with, whether
// its a primary instance, a tainted instance, or an orphan.
type ResourceFlag byte

const (
	FlagPrimary ResourceFlag = 1 << iota
	FlagTainted
	FlagOrphan
	FlagReplacePrimary
	FlagDeposed
)

// InstanceInfo is used to hold information about the instance and/or
// resource being modified.
type InstanceInfo struct {
	// Id is a unique name to represent this instance. This is not related
	// to InstanceState.ID in any way.
	Id string

	// ModulePath is the complete path of the module containing this
	// instance.
	ModulePath []string

	// Type is the resource type of this instance
	Type string
}

// HumanId is a unique Id that is human-friendly and useful for UI elements.
func (i *InstanceInfo) HumanId() string {
	if len(i.ModulePath) <= 1 {
		return i.Id
	}

	return fmt.Sprintf(
		"module.%s.%s",
		strings.Join(i.ModulePath[1:], "."),
		i.Id)
}

// ResourceConfig holds the configuration given for a resource. This is
// done instead of a raw `map[string]interface{}` type so that rich
// methods can be added to it to make dealing with it easier.
type ResourceConfig struct {
	ComputedKeys []string
	Raw          map[string]interface{}
	Config       map[string]interface{}

	raw *config.RawConfig
}

// NewResourceConfig creates a new ResourceConfig from a config.RawConfig.
func NewResourceConfig(c *config.RawConfig) *ResourceConfig {
	result := &ResourceConfig{raw: c}
	result.interpolateForce()
	return result
}

// CheckSet checks that the given list of configuration keys is
// properly set. If not, errors are returned for each unset key.
//
// This is useful to be called in the Validate method of a ResourceProvider.
func (c *ResourceConfig) CheckSet(keys []string) []error {
	var errs []error

	for _, k := range keys {
		if !c.IsSet(k) {
			errs = append(errs, fmt.Errorf("%s must be set", k))
		}
	}

	return errs
}

// Get looks up a configuration value by key and returns the value.
//
// The second return value is true if the get was successful. Get will
// not succeed if the value is being computed.
func (c *ResourceConfig) Get(k string) (interface{}, bool) {
	// First try to get it from c.Config since that has interpolated values
	result, ok := c.get(k, c.Config)
	if ok {
		return result, ok
	}

	// Otherwise, just get it from the raw config
	return c.get(k, c.Raw)
}

// GetRaw looks up a configuration value by key and returns the value,
// from the raw, uninterpolated config.
//
// The second return value is true if the get was successful. Get will
// not succeed if the value is being computed.
func (c *ResourceConfig) GetRaw(k string) (interface{}, bool) {
	return c.get(k, c.Raw)
}

// IsComputed returns whether the given key is computed or not.
func (c *ResourceConfig) IsComputed(k string) bool {
	_, ok := c.get(k, c.Config)
	_, okRaw := c.get(k, c.Raw)
	return !ok && okRaw
}

// IsSet checks if the key in the configuration is set. A key is set if
// it has a value or the value is being computed (is unknown currently).
//
// This function should be used rather than checking the keys of the
// raw configuration itself, since a key may be omitted from the raw
// configuration if it is being computed.
func (c *ResourceConfig) IsSet(k string) bool {
	if c == nil {
		return false
	}

	for _, ck := range c.ComputedKeys {
		if ck == k {
			return true
		}
	}

	if _, ok := c.Get(k); ok {
		return true
	}

	return false
}

func (c *ResourceConfig) get(
	k string, raw map[string]interface{}) (interface{}, bool) {
	parts := strings.Split(k, ".")
	if len(parts) == 1 && parts[0] == "" {
		parts = nil
	}

	var current interface{} = raw
	for _, part := range parts {
		if current == nil {
			return nil, false
		}

		cv := reflect.ValueOf(current)
		switch cv.Kind() {
		case reflect.Map:
			v := cv.MapIndex(reflect.ValueOf(part))
			if !v.IsValid() {
				return nil, false
			}
			current = v.Interface()
		case reflect.Slice:
			if part == "#" {
				current = cv.Len()
			} else {
				i, err := strconv.ParseInt(part, 0, 0)
				if err != nil {
					return nil, false
				}
				if i >= int64(cv.Len()) {
					return nil, false
				}
				current = cv.Index(int(i)).Interface()
			}
		default:
			panic(fmt.Sprintf("Unknown kind: %s", cv.Kind()))
		}
	}

	return current, true
}

// interpolateForce is a temporary thing. We want to get rid of interpolate
// above and likewise this, but it can only be done after the f-ast-graph
// refactor is complete.
func (c *ResourceConfig) interpolateForce() {
	if c.raw == nil {
		var err error
		c.raw, err = config.NewRawConfig(make(map[string]interface{}))
		if err != nil {
			panic(err)
		}
	}

	c.ComputedKeys = c.raw.UnknownKeys()
	c.Raw = c.raw.Raw
	c.Config = c.raw.Config()
}
<commit_msg>core: Allow "." character in map keys<commit_after>package terraform

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"

	"github.com/hashicorp/terraform/config"
)

// ResourceProvisionerConfig is used to pair a provisioner
// with its provided configuration. This allows us to use singleton
// instances of each ResourceProvisioner and to keep the relevant
// configuration instead of instantiating a new Provisioner for each
// resource.
type ResourceProvisionerConfig struct {
	Type        string
	Provisioner ResourceProvisioner
	Config      *ResourceConfig
	RawConfig   *config.RawConfig
	ConnInfo    *config.RawConfig
}

// Resource encapsulates a resource, its configuration, its provider,
// its current state, and potentially a desired diff from the state it
// wants to reach.
type Resource struct {
	// These are all used by the new EvalNode stuff.
	Name       string
	Type       string
	CountIndex int

	// These aren't really used anymore anywhere, but we keep them around
	// since we haven't done a proper cleanup yet.
	Id           string
	Info         *InstanceInfo
	Config       *ResourceConfig
	Dependencies []string
	Diff         *InstanceDiff
	Provider     ResourceProvider
	State        *InstanceState
	Provisioners []*ResourceProvisionerConfig
	Flags        ResourceFlag
}

// ResourceKind specifies what kind of instance we're working with, whether
// its a primary instance, a tainted instance, or an orphan.
type ResourceFlag byte

// InstanceInfo is used to hold information about the instance and/or
// resource being modified.
type InstanceInfo struct {
	// Id is a unique name to represent this instance. This is not related
	// to InstanceState.ID in any way.
	Id string

	// ModulePath is the complete path of the module containing this
	// instance.
	ModulePath []string

	// Type is the resource type of this instance
	Type string
}

// HumanId is a unique Id that is human-friendly and useful for UI elements.
func (i *InstanceInfo) HumanId() string {
	if len(i.ModulePath) <= 1 {
		return i.Id
	}

	return fmt.Sprintf(
		"module.%s.%s",
		strings.Join(i.ModulePath[1:], "."),
		i.Id)
}

// ResourceConfig holds the configuration given for a resource. This is
// done instead of a raw `map[string]interface{}` type so that rich
// methods can be added to it to make dealing with it easier.
type ResourceConfig struct {
	ComputedKeys []string
	Raw          map[string]interface{}
	Config       map[string]interface{}

	raw *config.RawConfig
}

// NewResourceConfig creates a new ResourceConfig from a config.RawConfig.
func NewResourceConfig(c *config.RawConfig) *ResourceConfig {
	result := &ResourceConfig{raw: c}
	result.interpolateForce()
	return result
}

// CheckSet checks that the given list of configuration keys is
// properly set. If not, errors are returned for each unset key.
//
// This is useful to be called in the Validate method of a ResourceProvider.
func (c *ResourceConfig) CheckSet(keys []string) []error {
	var errs []error

	for _, k := range keys {
		if !c.IsSet(k) {
			errs = append(errs, fmt.Errorf("%s must be set", k))
		}
	}

	return errs
}

// Get looks up a configuration value by key and returns the value.
//
// The second return value is true if the get was successful. Get will
// not succeed if the value is being computed.
func (c *ResourceConfig) Get(k string) (interface{}, bool) {
	// First try to get it from c.Config since that has interpolated values
	result, ok := c.get(k, c.Config)
	if ok {
		return result, ok
	}

	// Otherwise, just get it from the raw config
	return c.get(k, c.Raw)
}

// GetRaw looks up a configuration value by key and returns the value,
// from the raw, uninterpolated config.
//
// The second return value is true if the get was successful. Get will
// not succeed if the value is being computed.
func (c *ResourceConfig) GetRaw(k string) (interface{}, bool) {
	return c.get(k, c.Raw)
}

// IsComputed returns whether the given key is computed or not.
func (c *ResourceConfig) IsComputed(k string) bool {
	_, ok := c.get(k, c.Config)
	_, okRaw := c.get(k, c.Raw)
	return !ok && okRaw
}

// IsSet checks if the key in the configuration is set. A key is set if
// it has a value or the value is being computed (is unknown currently).
//
// This function should be used rather than checking the keys of the
// raw configuration itself, since a key may be omitted from the raw
// configuration if it is being computed.
func (c *ResourceConfig) IsSet(k string) bool {
	if c == nil {
		return false
	}

	for _, ck := range c.ComputedKeys {
		if ck == k {
			return true
		}
	}

	if _, ok := c.Get(k); ok {
		return true
	}

	return false
}

func (c *ResourceConfig) get(
	k string, raw map[string]interface{}) (interface{}, bool) {
	parts := strings.Split(k, ".")
	if len(parts) == 1 && parts[0] == "" {
		parts = nil
	}

	var current interface{} = raw
	var previous interface{} = nil
	for i, part := range parts {
		if current == nil {
			return nil, false
		}

		cv := reflect.ValueOf(current)
		switch cv.Kind() {
		case reflect.Map:
			previous = current
			v := cv.MapIndex(reflect.ValueOf(part))
			if !v.IsValid() {
				if i > 0 && i != (len(parts)-1) {
					tryKey := strings.Join(parts[i:], ".")
					v := cv.MapIndex(reflect.ValueOf(tryKey))
					if !v.IsValid() {
						return nil, false
					}
					return v.Interface(), true
				}

				return nil, false
			}
			current = v.Interface()
		case reflect.Slice:
			previous = current
			if part == "#" {
				current = cv.Len()
			} else {
				i, err := strconv.ParseInt(part, 0, 0)
				if err != nil {
					return nil, false
				}
				if i >= int64(cv.Len()) {
					return nil, false
				}
				current = cv.Index(int(i)).Interface()
			}
		case reflect.String:
			// This happens when map keys contain "." and have a common
			// prefix so were split as path components above.
			actualKey := strings.Join(parts[i-1:], ".")
			if prevMap, ok := previous.(map[string]interface{}); ok {
				return prevMap[actualKey], true
			}
			return nil, false
		default:
			panic(fmt.Sprintf("Unknown kind: %s", cv.Kind()))
		}
	}

	return current, true
}

// interpolateForce is a temporary thing. We want to get rid of interpolate
// above and likewise this, but it can only be done after the f-ast-graph
// refactor is complete.
func (c *ResourceConfig) interpolateForce() {
	if c.raw == nil {
		var err error
		c.raw, err = config.NewRawConfig(make(map[string]interface{}))
		if err != nil {
			panic(err)
		}
	}

	c.ComputedKeys = c.raw.UnknownKeys()
	c.Raw = c.raw.Raw
	c.Config = c.raw.Config()
}
<|endoftext|>


<commit_before>package nux

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/toolkits/file"
	"io"
	"io/ioutil"
	"strconv"
	"strings"
)

type Mem struct {
	Buffers   uint64
	Cached    uint64
	MemTotal  uint64
	MemFree   uint64
	SwapTotal uint64
	SwapUsed  uint64
	SwapFree  uint64
	MemAvailable  uint64
}

func (this *Mem) String() string {
	return fmt.Sprintf("<MemTotal:%d, MemFree:%d, MemAvailable:%s, Buffers:%d, Cached:%d...>", this.MemTotal, this.MemFree, this.MemAvailable, this.Buffers, this.Cached)
}

var Multi uint64 = 1024

var WANT = map[string]struct{}{
	"Buffers:":   struct{}{},
	"Cached:":    struct{}{},
	"MemTotal:":  struct{}{},
	"MemFree:":   struct{}{},
	"SwapTotal:": struct{}{},
	"SwapFree:":  struct{}{},
	"MemAvailable:":  struct{}{},
}

func MemInfo() (*Mem, error) {
	contents, err := ioutil.ReadFile("/proc/meminfo")
	if err != nil {
		return nil, err
	}

	memInfo := &Mem{}

	reader := bufio.NewReader(bytes.NewBuffer(contents))

	for {
		line, err := file.ReadLine(reader)
		if err == io.EOF {
			err = nil
			break
		} else if err != nil {
			return nil, err
		}

		fields := strings.Fields(string(line))
		fieldName := fields[0]

		_, ok := WANT[fieldName]
		if ok && len(fields) == 3 {
			val, numerr := strconv.ParseUint(fields[1], 10, 64)
			if numerr != nil {
				continue
			}
			switch fieldName {
			case "Buffers:":
				memInfo.Buffers = val * Multi
			case "Cached:":
				memInfo.Cached = val * Multi
			case "MemTotal:":
				memInfo.MemTotal = val * Multi
			case "MemFree:":
				memInfo.MemFree = val * Multi
			case "SwapTotal:":
				memInfo.SwapTotal = val * Multi
			case "SwapFree:":
				memInfo.SwapFree = val * Multi
			case "MemAvailable:":
				memInfo.MemAvailable = val * Multi
			}
		}
	}

	memInfo.SwapUsed = memInfo.SwapTotal - memInfo.SwapFree

	return memInfo, nil
}
<commit_msg>Update meminfo.go<commit_after>package nux

import (
	"bufio"
	"bytes"
	"fmt"
	"github.com/toolkits/file"
	"io"
	"io/ioutil"
	"strconv"
	"strings"
)

type Mem struct {
	Buffers      uint64
	Cached       uint64
	MemTotal     uint64
	MemFree      uint64
	MemAvailable uint64
	SwapTotal    uint64
	SwapUsed     uint64
	SwapFree     uint64
}

func (this *Mem) String() string {
	return fmt.Sprintf("<MemTotal:%d, MemFree:%d, MemAvailable:%d, Buffers:%d, Cached:%d...>", this.MemTotal, this.MemFree, this.MemAvailable, this.Buffers, this.Cached)
}

var Multi uint64 = 1024

var WANT = map[string]struct{}{
	"Buffers:":      struct{}{},
	"Cached:":       struct{}{},
	"MemTotal:":     struct{}{},
	"MemFree:":      struct{}{},
	"MemAvailable:": struct{}{},
	"SwapTotal:":    struct{}{},
	"SwapFree:":     struct{}{},
}

func MemInfo() (*Mem, error) {
	contents, err := ioutil.ReadFile("/proc/meminfo")
	if err != nil {
		return nil, err
	}

	memInfo := &Mem{}

	reader := bufio.NewReader(bytes.NewBuffer(contents))

	for {
		line, err := file.ReadLine(reader)
		if err == io.EOF {
			err = nil
			break
		} else if err != nil {
			return nil, err
		}

		fields := strings.Fields(string(line))
		fieldName := fields[0]

		_, ok := WANT[fieldName]
		if ok && len(fields) == 3 {
			val, numerr := strconv.ParseUint(fields[1], 10, 64)
			if numerr != nil {
				continue
			}
			switch fieldName {
			case "Buffers:":
				memInfo.Buffers = val * Multi
			case "Cached:":
				memInfo.Cached = val * Multi
			case "MemTotal:":
				memInfo.MemTotal = val * Multi
			case "MemFree:":
				memInfo.MemFree = val * Multi
			case "MemAvailable:":
				memInfo.MemAvailable = val * Multi
			case "SwapTotal:":
				memInfo.SwapTotal = val * Multi
			case "SwapFree:":
				memInfo.SwapFree = val * Multi
			}
		}
	}

	memInfo.SwapUsed = memInfo.SwapTotal - memInfo.SwapFree

	return memInfo, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"k8s.io/apimachinery/pkg/api/errors"

	"github.com/kubernetes-sigs/volcano/pkg/apis/batch/v1alpha1"
)

var _ = Describe("Job E2E Test: Test Admission service", func() {
	It("Duplicated Task Name", func() {
		jobName := "job-duplicated"
		namespace := "test"
		context := initTestContext()
		defer cleanupTestContext(context)
		rep := clusterSize(context, oneCPU)

		_, err := createJobInner(context, &jobSpec{
			namespace: namespace,
			name:      jobName,
			tasks: []taskSpec{
				{
					img:  defaultNginxImage,
					req:  oneCPU,
					min:  rep,
					rep:  rep,
					name: "duplicated",
				},
				{
					img:  defaultNginxImage,
					req:  oneCPU,
					min:  rep,
					rep:  rep,
					name: "duplicated",
				},
			},
		})
		Expect(err).To(HaveOccurred())
		stError, ok := err.(*errors.StatusError)
		Expect(ok).To(Equal(true))
		Expect(stError.ErrStatus.Code).To(Equal(int32(500)))
		Expect(stError.ErrStatus.Message).To(ContainSubstring("duplicated task name"))
	})

	It("Duplicated Policy Event", func() {
		jobName := "job-policy-duplicated"
		namespace := "test"
		context := initTestContext()
		defer cleanupTestContext(context)
		rep := clusterSize(context, oneCPU)

		_, err := createJobInner(context, &jobSpec{
			namespace: namespace,
			name:      jobName,
			tasks: []taskSpec{
				{
					img:  defaultNginxImage,
					req:  oneCPU,
					min:  rep,
					rep:  rep,
					name: "taskname",
				},
			},
			policies: []v1alpha1.LifecyclePolicy{
				{
					Event:  v1alpha1.PodFailedEvent,
					Action: v1alpha1.AbortJobAction,
				},
				{
					Event:  v1alpha1.PodFailedEvent,
					Action: v1alpha1.RestartJobAction,
				},
			},
		})
		Expect(err).To(HaveOccurred())
		stError, ok := err.(*errors.StatusError)
		Expect(ok).To(Equal(true))
		Expect(stError.ErrStatus.Code).To(Equal(int32(500)))
		Expect(stError.ErrStatus.Message).To(ContainSubstring("duplicated job event policies"))
	})

	It("Min Available illegal", func() {
		jobName := "job-min-illegal"
		namespace := "test"
		context := initTestContext()
		defer cleanupTestContext(context)
		rep := clusterSize(context, oneCPU)

		_, err := createJobInner(context, &jobSpec{
			min:       rep * 2,
			namespace: namespace,
			name:      jobName,
			tasks: []taskSpec{
				{
					img:  defaultNginxImage,
					req:  oneCPU,
					min:  rep,
					rep:  rep,
					name: "taskname",
				},
			},
		})
		Expect(err).To(HaveOccurred())
		stError, ok := err.(*errors.StatusError)
		Expect(ok).To(Equal(true))
		Expect(stError.ErrStatus.Code).To(Equal(int32(500)))
		Expect(stError.ErrStatus.Message).To(ContainSubstring("'minAvailable' should not be greater than total replicas in tasks"))
	})

	It("Job Plugin illegal", func() {
		jobName := "job-plugin-illegal"
		namespace := "test"
		context := initTestContext()
		defer cleanupTestContext(context)

		_, err := createJobInner(context, &jobSpec{
			min:       1,
			namespace: namespace,
			name:      jobName,
			plugins: map[string][]string{
				"big_plugin": {},
			},
			tasks: []taskSpec{
				{
					img:  defaultNginxImage,
					req:  oneCPU,
					min:  1,
					rep:  1,
					name: "taskname",
				},
			},
		})
		Expect(err).To(HaveOccurred())
		stError, ok := err.(*errors.StatusError)
		Expect(ok).To(Equal(true))
		Expect(stError.ErrStatus.Code).To(Equal(int32(500)))
		Expect(stError.ErrStatus.Message).To(ContainSubstring("unable to find job plugin: big_plugin"))
	})
})
<commit_msg>fix admission e2e<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"k8s.io/apimachinery/pkg/api/errors"

	"github.com/kubernetes-sigs/volcano/pkg/apis/batch/v1alpha1"
)

var _ = Describe("Job E2E Test: Test Admission service", func() {
	It("Duplicated Task Name", func() {
		jobName := "job-duplicated"
		namespace := "test"
		context := initTestContext()
		defer cleanupTestContext(context)
		rep := clusterSize(context, oneCPU)

		_, err := createJobInner(context, &jobSpec{
			namespace: namespace,
			name:      jobName,
			tasks: []taskSpec{
				{
					img:  defaultNginxImage,
					req:  oneCPU,
					min:  rep,
					rep:  rep,
					name: "duplicated",
				},
				{
					img:  defaultNginxImage,
					req:  oneCPU,
					min:  rep,
					rep:  rep,
					name: "duplicated",
				},
			},
		})
		Expect(err).To(HaveOccurred())
		stError, ok := err.(*errors.StatusError)
		Expect(ok).To(Equal(true))
		Expect(stError.ErrStatus.Code).To(Equal(int32(500)))
		Expect(stError.ErrStatus.Message).To(ContainSubstring("duplicated task name"))
	})

	It("Duplicated Policy Event", func() {
		jobName := "job-policy-duplicated"
		namespace := "test"
		context := initTestContext()
		defer cleanupTestContext(context)
		rep := clusterSize(context, oneCPU)

		_, err := createJobInner(context, &jobSpec{
			namespace: namespace,
			name:      jobName,
			tasks: []taskSpec{
				{
					img:  defaultNginxImage,
					req:  oneCPU,
					min:  rep,
					rep:  rep,
					name: "taskname",
				},
			},
			policies: []v1alpha1.LifecyclePolicy{
				{
					Event:  v1alpha1.PodFailedEvent,
					Action: v1alpha1.AbortJobAction,
				},
				{
					Event:  v1alpha1.PodFailedEvent,
					Action: v1alpha1.RestartJobAction,
				},
			},
		})
		Expect(err).To(HaveOccurred())
		stError, ok := err.(*errors.StatusError)
		Expect(ok).To(Equal(true))
		Expect(stError.ErrStatus.Code).To(Equal(int32(500)))
		Expect(stError.ErrStatus.Message).To(ContainSubstring("duplicate event PodFailed"))
	})

	It("Min Available illegal", func() {
		jobName := "job-min-illegal"
		namespace := "test"
		context := initTestContext()
		defer cleanupTestContext(context)
		rep := clusterSize(context, oneCPU)

		_, err := createJobInner(context, &jobSpec{
			min:       rep * 2,
			namespace: namespace,
			name:      jobName,
			tasks: []taskSpec{
				{
					img:  defaultNginxImage,
					req:  oneCPU,
					min:  rep,
					rep:  rep,
					name: "taskname",
				},
			},
		})
		Expect(err).To(HaveOccurred())
		stError, ok := err.(*errors.StatusError)
		Expect(ok).To(Equal(true))
		Expect(stError.ErrStatus.Code).To(Equal(int32(500)))
		Expect(stError.ErrStatus.Message).To(ContainSubstring("'minAvailable' should not be greater than total replicas in tasks"))
	})

	It("Job Plugin illegal", func() {
		jobName := "job-plugin-illegal"
		namespace := "test"
		context := initTestContext()
		defer cleanupTestContext(context)

		_, err := createJobInner(context, &jobSpec{
			min:       1,
			namespace: namespace,
			name:      jobName,
			plugins: map[string][]string{
				"big_plugin": {},
			},
			tasks: []taskSpec{
				{
					img:  defaultNginxImage,
					req:  oneCPU,
					min:  1,
					rep:  1,
					name: "taskname",
				},
			},
		})
		Expect(err).To(HaveOccurred())
		stError, ok := err.(*errors.StatusError)
		Expect(ok).To(Equal(true))
		Expect(stError.ErrStatus.Code).To(Equal(int32(500)))
		Expect(stError.ErrStatus.Message).To(ContainSubstring("unable to find job plugin: big_plugin"))
	})
})
<|endoftext|>


<commit_before>package s3api

import (
	"context"
	"encoding/xml"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"io"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
)

type ListBucketResultV2 struct {
	XMLName               xml.Name      `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
	Name                  string        `xml:"Name"`
	Prefix                string        `xml:"Prefix"`
	MaxKeys               int           `xml:"MaxKeys"`
	Delimiter             string        `xml:"Delimiter,omitempty"`
	IsTruncated           bool          `xml:"IsTruncated"`
	Contents              []ListEntry   `xml:"Contents,omitempty"`
	CommonPrefixes        []PrefixEntry `xml:"CommonPrefixes,omitempty"`
	ContinuationToken     string        `xml:"ContinuationToken,omitempty"`
	NextContinuationToken string        `xml:"NextContinuationToken,omitempty"`
	KeyCount              int           `xml:"KeyCount"`
	StartAfter            string        `xml:"StartAfter,omitempty"`
}

func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {

	// https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html

	// collect parameters
	bucket, _ := getBucketAndObject(r)
	glog.V(3).Infof("ListObjectsV2Handler %s", bucket)

	originalPrefix, continuationToken, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query())

	if maxKeys < 0 {
		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
		return
	}
	if delimiter != "" && delimiter != "/" {
		s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
		return
	}

	marker := continuationToken
	if continuationToken == "" {
		marker = startAfter
	}

	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)

	if err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
		return
	}

	if len(response.Contents) == 0 {
		if exists, existErr := s3a.exists(s3a.option.BucketsPath, bucket, true); existErr == nil && !exists {
			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
			return
		}
	}

	responseV2 := &ListBucketResultV2{
		XMLName:               response.XMLName,
		Name:                  response.Name,
		CommonPrefixes:        response.CommonPrefixes,
		Contents:              response.Contents,
		ContinuationToken:     continuationToken,
		Delimiter:             response.Delimiter,
		IsTruncated:           response.IsTruncated,
		KeyCount:              len(response.Contents) + len(response.CommonPrefixes),
		MaxKeys:               response.MaxKeys,
		NextContinuationToken: response.NextMarker,
		Prefix:                response.Prefix,
		StartAfter:            startAfter,
	}

	writeSuccessResponseXML(w, r, responseV2)
}

func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {

	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html

	// collect parameters
	bucket, _ := getBucketAndObject(r)
	glog.V(3).Infof("ListObjectsV1Handler %s", bucket)

	originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query())

	if maxKeys < 0 {
		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
		return
	}
	if delimiter != "" && delimiter != "/" {
		s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
		return
	}

	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)

	if err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
		return
	}

	if len(response.Contents) == 0 {
		if exists, existErr := s3a.exists(s3a.option.BucketsPath, bucket, true); existErr == nil && !exists {
			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
			return
		}
	}

	writeSuccessResponseXML(w, r, response)
}

func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, maxKeys int, marker string, delimiter string) (response ListBucketResult, err error) {
	// convert full path prefix into directory name and prefix for entry name
	reqDir, prefix := filepath.Split(originalPrefix)
	if strings.HasPrefix(reqDir, "/") {
		reqDir = reqDir[1:]
	}
	bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket)
	reqDir = fmt.Sprintf("%s%s", bucketPrefix, reqDir)
	if strings.HasSuffix(reqDir, "/") {
		// remove trailing "/"
		reqDir = reqDir[:len(reqDir)-1]
	}

	var contents []ListEntry
	var commonPrefixes []PrefixEntry
	var isTruncated bool
	var doErr error
	var nextMarker string

	// check filer
	err = s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		_, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, func(dir string, entry *filer_pb.Entry) {
			if entry.IsDirectory {
				if delimiter == "/" {
					commonPrefixes = append(commonPrefixes, PrefixEntry{
						Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
					})
				}
			} else {
				storageClass := "STANDARD"
				if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok {
					storageClass = string(v)
				}
				contents = append(contents, ListEntry{
					Key:          fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
					LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
					ETag:         "\"" + filer.ETag(entry) + "\"",
					Size:         int64(filer.FileSize(entry)),
					Owner: CanonicalUser{
						ID:          fmt.Sprintf("%x", entry.Attributes.Uid),
						DisplayName: entry.Attributes.UserName,
					},
					StorageClass: StorageClass(storageClass),
				})
			}
		})
		if doErr != nil {
			return doErr
		}

		if !isTruncated {
			nextMarker = ""
		}

		response = ListBucketResult{
			Name:           bucket,
			Prefix:         originalPrefix,
			Marker:         marker,
			NextMarker:     nextMarker,
			MaxKeys:        maxKeys,
			Delimiter:      delimiter,
			IsTruncated:    isTruncated,
			Contents:       contents,
			CommonPrefixes: commonPrefixes,
		}

		return nil
	})

	return
}

func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) {
	// invariants
	//   prefix and marker should be under dir, marker may contain "/"
	//   maxKeys should be updated for each recursion

	if prefix == "/" && delimiter == "/" {
		return
	}
	if maxKeys <= 0 {
		return
	}

	if strings.Contains(marker, "/") {
		sepIndex := strings.Index(marker, "/")
		subDir, subMarker := marker[0:sepIndex], marker[sepIndex+1:]
		// println("doListFilerEntries dir", dir+"/"+subDir, "subMarker", subMarker, "maxKeys", maxKeys)
		subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+subDir, "", maxKeys, subMarker, delimiter, eachEntryFn)
		if subErr != nil {
			err = subErr
			return
		}
		isTruncated = isTruncated || subIsTruncated
		maxKeys -= subCounter
		nextMarker = subDir + "/" + subNextMarker
		// finished processing this sub directory
		marker = subDir
	}

	// now marker is also a direct child of dir
	request := &filer_pb.ListEntriesRequest{
		Directory:          dir,
		Prefix:             prefix,
		Limit:              uint32(maxKeys + 1),
		StartFromFileName:  marker,
		InclusiveStartFrom: false,
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	stream, listErr := client.ListEntries(ctx, request)
	if listErr != nil {
		err = fmt.Errorf("list entires %+v: %v", request, listErr)
		return
	}

	for {
		resp, recvErr := stream.Recv()
		if recvErr != nil {
			if recvErr == io.EOF {
				break
			} else {
				err = fmt.Errorf("iterating entires %+v: %v", request, recvErr)
				return
			}
		}
		if counter >= maxKeys {
			isTruncated = true
			return
		}
		entry := resp.Entry
		nextMarker = entry.Name
		if entry.IsDirectory {
			// println("ListEntries", dir, "dir:", entry.Name)
			if entry.Name != ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys
				if delimiter != "/" {
					eachEntryFn(dir, entry)
					// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter)
					subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, eachEntryFn)
					if subErr != nil {
						err = fmt.Errorf("doListFilerEntries2: %v", subErr)
						return
					}
					// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter, "subCounter", subCounter, "subNextMarker", subNextMarker, "subIsTruncated", subIsTruncated)
					counter += subCounter
					nextMarker = entry.Name + "/" + subNextMarker
					if subIsTruncated {
						isTruncated = true
						return
					}
				} else {
					var isEmpty bool
					if !s3a.option.AllowEmptyFolder {
						if isEmpty, err = s3a.isDirectoryAllEmpty(client, dir, entry.Name); err != nil {
							glog.Errorf("check empty folder %s: %v", dir, err)
						}
					}
					if !isEmpty {
						eachEntryFn(dir, entry)
						counter++
					}
				}
			}
		} else {
			// println("ListEntries", dir, "file:", entry.Name)
			eachEntryFn(dir, entry)
			counter++
		}
	}
	return
}

func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int) {
	prefix = values.Get("prefix")
	token = values.Get("continuation-token")
	startAfter = values.Get("start-after")
	delimiter = values.Get("delimiter")
	if values.Get("max-keys") != "" {
		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
	} else {
		maxkeys = maxObjectListSizeLimit
	}
	fetchOwner = values.Get("fetch-owner") == "true"
	return
}

func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int) {
	prefix = values.Get("prefix")
	marker = values.Get("marker")
	delimiter = values.Get("delimiter")
	if values.Get("max-keys") != "" {
		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
	} else {
		maxkeys = maxObjectListSizeLimit
	}
	return
}

func (s3a *S3ApiServer) isDirectoryAllEmpty(filerClient filer_pb.SeaweedFilerClient, parentDir, name string) (isEmpty bool, err error) {
	// println("+ isDirectoryAllEmpty", dir, name)
	glog.V(4).Infof("+ isEmpty %s/%s", parentDir, name)
	defer glog.V(4).Infof("- isEmpty %s/%s %v", parentDir, name, isEmpty)
	var fileCounter int
	var subDirs []string
	currentDir := parentDir + "/" + name
	var startFrom string
	var isExhausted bool
	var foundEntry bool
	for fileCounter == 0 && !isExhausted && err == nil {
		err = filer_pb.SeaweedList(filerClient, currentDir, "", func(entry *filer_pb.Entry, isLast bool) error {
			foundEntry = true
			if entry.IsDirectory {
				subDirs = append(subDirs, entry.Name)
			} else {
				fileCounter++
			}
			startFrom = entry.Name
			isExhausted = isExhausted || isLast
			glog.V(4).Infof("    * %s/%s isLast: %t", currentDir, startFrom, isLast)
			return nil
		}, startFrom, false, 8)
		if !foundEntry {
			break
		}
	}

	if err != nil {
		return false, err
	}

	if fileCounter > 0 {
		return false, nil
	}

	for _, subDir := range subDirs {
		isSubEmpty, subErr := s3a.isDirectoryAllEmpty(filerClient, currentDir, subDir)
		if subErr != nil {
			return false, subErr
		}
		if !isSubEmpty {
			return false, nil
		}
	}

	glog.V(1).Infof("deleting empty folder %s", currentDir)
	if err = doDeleteEntry(filerClient, parentDir, name, true, true); err != nil {
		return
	}

	return true, nil
}
<commit_msg>s3: fix ListObject if more than 10000 objects<commit_after>package s3api

import (
	"context"
	"encoding/xml"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"io"
	"net/http"
	"net/url"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
	"github.com/chrislusf/seaweedfs/weed/s3api/s3err"
)

type ListBucketResultV2 struct {
	XMLName               xml.Name      `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult"`
	Name                  string        `xml:"Name"`
	Prefix                string        `xml:"Prefix"`
	MaxKeys               int           `xml:"MaxKeys"`
	Delimiter             string        `xml:"Delimiter,omitempty"`
	IsTruncated           bool          `xml:"IsTruncated"`
	Contents              []ListEntry   `xml:"Contents,omitempty"`
	CommonPrefixes        []PrefixEntry `xml:"CommonPrefixes,omitempty"`
	ContinuationToken     string        `xml:"ContinuationToken,omitempty"`
	NextContinuationToken string        `xml:"NextContinuationToken,omitempty"`
	KeyCount              int           `xml:"KeyCount"`
	StartAfter            string        `xml:"StartAfter,omitempty"`
}

func (s3a *S3ApiServer) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) {

	// https://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html

	// collect parameters
	bucket, _ := getBucketAndObject(r)
	glog.V(3).Infof("ListObjectsV2Handler %s", bucket)

	originalPrefix, continuationToken, startAfter, delimiter, _, maxKeys := getListObjectsV2Args(r.URL.Query())

	if maxKeys < 0 {
		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
		return
	}
	if delimiter != "" && delimiter != "/" {
		s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
		return
	}

	marker := continuationToken
	if continuationToken == "" {
		marker = startAfter
	}

	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)

	if err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
		return
	}

	if len(response.Contents) == 0 {
		if exists, existErr := s3a.exists(s3a.option.BucketsPath, bucket, true); existErr == nil && !exists {
			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
			return
		}
	}

	responseV2 := &ListBucketResultV2{
		XMLName:               response.XMLName,
		Name:                  response.Name,
		CommonPrefixes:        response.CommonPrefixes,
		Contents:              response.Contents,
		ContinuationToken:     continuationToken,
		Delimiter:             response.Delimiter,
		IsTruncated:           response.IsTruncated,
		KeyCount:              len(response.Contents) + len(response.CommonPrefixes),
		MaxKeys:               response.MaxKeys,
		NextContinuationToken: response.NextMarker,
		Prefix:                response.Prefix,
		StartAfter:            startAfter,
	}

	writeSuccessResponseXML(w, r, responseV2)
}

func (s3a *S3ApiServer) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) {

	// https://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html

	// collect parameters
	bucket, _ := getBucketAndObject(r)
	glog.V(3).Infof("ListObjectsV1Handler %s", bucket)

	originalPrefix, marker, delimiter, maxKeys := getListObjectsV1Args(r.URL.Query())

	if maxKeys < 0 {
		s3err.WriteErrorResponse(w, r, s3err.ErrInvalidMaxKeys)
		return
	}
	if delimiter != "" && delimiter != "/" {
		s3err.WriteErrorResponse(w, r, s3err.ErrNotImplemented)
		return
	}

	response, err := s3a.listFilerEntries(bucket, originalPrefix, maxKeys, marker, delimiter)

	if err != nil {
		s3err.WriteErrorResponse(w, r, s3err.ErrInternalError)
		return
	}

	if len(response.Contents) == 0 {
		if exists, existErr := s3a.exists(s3a.option.BucketsPath, bucket, true); existErr == nil && !exists {
			s3err.WriteErrorResponse(w, r, s3err.ErrNoSuchBucket)
			return
		}
	}

	writeSuccessResponseXML(w, r, response)
}

func (s3a *S3ApiServer) listFilerEntries(bucket string, originalPrefix string, maxKeys int, marker string, delimiter string) (response ListBucketResult, err error) {
	// convert full path prefix into directory name and prefix for entry name
	reqDir, prefix := filepath.Split(originalPrefix)
	if strings.HasPrefix(reqDir, "/") {
		reqDir = reqDir[1:]
	}
	bucketPrefix := fmt.Sprintf("%s/%s/", s3a.option.BucketsPath, bucket)
	reqDir = fmt.Sprintf("%s%s", bucketPrefix, reqDir)
	if strings.HasSuffix(reqDir, "/") {
		// remove trailing "/"
		reqDir = reqDir[:len(reqDir)-1]
	}

	var contents []ListEntry
	var commonPrefixes []PrefixEntry
	var isTruncated bool
	var doErr error
	var nextMarker string

	// check filer
	err = s3a.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		_, isTruncated, nextMarker, doErr = s3a.doListFilerEntries(client, reqDir, prefix, maxKeys, marker, delimiter, func(dir string, entry *filer_pb.Entry) {
			if entry.IsDirectory {
				if delimiter == "/" {
					commonPrefixes = append(commonPrefixes, PrefixEntry{
						Prefix: fmt.Sprintf("%s/%s/", dir, entry.Name)[len(bucketPrefix):],
					})
				}
			} else {
				storageClass := "STANDARD"
				if v, ok := entry.Extended[xhttp.AmzStorageClass]; ok {
					storageClass = string(v)
				}
				contents = append(contents, ListEntry{
					Key:          fmt.Sprintf("%s/%s", dir, entry.Name)[len(bucketPrefix):],
					LastModified: time.Unix(entry.Attributes.Mtime, 0).UTC(),
					ETag:         "\"" + filer.ETag(entry) + "\"",
					Size:         int64(filer.FileSize(entry)),
					Owner: CanonicalUser{
						ID:          fmt.Sprintf("%x", entry.Attributes.Uid),
						DisplayName: entry.Attributes.UserName,
					},
					StorageClass: StorageClass(storageClass),
				})
			}
		})
		if doErr != nil {
			return doErr
		}

		if !isTruncated {
			nextMarker = ""
		}

		response = ListBucketResult{
			Name:           bucket,
			Prefix:         originalPrefix,
			Marker:         marker,
			NextMarker:     nextMarker,
			MaxKeys:        maxKeys,
			Delimiter:      delimiter,
			IsTruncated:    isTruncated,
			Contents:       contents,
			CommonPrefixes: commonPrefixes,
		}

		return nil
	})

	return
}

func (s3a *S3ApiServer) doListFilerEntries(client filer_pb.SeaweedFilerClient, dir, prefix string, maxKeys int, marker, delimiter string, eachEntryFn func(dir string, entry *filer_pb.Entry)) (counter int, isTruncated bool, nextMarker string, err error) {
	// invariants
	//   prefix and marker should be under dir, marker may contain "/"
	//   maxKeys should be updated for each recursion

	if prefix == "/" && delimiter == "/" {
		return
	}
	if maxKeys <= 0 {
		return
	}

	if strings.Contains(marker, "/") {
		sepIndex := strings.Index(marker, "/")
		subDir, subMarker := marker[0:sepIndex], marker[sepIndex+1:]
		// println("doListFilerEntries dir", dir+"/"+subDir, "subMarker", subMarker, "maxKeys", maxKeys)
		subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+subDir, "", maxKeys, subMarker, delimiter, eachEntryFn)
		if subErr != nil {
			err = subErr
			return
		}
		counter += subCounter
		isTruncated = isTruncated || subIsTruncated
		maxKeys -= subCounter
		nextMarker = subDir + "/" + subNextMarker
		// finished processing this sub directory
		marker = subDir
	}
	if maxKeys <= 0 {
		return
	}

	// now marker is also a direct child of dir
	request := &filer_pb.ListEntriesRequest{
		Directory:          dir,
		Prefix:             prefix,
		Limit:              uint32(maxKeys + 1),
		StartFromFileName:  marker,
		InclusiveStartFrom: false,
	}

	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	stream, listErr := client.ListEntries(ctx, request)
	if listErr != nil {
		err = fmt.Errorf("list entires %+v: %v", request, listErr)
		return
	}

	for {
		resp, recvErr := stream.Recv()
		if recvErr != nil {
			if recvErr == io.EOF {
				break
			} else {
				err = fmt.Errorf("iterating entires %+v: %v", request, recvErr)
				return
			}
		}
		if counter >= maxKeys {
			isTruncated = true
			return
		}
		entry := resp.Entry
		nextMarker = entry.Name
		if entry.IsDirectory {
			// println("ListEntries", dir, "dir:", entry.Name)
			if entry.Name != ".uploads" { // FIXME no need to apply to all directories. this extra also affects maxKeys
				if delimiter != "/" {
					eachEntryFn(dir, entry)
					// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter)
					subCounter, subIsTruncated, subNextMarker, subErr := s3a.doListFilerEntries(client, dir+"/"+entry.Name, "", maxKeys-counter, "", delimiter, eachEntryFn)
					if subErr != nil {
						err = fmt.Errorf("doListFilerEntries2: %v", subErr)
						return
					}
					// println("doListFilerEntries2 dir", dir+"/"+entry.Name, "maxKeys", maxKeys-counter, "subCounter", subCounter, "subNextMarker", subNextMarker, "subIsTruncated", subIsTruncated)
					counter += subCounter
					nextMarker = entry.Name + "/" + subNextMarker
					if subIsTruncated {
						isTruncated = true
						return
					}
				} else {
					var isEmpty bool
					if !s3a.option.AllowEmptyFolder {
						if isEmpty, err = s3a.isDirectoryAllEmpty(client, dir, entry.Name); err != nil {
							glog.Errorf("check empty folder %s: %v", dir, err)
						}
					}
					if !isEmpty {
						eachEntryFn(dir, entry)
						counter++
					}
				}
			}
		} else {
			// println("ListEntries", dir, "file:", entry.Name)
			eachEntryFn(dir, entry)
			counter++
		}
	}
	return
}

func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int) {
	prefix = values.Get("prefix")
	token = values.Get("continuation-token")
	startAfter = values.Get("start-after")
	delimiter = values.Get("delimiter")
	if values.Get("max-keys") != "" {
		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
	} else {
		maxkeys = maxObjectListSizeLimit
	}
	fetchOwner = values.Get("fetch-owner") == "true"
	return
}

func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int) {
	prefix = values.Get("prefix")
	marker = values.Get("marker")
	delimiter = values.Get("delimiter")
	if values.Get("max-keys") != "" {
		maxkeys, _ = strconv.Atoi(values.Get("max-keys"))
	} else {
		maxkeys = maxObjectListSizeLimit
	}
	return
}

func (s3a *S3ApiServer) isDirectoryAllEmpty(filerClient filer_pb.SeaweedFilerClient, parentDir, name string) (isEmpty bool, err error) {
	// println("+ isDirectoryAllEmpty", dir, name)
	glog.V(4).Infof("+ isEmpty %s/%s", parentDir, name)
	defer glog.V(4).Infof("- isEmpty %s/%s %v", parentDir, name, isEmpty)
	var fileCounter int
	var subDirs []string
	currentDir := parentDir + "/" + name
	var startFrom string
	var isExhausted bool
	var foundEntry bool
	for fileCounter == 0 && !isExhausted && err == nil {
		err = filer_pb.SeaweedList(filerClient, currentDir, "", func(entry *filer_pb.Entry, isLast bool) error {
			foundEntry = true
			if entry.IsDirectory {
				subDirs = append(subDirs, entry.Name)
			} else {
				fileCounter++
			}
			startFrom = entry.Name
			isExhausted = isExhausted || isLast
			glog.V(4).Infof("    * %s/%s isLast: %t", currentDir, startFrom, isLast)
			return nil
		}, startFrom, false, 8)
		if !foundEntry {
			break
		}
	}

	if err != nil {
		return false, err
	}

	if fileCounter > 0 {
		return false, nil
	}

	for _, subDir := range subDirs {
		isSubEmpty, subErr := s3a.isDirectoryAllEmpty(filerClient, currentDir, subDir)
		if subErr != nil {
			return false, subErr
		}
		if !isSubEmpty {
			return false, nil
		}
	}

	glog.V(1).Infof("deleting empty folder %s", currentDir)
	if err = doDeleteEntry(filerClient, parentDir, name, true, true); err != nil {
		return
	}

	return true, nil
}
<|endoftext|>


<commit_before>package lnwallet

import (
	"encoding/binary"
	"io"
	"time"

	"li.lan/labs/plasma/shachain"

	"github.com/btcsuite/btcd/btcec"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcwallet/waddrmgr"
	"github.com/btcsuite/btcwallet/walletdb"
)

var (
	// Namespace bucket keys.
	lightningNamespaceKey = []byte("ln-wallet")
	waddrmgrNamespaceKey  = []byte("waddrmgr")
	wtxmgrNamespaceKey    = []byte("wtxmgr")

	openChannelBucket   = []byte("o-chans")
	closedChannelBucket = []byte("c-chans")
	fundingTxKey        = []byte("funding")

	endian = binary.BigEndian
)

// ChannelDB...
type ChannelDB struct {
	// TODO(roasbeef): caching, etc?
	wallet *LightningWallet

	namespace walletdb.Namespace
}

func NewChannelDB(wallet *LightningWallet, n walletdb.Namespace) *ChannelDB {
	return &ChannelDB{wallet, n}
}

// OpenChannelState...
// TODO(roasbeef): store only the essentials? optimize space...
// TODO(roasbeef): switch to "column store"
type OpenChannelState struct {
	// Hash? or Their current pubKey?
	// TODO(roasbeef): switch to Tadge's LNId
	TheirLNID [wire.HashSize]byte

	// The ID of a channel is the txid of the funding transaction.
	ChanID [wire.HashSize]byte

	MinFeePerKb btcutil.Amount
	// Our reserve. Assume symmetric reserve amounts. Only needed if the
	// funding type is CLTV.
	//ReserveAmount btcutil.Amount

	// Keys for both sides to be used for the commitment transactions.
	OurCommitKey   *btcec.PrivateKey
	TheirCommitKey *btcec.PublicKey

	// Tracking total channel capacity, and the amount of funds allocated
	// to each side.
	Capacity     btcutil.Amount
	OurBalance   btcutil.Amount
	TheirBalance btcutil.Amount

	// Commitment transactions for both sides (they're asymmetric). Also
	// their signature which lets us spend our version of the commitment
	// transaction.
	TheirCommitTx  *wire.MsgTx
	OurCommitTx    *wire.MsgTx // TODO(roasbeef): store hash instead?
	TheirCommitSig []byte      // TODO(roasbeef): fixed length?, same w/ redeem

	// The final funding transaction. Kept wallet-related records.
	FundingTx *wire.MsgTx

	MultiSigKey         *btcec.PrivateKey
	FundingRedeemScript []byte

	// Current revocation for their commitment transaction. However, since
	// this is the hash, and not the pre-image, we can't yet verify that
	// it's actually in the chain.
	TheirCurrentRevocation [wire.HashSize]byte
	TheirShaChain          *shachain.HyperShaChain
	OurShaChain            *shachain.HyperShaChain

	// Final delivery address
	OurDeliveryAddress   btcutil.Address
	TheirDeliveryAddress btcutil.Address

	// In blocks
	CsvDelay uint32

	// TODO(roasbeef): track fees, other stats?
	NumUpdates            uint64
	TotalSatoshisSent     uint64
	TotalSatoshisReceived uint64
	CreationTime          time.Time
}

// Encode...
// TODO(roasbeef): checksum
func (o *OpenChannelState) Encode(b io.Writer, addrManager *waddrmgr.Manager) error {
	if _, err := b.Write(o.TheirLNID[:]); err != nil {
		return err
	}
	if _, err := b.Write(o.ChanID[:]); err != nil {
		return err
	}

	if err := binary.Write(b, endian, uint64(o.MinFeePerKb)); err != nil {
		return err
	}

	encryptedPriv, err := addrManager.Encrypt(waddrmgr.CKTPrivate,
		o.OurCommitKey.Serialize())
	if err != nil {
		return err
	}
	if _, err := b.Write(encryptedPriv); err != nil {
		return err
	}
	if _, err := b.Write(o.TheirCommitKey.SerializeCompressed()); err != nil {
		return err
	}

	if err := binary.Write(b, endian, uint64(o.Capacity)); err != nil {
		return err
	}
	if err := binary.Write(b, endian, uint64(o.OurBalance)); err != nil {
		return err
	}
	if err := binary.Write(b, endian, uint64(o.TheirBalance)); err != nil {
		return err
	}

	if err := o.TheirCommitTx.Serialize(b); err != nil {
		return err
	}
	if err := o.OurCommitTx.Serialize(b); err != nil {
		return err
	}
	if _, err := b.Write(o.TheirCommitSig[:]); err != nil {
		return err
	}

	if err := o.FundingTx.Serialize(b); err != nil {
		return err
	}

	encryptedPriv, err = addrManager.Encrypt(waddrmgr.CKTPrivate,
		o.MultiSigKey.Serialize())
	if err != nil {
		return err
	}
	if _, err := b.Write(encryptedPriv); err != nil {
		return err
	}
	if _, err := b.Write(o.FundingRedeemScript); err != nil {
		return err
	}

	if _, err := b.Write(o.TheirCurrentRevocation[:]); err != nil {
		return err
	}
	// TODO(roasbeef): serialize shachains

	if _, err := b.Write([]byte(o.OurDeliveryAddress.EncodeAddress())); err != nil {
		return err
	}
	if _, err := b.Write([]byte(o.TheirDeliveryAddress.EncodeAddress())); err != nil {
		return err
	}

	if err := binary.Write(b, endian, o.CsvDelay); err != nil {
		return err
	}
	if err := binary.Write(b, endian, o.NumUpdates); err != nil {
		return err
	}
	if err := binary.Write(b, endian, o.TotalSatoshisSent); err != nil {
		return err
	}
	if err := binary.Write(b, endian, o.TotalSatoshisReceived); err != nil {
		return err
	}

	if err := binary.Write(b, endian, o.CreationTime.Unix()); err != nil {
		return err
	}

	return nil
}

// Decode...
func (o *OpenChannelState) Decode(b io.Reader, addrManager *waddrmgr.Manager) error {
	var scratch [8]byte

	if _, err := b.Read(o.TheirLNID[:]); err != nil {
		return err
	}
	if _, err := b.Read(o.ChanID[:]); err != nil {
		return err
	}

	if _, err := b.Read(scratch[:]); err != nil {
		return err
	}
	o.MinFeePerKb = btcutil.Amount(endian.Uint64(scratch[:]))

	// nonce + serPrivKey + mac
	var encryptedPriv [24 + 32 + 16]byte
	if _, err := b.Read(encryptedPriv[:]); err != nil {
		return err
	}
	decryptedPriv, err := addrManager.Decrypt(waddrmgr.CKTPrivate, encryptedPriv[:])
	if err != nil {
		return err
	}
	o.OurCommitKey, _ = btcec.PrivKeyFromBytes(btcec.S256(), decryptedPriv)

	var serPubKey [33]byte
	if _, err := b.Read(serPubKey[:]); err != nil {
		return err
	}
	o.TheirCommitKey, err = btcec.ParsePubKey(serPubKey[:], btcec.S256())
	if err != nil {
		return err
	}

	if _, err := b.Read(scratch[:]); err != nil {
		return err
	}
	o.Capacity = btcutil.Amount(endian.Uint64(scratch[:]))
	if _, err := b.Read(scratch[:]); err != nil {
		return err
	}
	o.OurBalance = btcutil.Amount(endian.Uint64(scratch[:]))
	if _, err := b.Read(scratch[:]); err != nil {
		return err
	}
	o.TheirBalance = btcutil.Amount(endian.Uint64(scratch[:]))

	o.TheirCommitTx = wire.NewMsgTx()
	if err := o.TheirCommitTx.Deserialize(b); err != nil {
		return err
	}
	o.OurCommitTx = wire.NewMsgTx()
	if err := o.OurCommitTx.Deserialize(b); err != nil {
		return err
	}

	var sig [64]byte
	if _, err := b.Read(sig[:]); err != nil {
		return err
	}
	o.TheirCommitSig = sig[:]
	if err != nil {
		return err
	}

	o.FundingTx = wire.NewMsgTx()
	if err := o.FundingTx.Deserialize(b); err != nil {
		return err
	}

	if _, err := b.Read(encryptedPriv[:]); err != nil {
		return err
	}
	decryptedPriv, err = addrManager.Decrypt(waddrmgr.CKTPrivate, encryptedPriv[:])
	if err != nil {
		return err
	}
	o.MultiSigKey, _ = btcec.PrivKeyFromBytes(btcec.S256(), decryptedPriv)

	var redeemScript [71]byte
	if _, err := b.Read(redeemScript[:]); err != nil {
		return err
	}
	o.FundingRedeemScript = redeemScript[:]

	if _, err := b.Read(o.TheirCurrentRevocation[:]); err != nil {
		return err
	}

	var addr [34]byte
	if _, err := b.Read(addr[:]); err != nil {
		return err
	}
	o.OurDeliveryAddress, err = btcutil.DecodeAddress(string(addr[:]), ActiveNetParams)
	if err != nil {
		return err
	}

	if _, err := b.Read(addr[:]); err != nil {
		return err
	}
	o.TheirDeliveryAddress, err = btcutil.DecodeAddress(string(addr[:]), ActiveNetParams)
	if err != nil {
		return err
	}

	if err := binary.Read(b, endian, &o.CsvDelay); err != nil {
		return err
	}
	if err := binary.Read(b, endian, &o.NumUpdates); err != nil {
		return err
	}
	if err := binary.Read(b, endian, &o.TotalSatoshisSent); err != nil {
		return err
	}
	if err := binary.Read(b, endian, &o.TotalSatoshisReceived); err != nil {
		return err
	}

	var unix int64
	if err := binary.Read(b, endian, &unix); err != nil {
		return err
	}
	o.CreationTime = time.Unix(unix, 0)

	return nil
}

func newOpenChannelState(ID [32]byte) *OpenChannelState {
	return &OpenChannelState{TheirLNID: ID}
}
<commit_msg>lnwallet: implement FetchOpenChannel and PutOpenChannel<commit_after>package lnwallet

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"time"

	"li.lan/labs/plasma/shachain"

	"github.com/btcsuite/btcd/btcec"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcwallet/waddrmgr"
	"github.com/btcsuite/btcwallet/walletdb"
)

var (
	// Namespace bucket keys.
	lightningNamespaceKey = []byte("ln-wallet")
	waddrmgrNamespaceKey  = []byte("waddrmgr")
	wtxmgrNamespaceKey    = []byte("wtxmgr")

	openChannelBucket   = []byte("o")
	closedChannelBucket = []byte("c")
	activeChanKey       = []byte("a")

	endian = binary.BigEndian
)

// ChannelDB...
// TODO(roasbeef): CHECKSUMS, REDUNDANCY, etc etc.
type ChannelDB struct {
	// TODO(roasbeef): caching, etc?
	addrmgr *waddrmgr.Manager

	namespace walletdb.Namespace
}

// PutOpenChannel...
func (c *ChannelDB) PutOpenChannel(channel *OpenChannelState) error {
	return c.namespace.Update(func(tx walletdb.Tx) error {
		// Get the bucket dedicated to storing the meta-data for open
		// channels.
		rootBucket := tx.RootBucket()
		openChanBucket, err := rootBucket.CreateBucketIfNotExists(openChannelBucket)
		if err != nil {
			return err
		}

		return dbPutOpenChannel(openChanBucket, channel, c.addrmgr)
	})
}

// GetOpenChannel...
// TODO(roasbeef): assumes only 1 active channel per-node
func (c *ChannelDB) FetchOpenChannel(nodeID [32]byte) (*OpenChannelState, error) {
	var channel *OpenChannelState
	var err error

	dbErr := c.namespace.View(func(tx walletdb.Tx) error {
		// Get the bucket dedicated to storing the meta-data for open
		// channels.
		rootBucket := tx.RootBucket()
		openChanBucket, err := rootBucket.CreateBucketIfNotExists(openChannelBucket)
		if err != nil {
			return err
		}

		channel, err = dbGetOpenChannel(openChanBucket, nodeID, c.addrmgr)
		if err != nil {
			return err
		}
		return nil
	})

	return channel, dbErr
}

// dbPutChannel...
func dbPutOpenChannel(activeChanBucket walletdb.Bucket, channel *OpenChannelState,
	addrmgr *waddrmgr.Manager) error {

	// Generate a serialized version of the open channel. The addrmgr is
	// required in order to encrypt densitive data.
	var b bytes.Buffer
	if err := channel.Encode(&b, addrmgr); err != nil {
		return err
	}

	// Grab the bucket dedicated to storing data related to this particular
	// node.
	nodeBucket, err := activeChanBucket.CreateBucketIfNotExists(channel.TheirLNID[:])
	if err != nil {
		return err
	}

	return nodeBucket.Put(activeChanKey, b.Bytes())
}

// dbPutChannel...
func dbGetOpenChannel(bucket walletdb.Bucket, nodeID [32]byte,
	addrmgr *waddrmgr.Manager) (*OpenChannelState, error) {
	// Grab the bucket dedicated to storing data related to this particular
	// node.
	nodeBucket, err := bucket.CreateBucketIfNotExists(nodeID[:])
	if err != nil {
		return nil, err
	}

	serializedChannel := nodeBucket.Get(activeChanKey)
	if serializedChannel == nil {
		// TODO(roasbeef): make proper in error.go
		return nil, fmt.Errorf("node has no open channels")
	}

	// Decode the serialized channel state, using the addrmgr to decrypt
	// sensitive information.
	channel := &OpenChannelState{}
	reader := bytes.NewReader(serializedChannel)
	if err := channel.Decode(reader, addrmgr); err != nil {
		return nil, err
	}

	return channel, nil
}

// NewChannelDB...
// TODO(roasbeef): re-visit this dependancy...
func NewChannelDB(addrmgr *waddrmgr.Manager, namespace walletdb.Namespace) *ChannelDB {
	// TODO(roasbeef): create buckets if not created?
	return &ChannelDB{addrmgr, namespace}
}

// OpenChannelState...
// TODO(roasbeef): store only the essentials? optimize space...
// TODO(roasbeef): switch to "column store"
type OpenChannelState struct {
	// Hash? or Their current pubKey?
	// TODO(roasbeef): switch to Tadge's LNId
	TheirLNID [wire.HashSize]byte

	// The ID of a channel is the txid of the funding transaction.
	ChanID [wire.HashSize]byte

	MinFeePerKb btcutil.Amount
	// Our reserve. Assume symmetric reserve amounts. Only needed if the
	// funding type is CLTV.
	//ReserveAmount btcutil.Amount

	// Keys for both sides to be used for the commitment transactions.
	OurCommitKey   *btcec.PrivateKey
	TheirCommitKey *btcec.PublicKey

	// Tracking total channel capacity, and the amount of funds allocated
	// to each side.
	Capacity     btcutil.Amount
	OurBalance   btcutil.Amount
	TheirBalance btcutil.Amount

	// Commitment transactions for both sides (they're asymmetric). Also
	// their signature which lets us spend our version of the commitment
	// transaction.
	TheirCommitTx  *wire.MsgTx
	OurCommitTx    *wire.MsgTx // TODO(roasbeef): store hash instead?
	TheirCommitSig []byte      // TODO(roasbeef): fixed length?, same w/ redeem

	// The final funding transaction. Kept wallet-related records.
	FundingTx *wire.MsgTx

	MultiSigKey         *btcec.PrivateKey
	FundingRedeemScript []byte

	// Current revocation for their commitment transaction. However, since
	// this is the hash, and not the pre-image, we can't yet verify that
	// it's actually in the chain.
	TheirCurrentRevocation [wire.HashSize]byte
	TheirShaChain          *shachain.HyperShaChain
	OurShaChain            *shachain.HyperShaChain

	// Final delivery address
	OurDeliveryAddress   btcutil.Address
	TheirDeliveryAddress btcutil.Address

	// In blocks
	CsvDelay uint32

	// TODO(roasbeef): track fees, other stats?
	NumUpdates            uint64
	TotalSatoshisSent     uint64
	TotalSatoshisReceived uint64
	CreationTime          time.Time
}

// Encode...
// TODO(roasbeef): checksum
func (o *OpenChannelState) Encode(b io.Writer, addrManager *waddrmgr.Manager) error {
	if _, err := b.Write(o.TheirLNID[:]); err != nil {
		return err
	}
	if _, err := b.Write(o.ChanID[:]); err != nil {
		return err
	}

	if err := binary.Write(b, endian, uint64(o.MinFeePerKb)); err != nil {
		return err
	}

	encryptedPriv, err := addrManager.Encrypt(waddrmgr.CKTPrivate,
		o.OurCommitKey.Serialize())
	if err != nil {
		return err
	}
	if _, err := b.Write(encryptedPriv); err != nil {
		return err
	}
	if _, err := b.Write(o.TheirCommitKey.SerializeCompressed()); err != nil {
		return err
	}

	if err := binary.Write(b, endian, uint64(o.Capacity)); err != nil {
		return err
	}
	if err := binary.Write(b, endian, uint64(o.OurBalance)); err != nil {
		return err
	}
	if err := binary.Write(b, endian, uint64(o.TheirBalance)); err != nil {
		return err
	}

	if err := o.TheirCommitTx.Serialize(b); err != nil {
		return err
	}
	if err := o.OurCommitTx.Serialize(b); err != nil {
		return err
	}
	if _, err := b.Write(o.TheirCommitSig[:]); err != nil {
		return err
	}

	if err := o.FundingTx.Serialize(b); err != nil {
		return err
	}

	encryptedPriv, err = addrManager.Encrypt(waddrmgr.CKTPrivate,
		o.MultiSigKey.Serialize())
	if err != nil {
		return err
	}
	if _, err := b.Write(encryptedPriv); err != nil {
		return err
	}
	if _, err := b.Write(o.FundingRedeemScript); err != nil {
		return err
	}

	if _, err := b.Write(o.TheirCurrentRevocation[:]); err != nil {
		return err
	}
	// TODO(roasbeef): serialize shachains

	if _, err := b.Write([]byte(o.OurDeliveryAddress.EncodeAddress())); err != nil {
		return err
	}
	if _, err := b.Write([]byte(o.TheirDeliveryAddress.EncodeAddress())); err != nil {
		return err
	}

	if err := binary.Write(b, endian, o.CsvDelay); err != nil {
		return err
	}
	if err := binary.Write(b, endian, o.NumUpdates); err != nil {
		return err
	}
	if err := binary.Write(b, endian, o.TotalSatoshisSent); err != nil {
		return err
	}
	if err := binary.Write(b, endian, o.TotalSatoshisReceived); err != nil {
		return err
	}

	if err := binary.Write(b, endian, o.CreationTime.Unix()); err != nil {
		return err
	}

	return nil
}

// Decode...
func (o *OpenChannelState) Decode(b io.Reader, addrManager *waddrmgr.Manager) error {
	var scratch [8]byte

	if _, err := b.Read(o.TheirLNID[:]); err != nil {
		return err
	}
	if _, err := b.Read(o.ChanID[:]); err != nil {
		return err
	}

	if _, err := b.Read(scratch[:]); err != nil {
		return err
	}
	o.MinFeePerKb = btcutil.Amount(endian.Uint64(scratch[:]))

	// nonce + serPrivKey + mac
	var encryptedPriv [24 + 32 + 16]byte
	if _, err := b.Read(encryptedPriv[:]); err != nil {
		return err
	}
	decryptedPriv, err := addrManager.Decrypt(waddrmgr.CKTPrivate, encryptedPriv[:])
	if err != nil {
		return err
	}
	o.OurCommitKey, _ = btcec.PrivKeyFromBytes(btcec.S256(), decryptedPriv)

	var serPubKey [33]byte
	if _, err := b.Read(serPubKey[:]); err != nil {
		return err
	}
	o.TheirCommitKey, err = btcec.ParsePubKey(serPubKey[:], btcec.S256())
	if err != nil {
		return err
	}

	if _, err := b.Read(scratch[:]); err != nil {
		return err
	}
	o.Capacity = btcutil.Amount(endian.Uint64(scratch[:]))
	if _, err := b.Read(scratch[:]); err != nil {
		return err
	}
	o.OurBalance = btcutil.Amount(endian.Uint64(scratch[:]))
	if _, err := b.Read(scratch[:]); err != nil {
		return err
	}
	o.TheirBalance = btcutil.Amount(endian.Uint64(scratch[:]))

	o.TheirCommitTx = wire.NewMsgTx()
	if err := o.TheirCommitTx.Deserialize(b); err != nil {
		return err
	}
	o.OurCommitTx = wire.NewMsgTx()
	if err := o.OurCommitTx.Deserialize(b); err != nil {
		return err
	}

	var sig [64]byte
	if _, err := b.Read(sig[:]); err != nil {
		return err
	}
	o.TheirCommitSig = sig[:]
	if err != nil {
		return err
	}

	o.FundingTx = wire.NewMsgTx()
	if err := o.FundingTx.Deserialize(b); err != nil {
		return err
	}

	if _, err := b.Read(encryptedPriv[:]); err != nil {
		return err
	}
	decryptedPriv, err = addrManager.Decrypt(waddrmgr.CKTPrivate, encryptedPriv[:])
	if err != nil {
		return err
	}
	o.MultiSigKey, _ = btcec.PrivKeyFromBytes(btcec.S256(), decryptedPriv)

	var redeemScript [71]byte
	if _, err := b.Read(redeemScript[:]); err != nil {
		return err
	}
	o.FundingRedeemScript = redeemScript[:]

	if _, err := b.Read(o.TheirCurrentRevocation[:]); err != nil {
		return err
	}

	var addr [34]byte
	if _, err := b.Read(addr[:]); err != nil {
		return err
	}
	o.OurDeliveryAddress, err = btcutil.DecodeAddress(string(addr[:]), ActiveNetParams)
	if err != nil {
		return err
	}

	if _, err := b.Read(addr[:]); err != nil {
		return err
	}
	o.TheirDeliveryAddress, err = btcutil.DecodeAddress(string(addr[:]), ActiveNetParams)
	if err != nil {
		return err
	}

	if err := binary.Read(b, endian, &o.CsvDelay); err != nil {
		return err
	}
	if err := binary.Read(b, endian, &o.NumUpdates); err != nil {
		return err
	}
	if err := binary.Read(b, endian, &o.TotalSatoshisSent); err != nil {
		return err
	}
	if err := binary.Read(b, endian, &o.TotalSatoshisReceived); err != nil {
		return err
	}

	var unix int64
	if err := binary.Read(b, endian, &unix); err != nil {
		return err
	}
	o.CreationTime = time.Unix(unix, 0)

	return nil
}
<|endoftext|>