<commit_before>package main

import (
	"fmt"
	"strings"

	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

func storageVolumePropertiesTranslate(targetConfig map[string]string, targetParentPoolDriver string) (map[string]string, error) {
	newConfig := make(map[string]string, len(targetConfig))
	for key, val := range targetConfig {
		// User keys are not validated.
		if strings.HasPrefix(key, "user.") {
			continue
		}

		// Validate storage volume config keys.
		validator, ok := storageVolumeConfigKeys[key]
		if !ok {
			return nil, fmt.Errorf("Invalid storage volume configuration key: %s", key)
		}

		validStorageDrivers, err := validator(val)
		if err != nil {
			return nil, err
		}

		// Drop invalid keys.
		if !shared.StringInSlice(targetParentPoolDriver, validStorageDrivers) {
			continue
		}

		newConfig[key] = val
	}

	return newConfig, nil
}

func updateStoragePoolVolumeError(unchangeable []string, driverName string) error {
	return fmt.Errorf(`The %v properties cannot be changed for "%s" `+
		`storage volumes`, unchangeable, driverName)
}

// For storage volumes of type storagePoolVolumeTypeContainer We only allow to
// change properties directly on the storage volume if there's not
// corresponding way to change it by other means. A good example is the "size"
// property which can be manipulated by setting a root disk device "size"
// property.
var changeableStoragePoolVolumeProperties = map[string][]string{
	"btrfs": {
		"security.unmapped",
		"size",
	},

	"ceph": {
		"block.mount_options",
		"security.unmapped",
		"size"},

	"dir": {
		"security.unmapped",
	},

	"lvm": {
		"block.mount_options",
		"security.unmapped",
		"size"},

	"zfs": {
		"security.unmapped",
		"size",
		"zfs.remove_snapshots",
		"zfs.use_refquota"},
}

// btrfs, ceph, dir, lvm, zfs
var storageVolumeConfigKeys = map[string]func(value string) ([]string, error){
	"block.filesystem": func(value string) ([]string, error) {
		err := shared.IsOneOf(value, []string{"btrfs", "ext4", "xfs"})
		if err != nil {
			return nil, err
		}

		return []string{"ceph", "lvm"}, nil
	},
	"block.mount_options": func(value string) ([]string, error) {
		return []string{"ceph", "lvm"}, shared.IsAny(value)
	},
	"security.unmapped": func(value string) ([]string, error) {
		return supportedPoolTypes, shared.IsBool(value)
	},
	"size": func(value string) ([]string, error) {
		if value == "" {
			return []string{"btrfs", "ceph", "lvm", "zfs"}, nil
		}

		_, err := shared.ParseByteSizeString(value)
		if err != nil {
			return nil, err
		}

		return []string{"btrfs", "ceph", "lvm", "zfs"}, nil
	},
	"volatile.idmap.last": func(value string) ([]string, error) {
		return supportedPoolTypes, shared.IsAny(value)
	},
	"volatile.idmap.next": func(value string) ([]string, error) {
		return supportedPoolTypes, shared.IsAny(value)
	},
	"zfs.remove_snapshots": func(value string) ([]string, error) {
		err := shared.IsBool(value)
		if err != nil {
			return nil, err
		}

		return []string{"zfs"}, nil
	},
	"zfs.use_refquota": func(value string) ([]string, error) {
		err := shared.IsBool(value)
		if err != nil {
			return nil, err
		}

		return []string{"zfs"}, nil
	},
}

func storageVolumeValidateConfig(name string, config map[string]string, parentPool *api.StoragePool) error {
	for key, val := range config {
		// User keys are not validated.
		if strings.HasPrefix(key, "user.") {
			continue
		}

		// Validate storage volume config keys.
		validator, ok := storageVolumeConfigKeys[key]
		if !ok {
			return fmt.Errorf("Invalid storage volume configuration key: %s", key)
		}

		_, err := validator(val)
		if err != nil {
			return err
		}

		if parentPool.Driver != "zfs" || parentPool.Driver == "dir" {
			if config["zfs.use_refquota"] != "" {
				return fmt.Errorf("the key volume.zfs.use_refquota cannot be used with non zfs storage volumes")
			}

			if config["zfs.remove_snapshots"] != "" {
				return fmt.Errorf("the key volume.zfs.remove_snapshots cannot be used with non zfs storage volumes")
			}
		}

		if parentPool.Driver == "dir" {
			if config["block.mount_options"] != "" {
				return fmt.Errorf("the key block.mount_options cannot be used with dir storage volumes")
			}

			if config["block.filesystem"] != "" {
				return fmt.Errorf("the key block.filesystem cannot be used with dir storage volumes")
			}

			if config["size"] != "" {
				return fmt.Errorf("the key size cannot be used with dir storage volumes")
			}
		}
	}

	return nil
}

func storageVolumeFillDefault(name string, config map[string]string, parentPool *api.StoragePool) error {
	if parentPool.Driver == "dir" {
		config["size"] = ""
	} else if parentPool.Driver == "lvm" || parentPool.Driver == "ceph" {
		if config["block.filesystem"] == "" {
			config["block.filesystem"] = parentPool.Config["volume.block.filesystem"]
		}
		if config["block.filesystem"] == "" {
			// Unchangeable volume property: Set unconditionally.
			config["block.filesystem"] = "ext4"
		}

		if config["block.mount_options"] == "" {
			config["block.mount_options"] = parentPool.Config["volume.block.mount_options"]
		}
		if config["block.mount_options"] == "" {
			// Unchangeable volume property: Set unconditionally.
			config["block.mount_options"] = "discard"
		}

		// Does the pool request a default size for new storage volumes?
		if config["size"] == "0" || config["size"] == "" {
			config["size"] = parentPool.Config["volume.size"]
		}
		// Does the user explicitly request a default size for new
		// storage volumes?
		if config["size"] == "0" || config["size"] == "" {
			config["size"] = "10GB"
		}
	} else {
		if config["size"] != "" {
			_, err := shared.ParseByteSizeString(config["size"])
			if err != nil {
				return err
			}
		}
	}

	return nil
}
<commit_msg>lxd/volumes: Allow quota on ext4<commit_after>package main

import (
	"fmt"
	"strings"

	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
)

func storageVolumePropertiesTranslate(targetConfig map[string]string, targetParentPoolDriver string) (map[string]string, error) {
	newConfig := make(map[string]string, len(targetConfig))
	for key, val := range targetConfig {
		// User keys are not validated.
		if strings.HasPrefix(key, "user.") {
			continue
		}

		// Validate storage volume config keys.
		validator, ok := storageVolumeConfigKeys[key]
		if !ok {
			return nil, fmt.Errorf("Invalid storage volume configuration key: %s", key)
		}

		validStorageDrivers, err := validator(val)
		if err != nil {
			return nil, err
		}

		// Drop invalid keys.
		if !shared.StringInSlice(targetParentPoolDriver, validStorageDrivers) {
			continue
		}

		newConfig[key] = val
	}

	return newConfig, nil
}

func updateStoragePoolVolumeError(unchangeable []string, driverName string) error {
	return fmt.Errorf(`The %v properties cannot be changed for "%s" `+
		`storage volumes`, unchangeable, driverName)
}

// For storage volumes of type storagePoolVolumeTypeContainer We only allow to
// change properties directly on the storage volume if there's not
// corresponding way to change it by other means. A good example is the "size"
// property which can be manipulated by setting a root disk device "size"
// property.
var changeableStoragePoolVolumeProperties = map[string][]string{
	"btrfs": {
		"security.unmapped",
		"size",
	},

	"ceph": {
		"block.mount_options",
		"security.unmapped",
		"size"},

	"dir": {
		"security.unmapped",
	},

	"lvm": {
		"block.mount_options",
		"security.unmapped",
		"size"},

	"zfs": {
		"security.unmapped",
		"size",
		"zfs.remove_snapshots",
		"zfs.use_refquota"},
}

// btrfs, ceph, dir, lvm, zfs
var storageVolumeConfigKeys = map[string]func(value string) ([]string, error){
	"block.filesystem": func(value string) ([]string, error) {
		err := shared.IsOneOf(value, []string{"btrfs", "ext4", "xfs"})
		if err != nil {
			return nil, err
		}

		return []string{"ceph", "lvm"}, nil
	},
	"block.mount_options": func(value string) ([]string, error) {
		return []string{"ceph", "lvm"}, shared.IsAny(value)
	},
	"security.unmapped": func(value string) ([]string, error) {
		return supportedPoolTypes, shared.IsBool(value)
	},
	"size": func(value string) ([]string, error) {
		if value == "" {
			return []string{"btrfs", "ceph", "lvm", "zfs"}, nil
		}

		_, err := shared.ParseByteSizeString(value)
		if err != nil {
			return nil, err
		}

		return []string{"btrfs", "ceph", "lvm", "zfs"}, nil
	},
	"volatile.idmap.last": func(value string) ([]string, error) {
		return supportedPoolTypes, shared.IsAny(value)
	},
	"volatile.idmap.next": func(value string) ([]string, error) {
		return supportedPoolTypes, shared.IsAny(value)
	},
	"zfs.remove_snapshots": func(value string) ([]string, error) {
		err := shared.IsBool(value)
		if err != nil {
			return nil, err
		}

		return []string{"zfs"}, nil
	},
	"zfs.use_refquota": func(value string) ([]string, error) {
		err := shared.IsBool(value)
		if err != nil {
			return nil, err
		}

		return []string{"zfs"}, nil
	},
}

func storageVolumeValidateConfig(name string, config map[string]string, parentPool *api.StoragePool) error {
	for key, val := range config {
		// User keys are not validated.
		if strings.HasPrefix(key, "user.") {
			continue
		}

		// Validate storage volume config keys.
		validator, ok := storageVolumeConfigKeys[key]
		if !ok {
			return fmt.Errorf("Invalid storage volume configuration key: %s", key)
		}

		_, err := validator(val)
		if err != nil {
			return err
		}

		if parentPool.Driver != "zfs" || parentPool.Driver == "dir" {
			if config["zfs.use_refquota"] != "" {
				return fmt.Errorf("the key volume.zfs.use_refquota cannot be used with non zfs storage volumes")
			}

			if config["zfs.remove_snapshots"] != "" {
				return fmt.Errorf("the key volume.zfs.remove_snapshots cannot be used with non zfs storage volumes")
			}
		}

		if parentPool.Driver == "dir" {
			if config["block.mount_options"] != "" {
				return fmt.Errorf("the key block.mount_options cannot be used with dir storage volumes")
			}

			if config["block.filesystem"] != "" {
				return fmt.Errorf("the key block.filesystem cannot be used with dir storage volumes")
			}
		}
	}

	return nil
}

func storageVolumeFillDefault(name string, config map[string]string, parentPool *api.StoragePool) error {
	if parentPool.Driver == "dir" {
		config["size"] = ""
	} else if parentPool.Driver == "lvm" || parentPool.Driver == "ceph" {
		if config["block.filesystem"] == "" {
			config["block.filesystem"] = parentPool.Config["volume.block.filesystem"]
		}
		if config["block.filesystem"] == "" {
			// Unchangeable volume property: Set unconditionally.
			config["block.filesystem"] = "ext4"
		}

		if config["block.mount_options"] == "" {
			config["block.mount_options"] = parentPool.Config["volume.block.mount_options"]
		}
		if config["block.mount_options"] == "" {
			// Unchangeable volume property: Set unconditionally.
			config["block.mount_options"] = "discard"
		}

		// Does the pool request a default size for new storage volumes?
		if config["size"] == "0" || config["size"] == "" {
			config["size"] = parentPool.Config["volume.size"]
		}
		// Does the user explicitly request a default size for new
		// storage volumes?
		if config["size"] == "0" || config["size"] == "" {
			config["size"] = "10GB"
		}
	} else {
		if config["size"] != "" {
			_, err := shared.ParseByteSizeString(config["size"])
			if err != nil {
				return err
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package security

import (
	"bytes"
	"crypto/rand"
	"encoding/base64"
	"fmt"
	"time"

	"veyron.io/veyron/veyron2/vlog"
	"veyron.io/veyron/veyron2/vom"
)

// NewCaveat returns a Caveat that requires validation by validator.
func NewCaveat(validator CaveatValidator) (Caveat, error) {
	var buf bytes.Buffer
	if err := vom.NewEncoder(&buf).Encode(validator); err != nil {
		return Caveat{}, err
	}
	return Caveat{buf.Bytes()}, nil
}

// ExpiryCaveat returns a Caveat that validates iff the current time is before t.
func ExpiryCaveat(t time.Time) (Caveat, error) {
	return NewCaveat(unixTimeExpiryCaveat(t.Unix()))
}

// MethodCaveat returns a Caveat that validates iff the method being invoked by
// the peer is listed in an argument to this function.
func MethodCaveat(method string, additionalMethods ...string) (Caveat, error) {
	return NewCaveat(methodCaveat(append(additionalMethods, method)))
}

// PeerBlessingsCaveat returns a Caveat that validates iff the peer has a blessing
// that matches one of the patterns provided as an argument to this function.
//
// For example, creating a blessing "alice/friend" with a PeerBlessingsCaveat("bob")
// will allow the blessing "alice/friend" to be used only when communicating with
// a principal that has the blessing "bob".
func PeerBlessingsCaveat(pattern BlessingPattern, additionalPatterns ...BlessingPattern) (Caveat, error) {
	return NewCaveat(peerBlessingsCaveat(append(additionalPatterns, pattern)))
}

// digest returns a hash of the contents of c.
func (c *Caveat) digest(hash Hash) []byte { return hash.sum(c.ValidatorVOM) }

func (c *Caveat) String() string {
	var validator CaveatValidator
	if err := vom.NewDecoder(bytes.NewReader(c.ValidatorVOM)).Decode(&validator); err != nil {
		return fmt.Sprintf("%T(%v)", validator, validator)
	}
	// If we could "peek" the type of the encoded object via the VOM-API, that may be a better message?
	return fmt.Sprintf("{Caveat(%d bytes) with the corresponding CaveatValidator not compiled into this binary}", len(c.ValidatorVOM))
}

func (c unixTimeExpiryCaveat) Validate(ctx Context) error {
	now := time.Now()
	expiry := time.Unix(int64(c), 0)
	if now.After(expiry) {
		return fmt.Errorf("%T(%v=%v) fails validation at %v", c, c, expiry, now)
	}
	return nil
}

func (c methodCaveat) Validate(ctx Context) error {
	methods := []string(c)
	if ctx.Method() == "" && len(methods) == 0 {
		return nil
	}
	for _, m := range methods {
		if ctx.Method() == m {
			return nil
		}
	}
	return fmt.Errorf("%T=%v fails validation for method %q", c, c, ctx.Method())
}

func (c peerBlessingsCaveat) Validate(ctx Context) error {
	patterns := []BlessingPattern(c)
	if ctx.LocalID() == nil {
		return fmt.Errorf("%T=%v fails validation since ctx.LocalID is nil", c, c)
	}
	peerblessings := ctx.LocalID().Names()
	for _, p := range patterns {
		if p.MatchedBy(peerblessings...) {
			return nil
		}
	}
	return fmt.Errorf("%T=%v fails validation for peer with blessings %v", c, c, peerblessings)
}

// DEPRECATED: TODO(ashankar,ataly): Remove when switching from PublicID/PrivateID to Blessings/Principal.
type Expiry struct {
	IssueTime  time.Time
	ExpiryTime time.Time
}

func (v *Expiry) Validate(context Context) error {
	now := time.Now()
	if now.Before(v.IssueTime) || now.After(v.ExpiryTime) {
		return fmt.Errorf("%#v forbids credential from being used at this time(%v)", v, now)
	}
	return nil
}

// UnconstrainedUse returns a Caveat implementation that never fails to
// validate. This is useful only for providing unconstrained blessings/discharges
// to another principal.
func UnconstrainedUse() Caveat { return Caveat{} }

func isUnconstrainedUseCaveat(c Caveat) bool { return c.ValidatorVOM == nil }

// NewPublicKeyCaveat returns a security.ThirdPartyCaveat which requires a
// discharge from a principal identified by the public key 'key' and present
// at the object name 'location'. This discharging principal is expected to
// validate all provided 'caveats' before issuing a discharge.
func NewPublicKeyCaveat(discharger PublicKey, location string, requirements ThirdPartyRequirements, caveat Caveat, additionalCaveats ...Caveat) (ThirdPartyCaveat, error) {
	cav := &publicKeyThirdPartyCaveat{
		Caveats:                append(additionalCaveats, caveat),
		DischargerLocation:     location,
		DischargerRequirements: requirements,
	}
	if _, err := rand.Read(cav.Nonce[:]); err != nil {
		return nil, err
	}
	var err error
	if cav.DischargerKey, err = discharger.MarshalBinary(); err != nil {
		return nil, err
	}
	return cav, nil
}

func (c *publicKeyThirdPartyCaveat) Validate(ctx Context) error {
	discharge, ok := ctx.Discharges()[c.ID()]
	if !ok {
		return fmt.Errorf("missing discharge for caveat(id=%v)", c.ID())
	}
	// Must be of the valid type.
	d, ok := discharge.(*publicKeyDischarge)
	if !ok {
		return fmt.Errorf("invalid discharge type(%T) for caveat(%T)", d, c)
	}
	// Must be signed by the principal designated by c.DischargerKey
	key, err := c.discharger()
	if err != nil {
		return err
	}
	if err := d.verify(key); err != nil {
		return err
	}
	// And all caveats on the discharge must be met.
	for _, cav := range d.Caveats {
		var validator CaveatValidator
		if err := vom.NewDecoder(bytes.NewReader(cav.ValidatorVOM)).Decode(&validator); err != nil {
			return fmt.Errorf("failed to interpret a caveat on the discharge: %v", err)
		}
		if err := validator.Validate(ctx); err != nil {
			return fmt.Errorf("a caveat(%T) on the discharge failed to validate: %v", validator, err)
		}
	}
	return nil
}

func (c *publicKeyThirdPartyCaveat) ID() string {
	key, err := c.discharger()
	if err != nil {
		vlog.Error(err)
		return ""
	}
	hash := key.hash()
	bytes := append(hash.sum(c.Nonce[:]), hash.sum(c.DischargerKey)...)
	for _, cav := range c.Caveats {
		bytes = append(bytes, cav.digest(hash)...)
	}
	return base64.StdEncoding.EncodeToString(hash.sum(bytes))
}

func (c *publicKeyThirdPartyCaveat) Location() string { return c.DischargerLocation }
func (c *publicKeyThirdPartyCaveat) Requirements() ThirdPartyRequirements {
	return c.DischargerRequirements
}

func (c *publicKeyThirdPartyCaveat) Dischargeable(context Context) error {
	// Validate the caveats embedded within this third-party caveat.
	for _, cav := range c.Caveats {
		var validator CaveatValidator
		if err := vom.NewDecoder(bytes.NewReader(cav.ValidatorVOM)).Decode(&validator); err != nil {
			return fmt.Errorf("failed to interpret restriction embedded in ThirdPartyCaveat: %v", err)
		}
		if err := validator.Validate(context); err != nil {
			return fmt.Errorf("could not validate embedded restriction %T: %v", validator, err)
		}
	}
	return nil
}

func (c *publicKeyThirdPartyCaveat) discharger() (PublicKey, error) {
	key, err := UnmarshalPublicKey(c.DischargerKey)
	if err != nil {
		return nil, fmt.Errorf("invalid %T: failed to unmarshal discharger's public key: %v", *c, err)
	}
	return key, nil
}

func (d *publicKeyDischarge) ID() string { return d.ThirdPartyCaveatID }
func (d *publicKeyDischarge) ThirdPartyCaveats() []ThirdPartyCaveat {
	var ret []ThirdPartyCaveat
	for _, cav := range d.Caveats {
		var tpcav ThirdPartyCaveat
		if err := vom.NewDecoder(bytes.NewReader(cav.ValidatorVOM)).Decode(&tpcav); err == nil {
			ret = append(ret, tpcav)
		}
	}
	return ret
}

func (d *publicKeyDischarge) digest(hash Hash) []byte {
	msg := hash.sum([]byte(d.ThirdPartyCaveatID))
	for _, cav := range d.Caveats {
		msg = append(msg, cav.digest(hash)...)
	}
	return hash.sum(msg)
}

func (d *publicKeyDischarge) verify(key PublicKey) error {
	if !bytes.Equal(d.Signature.Purpose, dischargePurpose) {
		return fmt.Errorf("signature on discharge for caveat %v was not intended for discharges(purpose=%q)", d.ThirdPartyCaveatID, d.Signature.Purpose)
	}
	if !d.Signature.Verify(key, d.digest(key.hash())) {
		return fmt.Errorf("signature verification on discharge for caveat %v failed", d.ThirdPartyCaveatID)
	}
	return nil
}

func (d *publicKeyDischarge) sign(signer Signer) error {
	var err error
	d.Signature, err = signer.Sign(dischargePurpose, d.digest(signer.PublicKey().hash()))
	return err
}
<commit_msg>veyron/runtimes/google/ipc: Check caveats on the server's blessings.<commit_after>package security

import (
	"bytes"
	"crypto/rand"
	"encoding/base64"
	"fmt"
	"time"

	"veyron.io/veyron/veyron2/vlog"
	"veyron.io/veyron/veyron2/vom"
)

// NewCaveat returns a Caveat that requires validation by validator.
func NewCaveat(validator CaveatValidator) (Caveat, error) {
	var buf bytes.Buffer
	if err := vom.NewEncoder(&buf).Encode(validator); err != nil {
		return Caveat{}, err
	}
	return Caveat{buf.Bytes()}, nil
}

// ExpiryCaveat returns a Caveat that validates iff the current time is before t.
func ExpiryCaveat(t time.Time) (Caveat, error) {
	return NewCaveat(unixTimeExpiryCaveat(t.Unix()))
}

// MethodCaveat returns a Caveat that validates iff the method being invoked by
// the peer is listed in an argument to this function.
func MethodCaveat(method string, additionalMethods ...string) (Caveat, error) {
	return NewCaveat(methodCaveat(append(additionalMethods, method)))
}

// PeerBlessingsCaveat returns a Caveat that validates iff the peer has a blessing
// that matches one of the patterns provided as an argument to this function.
//
// For example, creating a blessing "alice/friend" with a PeerBlessingsCaveat("bob")
// will allow the blessing "alice/friend" to be used only when communicating with
// a principal that has the blessing "bob".
func PeerBlessingsCaveat(pattern BlessingPattern, additionalPatterns ...BlessingPattern) (Caveat, error) {
	return NewCaveat(peerBlessingsCaveat(append(additionalPatterns, pattern)))
}

// digest returns a hash of the contents of c.
func (c *Caveat) digest(hash Hash) []byte { return hash.sum(c.ValidatorVOM) }

func (c *Caveat) String() string {
	var validator CaveatValidator
	if err := vom.NewDecoder(bytes.NewReader(c.ValidatorVOM)).Decode(&validator); err != nil {
		return fmt.Sprintf("%T(%v)", validator, validator)
	}
	// If we could "peek" the type of the encoded object via the VOM-API, that may be a better message?
	return fmt.Sprintf("{Caveat(%d bytes) with the corresponding CaveatValidator not compiled into this binary}", len(c.ValidatorVOM))
}

func (c unixTimeExpiryCaveat) Validate(ctx Context) error {
	now := time.Now()
	expiry := time.Unix(int64(c), 0)
	if now.After(expiry) {
		return fmt.Errorf("%T(%v=%v) fails validation at %v", c, c, expiry, now)
	}
	return nil
}

func (c methodCaveat) Validate(ctx Context) error {
	methods := []string(c)
	if ctx.Method() == "" && len(methods) == 0 {
		return nil
	}
	for _, m := range methods {
		if ctx.Method() == m {
			return nil
		}
	}
	return fmt.Errorf("%T=%v fails validation for method %q", c, c, ctx.Method())
}

func (c peerBlessingsCaveat) Validate(ctx Context) error {
	patterns := []BlessingPattern(c)
	if ctx.LocalID() == nil {
		return fmt.Errorf("%T=%v fails validation since ctx.LocalID is nil", c, c)
	}
	peerblessings := ctx.LocalID().Names()
	for _, p := range patterns {
		if p.MatchedBy(peerblessings...) {
			return nil
		}
	}
	return fmt.Errorf("%T=%v fails validation for peer with blessings %v", c, c, peerblessings)
}

// DEPRECATED: TODO(ashankar,ataly): Remove when switching from PublicID/PrivateID to Blessings/Principal.
type Expiry struct {
	IssueTime  time.Time
	ExpiryTime time.Time
}

func (v *Expiry) Validate(context Context) error {
	now := time.Now()
	if now.Before(v.IssueTime) || now.After(v.ExpiryTime) {
		return fmt.Errorf("%#v forbids credential from being used at this time(%v)", v, now)
	}
	return nil
}

// UnconstrainedUse returns a Caveat implementation that never fails to
// validate. This is useful only for providing unconstrained blessings/discharges
// to another principal.
func UnconstrainedUse() Caveat { return Caveat{} }

func isUnconstrainedUseCaveat(c Caveat) bool { return c.ValidatorVOM == nil }

// NewPublicKeyCaveat returns a security.ThirdPartyCaveat which requires a
// discharge from a principal identified by the public key 'key' and present
// at the object name 'location'. This discharging principal is expected to
// validate all provided 'caveats' before issuing a discharge.
func NewPublicKeyCaveat(discharger PublicKey, location string, requirements ThirdPartyRequirements, caveat Caveat, additionalCaveats ...Caveat) (ThirdPartyCaveat, error) {
	cav := &publicKeyThirdPartyCaveat{
		Caveats:                append(additionalCaveats, caveat),
		DischargerLocation:     location,
		DischargerRequirements: requirements,
	}
	if _, err := rand.Read(cav.Nonce[:]); err != nil {
		return nil, err
	}
	var err error
	if cav.DischargerKey, err = discharger.MarshalBinary(); err != nil {
		return nil, err
	}
	return cav, nil
}

func (c *publicKeyThirdPartyCaveat) Validate(ctx Context) error {
	// TODO(ashankar,ataly): REMOVE THIS HACK!
	// This test below effectively disables third-party caveat validation on "servers"
	// (i.e., when no method is known). Remove this hack after fixing the VC auth protocol
	// so that Discharges associated with a server's blessings are sent by the server to
	// the client.
	if len(ctx.Method()) == 0 && len(ctx.Name()) == 0 && len(ctx.Suffix()) == 0 {
		return nil
	}
	discharge, ok := ctx.Discharges()[c.ID()]
	if !ok {
		return fmt.Errorf("missing discharge for caveat(id=%v)", c.ID())
	}
	// Must be of the valid type.
	d, ok := discharge.(*publicKeyDischarge)
	if !ok {
		return fmt.Errorf("invalid discharge type(%T) for caveat(%T)", d, c)
	}
	// Must be signed by the principal designated by c.DischargerKey
	key, err := c.discharger()
	if err != nil {
		return err
	}
	if err := d.verify(key); err != nil {
		return err
	}
	// And all caveats on the discharge must be met.
	for _, cav := range d.Caveats {
		var validator CaveatValidator
		if err := vom.NewDecoder(bytes.NewReader(cav.ValidatorVOM)).Decode(&validator); err != nil {
			return fmt.Errorf("failed to interpret a caveat on the discharge: %v", err)
		}
		if err := validator.Validate(ctx); err != nil {
			return fmt.Errorf("a caveat(%T) on the discharge failed to validate: %v", validator, err)
		}
	}
	return nil
}

func (c *publicKeyThirdPartyCaveat) ID() string {
	key, err := c.discharger()
	if err != nil {
		vlog.Error(err)
		return ""
	}
	hash := key.hash()
	bytes := append(hash.sum(c.Nonce[:]), hash.sum(c.DischargerKey)...)
	for _, cav := range c.Caveats {
		bytes = append(bytes, cav.digest(hash)...)
	}
	return base64.StdEncoding.EncodeToString(hash.sum(bytes))
}

func (c *publicKeyThirdPartyCaveat) Location() string { return c.DischargerLocation }
func (c *publicKeyThirdPartyCaveat) Requirements() ThirdPartyRequirements {
	return c.DischargerRequirements
}

func (c *publicKeyThirdPartyCaveat) Dischargeable(context Context) error {
	// Validate the caveats embedded within this third-party caveat.
	for _, cav := range c.Caveats {
		var validator CaveatValidator
		if err := vom.NewDecoder(bytes.NewReader(cav.ValidatorVOM)).Decode(&validator); err != nil {
			return fmt.Errorf("failed to interpret restriction embedded in ThirdPartyCaveat: %v", err)
		}
		if err := validator.Validate(context); err != nil {
			return fmt.Errorf("could not validate embedded restriction %T: %v", validator, err)
		}
	}
	return nil
}

func (c *publicKeyThirdPartyCaveat) discharger() (PublicKey, error) {
	key, err := UnmarshalPublicKey(c.DischargerKey)
	if err != nil {
		return nil, fmt.Errorf("invalid %T: failed to unmarshal discharger's public key: %v", *c, err)
	}
	return key, nil
}

func (d *publicKeyDischarge) ID() string { return d.ThirdPartyCaveatID }
func (d *publicKeyDischarge) ThirdPartyCaveats() []ThirdPartyCaveat {
	var ret []ThirdPartyCaveat
	for _, cav := range d.Caveats {
		var tpcav ThirdPartyCaveat
		if err := vom.NewDecoder(bytes.NewReader(cav.ValidatorVOM)).Decode(&tpcav); err == nil {
			ret = append(ret, tpcav)
		}
	}
	return ret
}

func (d *publicKeyDischarge) digest(hash Hash) []byte {
	msg := hash.sum([]byte(d.ThirdPartyCaveatID))
	for _, cav := range d.Caveats {
		msg = append(msg, cav.digest(hash)...)
	}
	return hash.sum(msg)
}

func (d *publicKeyDischarge) verify(key PublicKey) error {
	if !bytes.Equal(d.Signature.Purpose, dischargePurpose) {
		return fmt.Errorf("signature on discharge for caveat %v was not intended for discharges(purpose=%q)", d.ThirdPartyCaveatID, d.Signature.Purpose)
	}
	if !d.Signature.Verify(key, d.digest(key.hash())) {
		return fmt.Errorf("signature verification on discharge for caveat %v failed", d.ThirdPartyCaveatID)
	}
	return nil
}

func (d *publicKeyDischarge) sign(signer Signer) error {
	var err error
	d.Signature, err = signer.Sign(dischargePurpose, d.digest(signer.PublicKey().hash()))
	return err
}
<|endoftext|>


<commit_before>package main

import "fmt"

type Graph interface {
	CalculatePath(start, end Node) ([]Node, error)
}

type Node interface {
	Neighbors() []Node
	Dist(neighbor Node) float64
	EstimatedCost(goal Node) float64
}

type astar struct {
	Nodes []Node
}

func (a *astar) CalculatePath(start, goal Node) ([]Node, error) {
	closedset := make(map[Node]bool)      // The set of nodes already evaluated.
	openset := map[Node]bool{start: true} // The set of tentative nodes to be evaluated, initially containing the start node
	came_from := make(map[Node]Node)      // The map of navigated nodes.

	g_score := make(map[Node]float64)
	f_score := make(map[Node]float64)
	g_score[start] = 0 // Cost from start along best known path.
	// Estimated total cost from start to goal through y.
	f_score[start] = start.EstimatedCost(goal)

	for len(openset) > 0 {
		current := lowest(f_score)
		if current == goal {
			return reconstructPath(came_from, goal), nil
		}
		delete(f_score, current)
		delete(openset, current)
		closedset[current] = true

		for _, neighbor := range current.Neighbors() {
			if _, ok := closedset[neighbor]; ok {
				continue
			}
			tentative_g_score := g_score[current] + current.Dist(neighbor)

			if _, ok := openset[neighbor]; !ok || tentative_g_score < g_score[neighbor] {
				came_from[neighbor] = current
				g_score[neighbor] = tentative_g_score
				f_score[neighbor] = g_score[neighbor] + neighbor.EstimatedCost(goal)
				if _, ok := openset[neighbor]; !ok {
					openset[neighbor] = true
				}
			}
		}

	}
	fmt.Println("Failed")
	return []Node{}, nil
}

func reconstructPath(came_from map[Node]Node, goal Node) []Node {
	res := make([]Node, 0, 500)
	res = append(res, goal)
	at := goal
	for {
		v, ok := came_from[at]
		if !ok {
			break
		}
		res = append(res, v)
		at = v

	}
	return res
}

func lowest(nodes map[Node]float64) Node {
	min := float64(-1)
	var res Node
	for k, v := range nodes {
		if min == -1 || v < min {
			min = v
			res = k
		}
	}
	return res
}

func NewAstar(nodes []Node) Graph {
	return &astar{Nodes: nodes}
}
<commit_msg>Remove fmt<commit_after>package main

type Graph interface {
	CalculatePath(start, end Node) ([]Node, error)
}

type Node interface {
	Neighbors() []Node
	Dist(neighbor Node) float64
	EstimatedCost(goal Node) float64
}

type astar struct {
	Nodes []Node
}

func (a *astar) CalculatePath(start, goal Node) ([]Node, error) {
	closedset := make(map[Node]bool)      // The set of nodes already evaluated.
	openset := map[Node]bool{start: true} // The set of tentative nodes to be evaluated, initially containing the start node
	came_from := make(map[Node]Node)      // The map of navigated nodes.

	g_score := make(map[Node]float64)
	f_score := make(map[Node]float64)
	g_score[start] = 0 // Cost from start along best known path.
	// Estimated total cost from start to goal through y.
	f_score[start] = start.EstimatedCost(goal)

	for len(openset) > 0 {
		current := lowest(f_score)
		if current == goal {
			return reconstructPath(came_from, goal), nil
		}
		delete(f_score, current)
		delete(openset, current)
		closedset[current] = true

		for _, neighbor := range current.Neighbors() {
			if _, ok := closedset[neighbor]; ok {
				continue
			}
			tentative_g_score := g_score[current] + current.Dist(neighbor)

			if _, ok := openset[neighbor]; !ok || tentative_g_score < g_score[neighbor] {
				came_from[neighbor] = current
				g_score[neighbor] = tentative_g_score
				f_score[neighbor] = g_score[neighbor] + neighbor.EstimatedCost(goal)
				if _, ok := openset[neighbor]; !ok {
					openset[neighbor] = true
				}
			}
		}

	}
	return []Node{}, nil
}

func reconstructPath(came_from map[Node]Node, goal Node) []Node {
	res := make([]Node, 0, 500)
	res = append(res, goal)
	at := goal
	for {
		v, ok := came_from[at]
		if !ok {
			break
		}
		res = append(res, v)
		at = v

	}
	return res
}

func lowest(nodes map[Node]float64) Node {
	min := float64(-1)
	var res Node
	for k, v := range nodes {
		if min == -1 || v < min {
			min = v
			res = k
		}
	}
	return res
}

func NewAstar(nodes []Node) Graph {
	return &astar{Nodes: nodes}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"

	fits "github.com/sbinet/go-cfitsio"
)

func main() {
	flag.Usage = func() {
		const msg = `Usage: go-cfitsio-imlist filename[ext][section filter]

List the the pixel values in a FITS image

Example:
  imlist image.fits                    - list the whole image
  imlist image.fits[100:110,400:410]   - list a section
  imlist table.fits[2][bin (x,y) = 32] - list the pixels in
         an image constructed from a 2D histogram of X and Y
         columns in a table with a binning factor = 32
`
		fmt.Fprintf(os.Stderr, "%v\n", msg)
		flag.PrintDefaults()
	}

	flag.Parse()
	if flag.NArg() != 1 {
		flag.Usage()
		os.Exit(1)
	}

	fname := flag.Arg(0)
	f, err := fits.Open(fname, fits.ReadOnly)
	if err != nil {
		panic(err)
	}
	defer f.Close()

	hdu := f.CHDU()
	hdr := hdu.Header()

	if len(hdr.Axes()) > 2 || len(hdr.Axes()) == 0 {
		fmt.Printf("Error: only 1D or 2D images are supported\n")
		os.Exit(1)
	}

	axes := hdr.Axes()
	data, err := hdu.Data()
	if err != nil {
		panic(err)
	}

	pixels := data.([]int16)

	// default output format string
	hdformat := " %15d"
	format := " %15.5f"
	if hdr.Bitpix() > 0 {
		hdformat = " %7d"
		format = " %7.0f"
	}
	// column header
	fmt.Printf("\n      ")
	row := int(axes[0])
	for ii := 0; ii < row; ii++ {
		fmt.Printf(hdformat, ii)
	}
	fmt.Printf("\n")

	// loop over all rows
	for jj := 0; jj < int(axes[1]); jj++ {
		fmt.Printf(" %4d ", jj)
		for ii := 0; ii < row; ii++ {
			fmt.Printf(format, float64(pixels[row*jj+ii]))
		}

		fmt.Printf("\n")
	}
	if err != nil && err != fits.END_OF_FILE {
		panic(err)
	}
}
<commit_msg>imlist: add FIXME<commit_after>package main

import (
	"flag"
	"fmt"
	"os"

	fits "github.com/sbinet/go-cfitsio"
)

func main() {
	flag.Usage = func() {
		const msg = `Usage: go-cfitsio-imlist filename[ext][section filter]

List the the pixel values in a FITS image

Example:
  imlist image.fits                    - list the whole image
  imlist image.fits[100:110,400:410]   - list a section
  imlist table.fits[2][bin (x,y) = 32] - list the pixels in
         an image constructed from a 2D histogram of X and Y
         columns in a table with a binning factor = 32
`
		fmt.Fprintf(os.Stderr, "%v\n", msg)
		flag.PrintDefaults()
	}

	flag.Parse()
	if flag.NArg() != 1 {
		flag.Usage()
		os.Exit(1)
	}

	fname := flag.Arg(0)
	f, err := fits.Open(fname, fits.ReadOnly)
	if err != nil {
		panic(err)
	}
	defer f.Close()

	hdu := f.CHDU()
	hdr := hdu.Header()

	if len(hdr.Axes()) > 2 || len(hdr.Axes()) == 0 {
		fmt.Printf("Error: only 1D or 2D images are supported\n")
		os.Exit(1)
	}

	axes := hdr.Axes()
	data, err := hdu.Data()
	if err != nil {
		panic(err)
	}

	pixels := data.([]int16) // FIXME

	// default output format string
	hdformat := " %15d"
	format := " %15.5f"
	if hdr.Bitpix() > 0 {
		hdformat = " %7d"
		format = " %7.0f"
	}
	// column header
	fmt.Printf("\n      ")
	row := int(axes[0])
	for ii := 0; ii < row; ii++ {
		fmt.Printf(hdformat, ii)
	}
	fmt.Printf("\n")

	// loop over all rows
	for jj := 0; jj < int(axes[1]); jj++ {
		fmt.Printf(" %4d ", jj)
		for ii := 0; ii < row; ii++ {
			fmt.Printf(format, float64(pixels[row*jj+ii]))
		}

		fmt.Printf("\n")
	}
	if err != nil && err != fits.END_OF_FILE {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package physical

import (
	"fmt"
	"math/rand"
	"os"
	"testing"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
)

func TestS3Backend(t *testing.T) {
	if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
		t.SkipNow()
	}

	credentialChain := aws.DefaultChainCredentials
	creds, err := credentialChain.Get()
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	region := os.Getenv("AWS_DEFAULT_REGION")
	if region == "" {
		region = "us-east-1"
	}

	s3conn := s3.New(&aws.Config{
		Credentials: aws.DefaultChainCredentials,
		Region:      aws.String(region),
	})

	var randInt = rand.New(rand.NewSource(time.Now().UnixNano())).Int()
	bucket := fmt.Sprintf("vault-s3-testacc-%d", randInt)

	_, err = s3conn.CreateBucket(&s3.CreateBucketInput{
		Bucket: aws.String(bucket),
	})
	if err != nil {
		t.Fatalf("unable to create test bucket: %s", err)
	}

	defer func() {
		// Gotta list all the objects and delete them
		// before being able to delete the bucket
		listResp, _ := s3conn.ListObjects(&s3.ListObjectsInput{
			Bucket: aws.String(bucket),
		})

		objects := &s3.Delete{}
		for _, key := range listResp.Contents {
			oi := &s3.ObjectIdentifier{Key: key.Key}
			objects.Objects = append(objects.Objects, oi)
		}

		s3conn.DeleteObjects(&s3.DeleteObjectsInput{
			Bucket: aws.String(bucket),
			Delete: objects,
		})

		_, err := s3conn.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucket)})
		if err != nil {
			t.Fatalf("err: %s", err)
		}
	}()

	b, err := NewBackend("s3", map[string]string{
		"access_key":    creds.AccessKeyID,
		"secret_key":    creds.SecretAccessKey,
		"session_token": creds.SessionToken,
		"bucket":        bucket,
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	testBackend(t, b)
	testBackend_ListPrefix(t, b)

}
<commit_msg>Fix tests with AWS changes.<commit_after>package physical

import (
	"fmt"
	"math/rand"
	"os"
	"testing"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/defaults"
	"github.com/aws/aws-sdk-go/service/s3"
)

func TestS3Backend(t *testing.T) {
	if os.Getenv("AWS_ACCESS_KEY_ID") == "" || os.Getenv("AWS_SECRET_ACCESS_KEY") == "" {
		t.SkipNow()
	}

	credentialChain := defaults.DefaultChainCredentials
	creds, err := credentialChain.Get()
	if err != nil {
		t.Fatalf("err: %v", err)
	}

	region := os.Getenv("AWS_DEFAULT_REGION")
	if region == "" {
		region = "us-east-1"
	}

	s3conn := s3.New(&aws.Config{
		Credentials: defaults.DefaultChainCredentials,
		Region:      aws.String(region),
	})

	var randInt = rand.New(rand.NewSource(time.Now().UnixNano())).Int()
	bucket := fmt.Sprintf("vault-s3-testacc-%d", randInt)

	_, err = s3conn.CreateBucket(&s3.CreateBucketInput{
		Bucket: aws.String(bucket),
	})
	if err != nil {
		t.Fatalf("unable to create test bucket: %s", err)
	}

	defer func() {
		// Gotta list all the objects and delete them
		// before being able to delete the bucket
		listResp, _ := s3conn.ListObjects(&s3.ListObjectsInput{
			Bucket: aws.String(bucket),
		})

		objects := &s3.Delete{}
		for _, key := range listResp.Contents {
			oi := &s3.ObjectIdentifier{Key: key.Key}
			objects.Objects = append(objects.Objects, oi)
		}

		s3conn.DeleteObjects(&s3.DeleteObjectsInput{
			Bucket: aws.String(bucket),
			Delete: objects,
		})

		_, err := s3conn.DeleteBucket(&s3.DeleteBucketInput{Bucket: aws.String(bucket)})
		if err != nil {
			t.Fatalf("err: %s", err)
		}
	}()

	b, err := NewBackend("s3", map[string]string{
		"access_key":    creds.AccessKeyID,
		"secret_key":    creds.SecretAccessKey,
		"session_token": creds.SessionToken,
		"bucket":        bucket,
	})
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	testBackend(t, b)
	testBackend_ListPrefix(t, b)

}
<|endoftext|>


<commit_before>package consts

import (
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/couchdb/mango"
)

// GlobalIndexes is the index list required on the global databases to run
// properly.
var GlobalIndexes = []*mango.Index{
	mango.IndexOnFields(Instances, "domain"),
}

// Indexes is the index list required by an instance to run properly.
var Indexes = []*mango.Index{
	// Permissions
	mango.IndexOnFields(Permissions, "source_id", "type"),

	// Used to lookup a file given its parent, and the children of a directory
	mango.IndexOnFields(Files, "dir_id", "name"),
	// Used to lookup a directory given its path
	mango.IndexOnFields(Files, "path"),
}

// DiskUsageView is the view used for computing the disk usage
var DiskUsageView = &couchdb.View{
	Name:    "disk-usage",
	Doctype: Files,
	Map: `
function(doc) {
  if (doc.type === 'file') {
    emit(doc._id, +doc.size);
  }
}
`,
	Reduce: "_sum",
}

// FilesReferencedByView is the view used for fetching files referenced by a
// given document
var FilesReferencedByView = &couchdb.View{
	Name:    "referenced-by",
	Doctype: Files,
	Map: `
function(doc) {
  if (doc.type === 'file' && isArray(doc.referenced_by)) {
    for (var i = 0; i < doc.referenced_by.length; i++) {
      emit([doc.referenced_by[i].type, doc.referenced_by[i].id]);
    }
  }
}`,
}

// PermissionsShareByCView is the view for fetching the permissions associated
// to a document via a token code.
var PermissionsShareByCView = &couchdb.View{
	Name:    "byToken",
	Doctype: Permissions,
	Map: `
function(doc) {
  if (doc.type === "share" && doc.codes) {
    Object.keys(doc.codes).forEach(function(k) {
      emit(doc.codes[k]);
    })
  }
}`,
}

// PermissionsShareByDocView is the view for fetching a list of permissions
// associated to a list of IDs.
var PermissionsShareByDocView = &couchdb.View{
	Name:    "byDoc",
	Doctype: Permissions,
	Map: `
function(doc){
  if (doc.type === "share" && doc.permissions) {
    Object.keys(doc.permissions).forEach(function(k) {
      var p = doc.permissions[k];
      var selector = p.selector || "_id";
      for (var i=0; i<p.values.length; i++) {
        emit([p.type, selector, p.values[i]], p.verbs);
      }
    });
  }
}`,
}

// Views is the required couchdb views for computing the disk usage
var Views = []*couchdb.View{
	DiskUsageView,
	FilesReferencedByView,
	PermissionsShareByCView,
}

// ViewsByDoctype returns the list of views for a specified doc type.
func ViewsByDoctype(doctype string) []*couchdb.View {
	var views []*couchdb.View
	for _, view := range Views {
		if view.Doctype == doctype {
			views = append(views, view)
		}
	}
	return views
}

// IndexesByDoctype returns the list of indexes for a specified doc type.
func IndexesByDoctype(doctype string) []*mango.Index {
	var indexes []*mango.Index
	for _, index := range Indexes {
		if index.Doctype == doctype {
			indexes = append(indexes, index)
		}
	}
	return indexes
}
<commit_msg>Permissions view was missing from the Views<commit_after>package consts

import (
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/cozy-stack/pkg/couchdb/mango"
)

// GlobalIndexes is the index list required on the global databases to run
// properly.
var GlobalIndexes = []*mango.Index{
	mango.IndexOnFields(Instances, "domain"),
}

// Indexes is the index list required by an instance to run properly.
var Indexes = []*mango.Index{
	// Permissions
	mango.IndexOnFields(Permissions, "source_id", "type"),

	// Used to lookup a file given its parent, and the children of a directory
	mango.IndexOnFields(Files, "dir_id", "name"),
	// Used to lookup a directory given its path
	mango.IndexOnFields(Files, "path"),
}

// DiskUsageView is the view used for computing the disk usage
var DiskUsageView = &couchdb.View{
	Name:    "disk-usage",
	Doctype: Files,
	Map: `
function(doc) {
  if (doc.type === 'file') {
    emit(doc._id, +doc.size);
  }
}
`,
	Reduce: "_sum",
}

// FilesReferencedByView is the view used for fetching files referenced by a
// given document
var FilesReferencedByView = &couchdb.View{
	Name:    "referenced-by",
	Doctype: Files,
	Map: `
function(doc) {
  if (doc.type === 'file' && isArray(doc.referenced_by)) {
    for (var i = 0; i < doc.referenced_by.length; i++) {
      emit([doc.referenced_by[i].type, doc.referenced_by[i].id]);
    }
  }
}`,
}

// PermissionsShareByCView is the view for fetching the permissions associated
// to a document via a token code.
var PermissionsShareByCView = &couchdb.View{
	Name:    "byToken",
	Doctype: Permissions,
	Map: `
function(doc) {
  if (doc.type === "share" && doc.codes) {
    Object.keys(doc.codes).forEach(function(k) {
      emit(doc.codes[k]);
    })
  }
}`,
}

// PermissionsShareByDocView is the view for fetching a list of permissions
// associated to a list of IDs.
var PermissionsShareByDocView = &couchdb.View{
	Name:    "byDoc",
	Doctype: Permissions,
	Map: `
function(doc){
  if (doc.type === "share" && doc.permissions) {
    Object.keys(doc.permissions).forEach(function(k) {
      var p = doc.permissions[k];
      var selector = p.selector || "_id";
      for (var i=0; i<p.values.length; i++) {
        emit([p.type, selector, p.values[i]], p.verbs);
      }
    });
  }
}`,
}

// Views is the required couchdb views for computing the disk usage
var Views = []*couchdb.View{
	DiskUsageView,
	FilesReferencedByView,
	PermissionsShareByCView,
	PermissionsShareByDocView,
}

// ViewsByDoctype returns the list of views for a specified doc type.
func ViewsByDoctype(doctype string) []*couchdb.View {
	var views []*couchdb.View
	for _, view := range Views {
		if view.Doctype == doctype {
			views = append(views, view)
		}
	}
	return views
}

// IndexesByDoctype returns the list of indexes for a specified doc type.
func IndexesByDoctype(doctype string) []*mango.Index {
	var indexes []*mango.Index
	for _, index := range Indexes {
		if index.Doctype == doctype {
			indexes = append(indexes, index)
		}
	}
	return indexes
}
<|endoftext|>


<commit_before>package eval

import (
	"fmt"
	"sort"
	"strconv"
	"strings"
	"unsafe"

	"src.elv.sh/pkg/diag"
	"src.elv.sh/pkg/eval/errs"
	"src.elv.sh/pkg/eval/vals"
	"src.elv.sh/pkg/eval/vars"
	"src.elv.sh/pkg/parse"
	"src.elv.sh/pkg/persistent/hash"
)

// A user-defined function in Elvish code. Each closure has its unique identity.
type closure struct {
	ArgNames []string
	// The index of the rest argument. -1 if there is no rest argument.
	RestArg     int
	OptNames    []string
	OptDefaults []interface{}
	Op          effectOp
	NewLocal    []staticVarInfo
	Captured    *Ns
	SrcMeta     parse.Source
	DefRange    diag.Ranging
}

var _ Callable = &closure{}

// Kind returns "fn".
func (*closure) Kind() string {
	return "fn"
}

// Equal compares by address.
func (c *closure) Equal(rhs interface{}) bool {
	return c == rhs
}

// Hash returns the hash of the address of the closure.
func (c *closure) Hash() uint32 {
	return hash.Pointer(unsafe.Pointer(c))
}

// Repr returns an opaque representation "<closure 0x23333333>".
func (c *closure) Repr(int) string {
	return fmt.Sprintf("<closure %p>", c)
}

// Call calls a closure.
func (c *closure) Call(fm *Frame, args []interface{}, opts map[string]interface{}) error {
	// Check number of arguments.
	if c.RestArg != -1 {
		if len(args) < len(c.ArgNames)-1 {
			return errs.ArityMismatch{What: "arguments",
				ValidLow: len(c.ArgNames) - 1, ValidHigh: -1, Actual: len(args)}
		}
	} else {
		if len(args) != len(c.ArgNames) {
			return errs.ArityMismatch{What: "arguments",
				ValidLow: len(c.ArgNames), ValidHigh: len(c.ArgNames), Actual: len(args)}
		}
	}
	// Check whether all supplied options are supported. This map contains the
	// subset of keys from opts that can be found in c.OptNames.
	optSupported := make(map[string]struct{})
	for _, name := range c.OptNames {
		_, ok := opts[name]
		if ok {
			optSupported[name] = struct{}{}
		}
	}
	if len(optSupported) < len(opts) {
		// Report all the options that are not supported.
		unsupported := make([]string, 0, len(opts)-len(optSupported))
		for name := range opts {
			_, supported := optSupported[name]
			if !supported {
				unsupported = append(unsupported, parse.Quote(name))
			}
		}
		sort.Strings(unsupported)
		return UnsupportedOptionsError{unsupported}
	}

	// This Frame is dedicated to the current form, so we can modify it in place.

	// BUG(xiaq): When evaluating closures, async access to global variables
	// and ports can be problematic.

	// Make upvalue namespace and capture variables.
	fm.up = c.Captured

	// Populate local scope with arguments, options, and newly created locals.
	localSize := len(c.ArgNames) + len(c.OptNames) + len(c.NewLocal)
	local := &Ns{make([]vars.Var, localSize), make([]staticVarInfo, localSize)}

	for i, name := range c.ArgNames {
		local.infos[i] = staticVarInfo{name, false, false}
	}
	if c.RestArg == -1 {
		for i := range c.ArgNames {
			local.slots[i] = vars.FromInit(args[i])
		}
	} else {
		for i := 0; i < c.RestArg; i++ {
			local.slots[i] = vars.FromInit(args[i])
		}
		restOff := len(args) - len(c.ArgNames)
		local.slots[c.RestArg] = vars.FromInit(
			vals.MakeList(args[c.RestArg : c.RestArg+restOff+1]...))
		for i := c.RestArg + 1; i < len(c.ArgNames); i++ {
			local.slots[i] = vars.FromInit(args[i+restOff])
		}
	}

	offset := len(c.ArgNames)
	for i, name := range c.OptNames {
		v, ok := opts[name]
		if !ok {
			v = c.OptDefaults[i]
		}
		local.infos[offset+i] = staticVarInfo{name, false, false}
		local.slots[offset+i] = vars.FromInit(v)
	}

	offset += len(c.OptNames)
	for i, info := range c.NewLocal {
		local.infos[offset+i] = info
		// TODO: Take info.readOnly into account too when creating variable
		local.slots[offset+i] = MakeVarFromName(info.name)
	}

	fm.local = local
	fm.srcMeta = c.SrcMeta
	return c.Op.exec(fm)
}

// MakeVarFromName creates a Var with a suitable type constraint inferred from
// the name.
func MakeVarFromName(name string) vars.Var {
	switch {
	case strings.HasSuffix(name, FnSuffix):
		val := NewGoFn("nop~", nop)
		return vars.FromPtr(&val)
	case strings.HasSuffix(name, NsSuffix):
		val := (*Ns)(nil)
		return vars.FromPtr(&val)
	default:
		return vars.FromInit(nil)
	}
}

// UnsupportedOptionsError is an error returned by a closure call when there are
// unsupported options.
type UnsupportedOptionsError struct {
	Options []string
}

func (er UnsupportedOptionsError) Error() string {
	if len(er.Options) == 1 {
		return fmt.Sprintf("unsupported option: %s", er.Options[0])
	}
	return fmt.Sprintf("unsupported options: %s", strings.Join(er.Options, ", "))
}

func (c *closure) Fields() vals.StructMap { return closureFields{c} }

type closureFields struct{ c *closure }

func (closureFields) IsStructMap() {}

func (cf closureFields) ArgNames() vals.List { return listOfStrings(cf.c.ArgNames) }
func (cf closureFields) RestArg() string     { return strconv.Itoa(cf.c.RestArg) }
func (cf closureFields) OptNames() vals.List { return listOfStrings(cf.c.OptNames) }
func (cf closureFields) Src() parse.Source   { return cf.c.SrcMeta }

func (cf closureFields) OptDefaults() vals.List {
	return vals.MakeList(cf.c.OptDefaults...)
}

func (cf closureFields) Body() string {
	r := cf.c.Op.(diag.Ranger).Range()
	return cf.c.SrcMeta.Code[r.From:r.To]
}

func (cf closureFields) Def() string {
	return cf.c.SrcMeta.Code[cf.c.DefRange.From:cf.c.DefRange.To]
}

func listOfStrings(ss []string) vals.List {
	list := vals.EmptyList
	for _, s := range ss {
		list = list.Cons(s)
	}
	return list
}
<commit_msg>pkg/eval: Initialize ns variables to an empty ns, rather than nil.<commit_after>package eval

import (
	"fmt"
	"sort"
	"strconv"
	"strings"
	"unsafe"

	"src.elv.sh/pkg/diag"
	"src.elv.sh/pkg/eval/errs"
	"src.elv.sh/pkg/eval/vals"
	"src.elv.sh/pkg/eval/vars"
	"src.elv.sh/pkg/parse"
	"src.elv.sh/pkg/persistent/hash"
)

// A user-defined function in Elvish code. Each closure has its unique identity.
type closure struct {
	ArgNames []string
	// The index of the rest argument. -1 if there is no rest argument.
	RestArg     int
	OptNames    []string
	OptDefaults []interface{}
	Op          effectOp
	NewLocal    []staticVarInfo
	Captured    *Ns
	SrcMeta     parse.Source
	DefRange    diag.Ranging
}

var _ Callable = &closure{}

// Kind returns "fn".
func (*closure) Kind() string {
	return "fn"
}

// Equal compares by address.
func (c *closure) Equal(rhs interface{}) bool {
	return c == rhs
}

// Hash returns the hash of the address of the closure.
func (c *closure) Hash() uint32 {
	return hash.Pointer(unsafe.Pointer(c))
}

// Repr returns an opaque representation "<closure 0x23333333>".
func (c *closure) Repr(int) string {
	return fmt.Sprintf("<closure %p>", c)
}

// Call calls a closure.
func (c *closure) Call(fm *Frame, args []interface{}, opts map[string]interface{}) error {
	// Check number of arguments.
	if c.RestArg != -1 {
		if len(args) < len(c.ArgNames)-1 {
			return errs.ArityMismatch{What: "arguments",
				ValidLow: len(c.ArgNames) - 1, ValidHigh: -1, Actual: len(args)}
		}
	} else {
		if len(args) != len(c.ArgNames) {
			return errs.ArityMismatch{What: "arguments",
				ValidLow: len(c.ArgNames), ValidHigh: len(c.ArgNames), Actual: len(args)}
		}
	}
	// Check whether all supplied options are supported. This map contains the
	// subset of keys from opts that can be found in c.OptNames.
	optSupported := make(map[string]struct{})
	for _, name := range c.OptNames {
		_, ok := opts[name]
		if ok {
			optSupported[name] = struct{}{}
		}
	}
	if len(optSupported) < len(opts) {
		// Report all the options that are not supported.
		unsupported := make([]string, 0, len(opts)-len(optSupported))
		for name := range opts {
			_, supported := optSupported[name]
			if !supported {
				unsupported = append(unsupported, parse.Quote(name))
			}
		}
		sort.Strings(unsupported)
		return UnsupportedOptionsError{unsupported}
	}

	// This Frame is dedicated to the current form, so we can modify it in place.

	// BUG(xiaq): When evaluating closures, async access to global variables
	// and ports can be problematic.

	// Make upvalue namespace and capture variables.
	fm.up = c.Captured

	// Populate local scope with arguments, options, and newly created locals.
	localSize := len(c.ArgNames) + len(c.OptNames) + len(c.NewLocal)
	local := &Ns{make([]vars.Var, localSize), make([]staticVarInfo, localSize)}

	for i, name := range c.ArgNames {
		local.infos[i] = staticVarInfo{name, false, false}
	}
	if c.RestArg == -1 {
		for i := range c.ArgNames {
			local.slots[i] = vars.FromInit(args[i])
		}
	} else {
		for i := 0; i < c.RestArg; i++ {
			local.slots[i] = vars.FromInit(args[i])
		}
		restOff := len(args) - len(c.ArgNames)
		local.slots[c.RestArg] = vars.FromInit(
			vals.MakeList(args[c.RestArg : c.RestArg+restOff+1]...))
		for i := c.RestArg + 1; i < len(c.ArgNames); i++ {
			local.slots[i] = vars.FromInit(args[i+restOff])
		}
	}

	offset := len(c.ArgNames)
	for i, name := range c.OptNames {
		v, ok := opts[name]
		if !ok {
			v = c.OptDefaults[i]
		}
		local.infos[offset+i] = staticVarInfo{name, false, false}
		local.slots[offset+i] = vars.FromInit(v)
	}

	offset += len(c.OptNames)
	for i, info := range c.NewLocal {
		local.infos[offset+i] = info
		// TODO: Take info.readOnly into account too when creating variable
		local.slots[offset+i] = MakeVarFromName(info.name)
	}

	fm.local = local
	fm.srcMeta = c.SrcMeta
	return c.Op.exec(fm)
}

var (
	fnDefault = NewGoFn("nop~", nop)
	nsDefault = &Ns{}
)

// MakeVarFromName creates a Var with a suitable type constraint inferred from
// the name.
func MakeVarFromName(name string) vars.Var {
	switch {
	case strings.HasSuffix(name, FnSuffix):
		val := fnDefault
		return vars.FromPtr(&val)
	case strings.HasSuffix(name, NsSuffix):
		val := nsDefault
		return vars.FromPtr(&val)
	default:
		return vars.FromInit(nil)
	}
}

// UnsupportedOptionsError is an error returned by a closure call when there are
// unsupported options.
type UnsupportedOptionsError struct {
	Options []string
}

func (er UnsupportedOptionsError) Error() string {
	if len(er.Options) == 1 {
		return fmt.Sprintf("unsupported option: %s", er.Options[0])
	}
	return fmt.Sprintf("unsupported options: %s", strings.Join(er.Options, ", "))
}

func (c *closure) Fields() vals.StructMap { return closureFields{c} }

type closureFields struct{ c *closure }

func (closureFields) IsStructMap() {}

func (cf closureFields) ArgNames() vals.List { return listOfStrings(cf.c.ArgNames) }
func (cf closureFields) RestArg() string     { return strconv.Itoa(cf.c.RestArg) }
func (cf closureFields) OptNames() vals.List { return listOfStrings(cf.c.OptNames) }
func (cf closureFields) Src() parse.Source   { return cf.c.SrcMeta }

func (cf closureFields) OptDefaults() vals.List {
	return vals.MakeList(cf.c.OptDefaults...)
}

func (cf closureFields) Body() string {
	r := cf.c.Op.(diag.Ranger).Range()
	return cf.c.SrcMeta.Code[r.From:r.To]
}

func (cf closureFields) Def() string {
	return cf.c.SrcMeta.Code[cf.c.DefRange.From:cf.c.DefRange.To]
}

func listOfStrings(ss []string) vals.List {
	list := vals.EmptyList
	for _, s := range ss {
		list = list.Cons(s)
	}
	return list
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package fs

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/minio/minio-xl/pkg/probe"
	"github.com/minio/minio/pkg/disk"
)

/// Bucket Operations

// DeleteBucket - delete bucket
func (fs Filesystem) DeleteBucket(bucket string) *probe.Error {
	fs.lock.Lock()
	defer fs.lock.Unlock()
	// verify bucket path legal
	if !IsValidBucket(bucket) {
		return probe.NewError(BucketNameInvalid{Bucket: bucket})
	}
	bucketDir := filepath.Join(fs.path, bucket)
	// check bucket exists
	if _, err := os.Stat(bucketDir); os.IsNotExist(err) {
		return probe.NewError(BucketNotFound{Bucket: bucket})
	}
	if err := RemoveAllDirs(bucketDir); err != nil {
		if err == ErrDirNotEmpty || strings.Contains(err.Error(), "directory not empty") {
			return probe.NewError(BucketNotEmpty{Bucket: bucket})
		}
		return probe.NewError(err)
	}
	if err := os.Remove(bucketDir); err != nil {
		if strings.Contains(err.Error(), "directory not empty") {
			return probe.NewError(BucketNotEmpty{Bucket: bucket})
		}
		return probe.NewError(err)
	}
	return nil
}

// ListBuckets - Get service
func (fs Filesystem) ListBuckets() ([]BucketMetadata, *probe.Error) {
	fs.lock.Lock()
	defer fs.lock.Unlock()

	files, err := ioutil.ReadDir(fs.path)
	if err != nil {
		return []BucketMetadata{}, probe.NewError(err)
	}

	var metadataList []BucketMetadata
	for _, file := range files {
		if !file.IsDir() {
			// if files found ignore them
			continue
		}
		if file.IsDir() {
			// if directories found with odd names, skip them too
			if !IsValidBucket(file.Name()) {
				continue
			}
		}
		metadata := BucketMetadata{
			Name:    file.Name(),
			Created: file.ModTime(),
		}
		metadataList = append(metadataList, metadata)
	}
	return metadataList, nil
}

// MakeBucket - PUT Bucket
func (fs Filesystem) MakeBucket(bucket, acl string) *probe.Error {
	fs.lock.Lock()
	defer fs.lock.Unlock()

	stfs, err := disk.Stat(fs.path)
	if err != nil {
		return probe.NewError(err)
	}

	// Remove 5% from total space for cumulative disk space used for journalling, inodes etc.
	availableDiskSpace := (float64(stfs.Free) / (float64(stfs.Total) - (0.05 * float64(stfs.Total)))) * 100
	if int64(availableDiskSpace) <= fs.minFreeDisk {
		return probe.NewError(RootPathFull{Path: fs.path})
	}

	// verify bucket path legal
	if !IsValidBucket(bucket) {
		return probe.NewError(BucketNameInvalid{Bucket: bucket})
	}

	// get bucket path
	bucketDir := filepath.Join(fs.path, bucket)

	// check if bucket exists
	if _, err = os.Stat(bucketDir); err == nil {
		return probe.NewError(BucketExists{
			Bucket: bucket,
		})
	}

	// make bucket
	err = os.Mkdir(bucketDir, aclToPerm(acl))
	if err != nil {
		return probe.NewError(err)
	}
	return nil
}

// GetBucketMetadata -
func (fs Filesystem) GetBucketMetadata(bucket string) (BucketMetadata, *probe.Error) {
	fs.lock.Lock()
	defer fs.lock.Unlock()
	if !IsValidBucket(bucket) {
		return BucketMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
	}
	// get bucket path
	bucketDir := filepath.Join(fs.path, bucket)
	bucketMetadata := BucketMetadata{}
	fi, err := os.Stat(bucketDir)
	// check if bucket exists
	if os.IsNotExist(err) {
		return BucketMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
	}
	if err != nil {
		return BucketMetadata{}, probe.NewError(err)
	}

	bucketMetadata.Name = fi.Name()
	bucketMetadata.Created = fi.ModTime()
	bucketMetadata.ACL = permToACL(fi.Mode())
	return bucketMetadata, nil
}

// permToACL - convert perm to meaningful ACL
func permToACL(mode os.FileMode) BucketACL {
	switch mode.Perm() {
	case os.FileMode(0700):
		return BucketACL("private")
	case os.FileMode(0500):
		return BucketACL("public-read")
	case os.FileMode(0777):
		return BucketACL("public-read-write")
	default:
		return BucketACL("private")
	}
}

// aclToPerm - convert acl to filesystem mode
func aclToPerm(acl string) os.FileMode {
	switch acl {
	case "private":
		return os.FileMode(0700)
	case "public-read":
		return os.FileMode(0500)
	case "public-read-write":
		return os.FileMode(0777)
	default:
		return os.FileMode(0700)
	}
}

// SetBucketMetadata -
func (fs Filesystem) SetBucketMetadata(bucket string, metadata map[string]string) *probe.Error {
	fs.lock.Lock()
	defer fs.lock.Unlock()
	if !IsValidBucket(bucket) {
		return probe.NewError(BucketNameInvalid{Bucket: bucket})
	}
	acl := metadata["acl"]
	if !IsValidBucketACL(acl) {
		return probe.NewError(InvalidACL{ACL: acl})
	}
	// get bucket path
	bucketDir := filepath.Join(fs.path, bucket)
	err := os.Chmod(bucketDir, aclToPerm(acl))
	if err != nil {
		return probe.NewError(err)
	}
	return nil
}

// ListObjects - GET bucket (list objects)
func (fs Filesystem) ListObjects(bucket string, resources BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, *probe.Error) {
	fs.lock.Lock()
	defer fs.lock.Unlock()
	if !IsValidBucket(bucket) {
		return nil, resources, probe.NewError(BucketNameInvalid{Bucket: bucket})
	}
	if resources.Prefix != "" && IsValidObjectName(resources.Prefix) == false {
		return nil, resources, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: resources.Prefix})
	}

	p := bucketDir{}
	rootPrefix := filepath.Join(fs.path, bucket)
	// check bucket exists
	if _, err := os.Stat(rootPrefix); os.IsNotExist(err) {
		return nil, resources, probe.NewError(BucketNotFound{Bucket: bucket})
	}

	p.root = rootPrefix
	/// automatically treat "/" delimiter as "\\" delimiter on windows due to its path constraints.
	if resources.Delimiter == "/" {
		if runtime.GOOS == "windows" {
			resources.Delimiter = string(os.PathSeparator)
		}
	}

	// if delimiter is supplied and not prefix then we are the very top level, list everything and move on.
	if resources.Delimiter != "" && resources.Prefix == "" {
		files, err := ioutil.ReadDir(rootPrefix)
		if err != nil {
			if os.IsNotExist(err) {
				return nil, resources, probe.NewError(BucketNotFound{Bucket: bucket})
			}
			return nil, resources, probe.NewError(err)
		}
		for _, fl := range files {
			p.files = append(p.files, contentInfo{
				Prefix:   fl.Name(),
				Size:     fl.Size(),
				Mode:     fl.Mode(),
				ModTime:  fl.ModTime(),
				FileInfo: fl,
			})
		}
	}

	// If delimiter and prefix is supplied make sure that paging doesn't go deep, treat it as simple directory listing.
	if resources.Delimiter != "" && resources.Prefix != "" {
		if !strings.HasSuffix(resources.Prefix, resources.Delimiter) {
			fl, err := os.Stat(filepath.Join(rootPrefix, resources.Prefix))
			if err != nil {
				if os.IsNotExist(err) {
					return nil, resources, probe.NewError(ObjectNotFound{Bucket: bucket, Object: resources.Prefix})
				}
				return nil, resources, probe.NewError(err)
			}
			p.files = append(p.files, contentInfo{
				Prefix:   resources.Prefix,
				Size:     fl.Size(),
				Mode:     os.ModeDir,
				ModTime:  fl.ModTime(),
				FileInfo: fl,
			})
		} else {
			files, err := ioutil.ReadDir(filepath.Join(rootPrefix, resources.Prefix))
			if err != nil {
				if os.IsNotExist(err) {
					return nil, resources, probe.NewError(ObjectNotFound{Bucket: bucket, Object: resources.Prefix})
				}
				return nil, resources, probe.NewError(err)
			}
			for _, fl := range files {
				prefix := fl.Name()
				if resources.Prefix != "" {
					prefix = filepath.Join(resources.Prefix, fl.Name())
				}
				p.files = append(p.files, contentInfo{
					Prefix:   prefix,
					Size:     fl.Size(),
					Mode:     fl.Mode(),
					ModTime:  fl.ModTime(),
					FileInfo: fl,
				})
			}
		}
	}
	if resources.Delimiter == "" {
		var files []contentInfo
		getAllFiles := func(fp string, fl os.FileInfo, err error) error {
			// If any error return back quickly
			if err != nil {
				return err
			}
			if strings.HasSuffix(fp, "$multiparts") {
				return nil
			}
			// if file pointer equals to rootPrefix - discard it
			if fp == p.root {
				return nil
			}
			if len(files) > resources.Maxkeys {
				return ErrSkipFile
			}
			// Split the root prefix from the incoming file pointer
			realFp := ""
			if runtime.GOOS == "windows" {
				if splits := strings.Split(fp, (p.root + string(os.PathSeparator))); len(splits) > 1 {
					realFp = splits[1]
				}
			} else {
				if splits := strings.Split(fp, (p.root + string(os.PathSeparator))); len(splits) > 1 {
					realFp = splits[1]
				}
			}
			// If path is a directory and has a prefix verify if the file pointer
			// has the prefix if it does not skip the directory.
			if fl.Mode().IsDir() {
				if resources.Prefix != "" {
					if !strings.HasPrefix(fp, filepath.Join(p.root, resources.Prefix)) {
						return ErrSkipDir
					}
				}
			}
			// If path is a directory and has a marker verify if the file split file pointer
			// is lesser than the Marker top level directory if yes skip it.
			if fl.Mode().IsDir() {
				if resources.Marker != "" {
					if realFp != "" {
						if runtime.GOOS == "windows" {
							if realFp < strings.Split(resources.Marker, string(os.PathSeparator))[0] {
								return ErrSkipDir
							}
						} else {
							if realFp < strings.Split(resources.Marker, string(os.PathSeparator))[0] {
								return ErrSkipDir
							}
						}
					}
				}
			}
			// If regular file verify
			if fl.Mode().IsRegular() {
				// If marker is present this will be used to check if filepointer is
				// lexically higher than then Marker
				if realFp != "" {
					if resources.Marker != "" {
						if realFp > resources.Marker {
							files = append(files, contentInfo{
								Prefix:   realFp,
								Size:     fl.Size(),
								Mode:     fl.Mode(),
								ModTime:  fl.ModTime(),
								FileInfo: fl,
							})
						}
					} else {
						files = append(files, contentInfo{
							Prefix:   realFp,
							Size:     fl.Size(),
							Mode:     fl.Mode(),
							ModTime:  fl.ModTime(),
							FileInfo: fl,
						})
					}
				}
			}
			// If file is a symlink follow it and populate values.
			if fl.Mode()&os.ModeSymlink == os.ModeSymlink {
				st, err := os.Stat(fp)
				if err != nil {
					return nil
				}
				// If marker is present this will be used to check if filepointer is
				// lexically higher than then Marker
				if realFp != "" {
					if resources.Marker != "" {
						if realFp > resources.Marker {
							files = append(files, contentInfo{
								Prefix:   realFp,
								Size:     st.Size(),
								Mode:     st.Mode(),
								ModTime:  st.ModTime(),
								FileInfo: st,
							})
						}
					} else {
						files = append(files, contentInfo{
							Prefix:   realFp,
							Size:     st.Size(),
							Mode:     st.Mode(),
							ModTime:  st.ModTime(),
							FileInfo: st,
						})
					}
				}
			}
			p.files = files
			return nil
		}
		// If no delimiter is specified, crawl through everything.
		err := Walk(rootPrefix, getAllFiles)
		if err != nil {
			if os.IsNotExist(err) {
				return nil, resources, probe.NewError(ObjectNotFound{Bucket: bucket, Object: resources.Prefix})
			}
			return nil, resources, probe.NewError(err)
		}
	}

	var metadataList []ObjectMetadata
	var metadata ObjectMetadata

	// Filter objects
	for _, content := range p.files {
		if len(metadataList) == resources.Maxkeys {
			resources.IsTruncated = true
			if resources.IsTruncated && resources.Delimiter != "" {
				resources.NextMarker = metadataList[len(metadataList)-1].Object
			}
			break
		}
		if content.Prefix > resources.Marker {
			var err *probe.Error
			metadata, resources, err = fs.filterObjects(bucket, content, resources)
			if err != nil {
				return nil, resources, err.Trace()
			}
			if metadata.Bucket != "" {
				metadataList = append(metadataList, metadata)
			}
		}
	}
	return metadataList, resources, nil
}
<commit_msg>On windows translate Prefix, Marker and Delimiter for paths<commit_after>/*
 * Minio Cloud Storage, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package fs

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/minio/minio-xl/pkg/probe"
	"github.com/minio/minio/pkg/disk"
)

/// Bucket Operations

// DeleteBucket - delete bucket
func (fs Filesystem) DeleteBucket(bucket string) *probe.Error {
	fs.lock.Lock()
	defer fs.lock.Unlock()
	// verify bucket path legal
	if !IsValidBucket(bucket) {
		return probe.NewError(BucketNameInvalid{Bucket: bucket})
	}
	bucketDir := filepath.Join(fs.path, bucket)
	// check bucket exists
	if _, err := os.Stat(bucketDir); os.IsNotExist(err) {
		return probe.NewError(BucketNotFound{Bucket: bucket})
	}
	if err := RemoveAllDirs(bucketDir); err != nil {
		if err == ErrDirNotEmpty || strings.Contains(err.Error(), "directory not empty") {
			return probe.NewError(BucketNotEmpty{Bucket: bucket})
		}
		return probe.NewError(err)
	}
	if err := os.Remove(bucketDir); err != nil {
		if strings.Contains(err.Error(), "directory not empty") {
			return probe.NewError(BucketNotEmpty{Bucket: bucket})
		}
		return probe.NewError(err)
	}
	return nil
}

// ListBuckets - Get service
func (fs Filesystem) ListBuckets() ([]BucketMetadata, *probe.Error) {
	fs.lock.Lock()
	defer fs.lock.Unlock()

	files, err := ioutil.ReadDir(fs.path)
	if err != nil {
		return []BucketMetadata{}, probe.NewError(err)
	}

	var metadataList []BucketMetadata
	for _, file := range files {
		if !file.IsDir() {
			// if files found ignore them
			continue
		}
		if file.IsDir() {
			// if directories found with odd names, skip them too
			if !IsValidBucket(file.Name()) {
				continue
			}
		}
		metadata := BucketMetadata{
			Name:    file.Name(),
			Created: file.ModTime(),
		}
		metadataList = append(metadataList, metadata)
	}
	return metadataList, nil
}

// MakeBucket - PUT Bucket
func (fs Filesystem) MakeBucket(bucket, acl string) *probe.Error {
	fs.lock.Lock()
	defer fs.lock.Unlock()

	stfs, err := disk.Stat(fs.path)
	if err != nil {
		return probe.NewError(err)
	}

	// Remove 5% from total space for cumulative disk space used for journalling, inodes etc.
	availableDiskSpace := (float64(stfs.Free) / (float64(stfs.Total) - (0.05 * float64(stfs.Total)))) * 100
	if int64(availableDiskSpace) <= fs.minFreeDisk {
		return probe.NewError(RootPathFull{Path: fs.path})
	}

	// verify bucket path legal
	if !IsValidBucket(bucket) {
		return probe.NewError(BucketNameInvalid{Bucket: bucket})
	}

	// get bucket path
	bucketDir := filepath.Join(fs.path, bucket)

	// check if bucket exists
	if _, err = os.Stat(bucketDir); err == nil {
		return probe.NewError(BucketExists{
			Bucket: bucket,
		})
	}

	// make bucket
	err = os.Mkdir(bucketDir, aclToPerm(acl))
	if err != nil {
		return probe.NewError(err)
	}
	return nil
}

// GetBucketMetadata -
func (fs Filesystem) GetBucketMetadata(bucket string) (BucketMetadata, *probe.Error) {
	fs.lock.Lock()
	defer fs.lock.Unlock()
	if !IsValidBucket(bucket) {
		return BucketMetadata{}, probe.NewError(BucketNameInvalid{Bucket: bucket})
	}
	// get bucket path
	bucketDir := filepath.Join(fs.path, bucket)
	bucketMetadata := BucketMetadata{}
	fi, err := os.Stat(bucketDir)
	// check if bucket exists
	if os.IsNotExist(err) {
		return BucketMetadata{}, probe.NewError(BucketNotFound{Bucket: bucket})
	}
	if err != nil {
		return BucketMetadata{}, probe.NewError(err)
	}

	bucketMetadata.Name = fi.Name()
	bucketMetadata.Created = fi.ModTime()
	bucketMetadata.ACL = permToACL(fi.Mode())
	return bucketMetadata, nil
}

// permToACL - convert perm to meaningful ACL
func permToACL(mode os.FileMode) BucketACL {
	switch mode.Perm() {
	case os.FileMode(0700):
		return BucketACL("private")
	case os.FileMode(0500):
		return BucketACL("public-read")
	case os.FileMode(0777):
		return BucketACL("public-read-write")
	default:
		return BucketACL("private")
	}
}

// aclToPerm - convert acl to filesystem mode
func aclToPerm(acl string) os.FileMode {
	switch acl {
	case "private":
		return os.FileMode(0700)
	case "public-read":
		return os.FileMode(0500)
	case "public-read-write":
		return os.FileMode(0777)
	default:
		return os.FileMode(0700)
	}
}

// SetBucketMetadata -
func (fs Filesystem) SetBucketMetadata(bucket string, metadata map[string]string) *probe.Error {
	fs.lock.Lock()
	defer fs.lock.Unlock()
	if !IsValidBucket(bucket) {
		return probe.NewError(BucketNameInvalid{Bucket: bucket})
	}
	acl := metadata["acl"]
	if !IsValidBucketACL(acl) {
		return probe.NewError(InvalidACL{ACL: acl})
	}
	// get bucket path
	bucketDir := filepath.Join(fs.path, bucket)
	err := os.Chmod(bucketDir, aclToPerm(acl))
	if err != nil {
		return probe.NewError(err)
	}
	return nil
}

// ListObjects - GET bucket (list objects)
func (fs Filesystem) ListObjects(bucket string, resources BucketResourcesMetadata) ([]ObjectMetadata, BucketResourcesMetadata, *probe.Error) {
	fs.lock.Lock()
	defer fs.lock.Unlock()
	if !IsValidBucket(bucket) {
		return nil, resources, probe.NewError(BucketNameInvalid{Bucket: bucket})
	}
	if resources.Prefix != "" && IsValidObjectName(resources.Prefix) == false {
		return nil, resources, probe.NewError(ObjectNameInvalid{Bucket: bucket, Object: resources.Prefix})
	}

	p := bucketDir{}
	rootPrefix := filepath.Join(fs.path, bucket)
	// check bucket exists
	if _, err := os.Stat(rootPrefix); os.IsNotExist(err) {
		return nil, resources, probe.NewError(BucketNotFound{Bucket: bucket})
	}

	p.root = rootPrefix
	/// automatically treat "/" delimiter as "\\" delimiter on windows due to its path constraints.
	if runtime.GOOS == "windows" {
		resources.Prefix = strings.Replace(resources.Prefix, "/", string(os.PathSeparator), -1)
		resources.Delimiter = strings.Replace(resources.Delimiter, "/", string(os.PathSeparator), -1)
		resources.Marker = strings.Replace(resources.Marker, "/", string(os.PathSeparator), -1)
	}

	// if delimiter is supplied and not prefix then we are the very top level, list everything and move on.
	if resources.Delimiter != "" && resources.Prefix == "" {
		files, err := ioutil.ReadDir(rootPrefix)
		if err != nil {
			if os.IsNotExist(err) {
				return nil, resources, probe.NewError(BucketNotFound{Bucket: bucket})
			}
			return nil, resources, probe.NewError(err)
		}
		for _, fl := range files {
			p.files = append(p.files, contentInfo{
				Prefix:   fl.Name(),
				Size:     fl.Size(),
				Mode:     fl.Mode(),
				ModTime:  fl.ModTime(),
				FileInfo: fl,
			})
		}
	}

	// If delimiter and prefix is supplied make sure that paging doesn't go deep, treat it as simple directory listing.
	if resources.Delimiter != "" && resources.Prefix != "" {
		if !strings.HasSuffix(resources.Prefix, resources.Delimiter) {
			fl, err := os.Stat(filepath.Join(rootPrefix, resources.Prefix))
			if err != nil {
				if os.IsNotExist(err) {
					return nil, resources, probe.NewError(ObjectNotFound{Bucket: bucket, Object: resources.Prefix})
				}
				return nil, resources, probe.NewError(err)
			}
			p.files = append(p.files, contentInfo{
				Prefix:   resources.Prefix,
				Size:     fl.Size(),
				Mode:     os.ModeDir,
				ModTime:  fl.ModTime(),
				FileInfo: fl,
			})
		} else {
			files, err := ioutil.ReadDir(filepath.Join(rootPrefix, resources.Prefix))
			if err != nil {
				if os.IsNotExist(err) {
					return nil, resources, probe.NewError(ObjectNotFound{Bucket: bucket, Object: resources.Prefix})
				}
				return nil, resources, probe.NewError(err)
			}
			for _, fl := range files {
				prefix := fl.Name()
				if resources.Prefix != "" {
					prefix = filepath.Join(resources.Prefix, fl.Name())
				}
				p.files = append(p.files, contentInfo{
					Prefix:   prefix,
					Size:     fl.Size(),
					Mode:     fl.Mode(),
					ModTime:  fl.ModTime(),
					FileInfo: fl,
				})
			}
		}
	}
	if resources.Delimiter == "" {
		var files []contentInfo
		getAllFiles := func(fp string, fl os.FileInfo, err error) error {
			// If any error return back quickly
			if err != nil {
				return err
			}
			if strings.HasSuffix(fp, "$multiparts") {
				return nil
			}
			// if file pointer equals to rootPrefix - discard it
			if fp == p.root {
				return nil
			}
			if len(files) > resources.Maxkeys {
				return ErrSkipFile
			}
			// Split the root prefix from the incoming file pointer
			realFp := ""
			if runtime.GOOS == "windows" {
				if splits := strings.Split(fp, (p.root + string(os.PathSeparator))); len(splits) > 1 {
					realFp = splits[1]
				}
			} else {
				if splits := strings.Split(fp, (p.root + string(os.PathSeparator))); len(splits) > 1 {
					realFp = splits[1]
				}
			}
			// If path is a directory and has a prefix verify if the file pointer
			// has the prefix if it does not skip the directory.
			if fl.Mode().IsDir() {
				if resources.Prefix != "" {
					if !strings.HasPrefix(fp, filepath.Join(p.root, resources.Prefix)) {
						return ErrSkipDir
					}
				}
			}
			// If path is a directory and has a marker verify if the file split file pointer
			// is lesser than the Marker top level directory if yes skip it.
			if fl.Mode().IsDir() {
				if resources.Marker != "" {
					if realFp != "" {
						if runtime.GOOS == "windows" {
							if realFp < strings.Split(resources.Marker, string(os.PathSeparator))[0] {
								return ErrSkipDir
							}
						} else {
							if realFp < strings.Split(resources.Marker, string(os.PathSeparator))[0] {
								return ErrSkipDir
							}
						}
					}
				}
			}
			// If regular file verify
			if fl.Mode().IsRegular() {
				// If marker is present this will be used to check if filepointer is
				// lexically higher than then Marker
				if realFp != "" {
					if resources.Marker != "" {
						if realFp > resources.Marker {
							files = append(files, contentInfo{
								Prefix:   realFp,
								Size:     fl.Size(),
								Mode:     fl.Mode(),
								ModTime:  fl.ModTime(),
								FileInfo: fl,
							})
						}
					} else {
						files = append(files, contentInfo{
							Prefix:   realFp,
							Size:     fl.Size(),
							Mode:     fl.Mode(),
							ModTime:  fl.ModTime(),
							FileInfo: fl,
						})
					}
				}
			}
			// If file is a symlink follow it and populate values.
			if fl.Mode()&os.ModeSymlink == os.ModeSymlink {
				st, err := os.Stat(fp)
				if err != nil {
					return nil
				}
				// If marker is present this will be used to check if filepointer is
				// lexically higher than then Marker
				if realFp != "" {
					if resources.Marker != "" {
						if realFp > resources.Marker {
							files = append(files, contentInfo{
								Prefix:   realFp,
								Size:     st.Size(),
								Mode:     st.Mode(),
								ModTime:  st.ModTime(),
								FileInfo: st,
							})
						}
					} else {
						files = append(files, contentInfo{
							Prefix:   realFp,
							Size:     st.Size(),
							Mode:     st.Mode(),
							ModTime:  st.ModTime(),
							FileInfo: st,
						})
					}
				}
			}
			p.files = files
			return nil
		}
		// If no delimiter is specified, crawl through everything.
		err := Walk(rootPrefix, getAllFiles)
		if err != nil {
			if os.IsNotExist(err) {
				return nil, resources, probe.NewError(ObjectNotFound{Bucket: bucket, Object: resources.Prefix})
			}
			return nil, resources, probe.NewError(err)
		}
	}

	var metadataList []ObjectMetadata
	var metadata ObjectMetadata

	// Filter objects
	for _, content := range p.files {
		if len(metadataList) == resources.Maxkeys {
			resources.IsTruncated = true
			if resources.IsTruncated && resources.Delimiter != "" {
				resources.NextMarker = metadataList[len(metadataList)-1].Object
			}
			break
		}
		if content.Prefix > resources.Marker {
			var err *probe.Error
			metadata, resources, err = fs.filterObjects(bucket, content, resources)
			if err != nil {
				return nil, resources, err.Trace()
			}
			if metadata.Bucket != "" {
				metadataList = append(metadataList, metadata)
			}
		}
	}
	return metadataList, resources, nil
}
<|endoftext|>