<commit_before>package drivers

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/lxd/migration"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/units"
)

var btrfsVersion string
var btrfsLoaded bool

type btrfs struct {
	common
}

// load is used to run one-time action per-driver rather than per-pool.
func (d *btrfs) load() error {
	// Register the patches.
	d.patches = map[string]func() error{
		"storage_create_vm":                        nil,
		"storage_zfs_mount":                        nil,
		"storage_create_vm_again":                  nil,
		"storage_zfs_volmode":                      nil,
		"storage_rename_custom_volume_add_project": nil,
	}

	// Done if previously loaded.
	if btrfsLoaded {
		return nil
	}

	// Validate the required binaries.
	for _, tool := range []string{"btrfs"} {
		_, err := exec.LookPath(tool)
		if err != nil {
			return fmt.Errorf("Required tool '%s' is missing", tool)
		}
	}

	// Detect and record the version.
	if btrfsVersion == "" {
		out, err := shared.RunCommand("btrfs", "version")
		if err != nil {
			return err
		}

		count, err := fmt.Sscanf(strings.SplitN(out, " ", 2)[1], "v%s\n", &btrfsVersion)
		if err != nil || count != 1 {
			return fmt.Errorf("The 'btrfs' tool isn't working properly")
		}
	}

	btrfsLoaded = true
	return nil
}

// Info returns info about the driver and its environment.
func (d *btrfs) Info() Info {
	return Info{
		Name:                  "btrfs",
		Version:               btrfsVersion,
		OptimizedImages:       true,
		OptimizedBackups:      true,
		PreservesInodes:       !d.state.OS.RunningInUserNS,
		Remote:                false,
		VolumeTypes:           []VolumeType{VolumeTypeCustom, VolumeTypeImage, VolumeTypeContainer, VolumeTypeVM},
		BlockBacking:          false,
		RunningQuotaResize:    true,
		RunningSnapshotFreeze: false,
		DirectIO:              true,
		MountedRoot:           true,
	}
}

// Create is called during pool creation and is effectively using an empty driver struct.
// WARNING: The Create() function cannot rely on any of the struct attributes being set.
func (d *btrfs) Create() error {
	// Store the provided source as we are likely to be mangling it.
	d.config["volatile.initial_source"] = d.config["source"]

	loopPath := loopFilePath(d.name)
	if d.config["source"] == "" || d.config["source"] == loopPath {
		// Create a loop based pool.
		d.config["source"] = loopPath

		// Create the loop file itself.
		size, err := units.ParseByteSizeString(d.config["size"])
		if err != nil {
			return err
		}

		err = createSparseFile(d.config["source"], size)
		if err != nil {
			return errors.Wrap(err, "Failed to create the sparse file")
		}

		// Format the file.
		_, err = makeFSType(d.config["source"], "btrfs", &mkfsOptions{Label: d.name})
		if err != nil {
			return errors.Wrap(err, "Failed to format sparse file")
		}
	} else if shared.IsBlockdevPath(d.config["source"]) {
		// Format the block device.
		_, err := makeFSType(d.config["source"], "btrfs", &mkfsOptions{Label: d.name})
		if err != nil {
			return errors.Wrap(err, "Failed to format block device")
		}

		// Record the UUID as the source.
		devUUID, err := fsUUID(d.config["source"])
		if err != nil {
			return err
		}

		// Confirm that the symlink is appearing (give it 10s).
		if tryExists(fmt.Sprintf("/dev/disk/by-uuid/%s", devUUID)) {
			// Override the config to use the UUID.
			d.config["source"] = devUUID
		}
	} else if d.config["source"] != "" {
		hostPath := shared.HostPath(d.config["source"])
		if d.isSubvolume(hostPath) {
			// Existing btrfs subvolume.
			subvols, err := d.getSubvolumes(hostPath)
			if err != nil {
				return errors.Wrap(err, "Could not determine if existing btrfs subvolume is empty")
			}

			// Check that the provided subvolume is empty.
			if len(subvols) > 0 {
				return fmt.Errorf("Requested btrfs subvolume exists but is not empty")
			}
		} else {
			// New btrfs subvolume on existing btrfs filesystem.
			cleanSource := filepath.Clean(hostPath)
			lxdDir := shared.VarPath()

			if shared.PathExists(hostPath) && !hasFilesystem(hostPath, util.FilesystemSuperMagicBtrfs) {
				return fmt.Errorf("Provided path does not reside on a btrfs filesystem")
			} else if strings.HasPrefix(cleanSource, lxdDir) {
				if cleanSource != GetPoolMountPath(d.name) {
					return fmt.Errorf("Only allowed source path under %s is %s", shared.VarPath(), GetPoolMountPath(d.name))
				} else if !hasFilesystem(shared.VarPath("storage-pools"), util.FilesystemSuperMagicBtrfs) {
					return fmt.Errorf("Provided path does not reside on a btrfs filesystem")
				}

				// Delete the current directory to replace by subvolume.
				err := os.Remove(cleanSource)
				if err != nil && !os.IsNotExist(err) {
					return errors.Wrapf(err, "Failed to remove '%s'", cleanSource)
				}
			}

			// Create the subvolume.
			_, err := shared.RunCommand("btrfs", "subvolume", "create", hostPath)
			if err != nil {
				return err
			}
		}
	} else {
		return fmt.Errorf("Invalid \"source\" property")
	}

	return nil
}

// Delete removes the storage pool from the storage device.
func (d *btrfs) Delete(op *operations.Operation) error {
	// If the user completely destroyed it, call it done.
	if !shared.PathExists(GetPoolMountPath(d.name)) {
		return nil
	}

	// Delete potential intermediate btrfs subvolumes.
	for _, volType := range d.Info().VolumeTypes {
		for _, dir := range BaseDirectories[volType] {
			path := filepath.Join(GetPoolMountPath(d.name), dir)
			if !shared.PathExists(path) {
				continue
			}

			if !d.isSubvolume(path) {
				continue
			}

			err := d.deleteSubvolume(path, true)
			if err != nil {
				return fmt.Errorf("Could not delete btrfs subvolume: %s", path)
			}
		}
	}

	// On delete, wipe everything in the directory.
	err := wipeDirectory(GetPoolMountPath(d.name))
	if err != nil {
		return err
	}

	// Unmount the path.
	_, err = d.Unmount()
	if err != nil {
		return err
	}

	// If the pool path is a subvolume itself, delete it.
	if d.isSubvolume(GetPoolMountPath(d.name)) {
		err := d.deleteSubvolume(GetPoolMountPath(d.name), false)
		if err != nil {
			return err
		}

		// And re-create as an empty directory to make the backend happy.
		err = os.Mkdir(GetPoolMountPath(d.name), 0700)
		if err != nil {
			return errors.Wrapf(err, "Failed to create directory '%s'", GetPoolMountPath(d.name))
		}
	}

	// Delete any loop file we may have used.
	loopPath := loopFilePath(d.name)
	err = os.Remove(loopPath)
	if err != nil && !os.IsNotExist(err) {
		return errors.Wrapf(err, "Failed to remove '%s'", loopPath)
	}

	return nil
}

// Validate checks that all provide keys are supported and that no conflicting or missing configuration is present.
func (d *btrfs) Validate(config map[string]string) error {
	rules := map[string]func(value string) error{
		"btrfs.mount_options": shared.IsAny,
	}

	return d.validatePool(config, rules)
}

// Update applies any driver changes required from a configuration change.
func (d *btrfs) Update(changedConfig map[string]string) error {
	// We only care about btrfs.mount_options.
	val, ok := changedConfig["btrfs.mount_options"]
	if !ok {
		return nil
	}

	// Custom mount options don't work inside containers
	if d.state.OS.RunningInUserNS {
		return nil
	}

	// Trigger a re-mount.
	d.config["btrfs.mount_options"] = val
	mntFlags, mntOptions := resolveMountOptions(d.getMountOptions())
	mntFlags |= unix.MS_REMOUNT

	err := TryMount("", GetPoolMountPath(d.name), "none", mntFlags, mntOptions)
	if err != nil {
		return err
	}

	return nil
}

// Mount mounts the storage pool.
func (d *btrfs) Mount() (bool, error) {
	// Check if already mounted.
	if shared.IsMountPoint(GetPoolMountPath(d.name)) {
		return false, nil
	}

	// Setup mount options.
	loopPath := loopFilePath(d.name)
	mntSrc := ""
	mntDst := GetPoolMountPath(d.name)
	mntFilesystem := "btrfs"
	if d.config["source"] == loopPath {
		// Bring up the loop device.
		loopF, err := PrepareLoopDev(d.config["source"], LoFlagsAutoclear)
		if err != nil {
			return false, err
		}
		defer loopF.Close()

		mntSrc = loopF.Name()
	} else if filepath.IsAbs(d.config["source"]) {
		// Bring up an existing device or path.
		mntSrc = shared.HostPath(d.config["source"])

		if !shared.IsBlockdevPath(mntSrc) {
			mntFilesystem = "none"

			if !hasFilesystem(mntSrc, util.FilesystemSuperMagicBtrfs) {
				return false, fmt.Errorf("Source path '%s' isn't btrfs", mntSrc)
			}
		}
	} else {
		// Mount using UUID.
		mntSrc = fmt.Sprintf("/dev/disk/by-uuid/%s", d.config["source"])
	}

	// Get the custom mount flags/options.
	mntFlags, mntOptions := resolveMountOptions(d.getMountOptions())

	// Handle bind-mounts first.
	if mntFilesystem == "none" {
		// Setup the bind-mount itself.
		err := TryMount(mntSrc, mntDst, mntFilesystem, unix.MS_BIND, "")
		if err != nil {
			return false, err
		}

		// Custom mount options don't work inside containers
		if d.state.OS.RunningInUserNS {
			return true, nil
		}

		// Now apply the custom options.
		mntFlags |= unix.MS_REMOUNT
		err = TryMount("", mntDst, mntFilesystem, mntFlags, mntOptions)
		if err != nil {
			return false, err
		}

		return true, nil
	}

	// Handle traditional mounts.
	err := TryMount(mntSrc, mntDst, mntFilesystem, mntFlags, mntOptions)
	if err != nil {
		return false, err
	}

	return true, nil
}

// Unmount unmounts the storage pool.
func (d *btrfs) Unmount() (bool, error) {
	// Unmount the pool.
	ourUnmount, err := forceUnmount(GetPoolMountPath(d.name))
	if err != nil {
		return false, err
	}

	// If loop backed, force release the loop device.
	loopPath := loopFilePath(d.name)
	if d.config["source"] == loopPath {
		releaseLoopDev(loopPath)
	}

	return ourUnmount, nil
}

// GetResources returns the pool resource usage information.
func (d *btrfs) GetResources() (*api.ResourcesStoragePool, error) {
	return genericVFSGetResources(d)
}

// MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order.
func (d *btrfs) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
	rsyncFeatures := []string{"xattrs", "delete", "compress", "bidirectional"}
	btrfsFeatures := []string{migration.BTRFSFeatureMigrationHeader, migration.BTRFSFeatureSubvolumes}

	// Only offer rsync for refreshes or if running in an unprivileged container.
	if refresh || d.state.OS.RunningInUserNS {
		var transportType migration.MigrationFSType

		if contentType == ContentTypeBlock {
			transportType = migration.MigrationFSType_BLOCK_AND_RSYNC
		} else {
			transportType = migration.MigrationFSType_RSYNC
		}

		return []migration.Type{
			{
				FSType:   transportType,
				Features: rsyncFeatures,
			},
		}
	}

	if contentType == ContentTypeBlock {
		return []migration.Type{
			{
				FSType:   migration.MigrationFSType_BTRFS,
				Features: btrfsFeatures,
			},
			{
				FSType:   migration.MigrationFSType_BLOCK_AND_RSYNC,
				Features: rsyncFeatures,
			},
		}
	}

	return []migration.Type{
		{
			FSType:   migration.MigrationFSType_BTRFS,
			Features: btrfsFeatures,
		},
		{
			FSType:   migration.MigrationFSType_RSYNC,
			Features: rsyncFeatures,
		},
	}
}
<commit_msg>lxd/storage/drivers/driver/btrfs: Sets OptimizedBackupHeader to true in Info struct response<commit_after>package drivers

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/lxd/migration"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/units"
)

var btrfsVersion string
var btrfsLoaded bool

type btrfs struct {
	common
}

// load is used to run one-time action per-driver rather than per-pool.
func (d *btrfs) load() error {
	// Register the patches.
	d.patches = map[string]func() error{
		"storage_create_vm":                        nil,
		"storage_zfs_mount":                        nil,
		"storage_create_vm_again":                  nil,
		"storage_zfs_volmode":                      nil,
		"storage_rename_custom_volume_add_project": nil,
	}

	// Done if previously loaded.
	if btrfsLoaded {
		return nil
	}

	// Validate the required binaries.
	for _, tool := range []string{"btrfs"} {
		_, err := exec.LookPath(tool)
		if err != nil {
			return fmt.Errorf("Required tool '%s' is missing", tool)
		}
	}

	// Detect and record the version.
	if btrfsVersion == "" {
		out, err := shared.RunCommand("btrfs", "version")
		if err != nil {
			return err
		}

		count, err := fmt.Sscanf(strings.SplitN(out, " ", 2)[1], "v%s\n", &btrfsVersion)
		if err != nil || count != 1 {
			return fmt.Errorf("The 'btrfs' tool isn't working properly")
		}
	}

	btrfsLoaded = true
	return nil
}

// Info returns info about the driver and its environment.
func (d *btrfs) Info() Info {
	return Info{
		Name:                  "btrfs",
		Version:               btrfsVersion,
		OptimizedImages:       true,
		OptimizedBackups:      true,
		OptimizedBackupHeader: true,
		PreservesInodes:       !d.state.OS.RunningInUserNS,
		Remote:                false,
		VolumeTypes:           []VolumeType{VolumeTypeCustom, VolumeTypeImage, VolumeTypeContainer, VolumeTypeVM},
		BlockBacking:          false,
		RunningQuotaResize:    true,
		RunningSnapshotFreeze: false,
		DirectIO:              true,
		MountedRoot:           true,
	}
}

// Create is called during pool creation and is effectively using an empty driver struct.
// WARNING: The Create() function cannot rely on any of the struct attributes being set.
func (d *btrfs) Create() error {
	// Store the provided source as we are likely to be mangling it.
	d.config["volatile.initial_source"] = d.config["source"]

	loopPath := loopFilePath(d.name)
	if d.config["source"] == "" || d.config["source"] == loopPath {
		// Create a loop based pool.
		d.config["source"] = loopPath

		// Create the loop file itself.
		size, err := units.ParseByteSizeString(d.config["size"])
		if err != nil {
			return err
		}

		err = createSparseFile(d.config["source"], size)
		if err != nil {
			return errors.Wrap(err, "Failed to create the sparse file")
		}

		// Format the file.
		_, err = makeFSType(d.config["source"], "btrfs", &mkfsOptions{Label: d.name})
		if err != nil {
			return errors.Wrap(err, "Failed to format sparse file")
		}
	} else if shared.IsBlockdevPath(d.config["source"]) {
		// Format the block device.
		_, err := makeFSType(d.config["source"], "btrfs", &mkfsOptions{Label: d.name})
		if err != nil {
			return errors.Wrap(err, "Failed to format block device")
		}

		// Record the UUID as the source.
		devUUID, err := fsUUID(d.config["source"])
		if err != nil {
			return err
		}

		// Confirm that the symlink is appearing (give it 10s).
		if tryExists(fmt.Sprintf("/dev/disk/by-uuid/%s", devUUID)) {
			// Override the config to use the UUID.
			d.config["source"] = devUUID
		}
	} else if d.config["source"] != "" {
		hostPath := shared.HostPath(d.config["source"])
		if d.isSubvolume(hostPath) {
			// Existing btrfs subvolume.
			subvols, err := d.getSubvolumes(hostPath)
			if err != nil {
				return errors.Wrap(err, "Could not determine if existing btrfs subvolume is empty")
			}

			// Check that the provided subvolume is empty.
			if len(subvols) > 0 {
				return fmt.Errorf("Requested btrfs subvolume exists but is not empty")
			}
		} else {
			// New btrfs subvolume on existing btrfs filesystem.
			cleanSource := filepath.Clean(hostPath)
			lxdDir := shared.VarPath()

			if shared.PathExists(hostPath) && !hasFilesystem(hostPath, util.FilesystemSuperMagicBtrfs) {
				return fmt.Errorf("Provided path does not reside on a btrfs filesystem")
			} else if strings.HasPrefix(cleanSource, lxdDir) {
				if cleanSource != GetPoolMountPath(d.name) {
					return fmt.Errorf("Only allowed source path under %s is %s", shared.VarPath(), GetPoolMountPath(d.name))
				} else if !hasFilesystem(shared.VarPath("storage-pools"), util.FilesystemSuperMagicBtrfs) {
					return fmt.Errorf("Provided path does not reside on a btrfs filesystem")
				}

				// Delete the current directory to replace by subvolume.
				err := os.Remove(cleanSource)
				if err != nil && !os.IsNotExist(err) {
					return errors.Wrapf(err, "Failed to remove '%s'", cleanSource)
				}
			}

			// Create the subvolume.
			_, err := shared.RunCommand("btrfs", "subvolume", "create", hostPath)
			if err != nil {
				return err
			}
		}
	} else {
		return fmt.Errorf("Invalid \"source\" property")
	}

	return nil
}

// Delete removes the storage pool from the storage device.
func (d *btrfs) Delete(op *operations.Operation) error {
	// If the user completely destroyed it, call it done.
	if !shared.PathExists(GetPoolMountPath(d.name)) {
		return nil
	}

	// Delete potential intermediate btrfs subvolumes.
	for _, volType := range d.Info().VolumeTypes {
		for _, dir := range BaseDirectories[volType] {
			path := filepath.Join(GetPoolMountPath(d.name), dir)
			if !shared.PathExists(path) {
				continue
			}

			if !d.isSubvolume(path) {
				continue
			}

			err := d.deleteSubvolume(path, true)
			if err != nil {
				return fmt.Errorf("Could not delete btrfs subvolume: %s", path)
			}
		}
	}

	// On delete, wipe everything in the directory.
	err := wipeDirectory(GetPoolMountPath(d.name))
	if err != nil {
		return err
	}

	// Unmount the path.
	_, err = d.Unmount()
	if err != nil {
		return err
	}

	// If the pool path is a subvolume itself, delete it.
	if d.isSubvolume(GetPoolMountPath(d.name)) {
		err := d.deleteSubvolume(GetPoolMountPath(d.name), false)
		if err != nil {
			return err
		}

		// And re-create as an empty directory to make the backend happy.
		err = os.Mkdir(GetPoolMountPath(d.name), 0700)
		if err != nil {
			return errors.Wrapf(err, "Failed to create directory '%s'", GetPoolMountPath(d.name))
		}
	}

	// Delete any loop file we may have used.
	loopPath := loopFilePath(d.name)
	err = os.Remove(loopPath)
	if err != nil && !os.IsNotExist(err) {
		return errors.Wrapf(err, "Failed to remove '%s'", loopPath)
	}

	return nil
}

// Validate checks that all provide keys are supported and that no conflicting or missing configuration is present.
func (d *btrfs) Validate(config map[string]string) error {
	rules := map[string]func(value string) error{
		"btrfs.mount_options": shared.IsAny,
	}

	return d.validatePool(config, rules)
}

// Update applies any driver changes required from a configuration change.
func (d *btrfs) Update(changedConfig map[string]string) error {
	// We only care about btrfs.mount_options.
	val, ok := changedConfig["btrfs.mount_options"]
	if !ok {
		return nil
	}

	// Custom mount options don't work inside containers
	if d.state.OS.RunningInUserNS {
		return nil
	}

	// Trigger a re-mount.
	d.config["btrfs.mount_options"] = val
	mntFlags, mntOptions := resolveMountOptions(d.getMountOptions())
	mntFlags |= unix.MS_REMOUNT

	err := TryMount("", GetPoolMountPath(d.name), "none", mntFlags, mntOptions)
	if err != nil {
		return err
	}

	return nil
}

// Mount mounts the storage pool.
func (d *btrfs) Mount() (bool, error) {
	// Check if already mounted.
	if shared.IsMountPoint(GetPoolMountPath(d.name)) {
		return false, nil
	}

	// Setup mount options.
	loopPath := loopFilePath(d.name)
	mntSrc := ""
	mntDst := GetPoolMountPath(d.name)
	mntFilesystem := "btrfs"
	if d.config["source"] == loopPath {
		// Bring up the loop device.
		loopF, err := PrepareLoopDev(d.config["source"], LoFlagsAutoclear)
		if err != nil {
			return false, err
		}
		defer loopF.Close()

		mntSrc = loopF.Name()
	} else if filepath.IsAbs(d.config["source"]) {
		// Bring up an existing device or path.
		mntSrc = shared.HostPath(d.config["source"])

		if !shared.IsBlockdevPath(mntSrc) {
			mntFilesystem = "none"

			if !hasFilesystem(mntSrc, util.FilesystemSuperMagicBtrfs) {
				return false, fmt.Errorf("Source path '%s' isn't btrfs", mntSrc)
			}
		}
	} else {
		// Mount using UUID.
		mntSrc = fmt.Sprintf("/dev/disk/by-uuid/%s", d.config["source"])
	}

	// Get the custom mount flags/options.
	mntFlags, mntOptions := resolveMountOptions(d.getMountOptions())

	// Handle bind-mounts first.
	if mntFilesystem == "none" {
		// Setup the bind-mount itself.
		err := TryMount(mntSrc, mntDst, mntFilesystem, unix.MS_BIND, "")
		if err != nil {
			return false, err
		}

		// Custom mount options don't work inside containers
		if d.state.OS.RunningInUserNS {
			return true, nil
		}

		// Now apply the custom options.
		mntFlags |= unix.MS_REMOUNT
		err = TryMount("", mntDst, mntFilesystem, mntFlags, mntOptions)
		if err != nil {
			return false, err
		}

		return true, nil
	}

	// Handle traditional mounts.
	err := TryMount(mntSrc, mntDst, mntFilesystem, mntFlags, mntOptions)
	if err != nil {
		return false, err
	}

	return true, nil
}

// Unmount unmounts the storage pool.
func (d *btrfs) Unmount() (bool, error) {
	// Unmount the pool.
	ourUnmount, err := forceUnmount(GetPoolMountPath(d.name))
	if err != nil {
		return false, err
	}

	// If loop backed, force release the loop device.
	loopPath := loopFilePath(d.name)
	if d.config["source"] == loopPath {
		releaseLoopDev(loopPath)
	}

	return ourUnmount, nil
}

// GetResources returns the pool resource usage information.
func (d *btrfs) GetResources() (*api.ResourcesStoragePool, error) {
	return genericVFSGetResources(d)
}

// MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order.
func (d *btrfs) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
	rsyncFeatures := []string{"xattrs", "delete", "compress", "bidirectional"}
	btrfsFeatures := []string{migration.BTRFSFeatureMigrationHeader, migration.BTRFSFeatureSubvolumes}

	// Only offer rsync for refreshes or if running in an unprivileged container.
	if refresh || d.state.OS.RunningInUserNS {
		var transportType migration.MigrationFSType

		if contentType == ContentTypeBlock {
			transportType = migration.MigrationFSType_BLOCK_AND_RSYNC
		} else {
			transportType = migration.MigrationFSType_RSYNC
		}

		return []migration.Type{
			{
				FSType:   transportType,
				Features: rsyncFeatures,
			},
		}
	}

	if contentType == ContentTypeBlock {
		return []migration.Type{
			{
				FSType:   migration.MigrationFSType_BTRFS,
				Features: btrfsFeatures,
			},
			{
				FSType:   migration.MigrationFSType_BLOCK_AND_RSYNC,
				Features: rsyncFeatures,
			},
		}
	}

	return []migration.Type{
		{
			FSType:   migration.MigrationFSType_BTRFS,
			Features: btrfsFeatures,
		},
		{
			FSType:   migration.MigrationFSType_RSYNC,
			Features: rsyncFeatures,
		},
	}
}
<|endoftext|>


<commit_before>package drivers

// Info represents information about a storage driver.
type Info struct {
	Name                  string
	Version               string
	VolumeTypes           []VolumeType // Supported volume types.
	Remote                bool         // Whether the driver uses a remote backing store.
	VolumeMultiNode       bool         // Whether volumes can be used on multiple nodes concurrently.
	OptimizedImages       bool         // Whether driver stores images as separate volume.
	OptimizedBackups      bool         // Whether driver supports optimized volume backups.
	OptimizedBackupHeader bool         // Whether driver generates an optimised backup header file in backup.
	PreservesInodes       bool         // Whether driver preserves inodes when volumes are moved hosts.
	BlockBacking          bool         // Whether driver uses block devices as backing store.
	RunningCopyFreeze     bool         // Whether instance should be frozen during snapshot if running.
	DirectIO              bool         // Whether the driver supports direct I/O.
	MountedRoot           bool         // Whether the pool directory itself is a mount.
}

// VolumeFiller provides a struct for filling a volume.
type VolumeFiller struct {
	Fill func(vol Volume, rootBlockPath string, allowUnsafeResize bool) (int64, error) // Function to fill the volume.
	Size int64                                                                         // Size of the unpacked volume in bytes.

	Fingerprint string // If the Filler will unpack an image, it should be this fingerprint.
}
<commit_msg>lxd/storage/drivers/driver/types: Adds Buckets field to Info<commit_after>package drivers

// Info represents information about a storage driver.
type Info struct {
	Name                  string
	Version               string
	VolumeTypes           []VolumeType // Supported volume types.
	Buckets               bool         // Buckets supported.
	Remote                bool         // Whether the driver uses a remote backing store.
	VolumeMultiNode       bool         // Whether volumes can be used on multiple nodes concurrently.
	OptimizedImages       bool         // Whether driver stores images as separate volume.
	OptimizedBackups      bool         // Whether driver supports optimized volume backups.
	OptimizedBackupHeader bool         // Whether driver generates an optimised backup header file in backup.
	PreservesInodes       bool         // Whether driver preserves inodes when volumes are moved hosts.
	BlockBacking          bool         // Whether driver uses block devices as backing store.
	RunningCopyFreeze     bool         // Whether instance should be frozen during snapshot if running.
	DirectIO              bool         // Whether the driver supports direct I/O.
	MountedRoot           bool         // Whether the pool directory itself is a mount.
}

// VolumeFiller provides a struct for filling a volume.
type VolumeFiller struct {
	Fill func(vol Volume, rootBlockPath string, allowUnsafeResize bool) (int64, error) // Function to fill the volume.
	Size int64                                                                         // Size of the unpacked volume in bytes.

	Fingerprint string // If the Filler will unpack an image, it should be this fingerprint.
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	Unknown = iota
	GO_1_0
	GO_1_1
)

var (
	regexes = map[int]*regexp.Regexp{
		GO_1_0: regexp.MustCompile(`gc(\d+)\((\d+)\):\s(\d+)\+(\d+)\+(\d+)\s\w+\s(\d+)\s->\s(\d+)\s\w+\s(\d+)\s->\s(\d+)\s\((\d+)-(\d+)\)\sobjects\s(\d+)\shandoff`),
		GO_1_1: regexp.MustCompile(`gc(\d+)\((\d+)\):\s(\d+)\+(\d+)\+(\d+)\s\w+,\s(\d+)\s->\s(\d+)\s\w+\s(\d+)\s->\s(\d+)\s\((\d+)-(\d+)\)\sobjects,\s(\d+)\((\d+)\)\shandoff,\s(\d+)\((\d+)\)\ssteal,\s(\d+)\/(\d+)\/(\d+)\syields`),
	}
	header = map[int]string{
		GO_1_0: "numgc,nproc,mark,sweep,cleanup,heap0,heap1,obj0,obj1,nmalloc,nfree,nhandoff",
		GO_1_1: "numgc,nproc,mark,sweep,cleanup,heap0,heap1,obj0,obj1,nmalloc,nfree,nhandoff,nhandoffcnt,nsteal,nstealcnt,nprocyield,nosyield,nsleep",
	}

	errVersionNotFound = errors.New("can't detected version")

	inputFile, outputFile string
	timestamp             bool
	isStdin               = false
)

func init() {
	flag.StringVar(&inputFile, "i", "", "The input file (default: Stdin)")
	flag.StringVar(&outputFile, "o", "", "The output file")
	flag.BoolVar(&timestamp, "t", false, "Add timestamp at line head(Stdin input only)")

	flag.Usage = func() {
		fmt.Println("Usage1: log2csv -i gc.log -o gc.csv")
		fmt.Println("Usage2: GOGCTRACE=1 your-go-program 2>&1 | log2csv -o gc.csv")
		flag.PrintDefaults()
		fmt.Println("  -h   : show help usage")
	}
}

func detectLogVersion(line string) (version int, err error) {
	for version, regexp := range regexes {
		if regexp.MatchString(line) {
			return version, nil
		}
	}

	err = errVersionNotFound
	return
}

func convert(input string, version int) (output string, err error) {
	defer func() {
		if e := recover(); e != nil {
			err = errors.New(fmt.Sprintf("unmatched uint string: %s => %s", input, e))
		}
	}()

	if matched := regexes[version].FindStringSubmatch(input); matched == nil {
		err = errors.New(fmt.Sprintf("unmatched string: %s", input))
	} else {
		output = strings.Join(matched[1:], ",")
	}
	return
}

func process(reader io.Reader, writer io.Writer) {
	bufReader := bufio.NewReader(reader)
	bufWriter := bufio.NewWriter(writer)

	currentLogVersion := Unknown
	for {

		filtered := false
		if line, err := bufReader.ReadString('\n'); err != nil {
			break
		} else {
			if currentLogVersion == Unknown {
				if version, err := detectLogVersion(line); err == nil {
					currentLogVersion = version
					err := writeHeader(bufWriter, currentLogVersion)
					checkError(err)
				}

			}

			if currentLogVersion != Unknown {
				if output, err := convert(line, currentLogVersion); err == nil {
					err := writeBody(bufWriter, output)
					checkError(err)
					filtered = true
				}
			}

			if isStdin && filtered == false {
				fmt.Print(line)
			}
		}

	}
	err := bufWriter.Flush()
	checkError(err)
}

func writeHeader(writer *bufio.Writer, version int) (err error) {
	prefix := ""
	if isStdin && timestamp {
		prefix = "unixtime,"
	}
	_, err = writer.WriteString(prefix + header[version] + "\n")
	return
}

func writeBody(writer *bufio.Writer, output string) (err error) {
	prefix := ""
	if isStdin && timestamp {
		prefix = fmtFrac(time.Now(), 6) + ","
	}

	_, err = writer.WriteString(prefix + output + "\n")
	return
}

func fmtFrac(t time.Time, prec int) string {
	unixNano := t.UnixNano()
	fmtStr := "%." + strconv.Itoa(prec) + "f"
	return fmt.Sprintf(fmtStr, float64(unixNano)/10e8)
}

func getReader(file string) (reader io.Reader, err error) {
	if file == "" {
		reader = os.Stdin
		isStdin = true
	} else {
		reader, err = os.Open(file)
	}
	return
}

func getWriter(file string) (writer io.Writer, err error) {
	if file == "" {
		err = errors.New("required output file parameter")
	} else {
		writer, err = os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	}
	return
}

func checkError(err error) {
	if err != nil {
		fmt.Fprint(os.Stderr, err)
		os.Exit(-1)
	}
}

func main() {
	flag.Parse()

	reader, err := getReader(inputFile)
	checkError(err)
	defer reader.(*os.File).Close()

	writer, err := getWriter(outputFile)
	checkError(err)
	defer writer.(*os.File).Close()

	process(reader, writer)
}
<commit_msg>enhance flag check<commit_after>package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	Unknown = iota
	GO_1_0
	GO_1_1
)

var (
	regexes = map[int]*regexp.Regexp{
		GO_1_0: regexp.MustCompile(`gc(\d+)\((\d+)\):\s(\d+)\+(\d+)\+(\d+)\s\w+\s(\d+)\s->\s(\d+)\s\w+\s(\d+)\s->\s(\d+)\s\((\d+)-(\d+)\)\sobjects\s(\d+)\shandoff`),
		GO_1_1: regexp.MustCompile(`gc(\d+)\((\d+)\):\s(\d+)\+(\d+)\+(\d+)\s\w+,\s(\d+)\s->\s(\d+)\s\w+\s(\d+)\s->\s(\d+)\s\((\d+)-(\d+)\)\sobjects,\s(\d+)\((\d+)\)\shandoff,\s(\d+)\((\d+)\)\ssteal,\s(\d+)\/(\d+)\/(\d+)\syields`),
	}
	header = map[int]string{
		GO_1_0: "numgc,nproc,mark,sweep,cleanup,heap0,heap1,obj0,obj1,nmalloc,nfree,nhandoff",
		GO_1_1: "numgc,nproc,mark,sweep,cleanup,heap0,heap1,obj0,obj1,nmalloc,nfree,nhandoff,nhandoffcnt,nsteal,nstealcnt,nprocyield,nosyield,nsleep",
	}

	errVersionNotFound = errors.New("can't detected version")

	inputFile, outputFile string
	timestamp             bool
	isStdin               = false
)

func init() {
	flag.StringVar(&inputFile, "i", "", "The input file (default: Stdin)")
	flag.StringVar(&outputFile, "o", "", "The output file")
	flag.BoolVar(&timestamp, "t", false, "Add timestamp at line head(Stdin input only)")

	flag.Usage = func() {
		fmt.Println("Usage1: log2csv -i gc.log -o gc.csv")
		fmt.Println("Usage2: GOGCTRACE=1 your-go-program 2>&1 | log2csv -o gc.csv")
		flag.PrintDefaults()
		fmt.Println("  -h   : show help usage")
	}
}

func detectLogVersion(line string) (version int, err error) {
	for version, regexp := range regexes {
		if regexp.MatchString(line) {
			return version, nil
		}
	}

	err = errVersionNotFound
	return
}

func convert(input string, version int) (output string, err error) {
	defer func() {
		if e := recover(); e != nil {
			err = errors.New(fmt.Sprintf("unmatched uint string: %s => %s", input, e))
		}
	}()

	if matched := regexes[version].FindStringSubmatch(input); matched == nil {
		err = errors.New(fmt.Sprintf("unmatched string: %s", input))
	} else {
		output = strings.Join(matched[1:], ",")
	}
	return
}

func process(reader io.Reader, writer io.Writer) {
	bufReader := bufio.NewReader(reader)
	bufWriter := bufio.NewWriter(writer)

	currentLogVersion := Unknown
	for {

		filtered := false
		if line, err := bufReader.ReadString('\n'); err != nil {
			break
		} else {
			if currentLogVersion == Unknown {
				if version, err := detectLogVersion(line); err == nil {
					currentLogVersion = version
					err := writeHeader(bufWriter, currentLogVersion)
					checkError(err)
				}

			}

			if currentLogVersion != Unknown {
				if output, err := convert(line, currentLogVersion); err == nil {
					err := writeBody(bufWriter, output)
					checkError(err)
					filtered = true
				}
			}

			if isStdin && filtered == false {
				fmt.Print(line)
			}
		}

	}
	err := bufWriter.Flush()
	checkError(err)
}

func writeHeader(writer *bufio.Writer, version int) (err error) {
	prefix := ""
	if isStdin && timestamp {
		prefix = "unixtime,"
	}
	_, err = writer.WriteString(prefix + header[version] + "\n")
	return
}

func writeBody(writer *bufio.Writer, output string) (err error) {
	prefix := ""
	if isStdin && timestamp {
		prefix = fmtFrac(time.Now(), 6) + ","
	}

	_, err = writer.WriteString(prefix + output + "\n")
	return
}

func fmtFrac(t time.Time, prec int) string {
	unixNano := t.UnixNano()
	fmtStr := "%." + strconv.Itoa(prec) + "f"
	return fmt.Sprintf(fmtStr, float64(unixNano)/10e8)
}

func getReader(file string) (reader io.Reader, err error) {
	if file == "" {
		reader = os.Stdin
		isStdin = true
	} else {
		reader, err = os.Open(file)
	}
	return
}

func getWriter(file string) (writer io.Writer, err error) {
	if file == "" {
		err = errors.New("required output file parameter")
	} else {
		writer, err = os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
	}
	return
}

func checkError(err error) {
	if err != nil {
		fmt.Fprint(os.Stderr, err)
		os.Exit(-1)
	}
}

func main() {
	flag.Parse()

	if flag.NArg() == 0 || flag.NFlag() == 0 {
		flag.Usage()
		os.Exit(-1)
	}

	reader, err := getReader(inputFile)
	checkError(err)
	defer reader.(*os.File).Close()

	writer, err := getWriter(outputFile)
	checkError(err)
	defer writer.(*os.File).Close()

	process(reader, writer)
}
<|endoftext|>


<commit_before>// Package stringutil contains utility functions for working with strings.
package qclient

// - Basic POST query
// - Basic GET query
// - Consistent hashing implementation equivalent to the python version

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
)

type QClient struct {
	nodes []string
}

type Clause []interface{}

type Query struct {
	Select   Clause   `json:"select,omitempty"`
	Where    Clause   `json:"where,omitempty"`
	OrderBy  []string `json:"order_by,omitempty"`
	GroupBy  []string `json:"group_by,omitempty"`
	Distinct []string `json:"distinct,omitempty"`
	Offset   int      `json:"offset,omitempty"`
	Limit    int      `json:"limit,omitempty"`
	From     *Query   `json:"from,omitempty"`
}

func And(clauses ...Clause) Clause {
	return clause("&", clauses...)
}

func clause(op string, clauses ...Clause) Clause {
	result := Clause{op}
	for _, clause := range clauses {
		result = append(result, clause)
	}

	return result
}

func Op(op string, operands ...interface{}) Clause {
	result := Clause{op}
	for _, operand := range operands {
		result = append(result, operand)
	}

	return result
}

func Eq(x, y interface{}) Clause {
	return Op("==", x, y)
}

func (c *QClient) Query(key string, q string) string {
	return "hello"
}

func (c *QClient) Get(key string, q Query) ([]byte, error) {
	jq, _ := json.Marshal(q)
	ujq := url.QueryEscape(string(jq[:]))
	response, err := http.Get(c.nodes[0] + "/qcache/dataset/" + key + "?q=" + ujq)

	if err != nil {
		log.Fatal("Error getting data: ", err)
		return nil, err
	}

	if response.StatusCode == 404 {
		// Not found in cache
		return nil, nil
	}

	defer response.Body.Close()
	contents, _ := ioutil.ReadAll(response.Body)

	if response.StatusCode != 200 {
		// Set error but also include content, it may provide valuable information about the error
		return contents, errors.New(fmt.Sprintf("Unexpected HTTP response code %s", response.StatusCode))
	}

	return contents, nil
}

func (c *QClient) Post(key string, bodyType string, body io.Reader) error {
	_, err := http.Post(c.nodes[0]+"/qcache/dataset/"+key, bodyType, body)
	return err
}

func NewClient(nodes []string) *QClient {
	return &QClient{nodes: nodes}
}
<commit_msg>Start using nodeRing in client<commit_after>// Package stringutil contains utility functions for working with strings.
package qclient

// - Basic POST query
// - Basic GET query
// - Consistent hashing implementation equivalent to the python version

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
)

type QClient struct {
	ring *nodeRing
}

type Clause []interface{}

type Query struct {
	Select   Clause   `json:"select,omitempty"`
	Where    Clause   `json:"where,omitempty"`
	OrderBy  []string `json:"order_by,omitempty"`
	GroupBy  []string `json:"group_by,omitempty"`
	Distinct []string `json:"distinct,omitempty"`
	Offset   int      `json:"offset,omitempty"`
	Limit    int      `json:"limit,omitempty"`
	From     *Query   `json:"from,omitempty"`
}

func And(clauses ...Clause) Clause {
	return clause("&", clauses...)
}

func clause(op string, clauses ...Clause) Clause {
	result := Clause{op}
	for _, clause := range clauses {
		result = append(result, clause)
	}

	return result
}

func Op(op string, operands ...interface{}) Clause {
	result := Clause{op}
	for _, operand := range operands {
		result = append(result, operand)
	}

	return result
}

func Eq(x, y interface{}) Clause {
	return Op("==", x, y)
}

func (c *QClient) Query(key string, q string) string {
	return "hello"
}

func (c *QClient) Get(key string, q Query) ([]byte, error) {
	jq, _ := json.Marshal(q)
	ujq := url.QueryEscape(string(jq[:]))
	node, err := c.ring.getNode(key)
	if err != nil {
		return nil, err
	}

	response, err := http.Get(node + "/qcache/dataset/" + key + "?q=" + ujq)

	if err != nil {
		log.Fatal("Error getting data: ", err)
		return nil, err
	}

	if response.StatusCode == 404 {
		// Not found in cache
		return nil, nil
	}

	defer response.Body.Close()
	contents, _ := ioutil.ReadAll(response.Body)

	if response.StatusCode != 200 {
		// Set error but also include content, it may provide valuable information about the error
		return contents, errors.New(fmt.Sprintf("Unexpected HTTP response code %s", response.StatusCode))
	}

	return contents, nil
}

func (c *QClient) Post(key string, bodyType string, body io.Reader) error {
	node, err := c.ring.getNode(key)
	if err != nil {
		return err
	}
	_, err = http.Post(node+"/qcache/dataset/"+key, bodyType, body)
	return err
}

func NewClient(nodes []string) *QClient {
	ring, _ := newNodeRing(nodes)
	return &QClient{ring: ring}
}
<|endoftext|>


<commit_before>package main

type LottoGame struct {
	Name   string
	Cost   int
	Prizes []Prize
}

type Prize struct {
	Value            int
	TotalTickets     int
	UnclaimedTickets int
}

func ParseLotteryGames(bytes []byte) (games []LottoGame, err error) {
	return
}
<commit_msg>Implementation of the Raw data structure parser<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"strings"
)

type LottoGame struct {
	Name   string
	Cost   int
	Prizes []Prize
}

type Prize struct {
	Value            int
	TotalTickets     int
	UnclaimedTickets int
}

func ParseLotteryGames(raw []byte) ([]LottoGame, error) {
	var games []LottoGame
	buf := bytes.NewBuffer(raw)

	for {
		game, err := ParseLotteryGame(buf)
		if err != nil {
			return nil, err
		}

		games = append(games, game)

		// Consume the blankline
		_, err = buf.ReadString('\n')

		// Check for EOF
		switch err {
		case io.EOF:
			return games, nil
		default:
			return nil, err
		case nil:
			continue
		}
	}

	panic("never reached")
}

func ParseLotteryGame(b *bytes.Buffer) (game LottoGame, err error) {

	// Consume Name
	name, err := b.ReadString('\n')
	if err != nil {
		return game, err
	}

	var cost, numPrizes int

	// Consume Game Cost and Total number of Prize Values
	n, err := fmt.Fscanln(b, &cost, &numPrizes)
	if n != 2 || err != nil {
		return game, err
	}

	prizes := make([]Prize, numPrizes)
	for i := 0; i < numPrizes; i++ {
		var value, totalTickets, unclaimedTickets int

		n, err := fmt.Fscanln(b, &value, &totalTickets, &unclaimedTickets)
		if n != 3 || err != nil {
			return game, err
		}

		prizes[i] = Prize{value, totalTickets, unclaimedTickets}
	}

	game = LottoGame{
		strings.TrimSpace(name),
		cost,
		prizes,
	}

	return game, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"os/user"
	"strconv"
	"syscall"
)

const MAXDIRREAD int = 50

var uwidth int
var gwidth int
var swidth int

var (
	usedir   = flag.Bool("d", false, "List a directory instead of its contents.")
	long     = flag.Bool("l", false, "Long format list.")
	usrname  = flag.Bool("m", false, "List the user who last modified the file.")
	nosort   = flag.Bool("n", false, "Don't sort the list.")
	fnlpath  = flag.Bool("p", false, "Only print the last path element.")
	reverse  = flag.Bool("r", false, "Reverse the sorting order.")
	kbytes   = flag.Bool("s", false, "Use KBytes for size.")
	timesort = flag.Bool("t", false, "Sort by latest-modified first.")
	useatime = flag.Bool("u", false, "If -t sort by access time; if -u print "+
		"last access time.")
	tlslash = flag.Bool("F", false, "Add / after all directories and * after "+
		"all executables.")
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: ls [-dlmnprstuF] [file ...]\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func error(s string) {
	fmt.Fprint(os.Stderr, s, "\n")
	os.Exit(1)
}

type dent struct {
	mode string
	p    string
	u    string
	g    uint32
	s    int64
	n    string
	d    bool
	e    bool
}

func (self *dent) getInfo(fi os.FileInfo) {
	self.n = fi.Name()
	self.s = fi.Size()
	self.mode = fi.Mode().String()
	self.d = fi.Mode().IsDir()

	if fi.Mode().Perm()&0111 != 0 && !self.d {
		self.e = true
	}

	if !*long {
		return
	}

	/* The following is probably not portable */
	l := len(strconv.Itoa(int(self.s)))
	if l > swidth {
		swidth = l
	}

	s := fi.Sys().(*syscall.Stat_t)
	u, _ := user.LookupId(strconv.Itoa(int(s.Uid)))

	if len(u.Username) > uwidth {
		uwidth = len(u.Username)
	}

	gl := strconv.Itoa(int(s.Gid))
	if len(gl) > gwidth {
		gwidth = len(gl)
	}

	self.u = u.Username
	self.g = s.Gid
}

func (self *dent) String() string {
	return fmt.Sprintf("%s %*s %*d %*d",
		self.mode,
		uwidth, self.u,
		gwidth, self.g,
		swidth, self.s)
}

func ls(path string) {
	dents := make([]*dent, 0, MAXDIRREAD)

	addDent := func(d *dent) {
		l := len(dents)
		if l+1 > cap(dents) {
			n_dents := make([]*dent, l+MAXDIRREAD)
			copy(n_dents, dents)
			dents = n_dents
		}
		dents = dents[0 : l+1]
		dents[l] = d
	}

	printFName := func(d *dent) {
		fmt.Printf("%s%s", d.p, d.n)
		if *tlslash {
			if d.d {
				fmt.Print("/")
			} else if d.e {
				fmt.Print("*")
			}
		}
		fmt.Print("\n")
	}

	var pth string
	if path == "" {
		pth = "."
	} else {
		pth = path
	}

	f, err := os.Open(pth)
	if err != nil {
		error(fmt.Sprint("%s", err))
	}

	for {
		fi, err := f.Readdir(MAXDIRREAD)
		if err != nil {
			if err == io.EOF {
				break
			} else {
				error(fmt.Sprint("%s", err))
			}
		}

		for _, file := range fi {
			d := new(dent)

			if path == "" {
				d.p = path
			} else {
				d.p = path + "/"
			}

			d.getInfo(file)
			addDent(d)
		}
	}

	for _, d := range dents {
		if *long {
			fmt.Printf("%s ", d.String())
		}
		printFName(d)
	}
}

func main() {
	flag.Usage = usage
	flag.Parse()
	args := flag.Args()

	if len(args) == 0 {
		ls("")
	} else {
		for _, path := range args {
			ls(path)
		}
	}

	os.Exit(0)
}
<commit_msg>-l time printing; -p implemented<commit_after>package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"os/user"
	"strconv"
	"syscall"
	"time"
)

const MAXDIRREAD int = 50

var uwidth int
var gwidth int
var swidth int

var (
	usedir   = flag.Bool("d", false, "List a directory instead of its contents.")
	long     = flag.Bool("l", false, "Long format list.")
	usrname  = flag.Bool("m", false, "List the user who last modified the file.")
	nosort   = flag.Bool("n", false, "Don't sort the list.")
	nopath   = flag.Bool("p", false, "Only print the last path element.")
	reverse  = flag.Bool("r", false, "Reverse the sorting order.")
	kbytes   = flag.Bool("s", false, "Give size in KBytes for each file.")
	timesort = flag.Bool("t", false, "Sort by latest-modified first.")
	useatime = flag.Bool("u", false, "If -t sort by access time; if -u print "+
		"last access time.")
	tlslash = flag.Bool("F", false, "Add / after all directories and * after "+
		"all executables.")
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: ls [-dlmnprstuF] [file ...]\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func error(s string) {
	fmt.Fprint(os.Stderr, s, "\n")
	os.Exit(1)
}

type dent struct {
	mode string
	p    string
	u    string
	m    string
	g    uint32
	s    int64
	n    string
	d    bool
	e    bool
}

func (self *dent) getInfo(fi os.FileInfo) {
	self.n = fi.Name()
	self.s = fi.Size()
	self.mode = fi.Mode().String()
	self.d = fi.Mode().IsDir()

	if fi.Mode().Perm()&0111 != 0 && !self.d {
		self.e = true
	}

	if !*long {
		return
	}

	yr := fi.ModTime().Year()
	mon := fi.ModTime().Format("Jan")
	day := fi.ModTime().Day()
	hr, min, _ := fi.ModTime().Clock()

	if time.Now().Year() == yr {
		self.m = fmt.Sprintf("%s %2d %02d:%02d", mon, day, hr, min)
	} else {
		self.m = fmt.Sprintf("%s %2d %5d", mon, day, yr)
	}

	/* The following is probably not portable */
	l := len(strconv.Itoa(int(self.s)))
	if l > swidth {
		swidth = l
	}

	s := fi.Sys().(*syscall.Stat_t)
	u, _ := user.LookupId(strconv.Itoa(int(s.Uid)))

	if len(u.Username) > uwidth {
		uwidth = len(u.Username)
	}

	gl := strconv.Itoa(int(s.Gid))
	if len(gl) > gwidth {
		gwidth = len(gl)
	}

	self.u = u.Username
	self.g = s.Gid
}

func (self *dent) String() string {
	return fmt.Sprintf("%s %*s %*d %*d %s",
		self.mode,
		uwidth, self.u,
		gwidth, self.g,
		swidth, self.s,
		self.m)
}

func ls(path string) {
	dents := make([]*dent, 0, MAXDIRREAD)

	addDent := func(d *dent) {
		l := len(dents)
		if l+1 > cap(dents) {
			n_dents := make([]*dent, l+MAXDIRREAD)
			copy(n_dents, dents)
			dents = n_dents
		}
		dents = dents[0 : l+1]
		dents[l] = d
	}

	printFName := func(d *dent) {
		fmt.Printf("%s%s", d.p, d.n)
		if *tlslash {
			if d.d {
				fmt.Print("/")
			} else if d.e {
				fmt.Print("*")
			}
		}
		fmt.Print("\n")
	}

	var pth string
	if path == "" {
		pth = "."
	} else {
		pth = path
	}

	f, err := os.Open(pth)
	if err != nil {
		error(fmt.Sprint("%s", err))
	}

	for {
		fi, err := f.Readdir(MAXDIRREAD)
		if err != nil {
			if err == io.EOF {
				break
			} else {
				error(fmt.Sprint("%s", err))
			}
		}

		for _, file := range fi {
			d := new(dent)

			if *nopath {
				path = ""
			}

			d.p = path

			if path != "" {
				d.p += "/"
			}

			d.getInfo(file)
			addDent(d)
		}
	}

	for _, d := range dents {
		if *long {
			fmt.Printf("%s ", d.String())
		}
		printFName(d)
	}
}

func main() {
	flag.Usage = usage
	flag.Parse()
	args := flag.Args()

	if len(args) == 0 {
		ls("")
	} else {
		for _, path := range args {
			ls(path)
		}
	}

	os.Exit(0)
}
<|endoftext|>


<commit_before><commit_msg>Switch to using errors.Wrapf rather then fmt.Errorsf<commit_after><|endoftext|>


<commit_before>// Copyright 2014 Unknown
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package macaron is a high productive and modular design web framework in Go.
package macaron

import (
	"log"
	"net/http"
	"os"
	"path/filepath"
	"reflect"

	"github.com/Unknwon/macaron/inject"
)

func Version() string {
	return "0.1.1.0717"
}

// Handler can be any callable function.
// Macaron attempts to inject services into the handler's argument list,
// and panics if an argument could not be fullfilled via dependency injection.
type Handler interface{}

// validateHandler makes sure a handler is a callable function,
// and panics if it is not.
func validateHandler(handler Handler) {
	if reflect.TypeOf(handler).Kind() != reflect.Func {
		panic("mocaron handler must be a callable function")
	}
}

// Macaron represents the top level web application.
// inject.Injector methods can be invoked to map services on a global level.
type Macaron struct {
	inject.Injector
	handlers []Handler
	action   Handler
	*Router
	logger *log.Logger
}

// New creates a bare bones Macaron instance.
// Use this method if you want to have full control over the middleware that is used.
func New() *Macaron {
	m := &Macaron{
		Injector: inject.New(),
		action:   func() {},
		Router:   NewRouter(),
		logger:   log.New(os.Stdout, "[Macaron] ", 0),
	}
	m.Router.m = m
	m.Map(m.logger)
	m.Map(defaultReturnHandler())
	m.notFound = func(resp http.ResponseWriter, req *http.Request) {
		c := m.createContext(resp, req)
		c.handlers = append(c.handlers, http.NotFound)
		c.run()
	}
	return m
}

// Classic creates a classic Macaron with some basic default middleware:
// mocaron.Logger, mocaron.Recovery and mocaron.Static.
func Classic() *Macaron {
	m := New()
	m.Use(Logger())
	m.Use(Recovery())
	m.Use(Static("public"))
	return m
}

// Handlers sets the entire middleware stack with the given Handlers.
// This will clear any current middleware handlers,
// and panics if any of the handlers is not a callable function
func (m *Macaron) Handlers(handlers ...Handler) {
	m.handlers = make([]Handler, 0)
	for _, handler := range handlers {
		m.Use(handler)
	}
}

// Action sets the handler that will be called after all the middleware has been invoked.
// This is set to macaron.Router in a macaron.Classic().
func (m *Macaron) Action(handler Handler) {
	validateHandler(handler)
	m.action = handler
}

// Use adds a middleware Handler to the stack,
// and panics if the handler is not a callable func.
// Middleware Handlers are invoked in the order that they are added.
func (m *Macaron) Use(handler Handler) {
	validateHandler(handler)
	m.handlers = append(m.handlers, handler)
}

func (m *Macaron) createContext(rw http.ResponseWriter, req *http.Request) *Context {
	c := &Context{
		Injector: inject.New(),
		handlers: m.handlers,
		action:   m.action,
		index:    0,
		Router:   m.Router,
		Req:      req,
		Resp:     NewResponseWriter(rw),
		Data:     make(map[string]interface{}),
	}
	c.SetParent(m)
	c.Map(c)
	c.MapTo(c.Resp, (*http.ResponseWriter)(nil))
	c.Map(req)
	return c
}

// ServeHTTP is the HTTP Entry point for a Macaron instance.
// Useful if you want to control your own HTTP server.
// Be aware that none of middleware will run without registering any router.
func (m *Macaron) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	m.Router.ServeHTTP(resp, req)
}

// getDefaultListenAddr returns default server listen address of Macaron.
func getDefaultListenAddr() string {
	port := os.Getenv("PORT")
	if port == "" {
		port = "4000"
	}
	host := os.Getenv("HOST")
	return host + ":" + port
}

// Run the http server. Listening on os.GetEnv("PORT") or 4000 by default.
func (m *Macaron) Run() {
	addr := getDefaultListenAddr()

	logger := m.Injector.GetVal(reflect.TypeOf(m.logger)).Interface().(*log.Logger)
	logger.Printf("listening on %s (%s)\n", addr, Env)
	logger.Fatalln(http.ListenAndServe(addr, m))
}

// \_   _____/ _______  __
//  |    __)_ /    \  \/ /
//  |        \   |  \   /
// /_______  /___|  /\_/
//         \/     \/

const (
	DEV  string = "development"
	PROD string = "production"
	TEST string = "test"
)

// Env is the environment that Macaron is executing in.
// The MACARON_ENV is read on initialization to set this variable.
var Env = DEV
var Root string

func setENV(e string) {
	if len(e) > 0 {
		Env = e
	}
}

func init() {
	setENV(os.Getenv("MACARON_ENV"))
	path, err := filepath.Abs(os.Args[0])
	if err != nil {
		panic(err)
	}
	Root = filepath.Dir(path)
}
<commit_msg>Fix router bug<commit_after>// Copyright 2014 Unknown
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package macaron is a high productive and modular design web framework in Go.
package macaron

import (
	"log"
	"net/http"
	"os"
	"path/filepath"
	"reflect"

	"github.com/Unknwon/macaron/inject"
)

func Version() string {
	return "0.1.1.0721"
}

// Handler can be any callable function.
// Macaron attempts to inject services into the handler's argument list,
// and panics if an argument could not be fullfilled via dependency injection.
type Handler interface{}

// validateHandler makes sure a handler is a callable function,
// and panics if it is not.
func validateHandler(handler Handler) {
	if reflect.TypeOf(handler).Kind() != reflect.Func {
		panic("mocaron handler must be a callable function")
	}
}

// Macaron represents the top level web application.
// inject.Injector methods can be invoked to map services on a global level.
type Macaron struct {
	inject.Injector
	handlers []Handler
	action   Handler
	*Router
	logger *log.Logger
}

// New creates a bare bones Macaron instance.
// Use this method if you want to have full control over the middleware that is used.
func New() *Macaron {
	m := &Macaron{
		Injector: inject.New(),
		action:   func() {},
		Router:   NewRouter(),
		logger:   log.New(os.Stdout, "[Macaron] ", 0),
	}
	m.Router.m = m
	m.Map(m.logger)
	m.Map(defaultReturnHandler())
	m.notFound = func(resp http.ResponseWriter, req *http.Request) {
		c := m.createContext(resp, req)
		c.handlers = append(c.handlers, http.NotFound)
		c.run()
	}
	return m
}

// Classic creates a classic Macaron with some basic default middleware:
// mocaron.Logger, mocaron.Recovery and mocaron.Static.
func Classic() *Macaron {
	m := New()
	m.Use(Logger())
	m.Use(Recovery())
	m.Use(Static("public"))
	return m
}

// Handlers sets the entire middleware stack with the given Handlers.
// This will clear any current middleware handlers,
// and panics if any of the handlers is not a callable function
func (m *Macaron) Handlers(handlers ...Handler) {
	m.handlers = make([]Handler, 0)
	for _, handler := range handlers {
		m.Use(handler)
	}
}

// Action sets the handler that will be called after all the middleware has been invoked.
// This is set to macaron.Router in a macaron.Classic().
func (m *Macaron) Action(handler Handler) {
	validateHandler(handler)
	m.action = handler
}

// Use adds a middleware Handler to the stack,
// and panics if the handler is not a callable func.
// Middleware Handlers are invoked in the order that they are added.
func (m *Macaron) Use(handler Handler) {
	validateHandler(handler)
	m.handlers = append(m.handlers, handler)
}

func (m *Macaron) createContext(rw http.ResponseWriter, req *http.Request) *Context {
	c := &Context{
		Injector: inject.New(),
		handlers: m.handlers,
		action:   m.action,
		index:    0,
		Router:   m.Router,
		Req:      req,
		Resp:     NewResponseWriter(rw),
		Data:     make(map[string]interface{}),
	}
	c.SetParent(m)
	c.Map(c)
	c.MapTo(c.Resp, (*http.ResponseWriter)(nil))
	c.Map(req)
	return c
}

// ServeHTTP is the HTTP Entry point for a Macaron instance.
// Useful if you want to control your own HTTP server.
// Be aware that none of middleware will run without registering any router.
func (m *Macaron) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	m.Router.ServeHTTP(resp, req)
}

// getDefaultListenAddr returns default server listen address of Macaron.
func getDefaultListenAddr() string {
	port := os.Getenv("PORT")
	if port == "" {
		port = "4000"
	}
	host := os.Getenv("HOST")
	return host + ":" + port
}

// Run the http server. Listening on os.GetEnv("PORT") or 4000 by default.
func (m *Macaron) Run() {
	addr := getDefaultListenAddr()

	logger := m.Injector.GetVal(reflect.TypeOf(m.logger)).Interface().(*log.Logger)
	logger.Printf("listening on %s (%s)\n", addr, Env)
	logger.Fatalln(http.ListenAndServe(addr, m))
}

// \_   _____/ _______  __
//  |    __)_ /    \  \/ /
//  |        \   |  \   /
// /_______  /___|  /\_/
//         \/     \/

const (
	DEV  string = "development"
	PROD string = "production"
	TEST string = "test"
)

// Env is the environment that Macaron is executing in.
// The MACARON_ENV is read on initialization to set this variable.
var Env = DEV
var Root string

func setENV(e string) {
	if len(e) > 0 {
		Env = e
	}
}

func init() {
	setENV(os.Getenv("MACARON_ENV"))
	path, err := filepath.Abs(os.Args[0])
	if err != nil {
		panic(err)
	}
	Root = filepath.Dir(path)
}
<|endoftext|>


<commit_before>/* bogomilter is a milter service for postfix */
package main

import (
	"bytes"
	"fmt"
	"sync"
	"time"
)

/* Mailbox object to keep outgoing rate */
type Mailbox struct {
	Name    string
	Blocked bool
	SentLog []time.Time
}

/* Size returns approximate size of memory consumed by Mailbox object */
func (m Mailbox) Size() uint64 {
	return uint64(len(m.Name) + len(m.SentLog)*24 + 1)
}

/* MarshalJSON implements the json.Marshaller interface */
func (m Mailbox) MarshalJSON() ([]byte, error) {
	buffer := new(bytes.Buffer)
	fmt.Fprintf(buffer,
		`{"name":"%s","blocked":"%v","count":%d}`,
		m.Name,
		m.Blocked,
		len(m.SentLog),
	)
	return buffer.Bytes(), nil
}

/* MailboxMemoryCache  */
type MailboxMemoryCache struct {
	Data  map[string]Mailbox
	Mutex sync.Mutex
}

/* IsBlocked returns true if mailbox is blocked from sending emails */
func (m *MailboxMemoryCache) IsBlocked(name string, RateLimit int, Duration time.Duration) bool {
	// acquire mutex lock
	m.Mutex.Lock()
	defer m.Mutex.Unlock()
	// get mailbox object
	mailbox, ok := m.Data[name]
	if !ok {
		// mailbox object does not exist, create a new one
		mailbox = Mailbox{
			Name:    name,
			Blocked: false,
		}
	}
	// check if mailbox is already blocked
	if mailbox.Blocked {
		return true
	}
	// clean up sent log
	threshold := time.Now().Add(-Duration)
	var SentLog []time.Time
	for _, ts := range mailbox.SentLog {
		if threshold.Before(ts) {
			SentLog = append(SentLog, ts)
		}
	}
	// add current timestamp and replace SentLog
	SentLog = append(SentLog, time.Now())
	mailbox.SentLog = SentLog
	// check if RateLimit is exceeded
	if len(mailbox.SentLog) >= RateLimit {
		mailbox.Blocked = true
		// TODO: sent notification to administrator
		// run in a goroutine to avoid excessive lock
	}
	m.Data[name] = mailbox
	return mailbox.Blocked
}

/* CleanUp unused memory */
func (m *MailboxMemoryCache) CleanUp(Duration time.Duration) {
	// acquire lock
	m.Mutex.Lock()
	defer m.Mutex.Unlock()
	// walk all mailboxes
	threshold := time.Now().Add(-Duration)
	for name, mailbox := range m.Data {
		// blocked mailboxes should stay blocked
		if mailbox.Blocked {
			continue
		}
		// clean up expired records
		var SentLog []time.Time
		for _, ts := range mailbox.SentLog {
			if threshold.Before(ts) {
				SentLog = append(SentLog, ts)
			}
		}
		// remove cache for expired records
		if len(SentLog) == 0 {
			delete(m.Data, name)
			continue
		}
		// set new log
		mailbox.SentLog = SentLog
		m.Data[name] = mailbox
	}
}

func (m *MailboxMemoryCache) Size() uint64 {
	var size uint64
	for _, mailbox := range m.Data {
		size += mailbox.Size()
	}
	return size
}

/* MarshalJSON implements json.Marshaler interface */
func (m *MailboxMemoryCache) MarshalJSON() ([]byte, error) {
	buffer := new(bytes.Buffer)
	// write opening bracket
	if _, err := buffer.WriteString(fmt.Sprintf(`{"cache":%d,"mailboxes":[`, m.Size())); err != nil {
		return nil, err
	}
	// walk mailboxes
	first := true
	for _, mailbox := range m.Data {
		if b, err := mailbox.MarshalJSON(); err != nil {
			return nil, err
		} else {
			// do not write delimiter for first item in the list
			if first {
				first = false
			} else {
				// write delimiter
				if _, err := buffer.WriteRune(','); err != nil {
					return nil, err
				}
			}
			// write data
			if _, err := buffer.Write(b); err != nil {
				return nil, err
			}
		}
	}
	// write closing bracket
	if _, err := buffer.WriteString("]}"); err != nil {
		return nil, err
	}
	return buffer.Bytes(), nil
}

func NewMailboxMemoryCache() *MailboxMemoryCache {
	return &MailboxMemoryCache{Data: make(map[string]Mailbox)}
}
<commit_msg>render proper boolean value in json output<commit_after>/* bogomilter is a milter service for postfix */
package main

import (
	"bytes"
	"fmt"
	"sync"
	"time"
)

/* Mailbox object to keep outgoing rate */
type Mailbox struct {
	Name    string
	Blocked bool
	SentLog []time.Time
}

/* Size returns approximate size of memory consumed by Mailbox object */
func (m Mailbox) Size() uint64 {
	return uint64(len(m.Name) + len(m.SentLog)*24 + 1)
}

/* MarshalJSON implements the json.Marshaller interface */
func (m Mailbox) MarshalJSON() ([]byte, error) {
	buffer := new(bytes.Buffer)
	fmt.Fprintf(buffer,
		`{"name":"%s","blocked":%v,"count":%d}`,
		m.Name,
		m.Blocked,
		len(m.SentLog),
	)
	return buffer.Bytes(), nil
}

/* MailboxMemoryCache  */
type MailboxMemoryCache struct {
	Data  map[string]Mailbox
	Mutex sync.Mutex
}

/* IsBlocked returns true if mailbox is blocked from sending emails */
func (m *MailboxMemoryCache) IsBlocked(name string, RateLimit int, Duration time.Duration) bool {
	// acquire mutex lock
	m.Mutex.Lock()
	defer m.Mutex.Unlock()
	// get mailbox object
	mailbox, ok := m.Data[name]
	if !ok {
		// mailbox object does not exist, create a new one
		mailbox = Mailbox{
			Name:    name,
			Blocked: false,
		}
	}
	// check if mailbox is already blocked
	if mailbox.Blocked {
		return true
	}
	// clean up sent log
	threshold := time.Now().Add(-Duration)
	var SentLog []time.Time
	for _, ts := range mailbox.SentLog {
		if threshold.Before(ts) {
			SentLog = append(SentLog, ts)
		}
	}
	// add current timestamp and replace SentLog
	SentLog = append(SentLog, time.Now())
	mailbox.SentLog = SentLog
	// check if RateLimit is exceeded
	if len(mailbox.SentLog) >= RateLimit {
		mailbox.Blocked = true
		// TODO: sent notification to administrator
		// run in a goroutine to avoid excessive lock
	}
	m.Data[name] = mailbox
	return mailbox.Blocked
}

/* CleanUp unused memory */
func (m *MailboxMemoryCache) CleanUp(Duration time.Duration) {
	// acquire lock
	m.Mutex.Lock()
	defer m.Mutex.Unlock()
	// walk all mailboxes
	threshold := time.Now().Add(-Duration)
	for name, mailbox := range m.Data {
		// blocked mailboxes should stay blocked
		if mailbox.Blocked {
			continue
		}
		// clean up expired records
		var SentLog []time.Time
		for _, ts := range mailbox.SentLog {
			if threshold.Before(ts) {
				SentLog = append(SentLog, ts)
			}
		}
		// remove cache for expired records
		if len(SentLog) == 0 {
			delete(m.Data, name)
			continue
		}
		// set new log
		mailbox.SentLog = SentLog
		m.Data[name] = mailbox
	}
}

func (m *MailboxMemoryCache) Size() uint64 {
	var size uint64
	for _, mailbox := range m.Data {
		size += mailbox.Size()
	}
	return size
}

/* MarshalJSON implements json.Marshaler interface */
func (m *MailboxMemoryCache) MarshalJSON() ([]byte, error) {
	buffer := new(bytes.Buffer)
	// write opening bracket
	if _, err := buffer.WriteString(fmt.Sprintf(`{"cache":%d,"mailboxes":[`, m.Size())); err != nil {
		return nil, err
	}
	// walk mailboxes
	first := true
	for _, mailbox := range m.Data {
		if b, err := mailbox.MarshalJSON(); err != nil {
			return nil, err
		} else {
			// do not write delimiter for first item in the list
			if first {
				first = false
			} else {
				// write delimiter
				if _, err := buffer.WriteRune(','); err != nil {
					return nil, err
				}
			}
			// write data
			if _, err := buffer.Write(b); err != nil {
				return nil, err
			}
		}
	}
	// write closing bracket
	if _, err := buffer.WriteString("]}"); err != nil {
		return nil, err
	}
	return buffer.Bytes(), nil
}

func NewMailboxMemoryCache() *MailboxMemoryCache {
	return &MailboxMemoryCache{Data: make(map[string]Mailbox)}
}
<|endoftext|>


<commit_before>package avronats

import (
	"reflect"
	"testing"

	avro "github.com/elodina/go-avro"
	"github.com/golang/protobuf/proto"
	"github.com/nats-io/nats"
	"github.com/nats-io/nats/encoders/protobuf"
)

const (
	schemaRepositoryURL = "http://localhost:8081"
	rawMetricsSchema    = `{"namespace": "metrics","type": "record","name": "Timings","fields": [{"name": "id", "type": "long"},{"name": "timings",  "type": {"type":"array", "items": "long"} }]}`
)

var parsedSchema, _ = avro.ParseSchema(rawMetricsSchema)

type Metric struct {
	ID      int64   `protobuf:"varint,1,opt,name=id" json:"id,omitempty"`
	Timings []int64 `protobuf:"varint,2,rep,name=timings" json:"timings,omitempty"`
}

func (*Metric) Schema() avro.Schema {
	return parsedSchema
}

func TestEncoderDecoder(t *testing.T) {
	encoder := NewAvroEncoder(schemaRepositoryURL)

	record := &Metric{
		Id:      int64(3),
		Timings: []int64{123456, 654321},
	}

	bytes, err := encoder.Encode("test", record)
	assert(t, err, nil)

	decoded := &Metric{}
	err = encoder.Decode("test", bytes, decoded)
	assert(t, err, nil)

	assert(t, decoded.Id, record.Id)
	assert(t, decoded.Timings, []int64{123456, 654321})
}

func TestNatsIntegration(t *testing.T) {
	nc, _ := nats.Connect(nats.DefaultURL)
	encoder := NewAvroEncoder(schemaRepositoryURL)
	nats.RegisterEncoder("avro", encoder)
	c, err := nats.NewEncodedConn(nc, "avro")
	assert(t, err, nil)
	metric := &Metric{
		Id:      int64(3),
		Timings: []int64{123456, 654321},
	}

	done := make(chan struct{})
	_, err = c.Subscribe("foo", func(m *Metric) {
		assert(t, m.Id, metric.Id)
		assert(t, m.Timings, metric.Timings)
		done <- struct{}{}
	})
	assert(t, err, nil)

	err = c.Publish("foo", metric)
	assert(t, err, nil)
	<-done
}

func BenchmarkAvroEncode(b *testing.B) {
	encoder := NewAvroEncoder(schemaRepositoryURL)

	metric := &Metric{
		Id:      int64(3),
		Timings: []int64{123456, 654321},
	}

	for i := 0; i < b.N; i++ {
		encoder.Encode("test", metric)
	}
}

func BenchmarkProtoEncode(b *testing.B) {
	encoder := &protobuf.ProtobufEncoder{}

	metric := &Metric{
		Id:      int64(3),
		Timings: []int64{123456, 654321},
	}

	for i := 0; i < b.N; i++ {
		encoder.Encode("test", metric)
	}
}

func BenchmarkAvroDecode(b *testing.B) {
	encoder := NewAvroEncoder(schemaRepositoryURL)

	metric := &Metric{
		Id:      int64(3),
		Timings: []int64{123456, 654321},
	}

	newMetric := Metric{}
	encoded, _ := encoder.Encode("test", metric)

	for i := 0; i < b.N; i++ {
		encoder.Decode("test", encoded, &newMetric)
	}
}

func BenchmarkProtoDecode(b *testing.B) {
	encoder := &protobuf.ProtobufEncoder{}

	metric := &Metric{
		Id:      int64(3),
		Timings: []int64{123456, 654321},
	}

	newMetric := Metric{}
	encoded, _ := encoder.Encode("test", metric)

	for i := 0; i < b.N; i++ {
		encoder.Decode("test", encoded, &newMetric)
	}
}

func assert(t *testing.T, actual interface{}, expected interface{}) {
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("Expected %v, actual %v", expected, actual)
	}
}

func assertNot(t *testing.T, actual interface{}, expected interface{}) {
	if reflect.DeepEqual(actual, expected) {
		t.Errorf("%v should not be %v", actual, expected)
	}
}

func (m *Metric) Reset()                    { *m = Metric{} }
func (m *Metric) String() string            { return proto.CompactTextString(m) }
func (*Metric) ProtoMessage()               {}
func (*Metric) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }

var fileDescriptor0 = []byte{
	// 87 bytes of a gzipped FileDescriptorProto
	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xc9, 0x4d, 0x2d, 0x29,
	0xca, 0x4c, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x05, 0x53, 0x4a, 0x46, 0x5c, 0x6c,
	0xbe, 0x60, 0x61, 0x21, 0x3e, 0x2e, 0xa6, 0xcc, 0x14, 0x09, 0x46, 0x05, 0x46, 0x0d, 0xe6, 0x20,
	0x20, 0x4b, 0x48, 0x82, 0x8b, 0xbd, 0x24, 0x33, 0x37, 0x33, 0x2f, 0xbd, 0x58, 0x82, 0x49, 0x81,
	0x19, 0x28, 0x08, 0xe3, 0x26, 0xb1, 0x81, 0xb5, 0x1a, 0x03, 0x02, 0x00, 0x00, 0xff, 0xff, 0x28,
	0x05, 0xd1, 0xa7, 0x51, 0x00, 0x00, 0x00,
}
<commit_msg>Fix struct field.<commit_after>package avronats

import (
	"reflect"
	"testing"

	avro "github.com/elodina/go-avro"
	"github.com/golang/protobuf/proto"
	"github.com/nats-io/nats"
	"github.com/nats-io/nats/encoders/protobuf"
)

const (
	schemaRepositoryURL = "http://localhost:8081"
	rawMetricsSchema    = `{"namespace": "metrics","type": "record","name": "Timings","fields": [{"name": "id", "type": "long"},{"name": "timings",  "type": {"type":"array", "items": "long"} }]}`
)

var parsedSchema, _ = avro.ParseSchema(rawMetricsSchema)

type Metric struct {
	Id      int64   `protobuf:"varint,1,opt,name=id" json:"id,omitempty"`
	Timings []int64 `protobuf:"varint,2,rep,name=timings" json:"timings,omitempty"`
}

func (*Metric) Schema() avro.Schema {
	return parsedSchema
}

func TestEncoderDecoder(t *testing.T) {
	encoder := NewAvroEncoder(schemaRepositoryURL)

	record := &Metric{
		Id:      int64(3),
		Timings: []int64{123456, 654321},
	}

	bytes, err := encoder.Encode("test", record)
	assert(t, err, nil)

	decoded := &Metric{}
	err = encoder.Decode("test", bytes, decoded)
	assert(t, err, nil)

	assert(t, decoded.Id, record.Id)
	assert(t, decoded.Timings, []int64{123456, 654321})
}

func TestNatsIntegration(t *testing.T) {
	nc, _ := nats.Connect(nats.DefaultURL)
	encoder := NewAvroEncoder(schemaRepositoryURL)
	nats.RegisterEncoder("avro", encoder)
	c, err := nats.NewEncodedConn(nc, "avro")
	assert(t, err, nil)
	metric := &Metric{
		Id:      int64(3),
		Timings: []int64{123456, 654321},
	}

	done := make(chan struct{})
	_, err = c.Subscribe("foo", func(m *Metric) {
		assert(t, m.Id, metric.Id)
		assert(t, m.Timings, metric.Timings)
		done <- struct{}{}
	})
	assert(t, err, nil)

	err = c.Publish("foo", metric)
	assert(t, err, nil)
	<-done
}

func BenchmarkAvroEncode(b *testing.B) {
	encoder := NewAvroEncoder(schemaRepositoryURL)

	metric := &Metric{
		Id:      int64(3),
		Timings: []int64{123456, 654321},
	}

	for i := 0; i < b.N; i++ {
		encoder.Encode("test", metric)
	}
}

func BenchmarkProtoEncode(b *testing.B) {
	encoder := &protobuf.ProtobufEncoder{}

	metric := &Metric{
		Id:      int64(3),
		Timings: []int64{123456, 654321},
	}

	for i := 0; i < b.N; i++ {
		encoder.Encode("test", metric)
	}
}

func BenchmarkAvroDecode(b *testing.B) {
	encoder := NewAvroEncoder(schemaRepositoryURL)

	metric := &Metric{
		Id:      int64(3),
		Timings: []int64{123456, 654321},
	}

	newMetric := Metric{}
	encoded, _ := encoder.Encode("test", metric)

	for i := 0; i < b.N; i++ {
		encoder.Decode("test", encoded, &newMetric)
	}
}

func BenchmarkProtoDecode(b *testing.B) {
	encoder := &protobuf.ProtobufEncoder{}

	metric := &Metric{
		Id:      int64(3),
		Timings: []int64{123456, 654321},
	}

	newMetric := Metric{}
	encoded, _ := encoder.Encode("test", metric)

	for i := 0; i < b.N; i++ {
		encoder.Decode("test", encoded, &newMetric)
	}
}

func assert(t *testing.T, actual interface{}, expected interface{}) {
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("Expected %v, actual %v", expected, actual)
	}
}

func assertNot(t *testing.T, actual interface{}, expected interface{}) {
	if reflect.DeepEqual(actual, expected) {
		t.Errorf("%v should not be %v", actual, expected)
	}
}

func (m *Metric) Reset()                    { *m = Metric{} }
func (m *Metric) String() string            { return proto.CompactTextString(m) }
func (*Metric) ProtoMessage()               {}
func (*Metric) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }

var fileDescriptor0 = []byte{
	// 87 bytes of a gzipped FileDescriptorProto
	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xe2, 0xe2, 0xc9, 0x4d, 0x2d, 0x29,
	0xca, 0x4c, 0xd6, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0x62, 0x05, 0x53, 0x4a, 0x46, 0x5c, 0x6c,
	0xbe, 0x60, 0x61, 0x21, 0x3e, 0x2e, 0xa6, 0xcc, 0x14, 0x09, 0x46, 0x05, 0x46, 0x0d, 0xe6, 0x20,
	0x20, 0x4b, 0x48, 0x82, 0x8b, 0xbd, 0x24, 0x33, 0x37, 0x33, 0x2f, 0xbd, 0x58, 0x82, 0x49, 0x81,
	0x19, 0x28, 0x08, 0xe3, 0x26, 0xb1, 0x81, 0xb5, 0x1a, 0x03, 0x02, 0x00, 0x00, 0xff, 0xff, 0x28,
	0x05, 0xd1, 0xa7, 0x51, 0x00, 0x00, 0x00,
}
<|endoftext|>


<commit_before>package engine

import (
	"errors"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	log "github.com/sirupsen/logrus"
	"github.com/tylerb/graceful"
	"golang.org/x/net/http2"
)

// List of functions to run at shutdown
var (
	shutdownFunctions [](func())
	mut               sync.Mutex
	completed         bool
)

// AtShutdown adds a function to the list of functions that will be ran at shutdown
func AtShutdown(shutdownFunction func()) {
	mut.Lock()
	defer mut.Unlock()
	shutdownFunctions = append(shutdownFunctions, shutdownFunction)
}

// GenerateShutdownFunction generates a function that will run the postponed
// shutdown functions.  Note that gracefulServer can be nil. It's only used for
// finding out if the server was interrupted (ctrl-c or killed, SIGINT/SIGTERM)
func (ac *Config) GenerateShutdownFunction(gracefulServer *graceful.Server) func() {
	return func() {
		mut.Lock()
		defer mut.Unlock()

		if completed {
			// The shutdown functions have already been called
			return
		}

		if ac.verboseMode {
			log.Info("Initiating shutdown")
		}

		// Call the shutdown functions in chronological order (FIFO)
		for _, shutdownFunction := range shutdownFunctions {
			shutdownFunction()
		}

		completed = true

		if ac.verboseMode {
			log.Info("Shutdown complete")
		}

		// Forced shutdown
		if gracefulServer != nil {
			if gracefulServer.Interrupted {
				//gracefulServer.Stop(forcedShutdownTimeout)
				ac.fatalExit(errors.New("Interrupted"))
			}
		}

		// One final flush
		os.Stdout.Sync()
	}
}

// NewGracefulServer creates a new graceful server configuration
func (ac *Config) NewGracefulServer(mux *http.ServeMux, http2support bool, addr string) *graceful.Server {
	// Server configuration
	s := &http.Server{
		Addr:    addr,
		Handler: mux,

		// The timeout values is also the maximum time it can take
		// for a complete page of Server-Sent Events (SSE).
		ReadTimeout:  10 * time.Second,
		WriteTimeout: 10 * time.Second,

		MaxHeaderBytes: 1 << 20,
	}
	if http2support {
		// Enable HTTP/2 support
		http2.ConfigureServer(s, nil)
	}
	gracefulServer := &graceful.Server{
		Server:  s,
		Timeout: ac.shutdownTimeout,
	}
	// Handle ctrl-c
	gracefulServer.ShutdownInitiated = ac.GenerateShutdownFunction(gracefulServer) // for investigating gracefulServer.Interrupted
	return gracefulServer
}

// Serve HTTP, HTTP/2 and/or HTTPS. Returns an error if unable to serve, or nil when done serving.
func (ac *Config) Serve(mux *http.ServeMux, done, ready chan bool) error {

	// If we are not writing internal logs to a file, reduce the verbosity
	http2.VerboseLogs = (ac.internalLogFilename != os.DevNull)

	// Channel to wait and see if we should just serve regular HTTP instead
	justServeRegularHTTP := make(chan bool)

	// Goroutine that wait for a message to just serve regular HTTP, if needed
	go func() {
		<-justServeRegularHTTP // Wait for a message to just serve regular HTTP
		if strings.HasPrefix(ac.serverAddr, ":") {
			log.Info("Serving HTTP on http://localhost" + ac.serverAddr + "/")
		} else {
			log.Info("Serving HTTP on http://" + ac.serverAddr + "/")
		}
		HTTPserver := ac.NewGracefulServer(mux, false, ac.serverAddr)
		// Start serving. Shut down gracefully at exit.
		if err := HTTPserver.ListenAndServe(); err != nil {
			// If we can't serve regular HTTP on port 80, give up
			ac.fatalExit(err)
		}
	}()

	// Decide which protocol to listen to
	switch {
	case ac.productionMode:
		// Listen for both HTTPS+HTTP/2 and HTTP requests, on different ports
		log.Info("Serving HTTP/2 on https://" + ac.serverHost + "/")
		go func() {
			// Start serving. Shut down gracefully at exit.
			// Listen for HTTPS + HTTP/2 requests
			HTTPS2server := ac.NewGracefulServer(mux, true, ac.serverHost+":443")
			// Start serving. Shut down gracefully at exit.
			if err := HTTPS2server.ListenAndServeTLS(ac.serverCert, ac.serverKey); err != nil {
				log.Error(err)
			}
		}()
		log.Info("Serving HTTP on http://" + ac.serverHost + "/")
		go func() {
			HTTPserver := ac.NewGracefulServer(mux, false, ac.serverHost+":80")
			if err := HTTPserver.ListenAndServe(); err != nil {
				// If we can't serve regular HTTP on port 80, give up
				ac.fatalExit(err)
			}
		}()
	case ac.serveJustHTTP2: // It's unusual to serve HTTP/2 without HTTPS
		if strings.HasPrefix(ac.serverAddr, ":") {
			log.Warn("Serving HTTP/2 without HTTPS (not recommended!) on http://localhost" + ac.serverAddr + "/")
		} else {
			log.Warn("Serving HTTP/2 without HTTPS (not recommended!) on http://" + ac.serverAddr + "/")
		}
		go func() {
			// Listen for HTTP/2 requests
			HTTP2server := ac.NewGracefulServer(mux, true, ac.serverAddr)
			// Start serving. Shut down gracefully at exit.
			if err := HTTP2server.ListenAndServe(); err != nil {
				justServeRegularHTTP <- true
			}
		}()
	case !(ac.serveJustHTTP2 || ac.serveJustHTTP):
		if strings.HasPrefix(ac.serverAddr, ":") {
			log.Info("Serving HTTP/2 on https://localhost" + ac.serverAddr + "/")
		} else {
			log.Info("Serving HTTP/2 on https://" + ac.serverAddr + "/")
		}
		// Listen for HTTPS + HTTP/2 requests
		HTTPS2server := ac.NewGracefulServer(mux, true, ac.serverAddr)
		// Start serving. Shut down gracefully at exit.
		go func() {
			if err := HTTPS2server.ListenAndServeTLS(ac.serverCert, ac.serverKey); err != nil {
				log.Error("Not serving HTTPS: ", err)
				log.Info("Use the -t flag for serving regular HTTP")
				// If HTTPS failed (perhaps the key + cert are missing),
				// serve plain HTTP instead
				justServeRegularHTTP <- true
			}
		}()
	default:
		justServeRegularHTTP <- true
	}

	// Wait just a tiny bit
	time.Sleep(20 * time.Millisecond)

	ready <- true // Send a "ready" message to the REPL

	// Open the URL, if specified
	if ac.openURLAfterServing {
		// TODO: Better check for HTTP vs HTTPS when selecting the URL to open
		//       when both are being served.
		ac.OpenURL(ac.serverHost, ac.serverAddr, !ac.serveJustHTTP2)
	}

	<-done // Wait for a "done" message from the REPL (or just keep waiting)

	return nil // Done serving
}
<commit_msg>Minor improvement to console output<commit_after>package engine

import (
	"errors"
	"net/http"
	"os"
	"strings"
	"sync"
	"time"

	log "github.com/sirupsen/logrus"
	"github.com/tylerb/graceful"
	"golang.org/x/net/http2"
)

// List of functions to run at shutdown
var (
	shutdownFunctions [](func())
	mut               sync.Mutex
	completed         bool
)

// AtShutdown adds a function to the list of functions that will be ran at shutdown
func AtShutdown(shutdownFunction func()) {
	mut.Lock()
	defer mut.Unlock()
	shutdownFunctions = append(shutdownFunctions, shutdownFunction)
}

// GenerateShutdownFunction generates a function that will run the postponed
// shutdown functions.  Note that gracefulServer can be nil. It's only used for
// finding out if the server was interrupted (ctrl-c or killed, SIGINT/SIGTERM)
func (ac *Config) GenerateShutdownFunction(gracefulServer *graceful.Server) func() {
	return func() {
		mut.Lock()
		defer mut.Unlock()

		if completed {
			// The shutdown functions have already been called
			return
		}

		if ac.verboseMode {
			log.Info("Initiating shutdown")
		}

		// Call the shutdown functions in chronological order (FIFO)
		for _, shutdownFunction := range shutdownFunctions {
			shutdownFunction()
		}

		completed = true

		if ac.verboseMode {
			log.Info("Shutdown complete")
		}

		// Forced shutdown
		if gracefulServer != nil {
			if gracefulServer.Interrupted {
				//gracefulServer.Stop(forcedShutdownTimeout)
				ac.fatalExit(errors.New("Interrupted"))
			}
		}

		// One final flush
		os.Stdout.Sync()
	}
}

// NewGracefulServer creates a new graceful server configuration
func (ac *Config) NewGracefulServer(mux *http.ServeMux, http2support bool, addr string) *graceful.Server {
	// Server configuration
	s := &http.Server{
		Addr:    addr,
		Handler: mux,

		// The timeout values is also the maximum time it can take
		// for a complete page of Server-Sent Events (SSE).
		ReadTimeout:  10 * time.Second,
		WriteTimeout: 10 * time.Second,

		MaxHeaderBytes: 1 << 20,
	}
	if http2support {
		// Enable HTTP/2 support
		http2.ConfigureServer(s, nil)
	}
	gracefulServer := &graceful.Server{
		Server:  s,
		Timeout: ac.shutdownTimeout,
	}
	// Handle ctrl-c
	gracefulServer.ShutdownInitiated = ac.GenerateShutdownFunction(gracefulServer) // for investigating gracefulServer.Interrupted
	return gracefulServer
}

// Serve HTTP, HTTP/2 and/or HTTPS. Returns an error if unable to serve, or nil when done serving.
func (ac *Config) Serve(mux *http.ServeMux, done, ready chan bool) error {

	// If we are not writing internal logs to a file, reduce the verbosity
	http2.VerboseLogs = (ac.internalLogFilename != os.DevNull)

	// Channel to wait and see if we should just serve regular HTTP instead
	justServeRegularHTTP := make(chan bool)

	// Goroutine that wait for a message to just serve regular HTTP, if needed
	go func() {
		<-justServeRegularHTTP // Wait for a message to just serve regular HTTP
		if strings.HasPrefix(ac.serverAddr, ":") {
			log.Info("Serving HTTP on http://localhost" + ac.serverAddr + "/")
		} else {
			log.Info("Serving HTTP on http://" + ac.serverAddr + "/")
		}
		HTTPserver := ac.NewGracefulServer(mux, false, ac.serverAddr)
		// Start serving. Shut down gracefully at exit.
		if err := HTTPserver.ListenAndServe(); err != nil {
			// If we can't serve regular HTTP on port 80, give up
			ac.fatalExit(err)
		}
	}()

	// Decide which protocol to listen to
	switch {
	case ac.productionMode:
		// Listen for both HTTPS+HTTP/2 and HTTP requests, on different ports
		log.Info("Serving HTTP/2 on https://" + ac.serverHost + "/")
		go func() {
			// Start serving. Shut down gracefully at exit.
			// Listen for HTTPS + HTTP/2 requests
			HTTPS2server := ac.NewGracefulServer(mux, true, ac.serverHost+":443")
			// Start serving. Shut down gracefully at exit.
			if err := HTTPS2server.ListenAndServeTLS(ac.serverCert, ac.serverKey); err != nil {
				log.Error(err)
			}
		}()
		log.Info("Serving HTTP on http://" + ac.serverHost + "/")
		go func() {
			HTTPserver := ac.NewGracefulServer(mux, false, ac.serverHost+":80")
			if err := HTTPserver.ListenAndServe(); err != nil {
				// If we can't serve regular HTTP on port 80, give up
				ac.fatalExit(err)
			}
		}()
	case ac.serveJustHTTP2: // It's unusual to serve HTTP/2 without HTTPS
		if strings.HasPrefix(ac.serverAddr, ":") {
			log.Warn("Serving HTTP/2 without HTTPS (not recommended!) on http://localhost" + ac.serverAddr + "/")
		} else {
			log.Warn("Serving HTTP/2 without HTTPS (not recommended!) on http://" + ac.serverAddr + "/")
		}
		go func() {
			// Listen for HTTP/2 requests
			HTTP2server := ac.NewGracefulServer(mux, true, ac.serverAddr)
			// Start serving. Shut down gracefully at exit.
			if err := HTTP2server.ListenAndServe(); err != nil {
				justServeRegularHTTP <- true
			}
		}()
	case !(ac.serveJustHTTP2 || ac.serveJustHTTP):
		if strings.HasPrefix(ac.serverAddr, ":") {
			log.Info("Serving HTTP/2 on https://localhost" + ac.serverAddr + "/")
		} else {
			log.Info("Serving HTTP/2 on https://" + ac.serverAddr + "/")
		}
		// Listen for HTTPS + HTTP/2 requests
		HTTPS2server := ac.NewGracefulServer(mux, true, ac.serverAddr)
		// Start serving. Shut down gracefully at exit.
		go func() {
			if err := HTTPS2server.ListenAndServeTLS(ac.serverCert, ac.serverKey); err != nil {
				log.Error("Not serving HTTP/2 after all. Error: ", err)
				log.Info("Use the -t flag for serving regular HTTP instead")
				// If HTTPS failed (perhaps the key + cert are missing),
				// serve plain HTTP instead
				justServeRegularHTTP <- true
			}
		}()
	default:
		justServeRegularHTTP <- true
	}

	// Wait just a tiny bit
	time.Sleep(20 * time.Millisecond)

	ready <- true // Send a "ready" message to the REPL

	// Open the URL, if specified
	if ac.openURLAfterServing {
		// TODO: Better check for HTTP vs HTTPS when selecting the URL to open
		//       when both are being served.
		ac.OpenURL(ac.serverHost, ac.serverAddr, !ac.serveJustHTTP2)
	}

	<-done // Wait for a "done" message from the REPL (or just keep waiting)

	return nil // Done serving
}
<|endoftext|>