<commit_before>package drivers

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/lxd/migration"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/units"
	"github.com/lxc/lxd/shared/validate"
)

var btrfsVersion string
var btrfsLoaded bool

type btrfs struct {
	common
}

// load is used to run one-time action per-driver rather than per-pool.
func (d *btrfs) load() error {
	// Register the patches.
	d.patches = map[string]func() error{
		"storage_create_vm":                        nil,
		"storage_zfs_mount":                        nil,
		"storage_create_vm_again":                  nil,
		"storage_zfs_volmode":                      nil,
		"storage_rename_custom_volume_add_project": nil,
		"storage_lvm_skipactivation":               nil,
	}

	// Done if previously loaded.
	if btrfsLoaded {
		return nil
	}

	// Validate the required binaries.
	for _, tool := range []string{"btrfs"} {
		_, err := exec.LookPath(tool)
		if err != nil {
			return fmt.Errorf("Required tool '%s' is missing", tool)
		}
	}

	// Detect and record the version.
	if btrfsVersion == "" {
		out, err := shared.RunCommand("btrfs", "version")
		if err != nil {
			return err
		}

		count, err := fmt.Sscanf(strings.SplitN(out, " ", 2)[1], "v%s\n", &btrfsVersion)
		if err != nil || count != 1 {
			return fmt.Errorf("The 'btrfs' tool isn't working properly")
		}
	}

	btrfsLoaded = true
	return nil
}

// Info returns info about the driver and its environment.
func (d *btrfs) Info() Info {
	return Info{
		Name:                  "btrfs",
		Version:               btrfsVersion,
		OptimizedImages:       true,
		OptimizedBackups:      true,
		OptimizedBackupHeader: true,
		PreservesInodes:       !d.state.OS.RunningInUserNS,
		Remote:                d.isRemote(),
		VolumeTypes:           []VolumeType{VolumeTypeCustom, VolumeTypeImage, VolumeTypeContainer, VolumeTypeVM},
		BlockBacking:          false,
		RunningCopyFreeze:     false,
		DirectIO:              true,
		MountedRoot:           true,
	}
}

// Create is called during pool creation and is effectively using an empty driver struct.
// WARNING: The Create() function cannot rely on any of the struct attributes being set.
func (d *btrfs) Create() error {
	// Store the provided source as we are likely to be mangling it.
	d.config["volatile.initial_source"] = d.config["source"]

	loopPath := loopFilePath(d.name)
	if d.config["source"] == "" || d.config["source"] == loopPath {
		// Create a loop based pool.
		d.config["source"] = loopPath

		// Create the loop file itself.
		size, err := units.ParseByteSizeString(d.config["size"])
		if err != nil {
			return err
		}

		err = ensureSparseFile(d.config["source"], size)
		if err != nil {
			return errors.Wrap(err, "Failed to create the sparse file")
		}

		// Format the file.
		_, err = makeFSType(d.config["source"], "btrfs", &mkfsOptions{Label: d.name})
		if err != nil {
			return errors.Wrap(err, "Failed to format sparse file")
		}
	} else if shared.IsBlockdevPath(d.config["source"]) {
		// Unset size property since it's irrelevant.
		d.config["size"] = ""

		// Format the block device.
		_, err := makeFSType(d.config["source"], "btrfs", &mkfsOptions{Label: d.name})
		if err != nil {
			return errors.Wrap(err, "Failed to format block device")
		}

		// Record the UUID as the source.
		devUUID, err := fsUUID(d.config["source"])
		if err != nil {
			return err
		}

		// Confirm that the symlink is appearing (give it 10s).
		if tryExists(fmt.Sprintf("/dev/disk/by-uuid/%s", devUUID)) {
			// Override the config to use the UUID.
			d.config["source"] = devUUID
		}
	} else if d.config["source"] != "" {
		// Unset size property since it's irrelevant.
		d.config["size"] = ""

		hostPath := shared.HostPath(d.config["source"])
		if d.isSubvolume(hostPath) {
			// Existing btrfs subvolume.
			subvols, err := d.getSubvolumes(hostPath)
			if err != nil {
				return errors.Wrap(err, "Could not determine if existing btrfs subvolume is empty")
			}

			// Check that the provided subvolume is empty.
			if len(subvols) > 0 {
				return fmt.Errorf("Requested btrfs subvolume exists but is not empty")
			}
		} else {
			// New btrfs subvolume on existing btrfs filesystem.
			cleanSource := filepath.Clean(hostPath)
			lxdDir := shared.VarPath()

			if shared.PathExists(hostPath) && !hasFilesystem(hostPath, util.FilesystemSuperMagicBtrfs) {
				return fmt.Errorf("Provided path does not reside on a btrfs filesystem")
			} else if strings.HasPrefix(cleanSource, lxdDir) {
				if cleanSource != GetPoolMountPath(d.name) {
					return fmt.Errorf("Only allowed source path under %q is %q", shared.VarPath(), GetPoolMountPath(d.name))
				} else if !hasFilesystem(shared.VarPath("storage-pools"), util.FilesystemSuperMagicBtrfs) {
					return fmt.Errorf("Provided path does not reside on a btrfs filesystem")
				}

				// Delete the current directory to replace by subvolume.
				err := os.Remove(cleanSource)
				if err != nil && !os.IsNotExist(err) {
					return errors.Wrapf(err, "Failed to remove %q", cleanSource)
				}
			}

			// Create the subvolume.
			_, err := shared.RunCommand("btrfs", "subvolume", "create", hostPath)
			if err != nil {
				return err
			}
		}
	} else {
		return fmt.Errorf(`Invalid "source" property`)
	}

	return nil
}

// Delete removes the storage pool from the storage device.
func (d *btrfs) Delete(op *operations.Operation) error {
	// If the user completely destroyed it, call it done.
	if !shared.PathExists(GetPoolMountPath(d.name)) {
		return nil
	}

	// Delete potential intermediate btrfs subvolumes.
	for _, volType := range d.Info().VolumeTypes {
		for _, dir := range BaseDirectories[volType] {
			path := filepath.Join(GetPoolMountPath(d.name), dir)
			if !shared.PathExists(path) {
				continue
			}

			if !d.isSubvolume(path) {
				continue
			}

			err := d.deleteSubvolume(path, true)
			if err != nil {
				return fmt.Errorf("Failed deleting btrfs subvolume %q", path)
			}
		}
	}

	// On delete, wipe everything in the directory.
	mountPath := GetPoolMountPath(d.name)
	err := wipeDirectory(mountPath)
	if err != nil {
		return errors.Wrapf(err, "Failed removing mount path %q", mountPath)
	}

	// Unmount the path.
	_, err = d.Unmount()
	if err != nil {
		return err
	}

	// If the pool path is a subvolume itself, delete it.
	if d.isSubvolume(mountPath) {
		err := d.deleteSubvolume(mountPath, false)
		if err != nil {
			return err
		}

		// And re-create as an empty directory to make the backend happy.
		err = os.Mkdir(mountPath, 0700)
		if err != nil {
			return errors.Wrapf(err, "Failed creating directory %q", mountPath)
		}
	}

	// Delete any loop file we may have used.
	loopPath := loopFilePath(d.name)
	err = os.Remove(loopPath)
	if err != nil && !os.IsNotExist(err) {
		return errors.Wrapf(err, "Failed removing loop file %q", loopPath)
	}

	return nil
}

// Validate checks that all provide keys are supported and that no conflicting or missing configuration is present.
func (d *btrfs) Validate(config map[string]string) error {
	rules := map[string]func(value string) error{
		"btrfs.mount_options": validate.IsAny,
	}

	return d.validatePool(config, rules)
}

// Update applies any driver changes required from a configuration change.
func (d *btrfs) Update(changedConfig map[string]string) error {
	// We only care about btrfs.mount_options.
	val, ok := changedConfig["btrfs.mount_options"]
	if !ok {
		return nil
	}

	// Custom mount options don't work inside containers
	if d.state.OS.RunningInUserNS {
		return nil
	}

	// Trigger a re-mount.
	d.config["btrfs.mount_options"] = val
	mntFlags, mntOptions := resolveMountOptions(d.getMountOptions())
	mntFlags |= unix.MS_REMOUNT

	err := TryMount("", GetPoolMountPath(d.name), "none", mntFlags, mntOptions)
	if err != nil {
		return err
	}

	return nil
}

// Mount mounts the storage pool.
func (d *btrfs) Mount() (bool, error) {
	// Check if already mounted.
	if shared.IsMountPoint(GetPoolMountPath(d.name)) {
		return false, nil
	}

	// Setup mount options.
	loopPath := loopFilePath(d.name)
	mntSrc := ""
	mntDst := GetPoolMountPath(d.name)
	mntFilesystem := "btrfs"
	if d.config["source"] == loopPath {
		// Bring up the loop device.
		loopF, err := PrepareLoopDev(d.config["source"], LoFlagsAutoclear)
		if err != nil {
			return false, err
		}
		defer loopF.Close()

		mntSrc = loopF.Name()
	} else if filepath.IsAbs(d.config["source"]) {
		// Bring up an existing device or path.
		mntSrc = shared.HostPath(d.config["source"])

		if !shared.IsBlockdevPath(mntSrc) {
			mntFilesystem = "none"

			if !hasFilesystem(mntSrc, util.FilesystemSuperMagicBtrfs) {
				return false, fmt.Errorf("Source path '%s' isn't btrfs", mntSrc)
			}
		}
	} else {
		// Mount using UUID.
		mntSrc = fmt.Sprintf("/dev/disk/by-uuid/%s", d.config["source"])
	}

	// Get the custom mount flags/options.
	mntFlags, mntOptions := resolveMountOptions(d.getMountOptions())

	// Handle bind-mounts first.
	if mntFilesystem == "none" {
		// Setup the bind-mount itself.
		err := TryMount(mntSrc, mntDst, mntFilesystem, unix.MS_BIND, "")
		if err != nil {
			return false, err
		}

		// Custom mount options don't work inside containers
		if d.state.OS.RunningInUserNS {
			return true, nil
		}

		// Now apply the custom options.
		mntFlags |= unix.MS_REMOUNT
		err = TryMount("", mntDst, mntFilesystem, mntFlags, mntOptions)
		if err != nil {
			return false, err
		}

		return true, nil
	}

	// Handle traditional mounts.
	err := TryMount(mntSrc, mntDst, mntFilesystem, mntFlags, mntOptions)
	if err != nil {
		return false, err
	}

	return true, nil
}

// Unmount unmounts the storage pool.
func (d *btrfs) Unmount() (bool, error) {
	// Unmount the pool.
	ourUnmount, err := forceUnmount(GetPoolMountPath(d.name))
	if err != nil {
		return false, err
	}

	// If loop backed, force release the loop device.
	loopPath := loopFilePath(d.name)
	if d.config["source"] == loopPath {
		releaseLoopDev(loopPath)
	}

	return ourUnmount, nil
}

// GetResources returns the pool resource usage information.
func (d *btrfs) GetResources() (*api.ResourcesStoragePool, error) {
	return genericVFSGetResources(d)
}

// MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order.
func (d *btrfs) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
	var rsyncFeatures []string
	btrfsFeatures := []string{migration.BTRFSFeatureMigrationHeader, migration.BTRFSFeatureSubvolumes}

	// Do not pass compression argument to rsync if the associated
	// config key, that is rsync.compression, is set to false.
	if d.Config()["rsync.compression"] != "" && !shared.IsTrue(d.Config()["rsync.compression"]) {
		rsyncFeatures = []string{"xattrs", "delete", "bidirectional"}
	} else {
		rsyncFeatures = []string{"xattrs", "delete", "compress", "bidirectional"}
	}

	// Only offer rsync for refreshes or if running in an unprivileged container.
	if refresh || d.state.OS.RunningInUserNS {
		var transportType migration.MigrationFSType

		if contentType == ContentTypeBlock {
			transportType = migration.MigrationFSType_BLOCK_AND_RSYNC
		} else {
			transportType = migration.MigrationFSType_RSYNC
		}

		return []migration.Type{
			{
				FSType:   transportType,
				Features: rsyncFeatures,
			},
		}
	}

	if contentType == ContentTypeBlock {
		return []migration.Type{
			{
				FSType:   migration.MigrationFSType_BTRFS,
				Features: btrfsFeatures,
			},
			{
				FSType:   migration.MigrationFSType_BLOCK_AND_RSYNC,
				Features: rsyncFeatures,
			},
		}
	}

	return []migration.Type{
		{
			FSType:   migration.MigrationFSType_BTRFS,
			Features: btrfsFeatures,
		},
		{
			FSType:   migration.MigrationFSType_RSYNC,
			Features: rsyncFeatures,
		},
	}
}
<commit_msg>lxd/storage/drivers/driver/btrfs: Replace hasFilesystem usage with filesystem.Detect<commit_after>package drivers

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/pkg/errors"
	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/lxd/migration"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/storage/filesystem"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/units"
	"github.com/lxc/lxd/shared/validate"
)

var btrfsVersion string
var btrfsLoaded bool

type btrfs struct {
	common
}

// load is used to run one-time action per-driver rather than per-pool.
func (d *btrfs) load() error {
	// Register the patches.
	d.patches = map[string]func() error{
		"storage_create_vm":                        nil,
		"storage_zfs_mount":                        nil,
		"storage_create_vm_again":                  nil,
		"storage_zfs_volmode":                      nil,
		"storage_rename_custom_volume_add_project": nil,
		"storage_lvm_skipactivation":               nil,
	}

	// Done if previously loaded.
	if btrfsLoaded {
		return nil
	}

	// Validate the required binaries.
	for _, tool := range []string{"btrfs"} {
		_, err := exec.LookPath(tool)
		if err != nil {
			return fmt.Errorf("Required tool '%s' is missing", tool)
		}
	}

	// Detect and record the version.
	if btrfsVersion == "" {
		out, err := shared.RunCommand("btrfs", "version")
		if err != nil {
			return err
		}

		count, err := fmt.Sscanf(strings.SplitN(out, " ", 2)[1], "v%s\n", &btrfsVersion)
		if err != nil || count != 1 {
			return fmt.Errorf("The 'btrfs' tool isn't working properly")
		}
	}

	btrfsLoaded = true
	return nil
}

// Info returns info about the driver and its environment.
func (d *btrfs) Info() Info {
	return Info{
		Name:                  "btrfs",
		Version:               btrfsVersion,
		OptimizedImages:       true,
		OptimizedBackups:      true,
		OptimizedBackupHeader: true,
		PreservesInodes:       !d.state.OS.RunningInUserNS,
		Remote:                d.isRemote(),
		VolumeTypes:           []VolumeType{VolumeTypeCustom, VolumeTypeImage, VolumeTypeContainer, VolumeTypeVM},
		BlockBacking:          false,
		RunningCopyFreeze:     false,
		DirectIO:              true,
		MountedRoot:           true,
	}
}

// Create is called during pool creation and is effectively using an empty driver struct.
// WARNING: The Create() function cannot rely on any of the struct attributes being set.
func (d *btrfs) Create() error {
	// Store the provided source as we are likely to be mangling it.
	d.config["volatile.initial_source"] = d.config["source"]

	loopPath := loopFilePath(d.name)
	if d.config["source"] == "" || d.config["source"] == loopPath {
		// Create a loop based pool.
		d.config["source"] = loopPath

		// Create the loop file itself.
		size, err := units.ParseByteSizeString(d.config["size"])
		if err != nil {
			return err
		}

		err = ensureSparseFile(d.config["source"], size)
		if err != nil {
			return errors.Wrap(err, "Failed to create the sparse file")
		}

		// Format the file.
		_, err = makeFSType(d.config["source"], "btrfs", &mkfsOptions{Label: d.name})
		if err != nil {
			return errors.Wrap(err, "Failed to format sparse file")
		}
	} else if shared.IsBlockdevPath(d.config["source"]) {
		// Unset size property since it's irrelevant.
		d.config["size"] = ""

		// Format the block device.
		_, err := makeFSType(d.config["source"], "btrfs", &mkfsOptions{Label: d.name})
		if err != nil {
			return errors.Wrap(err, "Failed to format block device")
		}

		// Record the UUID as the source.
		devUUID, err := fsUUID(d.config["source"])
		if err != nil {
			return err
		}

		// Confirm that the symlink is appearing (give it 10s).
		if tryExists(fmt.Sprintf("/dev/disk/by-uuid/%s", devUUID)) {
			// Override the config to use the UUID.
			d.config["source"] = devUUID
		}
	} else if d.config["source"] != "" {
		// Unset size property since it's irrelevant.
		d.config["size"] = ""

		hostPath := shared.HostPath(d.config["source"])
		if d.isSubvolume(hostPath) {
			// Existing btrfs subvolume.
			subvols, err := d.getSubvolumes(hostPath)
			if err != nil {
				return errors.Wrap(err, "Could not determine if existing btrfs subvolume is empty")
			}

			// Check that the provided subvolume is empty.
			if len(subvols) > 0 {
				return fmt.Errorf("Requested btrfs subvolume exists but is not empty")
			}
		} else {
			// New btrfs subvolume on existing btrfs filesystem.
			cleanSource := filepath.Clean(hostPath)
			lxdDir := shared.VarPath()

			if shared.PathExists(hostPath) {
				hostPathFS, _ := filesystem.Detect(hostPath)
				if hostPathFS != "btrfs" {
					return fmt.Errorf("Provided path does not reside on a btrfs filesystem")
				}
			}

			if strings.HasPrefix(cleanSource, lxdDir) {
				if cleanSource != GetPoolMountPath(d.name) {
					return fmt.Errorf("Only allowed source path under %q is %q", shared.VarPath(), GetPoolMountPath(d.name))
				}

				storagePoolDirFS, _ := filesystem.Detect(shared.VarPath("storage-pools"))
				if storagePoolDirFS != "btrfs" {
					return fmt.Errorf("Provided path does not reside on a btrfs filesystem")
				}

				// Delete the current directory to replace by subvolume.
				err := os.Remove(cleanSource)
				if err != nil && !os.IsNotExist(err) {
					return errors.Wrapf(err, "Failed to remove %q", cleanSource)
				}
			}

			// Create the subvolume.
			_, err := shared.RunCommand("btrfs", "subvolume", "create", hostPath)
			if err != nil {
				return err
			}
		}
	} else {
		return fmt.Errorf(`Invalid "source" property`)
	}

	return nil
}

// Delete removes the storage pool from the storage device.
func (d *btrfs) Delete(op *operations.Operation) error {
	// If the user completely destroyed it, call it done.
	if !shared.PathExists(GetPoolMountPath(d.name)) {
		return nil
	}

	// Delete potential intermediate btrfs subvolumes.
	for _, volType := range d.Info().VolumeTypes {
		for _, dir := range BaseDirectories[volType] {
			path := filepath.Join(GetPoolMountPath(d.name), dir)
			if !shared.PathExists(path) {
				continue
			}

			if !d.isSubvolume(path) {
				continue
			}

			err := d.deleteSubvolume(path, true)
			if err != nil {
				return fmt.Errorf("Failed deleting btrfs subvolume %q", path)
			}
		}
	}

	// On delete, wipe everything in the directory.
	mountPath := GetPoolMountPath(d.name)
	err := wipeDirectory(mountPath)
	if err != nil {
		return errors.Wrapf(err, "Failed removing mount path %q", mountPath)
	}

	// Unmount the path.
	_, err = d.Unmount()
	if err != nil {
		return err
	}

	// If the pool path is a subvolume itself, delete it.
	if d.isSubvolume(mountPath) {
		err := d.deleteSubvolume(mountPath, false)
		if err != nil {
			return err
		}

		// And re-create as an empty directory to make the backend happy.
		err = os.Mkdir(mountPath, 0700)
		if err != nil {
			return errors.Wrapf(err, "Failed creating directory %q", mountPath)
		}
	}

	// Delete any loop file we may have used.
	loopPath := loopFilePath(d.name)
	err = os.Remove(loopPath)
	if err != nil && !os.IsNotExist(err) {
		return errors.Wrapf(err, "Failed removing loop file %q", loopPath)
	}

	return nil
}

// Validate checks that all provide keys are supported and that no conflicting or missing configuration is present.
func (d *btrfs) Validate(config map[string]string) error {
	rules := map[string]func(value string) error{
		"btrfs.mount_options": validate.IsAny,
	}

	return d.validatePool(config, rules)
}

// Update applies any driver changes required from a configuration change.
func (d *btrfs) Update(changedConfig map[string]string) error {
	// We only care about btrfs.mount_options.
	val, ok := changedConfig["btrfs.mount_options"]
	if !ok {
		return nil
	}

	// Custom mount options don't work inside containers
	if d.state.OS.RunningInUserNS {
		return nil
	}

	// Trigger a re-mount.
	d.config["btrfs.mount_options"] = val
	mntFlags, mntOptions := resolveMountOptions(d.getMountOptions())
	mntFlags |= unix.MS_REMOUNT

	err := TryMount("", GetPoolMountPath(d.name), "none", mntFlags, mntOptions)
	if err != nil {
		return err
	}

	return nil
}

// Mount mounts the storage pool.
func (d *btrfs) Mount() (bool, error) {
	// Check if already mounted.
	if shared.IsMountPoint(GetPoolMountPath(d.name)) {
		return false, nil
	}

	// Setup mount options.
	loopPath := loopFilePath(d.name)
	mntSrc := ""
	mntDst := GetPoolMountPath(d.name)
	mntFilesystem := "btrfs"
	if d.config["source"] == loopPath {
		// Bring up the loop device.
		loopF, err := PrepareLoopDev(d.config["source"], LoFlagsAutoclear)
		if err != nil {
			return false, err
		}
		defer loopF.Close()

		mntSrc = loopF.Name()
	} else if filepath.IsAbs(d.config["source"]) {
		// Bring up an existing device or path.
		mntSrc = shared.HostPath(d.config["source"])

		if !shared.IsBlockdevPath(mntSrc) {
			mntFilesystem = "none"

			mntSrcFS, _ := filesystem.Detect(mntSrc)
			if mntSrcFS != "btrfs" {
				return false, fmt.Errorf("Source path %q isn't btrfs", mntSrc)
			}
		}
	} else {
		// Mount using UUID.
		mntSrc = fmt.Sprintf("/dev/disk/by-uuid/%s", d.config["source"])
	}

	// Get the custom mount flags/options.
	mntFlags, mntOptions := resolveMountOptions(d.getMountOptions())

	// Handle bind-mounts first.
	if mntFilesystem == "none" {
		// Setup the bind-mount itself.
		err := TryMount(mntSrc, mntDst, mntFilesystem, unix.MS_BIND, "")
		if err != nil {
			return false, err
		}

		// Custom mount options don't work inside containers
		if d.state.OS.RunningInUserNS {
			return true, nil
		}

		// Now apply the custom options.
		mntFlags |= unix.MS_REMOUNT
		err = TryMount("", mntDst, mntFilesystem, mntFlags, mntOptions)
		if err != nil {
			return false, err
		}

		return true, nil
	}

	// Handle traditional mounts.
	err := TryMount(mntSrc, mntDst, mntFilesystem, mntFlags, mntOptions)
	if err != nil {
		return false, err
	}

	return true, nil
}

// Unmount unmounts the storage pool.
func (d *btrfs) Unmount() (bool, error) {
	// Unmount the pool.
	ourUnmount, err := forceUnmount(GetPoolMountPath(d.name))
	if err != nil {
		return false, err
	}

	// If loop backed, force release the loop device.
	loopPath := loopFilePath(d.name)
	if d.config["source"] == loopPath {
		releaseLoopDev(loopPath)
	}

	return ourUnmount, nil
}

// GetResources returns the pool resource usage information.
func (d *btrfs) GetResources() (*api.ResourcesStoragePool, error) {
	return genericVFSGetResources(d)
}

// MigrationType returns the type of transfer methods to be used when doing migrations between pools in preference order.
func (d *btrfs) MigrationTypes(contentType ContentType, refresh bool) []migration.Type {
	var rsyncFeatures []string
	btrfsFeatures := []string{migration.BTRFSFeatureMigrationHeader, migration.BTRFSFeatureSubvolumes}

	// Do not pass compression argument to rsync if the associated
	// config key, that is rsync.compression, is set to false.
	if d.Config()["rsync.compression"] != "" && !shared.IsTrue(d.Config()["rsync.compression"]) {
		rsyncFeatures = []string{"xattrs", "delete", "bidirectional"}
	} else {
		rsyncFeatures = []string{"xattrs", "delete", "compress", "bidirectional"}
	}

	// Only offer rsync for refreshes or if running in an unprivileged container.
	if refresh || d.state.OS.RunningInUserNS {
		var transportType migration.MigrationFSType

		if contentType == ContentTypeBlock {
			transportType = migration.MigrationFSType_BLOCK_AND_RSYNC
		} else {
			transportType = migration.MigrationFSType_RSYNC
		}

		return []migration.Type{
			{
				FSType:   transportType,
				Features: rsyncFeatures,
			},
		}
	}

	if contentType == ContentTypeBlock {
		return []migration.Type{
			{
				FSType:   migration.MigrationFSType_BTRFS,
				Features: btrfsFeatures,
			},
			{
				FSType:   migration.MigrationFSType_BLOCK_AND_RSYNC,
				Features: rsyncFeatures,
			},
		}
	}

	return []migration.Type{
		{
			FSType:   migration.MigrationFSType_BTRFS,
			Features: btrfsFeatures,
		},
		{
			FSType:   migration.MigrationFSType_RSYNC,
			Features: rsyncFeatures,
		},
	}
}
<|endoftext|>


<commit_before>// +build !android

package gozmo

import (
	"github.com/go-gl/glfw/v3.1/glfw"
	"log"
	"runtime"
)

type Window struct {
	width        int32
	height       int32
	title        string
	glfwWindow   *glfw.Window
	scenes       map[string]*Scene
	currentScene *Scene
}

func OpenWindowVersion(width int32, height int32, title string, major int, minor int) *Window {
	runtime.LockOSThread()
	window := Window{width: width, height: height, title: title}

	window.scenes = make(map[string]*Scene)

	if err := glfw.Init(); err != nil {
		log.Fatalln("failed to initialize glfw:", err)
	}

	glfw.WindowHint(glfw.Resizable, glfw.False)
	glfw.WindowHint(glfw.ContextVersionMajor, major)
	glfw.WindowHint(glfw.ContextVersionMinor, minor)
	if major >= 4 || (major >= 3 && minor >= 2) {
		glfw.WindowHint(glfw.OpenGLProfile, glfw.OpenGLCoreProfile)
	}
	glfw.WindowHint(glfw.OpenGLForwardCompatible, glfw.True)
	glfw.WindowHint(glfw.DoubleBuffer, 1)

	//glfw.WindowHint(glfw.Samples, 2)

	// TODO support full screen
	//glfwin, err := glfw.CreateWindow(window.width, window.height, window.title, glfw.GetPrimaryMonitor(), nil)
	glfwin, err := glfw.CreateWindow(int(window.width), int(window.height), window.title, nil, nil)
	if err != nil {
		panic(err)
	}

	glfwin.MakeContextCurrent()

	window.glfwWindow = glfwin

	glfw.SwapInterval(1)
	//glfw.SwapInterval(0)

	fbWidth, fbHeight := glfwin.GetFramebufferSize()
	GLInit(int32(fbWidth), int32(fbHeight))

	Engine.Window = &window

	return &window
}

func OpenWindow(width int32, height int32, title string) *Window {
	return OpenWindowVersion(width, height, title, 3, 3)
}

func (window *Window) Run() {
	win := window.glfwWindow

	glfw.SetTime(0.0)

	for !win.ShouldClose() {

		GLClear()

		scene := window.currentScene
		if scene != nil {
			scene.Update(glfw.GetTime())
		}

		win.SwapBuffers()
		glfw.PollEvents()
	}
	glfw.Terminate()
}

func (window *Window) getKey(kc Key) bool {
	return window.glfwWindow.GetKey(glfw.Key(kc)) == glfw.Press
}
<commit_msg>no need to make a reference to window in scene<commit_after>// +build !android

package gozmo

import (
	"github.com/go-gl/glfw/v3.1/glfw"
	"log"
	"runtime"
)

type Window struct {
	width        int32
	height       int32
	title        string
	glfwWindow   *glfw.Window
	scenes       map[string]*Scene
	currentScene *Scene
}

func OpenWindowVersion(width int32, height int32, title string, major int, minor int) *Window {
	if Engine.Window != nil {
		panic("a window is already active")
	}
	runtime.LockOSThread()
	window := Window{width: width, height: height, title: title}

	window.scenes = make(map[string]*Scene)

	if err := glfw.Init(); err != nil {
		log.Fatalln("failed to initialize glfw:", err)
	}

	glfw.WindowHint(glfw.Resizable, glfw.False)
	glfw.WindowHint(glfw.ContextVersionMajor, major)
	glfw.WindowHint(glfw.ContextVersionMinor, minor)
	if major >= 4 || (major >= 3 && minor >= 2) {
		glfw.WindowHint(glfw.OpenGLProfile, glfw.OpenGLCoreProfile)
	}
	glfw.WindowHint(glfw.OpenGLForwardCompatible, glfw.True)
	glfw.WindowHint(glfw.DoubleBuffer, 1)

	//glfw.WindowHint(glfw.Samples, 2)

	// TODO support full screen
	//glfwin, err := glfw.CreateWindow(window.width, window.height, window.title, glfw.GetPrimaryMonitor(), nil)
	glfwin, err := glfw.CreateWindow(int(window.width), int(window.height), window.title, nil, nil)
	if err != nil {
		panic(err)
	}

	glfwin.MakeContextCurrent()

	window.glfwWindow = glfwin

	glfw.SwapInterval(1)
	//glfw.SwapInterval(0)

	fbWidth, fbHeight := glfwin.GetFramebufferSize()
	GLInit(int32(fbWidth), int32(fbHeight))

	Engine.Window = &window

	return &window
}

func OpenWindow(width int32, height int32, title string) *Window {
	return OpenWindowVersion(width, height, title, 3, 3)
}

func (window *Window) Run() {
	win := window.glfwWindow

	glfw.SetTime(0.0)

	for !win.ShouldClose() {

		GLClear()

		scene := window.currentScene
		if scene != nil {
			scene.Update(glfw.GetTime())
		}

		win.SwapBuffers()
		glfw.PollEvents()
	}
	glfw.Terminate()
}

func (window *Window) getKey(kc Key) bool {
	return window.glfwWindow.GetKey(glfw.Key(kc)) == glfw.Press
}
<|endoftext|>


<commit_before><commit_msg>worker<commit_after>package main
<|endoftext|>


<commit_before>// delayd2 is an available setTimeout() service for scheduling message sends
package delayd2

import (
	"log"
	"time"

	"github.com/nabeken/aws-go-sqs/queue"
)

// QueueMessage is a message queued in the database.
type QueueMessage struct {
	QueueID   string
	WorkerID  string
	ReleaseAt time.Time
	RelayTo   string
	Payload   string
}

// Worker is the delayd2 worker.
type Worker struct {
	id       string
	driver   Driver
	consumer *Consumer
	relay    *Relay

	// signaled from external
	shutdownCh chan struct{}

	// notified when handleRelease() finishes its task
	stoppedCh chan struct{}
}

// NewWorker creates a new worker.
func NewWorker(workerID string, driver Driver, consumer *Consumer, relay *Relay) *Worker {
	return &Worker{
		id:       workerID,
		driver:   driver,
		consumer: consumer,
		relay:    relay,

		shutdownCh: make(chan struct{}),
		stoppedCh:  make(chan struct{}),
	}
}

// Run starts the worker process.
func (w *Worker) Run() error {
	log.Print("worker: starting delayd2 worker process")

	log.Print("worker: resetting aborted messages remaining in active queue")
	n, err := w.driver.ResetActive()
	if err != nil {
		return err
	}
	log.Printf("worker: %d aborted messages in active queue resetted", n)

	log.Print("worker: starting delayd2 process")
	go w.handleConsume()
	go w.handleMarkActive()
	go w.handleRelease()

	select {
	case <-w.shutdownCh:
		log.Print("worker: receiving shutdown signal. waiting for the process finished.")
	}

	select {
	case <-w.stoppedCh:
		log.Print("worker: the process finished.")
	}
	return nil
}

func (w *Worker) Stop() {
	close(w.shutdownCh)
}

func (w *Worker) consume() (int, error) {
	return w.consumer.ConsumeMessages()
}

// handleConsume consumes messages in SQS.
func (w *Worker) handleConsume() {
	for {
		begin := time.Now()
		n, err := w.consume()
		end := time.Now()

		if err != nil {
			log.Printf("worker: unable to consume messages: %s", err)
			continue
		}

		if n > 0 {
			log.Printf("worker: %d messages consumed in %s", n, end.Sub(begin))
		}
	}
}

func (w *Worker) markActive(begin time.Time) (int64, error) {
	return w.driver.MarkActive(begin)
}

// handleMarkActive marks coming messages in queue.
func (w *Worker) handleMarkActive() {
	for range time.Tick(1 * time.Second) {
		begin := time.Now()

		n, err := w.markActive(begin)

		end := time.Now()

		if err != nil {
			log.Printf("worker: unable to mark messages active: %s", err)
			continue
		}

		if n > 0 {
			log.Printf("worker: %d messages marked as active in %s", n, end.Sub(begin))
		}
	}
}

// handleRelease releases messages that is ready to fire.
func (w *Worker) handleRelease() {
	for range time.Tick(1 * time.Second) {
		select {
		case <-w.shutdownCh:
			close(w.stoppedCh)
			return
		default:
		}

		begin := time.Now()

		n, err := w.release()
		if err != nil {
			log.Printf("worker: unable to release messages: %s", err)
			continue
		}

		end := time.Now()

		if n > 0 {
			log.Printf("worker: %d messages removed from queue in %s", n, end.Sub(begin))
		}
	}
}

// release releases active messages. It returns the number of released message n.
func (w *Worker) release() (int64, error) {
	messages, err := w.driver.GetActiveMessages()

	if err != nil {
		return 0, err
	}

	var n int64
	messagesMap := make(map[string][]*QueueMessage)
	for _, m := range messages {
		messagesMap[m.RelayTo] = append(messagesMap[m.RelayTo], m)
	}

	for r, ms := range messagesMap {
		// FIXME: should be in parallel
		payloads := make([]string, 0, len(ms))
		for _, m := range ms {
			payloads = append(payloads, m.Payload)
		}

		failedIndex := make(map[int]struct{})
		err := w.relay.Relay(payloads, r)
		if err != nil {
			// only print log here in batch operation
			// You should setup a dead letter queue to detect unrecoverable errors.
			berrs, batchOk := queue.IsBatchError(err)
			if !batchOk {
				log.Printf("worker: unable to send message due to non batch error. skipping: %s", err)
				continue
			}

			for _, berr := range berrs {
				if berr.SenderFault {
					log.Printf("worker: unable to send message due to sender's fault: skipping: %s", berr.Message)
				}
				failedIndex[berr.Index] = struct{}{}
			}
		}

		for i, m := range ms {
			if _, failed := failedIndex[i]; failed {
				continue
			}

			if err := w.driver.RemoveMessage(m.QueueID); err != nil {
				log.Printf("worker: unable to remove messages from queue: %s", err)
				continue
			}
			n++
		}
	}
	return n, nil
}
<commit_msg>worker: Add TODO<commit_after>// delayd2 is an available setTimeout() service for scheduling message sends
package delayd2

import (
	"log"
	"time"

	"github.com/nabeken/aws-go-sqs/queue"
)

// QueueMessage is a message queued in the database.
type QueueMessage struct {
	QueueID   string
	WorkerID  string
	ReleaseAt time.Time
	RelayTo   string
	Payload   string
}

// Worker is the delayd2 worker.
type Worker struct {
	id       string
	driver   Driver
	consumer *Consumer
	relay    *Relay

	// signaled from external
	shutdownCh chan struct{}

	// notified when handleRelease() finishes its task
	stoppedCh chan struct{}
}

// NewWorker creates a new worker.
func NewWorker(workerID string, driver Driver, consumer *Consumer, relay *Relay) *Worker {
	return &Worker{
		id:       workerID,
		driver:   driver,
		consumer: consumer,
		relay:    relay,

		shutdownCh: make(chan struct{}),
		stoppedCh:  make(chan struct{}),
	}
}

// Run starts the worker process.
func (w *Worker) Run() error {
	log.Print("worker: starting delayd2 worker process")

	log.Print("worker: resetting aborted messages remaining in active queue")
	n, err := w.driver.ResetActive()
	if err != nil {
		return err
	}
	log.Printf("worker: %d aborted messages in active queue resetted", n)

	log.Print("worker: starting delayd2 process")
	go w.handleConsume()
	go w.handleMarkActive()
	go w.handleRelease()

	select {
	case <-w.shutdownCh:
		log.Print("worker: receiving shutdown signal. waiting for the process finished.")
	}

	select {
	case <-w.stoppedCh:
		log.Print("worker: the process finished.")
	}
	return nil
}

func (w *Worker) Stop() {
	close(w.shutdownCh)
}

func (w *Worker) consume() (int, error) {
	return w.consumer.ConsumeMessages()
}

// handleConsume consumes messages in SQS.
func (w *Worker) handleConsume() {
	for {
		begin := time.Now()
		n, err := w.consume()
		end := time.Now()

		if err != nil {
			log.Printf("worker: unable to consume messages: %s", err)
			continue
		}

		if n > 0 {
			log.Printf("worker: %d messages consumed in %s", n, end.Sub(begin))
		}
	}
}

func (w *Worker) markActive(begin time.Time) (int64, error) {
	return w.driver.MarkActive(begin)
}

// handleMarkActive marks coming messages in queue.
func (w *Worker) handleMarkActive() {
	for range time.Tick(1 * time.Second) {
		begin := time.Now()

		n, err := w.markActive(begin)

		end := time.Now()

		if err != nil {
			log.Printf("worker: unable to mark messages active: %s", err)
			continue
		}

		if n > 0 {
			log.Printf("worker: %d messages marked as active in %s", n, end.Sub(begin))
		}
	}
}

// handleRelease releases messages that is ready to fire.
func (w *Worker) handleRelease() {
	for range time.Tick(1 * time.Second) {
		select {
		case <-w.shutdownCh:
			close(w.stoppedCh)
			return
		default:
		}

		begin := time.Now()

		n, err := w.release()
		if err != nil {
			log.Printf("worker: unable to release messages: %s", err)
			continue
		}

		end := time.Now()

		if n > 0 {
			log.Printf("worker: %d messages removed from queue in %s", n, end.Sub(begin))
		}
	}
}

// release releases active messages. It returns the number of released message n.
func (w *Worker) release() (int64, error) {
	messages, err := w.driver.GetActiveMessages()

	if err != nil {
		return 0, err
	}

	var n int64
	messagesMap := make(map[string][]*QueueMessage)
	for _, m := range messages {
		messagesMap[m.RelayTo] = append(messagesMap[m.RelayTo], m)
	}

	for r, ms := range messagesMap {
		// FIXME: should be in parallel
		payloads := make([]string, 0, len(ms))
		for _, m := range ms {
			payloads = append(payloads, m.Payload)
		}

		failedIndex := make(map[int]struct{})
		err := w.relay.Relay(payloads, r)
		if err != nil {
			// only print log here in batch operation
			// TODO: a dead letter queue support
			berrs, batchOk := queue.IsBatchError(err)
			if !batchOk {
				log.Printf("worker: unable to send message due to non batch error. skipping: %s", err)
				continue
			}

			for _, berr := range berrs {
				if berr.SenderFault {
					log.Printf("worker: unable to send message due to sender's fault: skipping: %s", berr.Message)
				}
				failedIndex[berr.Index] = struct{}{}
			}
		}

		for i, m := range ms {
			if _, failed := failedIndex[i]; failed {
				continue
			}

			if err := w.driver.RemoveMessage(m.QueueID); err != nil {
				log.Printf("worker: unable to remove messages from queue: %s", err)
				continue
			}
			n++
		}
	}
	return n, nil
}
<|endoftext|>


<commit_before>// delayd2 is an available setTimeout() service for scheduling message sends
package delayd2

import (
	"log"
	"time"

	"github.com/nabeken/aws-go-sqs/queue"
)

// QueueMessage is a message queued in the database.
type QueueMessage struct {
	QueueID   string
	WorkerID  string
	ReleaseAt time.Time
	RelayTo   string
	Payload   string
}

// Worker is the delayd2 worker.
type Worker struct {
	id       string
	driver   Driver
	consumer *Consumer
	relay    *Relay

	// signaled from external
	shutdownCh chan struct{}

	// notified when handleRelease() finishes its task
	stoppedCh chan struct{}
}

// NewWorker creates a new worker.
func NewWorker(workerID string, driver Driver, consumer *Consumer, relay *Relay) *Worker {
	return &Worker{
		id:       workerID,
		driver:   driver,
		consumer: consumer,
		relay:    relay,

		shutdownCh: make(chan struct{}),
		stoppedCh:  make(chan struct{}),
	}
}

// Run starts the worker process.
func (w *Worker) Run() error {
	log.Print("worker: starting delayd2 worker process")

	log.Print("worker: resetting aborted messages remaining in active queue")
	n, err := w.driver.ResetActive()
	if err != nil {
		return err
	}
	log.Printf("worker: %d aborted messages in active queue resetted", n)

	log.Print("worker: starting delayd2 process")
	go w.handleIncoming()
	go w.handleMarkActive()
	go w.handleRelease()

	select {
	case <-w.shutdownCh:
		log.Print("worker: receiving shutdown signal. waiting for the process finished.")
	}

	select {
	case <-w.stoppedCh:
		log.Print("worker: the process finished.")
	}
	return nil
}

func (w *Worker) Stop() {
	close(w.shutdownCh)
}

// handleIncoming consumes messages in SQS.
func (w *Worker) handleIncoming() {
	for {
		begin := time.Now()
		n, err := w.consumer.ConsumeMessages()
		end := time.Now()

		if err != nil {
			log.Printf("worker: unable to consume messages: %s", err)
			continue
		}

		if n > 0 {
			log.Printf("worker: %d messages consumed in %s", n, end.Sub(begin))
		}
	}
}

// handleMarkActive marks coming messages in queue.
func (w *Worker) handleMarkActive() {
	for range time.Tick(1 * time.Second) {
		begin := time.Now()

		n, err := w.driver.MarkActive(begin)

		end := time.Now()

		if err != nil {
			log.Printf("worker: unable to mark messages active: %s", err)
			continue
		}

		if n > 0 {
			log.Printf("worker: %d messages marked as active in %s", n, end.Sub(begin))
		}
	}
}

// handleRelease releases messages that is ready to fire.
func (w *Worker) handleRelease() {
	for range time.Tick(1 * time.Second) {
		select {
		case <-w.shutdownCh:
			close(w.stoppedCh)
			return
		default:
		}

		begin := time.Now()
		messages, err := w.driver.GetActiveMessages()
		end := time.Now()

		if err != nil {
			log.Printf("worker: unable to retrieve messages to be released: %s", err)
			continue
		}

		var n int
		messagesMap := make(map[string][]*QueueMessage)
		for _, m := range messages {
			messagesMap[m.RelayTo] = append(messagesMap[m.RelayTo], m)
		}

		for r, ms := range messagesMap {
			// FIXME: should be in parallel
			payloads := make([]string, 0, len(ms))
			for _, m := range ms {
				payloads = append(payloads, m.Payload)
			}

			failedIndex := make(map[int]struct{})
			err := w.relay.Relay(payloads, r)
			if err != nil {
				if berrs, ok := queue.IsBatchError(err); ok {
					for _, berr := range berrs {
						if berr.SenderFault {
							// TODO: remove? log?
							log.Printf("worker: unable to send message batch due to sender's fault: %s", berr.Message)
						}
						failedIndex[berr.Index] = struct{}{}
					}
				}
			}

			for i, m := range ms {
				_, failed := failedIndex[i]
				if failed {
					continue
				}

				if err := w.driver.RemoveMessage(m.QueueID); err != nil {
					log.Printf("worker: unable to remove messages from queue: %s", err)
					continue
				}
				n++
			}
		}

		if n > 0 {
			log.Printf("worker: %d messages removed from queue in %s", n, end.Sub(begin))
		}
	}
}
<commit_msg>worker: cleanup<commit_after>// delayd2 is an available setTimeout() service for scheduling message sends
package delayd2

import (
	"log"
	"time"

	"github.com/nabeken/aws-go-sqs/queue"
)

// QueueMessage is a message queued in the database.
type QueueMessage struct {
	QueueID   string
	WorkerID  string
	ReleaseAt time.Time
	RelayTo   string
	Payload   string
}

// Worker is the delayd2 worker.
type Worker struct {
	id       string
	driver   Driver
	consumer *Consumer
	relay    *Relay

	// signaled from external
	shutdownCh chan struct{}

	// notified when handleRelease() finishes its task
	stoppedCh chan struct{}
}

// NewWorker creates a new worker.
func NewWorker(workerID string, driver Driver, consumer *Consumer, relay *Relay) *Worker {
	return &Worker{
		id:       workerID,
		driver:   driver,
		consumer: consumer,
		relay:    relay,

		shutdownCh: make(chan struct{}),
		stoppedCh:  make(chan struct{}),
	}
}

// Run starts the worker process.
func (w *Worker) Run() error {
	log.Print("worker: starting delayd2 worker process")

	log.Print("worker: resetting aborted messages remaining in active queue")
	n, err := w.driver.ResetActive()
	if err != nil {
		return err
	}
	log.Printf("worker: %d aborted messages in active queue resetted", n)

	log.Print("worker: starting delayd2 process")
	go w.handleIncoming()
	go w.handleMarkActive()
	go w.handleRelease()

	select {
	case <-w.shutdownCh:
		log.Print("worker: receiving shutdown signal. waiting for the process finished.")
	}

	select {
	case <-w.stoppedCh:
		log.Print("worker: the process finished.")
	}
	return nil
}

func (w *Worker) Stop() {
	close(w.shutdownCh)
}

// handleIncoming consumes messages in SQS.
func (w *Worker) handleIncoming() {
	for {
		begin := time.Now()
		n, err := w.consumer.ConsumeMessages()
		end := time.Now()

		if err != nil {
			log.Printf("worker: unable to consume messages: %s", err)
			continue
		}

		if n > 0 {
			log.Printf("worker: %d messages consumed in %s", n, end.Sub(begin))
		}
	}
}

// handleMarkActive marks coming messages in queue.
func (w *Worker) handleMarkActive() {
	for range time.Tick(1 * time.Second) {
		begin := time.Now()

		n, err := w.driver.MarkActive(begin)

		end := time.Now()

		if err != nil {
			log.Printf("worker: unable to mark messages active: %s", err)
			continue
		}

		if n > 0 {
			log.Printf("worker: %d messages marked as active in %s", n, end.Sub(begin))
		}
	}
}

// handleRelease releases messages that is ready to fire.
func (w *Worker) handleRelease() {
	for range time.Tick(1 * time.Second) {
		select {
		case <-w.shutdownCh:
			close(w.stoppedCh)
			return
		default:
		}

		begin := time.Now()

		n, err := w.release()
		if err != nil {
			log.Printf("worker: unable to release messages: %s", err)
			continue
		}

		end := time.Now()

		if n > 0 {
			log.Printf("worker: %d messages removed from queue in %s", n, end.Sub(begin))
		}
	}
}

// release releases active messages. It returns the number of released message n.
func (w *Worker) release() (int64, error) {
	messages, err := w.driver.GetActiveMessages()

	if err != nil {
		return 0, err
	}

	var n int64
	messagesMap := make(map[string][]*QueueMessage)
	for _, m := range messages {
		messagesMap[m.RelayTo] = append(messagesMap[m.RelayTo], m)
	}

	for r, ms := range messagesMap {
		// FIXME: should be in parallel
		payloads := make([]string, 0, len(ms))
		for _, m := range ms {
			payloads = append(payloads, m.Payload)
		}

		failedIndex := make(map[int]struct{})
		err := w.relay.Relay(payloads, r)
		if err != nil {
			berrs, batchOk := queue.IsBatchError(err)
			if !batchOk {
				return 0, err
			}

			for _, berr := range berrs {
				if berr.SenderFault {
					// only print log here
					// You should setup a dead queue to detect unrecoverable errors.
					log.Printf("worker: unable to send message batch due to sender's fault: %s", berr.Message)
				}
				failedIndex[berr.Index] = struct{}{}
			}
		}

		for i, m := range ms {
			if _, failed := failedIndex[i]; failed {
				continue
			}

			if err := w.driver.RemoveMessage(m.QueueID); err != nil {
				log.Printf("worker: unable to remove messages from queue: %s", err)
				continue
			}
			n++
		}
	}
	return n, nil
}
<|endoftext|>


<commit_before>/*
 * Copyright 2014 Matthew Endsley
 * All rights reserved
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted providing that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

package queuedwriter

import (
	"bytes"
	"io"
	"sync"
)

type W struct {
	W       io.Writer
	out     *bytes.Buffer
	wg      sync.WaitGroup
	lock    sync.Mutex
	err     error
	running bool
}

// Create a new queued writer
func New(w io.Writer) *W {
	qw := &W{
		W:   w,
		out: new(bytes.Buffer),
	}
	return qw
}

// Wait for background process to exit
func (w *W) Wait() {
	w.wg.Wait()
}

// Write data to the queue
func (w *W) Write(p []byte) (int, error) {
	w.lock.Lock()
	defer w.lock.Unlock()

	if w.err != nil {
		return 0, w.err
	}

	if !w.running {
		w.wg.Add(1)
		w.running = true
		go w.proc()
	}

	return w.out.Write(p)
}

// Write a string to the queue
func (w *W) WriteString(s string) (int, error) {
	w.lock.Lock()
	defer w.lock.Unlock()

	if w.err != nil {
		return 0, w.err
	}

	if !w.running {
		w.wg.Add(1)
		w.running = true
		go w.proc()
	}

	return w.out.WriteString(s)
}

// Copy an io.Reader to the queue
func (w *W) ReadFrom(r io.Reader) (int64, error) {
	w.lock.Lock()
	defer w.lock.Unlock()

	var total int64

	// copy in blocks, unlocking the mutex
	// to allow the background proc to flush data out
	for {
		if w.err != nil {
			return total, w.err
		}

		if !w.running {
			w.wg.Add(1)
			w.running = true
			go w.proc()
		}

		n, err := io.CopyN(w.out, r, 4096)
		total += n
		if err != nil {
			if err == io.EOF {
				err = nil
			}
			return total, err
		}

		// pulse the lock to allow the background proc
		// a chance to flush data
		w.lock.Unlock()
		w.lock.Lock()
	}
}

// background proc to write data to the underlying io.Writer
func (w *W) proc() {
	back := new(bytes.Buffer)

	w.lock.Lock()
	defer func() {
		w.running = false
		w.lock.Unlock()
		w.wg.Done()
	}()

	for {
		if w.out.Len() == 0 {
			return
		}

		// swap buffers; no need to hold lock during I/O
		back.Reset()
		w.out, back = back, w.out

		var err error
		if back.Len() != 0 {
			// BEGIN --- unlocked for I/O ---
			w.lock.Unlock()
			_, err = w.W.Write(back.Bytes())
			w.lock.Lock()
			// END   --- unlocked for I/O ---

			if err != nil {
				w.err = err
				return
			}
		}
	}
}
<commit_msg>Add max buffered size to writer<commit_after>/*
 * Copyright 2014 Matthew Endsley
 * All rights reserved
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted providing that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
 * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

package queuedwriter

import (
	"bytes"
	"errors"
	"io"
	"sync"
)

var ErrTooLarge = errors.New("queuedWriter.W: too large")

type W struct {
	W       io.Writer
	out     *bytes.Buffer
	wg      sync.WaitGroup
	lock    sync.Mutex
	maxSize int64
	err     error
	running bool
}

// Create a new queued writer
func New(w io.Writer) *W {
	return NewSize(w, 0x7fffffffffffffff)
}

// Craete a new queued writer with a max size
func NewSize(w io.Writer, maxSize int64) *W {
	qw := &W{
		W:       w,
		out:     new(bytes.Buffer),
		maxSize: maxSize,
	}
	return qw
}

// Wait for background process to exit
func (w *W) Wait() {
	w.wg.Wait()
}

// Write data to the queue
func (w *W) Write(p []byte) (int, error) {
	w.lock.Lock()
	defer w.lock.Unlock()

	if int64(w.out.Len())+int64(len(p)) > w.maxSize {
		w.err = ErrTooLarge
	}

	if w.err != nil {
		return 0, w.err
	}

	if !w.running {
		w.wg.Add(1)
		w.running = true
		go w.proc()
	}

	return w.out.Write(p)
}

// Write a string to the queue
func (w *W) WriteString(s string) (int, error) {
	w.lock.Lock()
	defer w.lock.Unlock()

	if int64(w.out.Len())+int64(len(s)) > w.maxSize {
		w.err = ErrTooLarge
	}

	if w.err != nil {
		return 0, w.err
	}

	if !w.running {
		w.wg.Add(1)
		w.running = true
		go w.proc()
	}

	return w.out.WriteString(s)
}

// Copy an io.Reader to the queue
func (w *W) ReadFrom(r io.Reader) (int64, error) {
	w.lock.Lock()
	defer w.lock.Unlock()

	var total int64

	// copy in blocks, unlocking the mutex
	// to allow the background proc to flush data out
	for {
		if w.err != nil {
			return total, w.err
		}

		if !w.running {
			w.wg.Add(1)
			w.running = true
			go w.proc()
		}

		n, err := io.CopyN(w.out, r, 4096)
		total += n
		if err != nil {
			if err == io.EOF {
				err = nil
			}
			return total, err
		}

		if int64(w.out.Len()) > w.maxSize {
			w.err = ErrTooLarge
		}

		// pulse the lock to allow the background proc
		// a chance to flush data
		w.lock.Unlock()
		w.lock.Lock()
	}
}

// background proc to write data to the underlying io.Writer
func (w *W) proc() {
	back := new(bytes.Buffer)

	w.lock.Lock()
	defer func() {
		w.running = false
		w.lock.Unlock()
		w.wg.Done()
	}()

	for {
		if w.out.Len() == 0 {
			return
		}

		// swap buffers; no need to hold lock during I/O
		back.Reset()
		w.out, back = back, w.out

		var err error
		if back.Len() != 0 {
			// BEGIN --- unlocked for I/O ---
			w.lock.Unlock()
			_, err = w.W.Write(back.Bytes())
			w.lock.Lock()
			// END   --- unlocked for I/O ---

			if err != nil {
				w.err = err
				return
			}
		}
	}
}
<|endoftext|>


<commit_before>package compactcorpus

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"
)

var (
	errEmptyName = errors.New("name is empty")
)

type Writer struct {
	size   uint64
	idx    *os.File
	dz     *io.PipeWriter
	chRet  chan error
	err    error
	opened bool
}

// Start creating a new corpus.
// The corpus will not be written completely until Close() is called.
func NewCorpus(filename string) (*Writer, error) {
	name := root(filename)

	fp, err := os.Create(name + ".index")
	if err != nil {
		return nil, err
	}

	rdr, wtr := io.Pipe()

	ret := make(chan error, 1)

	go func() {
		err := write(rdr, name+".data.dz", 9)
		ret <- err
	}()

	w := &Writer{
		idx:    fp,
		dz:     wtr,
		chRet:  ret,
		opened: true,
	}
	return w, nil
}

func (w *Writer) Close() error {
	if w.opened {
		w.opened = false
		w.idx.Close()
		w.dz.Close()
		w.err = <-w.chRet
	}
	return w.err
}

func (w *Writer) Write(name string, data []byte) error {
	name = strings.TrimSpace(name)
	if len(name) == 0 {
		return errEmptyName
	}

	w.dz.Write(data)

	ln := uint64(len(data))
	off := encode(w.size)
	size := encode(ln)
	fmt.Fprintf(w.idx, "%s\t%s\t%s\n", name, off, size)

	w.size += ln

	return nil
}

func (w *Writer) WriteString(name string, data string) error {
	return w.Write(name, []byte(data))
}

func (w *Writer) WriteFile(filename string) error {
	data, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}
	return w.Write(filename, data)
}
<commit_msg>argument names<commit_after>package compactcorpus

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"
)

var (
	errEmptyName = errors.New("name is empty")
)

type Writer struct {
	size   uint64
	idx    *os.File
	dz     *io.PipeWriter
	chRet  chan error
	err    error
	opened bool
}

// Start creating a new corpus.
// The corpus will not be written completely until Close() is called.
func NewCorpus(filename string) (*Writer, error) {
	name := root(filename)

	fp, err := os.Create(name + ".index")
	if err != nil {
		return nil, err
	}

	rdr, wtr := io.Pipe()

	ret := make(chan error, 1)

	go func() {
		err := write(rdr, name+".data.dz", 9)
		ret <- err
	}()

	w := &Writer{
		idx:    fp,
		dz:     wtr,
		chRet:  ret,
		opened: true,
	}
	return w, nil
}

func (w *Writer) Close() error {
	if w.opened {
		w.opened = false
		w.idx.Close()
		w.dz.Close()
		w.err = <-w.chRet
	}
	return w.err
}

func (w *Writer) Write(name string, xml []byte) error {
	name = strings.TrimSpace(name)
	if len(name) == 0 {
		return errEmptyName
	}

	w.dz.Write(xml)

	ln := uint64(len(xml))
	off := encode(w.size)
	size := encode(ln)
	fmt.Fprintf(w.idx, "%s\t%s\t%s\n", name, off, size)

	w.size += ln

	return nil
}

func (w *Writer) WriteString(name string, xml string) error {
	return w.Write(name, []byte(xml))
}

func (w *Writer) WriteFile(filename string) error {
	xml, err := ioutil.ReadFile(filename)
	if err != nil {
		return err
	}
	return w.Write(filename, xml)
}
<|endoftext|>


<commit_before>package dedup

import (
	"bytes"
	"crypto/sha1"
	"encoding/binary"
	"errors"
	"io"
	"runtime"
	"sync"
)

type writer struct {
	blks    io.Writer
	idx     io.Writer
	size    int
	index   map[[sha1.Size]byte]int
	input   chan *block
	write   chan *block
	exited  chan struct{}
	cur     []byte
	off     int
	buffers chan *block
	vari64  []byte
	err     error
	mu      sync.Mutex
	nblocks int
}

type block struct {
	data     []byte
	sha1Hash [sha1.Size]byte
	hashDone chan error
	N        int
}

var ErrSizeTooSmall = errors.New("block size too small")

func NewWriter(index io.Writer, blocks io.Writer, size uint) (io.WriteCloser, error) {
	ncpu := runtime.GOMAXPROCS(0)
	// For small block sizes we need to keep a pretty big buffer to keep input fed.
	// Constant below appears to be sweet spot measured with 4K blocks.
	var bufmul = 256 << 10 / int(size)
	if bufmul < 2 {
		bufmul = 2
	}
	r := &writer{
		blks:    blocks,
		idx:     index,
		size:    int(size),
		index:   make(map[[sha1.Size]byte]int),
		input:   make(chan *block, ncpu*bufmul),
		write:   make(chan *block, ncpu*bufmul),
		exited:  make(chan struct{}, 0),
		cur:     make([]byte, size),
		vari64:  make([]byte, binary.MaxVarintLen64),
		buffers: make(chan *block, ncpu*bufmul),
	}

	if r.size <= sha1.Size {
		return nil, ErrSizeTooSmall
	}

	r.putUint64(1) // Format
	r.putUint64(uint64(size))

	// Start one goroutine per core
	for i := 0; i < ncpu; i++ {
		go r.hasher()
	}
	// Insert the buffers we will use
	for i := 0; i < ncpu*bufmul; i++ {
		r.buffers <- &block{data: make([]byte, size), hashDone: make(chan error, 1)}
	}
	go r.writer()
	return r, nil
}

// putUint64 will Write uint64 value to index stream.
func (r *writer) putUint64(v uint64) error {
	n := binary.PutUvarint(r.vari64, v)
	n2, err := r.idx.Write(r.vari64[:n])
	if err != nil {
		return err
	}
	if n2 != n {
		return io.ErrShortWrite
	}
	return nil
}

func (r *writer) Write(b []byte) (n int, err error) {
	written := 0
	for len(b) > 0 {
		n := copy(r.cur[r.off:], b)
		b = b[n:]
		r.off += n
		written += n
		if r.off == r.size {
			b := <-r.buffers
			// Swap block with current
			r.cur, b.data = b.data, r.cur
			b.N = r.nblocks

			r.input <- b
			r.write <- b
			r.nblocks++
			r.off = 0
		}
	}
	return written, nil
}

func (r *writer) setErr(err error) {
	r.mu.Lock()
	r.err = err
	r.mu.Unlock()
}

var emptyHash = [sha1.Size]byte{}

func (r *writer) Close() (err error) {
	select {
	case <-r.exited:
		return nil
	default:
	}
	close(r.input)
	close(r.write)
	<-r.exited

	// Insert empty hash into index to indicate EOF
	buf := bytes.NewBuffer(emptyHash[:])
	n, err := io.Copy(r.idx, buf)
	if err != nil {
		return err
	}
	if n != sha1.Size {
		return errors.New("close short copy")
	}
	// Insert length of remaining data into index
	r.putUint64(uint64(r.off))

	buf = bytes.NewBuffer(r.cur[0:r.off])
	n, err = io.Copy(r.blks, buf)
	if err != nil {
		return err
	}
	if int(n) != r.off {
		return errors.New("r.cur short copy")
	}

	return nil
}

func (r *writer) hasher() {
	hasher := sha1.New()
	for b := range r.input {
		buf := bytes.NewBuffer(b.data)
		hasher.Reset()
		n, err := io.Copy(hasher, buf)
		if err != nil {
			r.setErr(err)
			return
		}
		if int(n) != len(b.data) {
			panic("short write")
		}
		_ = hasher.Sum(b.sha1Hash[:0])
		b.hashDone <- nil
	}
}

func (r *writer) writer() {
	defer close(r.exited)
	for b := range r.write {
		_ = <-b.hashDone
		_, ok := r.index[b.sha1Hash]
		if !ok {
			r.index[b.sha1Hash] = b.N
			buf := bytes.NewBuffer(b.data)
			n, err := io.Copy(r.blks, buf)
			if err != nil {
				r.setErr(err)
				return
			}
			if int(n) != len(b.data) {
				panic("short write")
			}
		}
		n, err := r.idx.Write(b.sha1Hash[:])
		if err != nil {
			r.setErr(err)
			return
		}
		if n != sha1.Size {
			panic("short write")
		}
		// Done, reinsert buffer
		r.buffers <- b
	}
}

/*
ZPAQ: (public domain)

  // Set block and fragment sizes
  // -fragment N     Set average dedupe fragment size = 2^N KiB (default: 6)
  if (fragment<0) fragment=0;

  const unsigned blocksize=(1u<<(20+atoi(method.c_str()+1)))-4096;
  const unsigned MAX_FRAGMENT=fragment>19 || (8128u<<fragment)>blocksize-12
      ? blocksize-12 : 8128u<<fragment;
  const unsigned MIN_FRAGMENT=fragment>25 || (64u<<fragment)>MAX_FRAGMENT
      ? MAX_FRAGMENT : 64u<<fragment;

		unsigned char o1[256]={0}; // order 1 context -> predicted byte
		int c=EOF;  // current byte
		int hits = 0;  // hits - we can use this to determine
        while (true) {
          c=in.get();
          if (c!=EOF) {
            if (c==o1[c1]) h=(h+c+1)*314159265u; hits++
            else h=(h+c+1)*271828182u;
            o1[c1]=c;
            c1=c;
            sha1.put(c);
            fragbuf[sz++]=c;
          	if ( sz>=MAX_FRAGMENT || (fragment<=22 && h<(1u<<(22-fragment)) && sz>=MIN_FRAGMENT))
            	break;
          } else {
          	break;
          }
        }

*/
<commit_msg>Write offsets to index instead of full hashes.<commit_after>package dedup

import (
	"bytes"
	"crypto/sha1"
	"encoding/binary"
	"errors"
	"io"
	"math"
	"runtime"
	"sync"
)

type writer struct {
	blks    io.Writer
	idx     io.Writer
	size    int
	index   map[[sha1.Size]byte]int
	input   chan *block
	write   chan *block
	exited  chan struct{}
	cur     []byte
	off     int
	buffers chan *block
	vari64  []byte
	err     error
	mu      sync.Mutex
	nblocks int
}

type block struct {
	data     []byte
	sha1Hash [sha1.Size]byte
	hashDone chan error
	N        int
}

var ErrSizeTooSmall = errors.New("block size too small")

func NewWriter(index io.Writer, blocks io.Writer, size uint) (io.WriteCloser, error) {
	ncpu := runtime.GOMAXPROCS(0)
	// For small block sizes we need to keep a pretty big buffer to keep input fed.
	// Constant below appears to be sweet spot measured with 4K blocks.
	var bufmul = 256 << 10 / int(size)
	if bufmul < 2 {
		bufmul = 2
	}
	r := &writer{
		blks:    blocks,
		idx:     index,
		size:    int(size),
		index:   make(map[[sha1.Size]byte]int),
		input:   make(chan *block, ncpu*bufmul),
		write:   make(chan *block, ncpu*bufmul),
		exited:  make(chan struct{}, 0),
		cur:     make([]byte, size),
		vari64:  make([]byte, binary.MaxVarintLen64),
		buffers: make(chan *block, ncpu*bufmul),
	}

	if r.size <= sha1.Size {
		return nil, ErrSizeTooSmall
	}

	r.putUint64(1) // Format
	r.putUint64(uint64(size))

	// Start one goroutine per core
	for i := 0; i < ncpu; i++ {
		go r.hasher()
	}
	// Insert the buffers we will use
	for i := 0; i < ncpu*bufmul; i++ {
		r.buffers <- &block{data: make([]byte, size), hashDone: make(chan error, 1)}
	}
	go r.writer()
	return r, nil
}

// putUint64 will Write uint64 value to index stream.
func (r *writer) putUint64(v uint64) error {
	n := binary.PutUvarint(r.vari64, v)
	n2, err := r.idx.Write(r.vari64[:n])
	if err != nil {
		return err
	}
	if n2 != n {
		return io.ErrShortWrite
	}
	return nil
}

func (r *writer) Write(b []byte) (n int, err error) {
	written := 0
	for len(b) > 0 {
		n := copy(r.cur[r.off:], b)
		b = b[n:]
		r.off += n
		written += n
		if r.off == r.size {
			b := <-r.buffers
			// Swap block with current
			r.cur, b.data = b.data, r.cur
			b.N = r.nblocks

			r.input <- b
			r.write <- b
			r.nblocks++
			r.off = 0
		}
	}
	return written, nil
}

func (r *writer) setErr(err error) {
	r.mu.Lock()
	r.err = err
	r.mu.Unlock()
}

var emptyHash = [sha1.Size]byte{}

func (r *writer) Close() (err error) {
	select {
	case <-r.exited:
		return nil
	default:
	}
	close(r.input)
	close(r.write)
	<-r.exited

	// Insert empty hash into index to indicate EOF
	buf := bytes.NewBuffer(emptyHash[:])
	n, err := io.Copy(r.idx, buf)
	if err != nil {
		return err
	}
	if n != sha1.Size {
		return errors.New("close short copy")
	}
	// Insert length of remaining data into index
	r.putUint64(uint64(math.MaxUint64))
	r.putUint64(uint64(r.off))

	buf = bytes.NewBuffer(r.cur[0:r.off])
	n, err = io.Copy(r.blks, buf)
	if err != nil {
		return err
	}
	if int(n) != r.off {
		return errors.New("r.cur short copy")
	}

	return nil
}

func (r *writer) hasher() {
	hasher := sha1.New()
	for b := range r.input {
		buf := bytes.NewBuffer(b.data)
		hasher.Reset()
		n, err := io.Copy(hasher, buf)
		if err != nil {
			r.setErr(err)
			return
		}
		if int(n) != len(b.data) {
			panic("short write")
		}
		_ = hasher.Sum(b.sha1Hash[:0])
		b.hashDone <- nil
	}
}

func (r *writer) writer() {
	defer close(r.exited)
	for b := range r.write {
		_ = <-b.hashDone
		match, ok := r.index[b.sha1Hash]
		if !ok {
			buf := bytes.NewBuffer(b.data)
			n, err := io.Copy(r.blks, buf)
			if err != nil {
				r.setErr(err)
				return
			}
			if int(n) != len(b.data) {
				panic("short write")
			}
			r.putUint64(0)
		} else {
			offset := b.N - match
			if offset <= 0 {
				panic("negative offset, should be impossible")
			}
			r.putUint64(uint64(offset))
		}
		// Update hash to latest match
		r.index[b.sha1Hash] = b.N

		// Done, reinsert buffer
		r.buffers <- b
	}
}

/*
ZPAQ: (public domain)

  // Set block and fragment sizes
  // -fragment N     Set average dedupe fragment size = 2^N KiB (default: 6)
  if (fragment<0) fragment=0;

  const unsigned blocksize=(1u<<(20+atoi(method.c_str()+1)))-4096;
  const unsigned MAX_FRAGMENT=fragment>19 || (8128u<<fragment)>blocksize-12
      ? blocksize-12 : 8128u<<fragment;
  const unsigned MIN_FRAGMENT=fragment>25 || (64u<<fragment)>MAX_FRAGMENT
      ? MAX_FRAGMENT : 64u<<fragment;

		unsigned char o1[256]={0}; // order 1 context -> predicted byte
		int c=EOF;  // current byte
		int hits = 0;  // hits - we can use this to determine
        while (true) {
          c=in.get();
          if (c!=EOF) {
            if (c==o1[c1]) h=(h+c+1)*314159265u; hits++
            else h=(h+c+1)*271828182u;
            o1[c1]=c;
            c1=c;
            sha1.put(c);
            fragbuf[sz++]=c;
          	if ( sz>=MAX_FRAGMENT || (fragment<=22 && h<(1u<<(22-fragment)) && sz>=MIN_FRAGMENT))
            	break;
          } else {
          	break;
          }
        }

*/
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ast

import (
	"datafmt";
	"go/ast";
	"go/token";
	"io";
	"os";
)

// Format is a customized datafmt.Format for printing of ASTs.
type Format datafmt.Format;

// ----------------------------------------------------------------------------
// Custom formatters

// The AST-specific formatting state is maintained by a state variable.
type state struct {
	// for now we have very little state
	// TODO maintain list of unassociated comments
	optSemi *bool
}

func (s *state) Copy() datafmt.Environment {
	optSemi := *s.optSemi;
	return &state{&optSemi};
}

func isValidPos(s *datafmt.State, value interface{}, ruleName string) bool {
	pos := value.(token.Position);
	return pos.IsValid();
}

func isSend(s *datafmt.State, value interface{}, ruleName string) bool {
	return value.(ast.ChanDir) & ast.SEND != 0;
}

func isRecv(s *datafmt.State, value interface{}, ruleName string) bool {
	return value.(ast.ChanDir) & ast.RECV != 0;
}

func isMultiLineComment(s *datafmt.State, value interface{}, ruleName string) bool {
	return value.([]byte)[1] == '*';
}

func clearOptSemi(s *datafmt.State, value interface{}, ruleName string) bool {
	*s.Env().(*state).optSemi = false;
	return true;
}

func setOptSemi(s *datafmt.State, value interface{}, ruleName string) bool {
	*s.Env().(*state).optSemi = true;
	return true;
}

func optSemi(s *datafmt.State, value interface{}, ruleName string) bool {
	if !*s.Env().(*state).optSemi {
		s.Write([]byte{';'});
	}
	return true;
}

var fmap = datafmt.FormatterMap {
	"isValidPos": isValidPos,
	"isSend": isSend,
	"isRecv": isRecv,
	"isMultiLineComment": isMultiLineComment,
	"/": clearOptSemi,
	"clearOptSemi": clearOptSemi,
	"setOptSemi": setOptSemi,
	"optSemi": optSemi,
}

// ----------------------------------------------------------------------------
// Printing

// NewFormat parses a datafmt format specification from a file
// and adds AST-specific custom formatter rules. The result is
// the customized format or an os.Error, if any.
//
func NewFormat(filename string) (Format, os.Error) {
	src, err := io.ReadFile(filename);
	if err != nil {
		return nil, err;
	}
	f, err := datafmt.Parse(src, fmap);
	return Format(f), err;
}

// Fprint formats each AST node provided as argument according to the
// format f and writes to standard output. The result is the total number
// of bytes written and an os.Error, if any.
//
func (f Format) Fprint(w io.Writer, nodes ...) (int, os.Error) {
	s := state{new(bool)};
	return datafmt.Format(f).Fprint(w, &s, nodes);
}

// Fprint formats each AST node provided as argument according to the
// format f and writes to w. The result is the total number of bytes
// written and an os.Error, if any.
//
func (f Format) Print(nodes ...) (int, os.Error) {
	return f.Fprint(os.Stdout, nodes);
}
<commit_msg>remove superfluous indirection<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ast

import (
	"datafmt";
	"go/ast";
	"go/token";
	"io";
	"os";
)

// Format is a customized datafmt.Format for printing of ASTs.
type Format datafmt.Format;

// ----------------------------------------------------------------------------
// Custom formatters

// The AST-specific formatting state is maintained by a state variable.
type state struct {
	// for now we have very little state
	// TODO maintain list of unassociated comments
	optSemi bool
}

func (s *state) Copy() datafmt.Environment {
	copy := *s;
	return ©
}

func isValidPos(s *datafmt.State, value interface{}, ruleName string) bool {
	pos := value.(token.Position);
	return pos.IsValid();
}

func isSend(s *datafmt.State, value interface{}, ruleName string) bool {
	return value.(ast.ChanDir) & ast.SEND != 0;
}

func isRecv(s *datafmt.State, value interface{}, ruleName string) bool {
	return value.(ast.ChanDir) & ast.RECV != 0;
}

func isMultiLineComment(s *datafmt.State, value interface{}, ruleName string) bool {
	return value.([]byte)[1] == '*';
}

func clearOptSemi(s *datafmt.State, value interface{}, ruleName string) bool {
	s.Env().(*state).optSemi = false;
	return true;
}

func setOptSemi(s *datafmt.State, value interface{}, ruleName string) bool {
	s.Env().(*state).optSemi = true;
	return true;
}

func optSemi(s *datafmt.State, value interface{}, ruleName string) bool {
	if !s.Env().(*state).optSemi {
		s.Write([]byte{';'});
	}
	return true;
}

var fmap = datafmt.FormatterMap {
	"isValidPos": isValidPos,
	"isSend": isSend,
	"isRecv": isRecv,
	"isMultiLineComment": isMultiLineComment,
	"/": clearOptSemi,
	"clearOptSemi": clearOptSemi,
	"setOptSemi": setOptSemi,
	"optSemi": optSemi,
}

// ----------------------------------------------------------------------------
// Printing

// NewFormat parses a datafmt format specification from a file
// and adds AST-specific custom formatter rules. The result is
// the customized format or an os.Error, if any.
//
func NewFormat(filename string) (Format, os.Error) {
	src, err := io.ReadFile(filename);
	if err != nil {
		return nil, err;
	}
	f, err := datafmt.Parse(src, fmap);
	return Format(f), err;
}

// Fprint formats each AST node provided as argument according to the
// format f and writes to standard output. The result is the total number
// of bytes written and an os.Error, if any.
//
func (f Format) Fprint(w io.Writer, nodes ...) (int, os.Error) {
	var s state;
	return datafmt.Format(f).Fprint(w, &s, nodes);
}

// Fprint formats each AST node provided as argument according to the
// format f and writes to w. The result is the total number of bytes
// written and an os.Error, if any.
//
func (f Format) Print(nodes ...) (int, os.Error) {
	return f.Fprint(os.Stdout, nodes);
}
<|endoftext|>


<commit_before>// this is used to give the ability to NULL times in the database
// source: https://github.com/jinzhu/gorm/issues/10

package nullables

import (
	"database/sql/driver"
	"encoding/json"
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"time"
)

const (
	HTMLFormTime         = "15:04"
	HTMLFormDate         = "2006-01-02"
	HTMLFormDateTime     = "2006-01-02 3:04 PM"
	NZHTMLFormDate       = "02-01-2006"
	NZHTMLFormDateTime   = "02-01-2006 3:04 PM"
	DBTime               = "15:04:05:000"
	HTMLFormDateTime24Hr = "2006-01-02 15:04"
	DBDateTime           = "2006-01-02 15:04:05:000"
	TimeWithSeconds      = "15:04:05"
)

type NullTime struct {
	Time  time.Time
	Valid bool // Valid is true if Time is not NULL
}

// Scan implements the Scanner interface.
// Can scan from both time.Time and NullTime interfaces
func (nt *NullTime) Scan(value interface{}) error {
	if value == nil {
		nt.Valid = false
		return nil
	}

	switch iface := value.(type) {
	case time.Time:
		nt.Time, nt.Valid = iface, true
		return nil
	case NullTime:
		nt.Time, nt.Valid = iface.Time, iface.Valid
		return nil
	default:
		return nil
	}
	return nil
}

// Value implements the driver Valuer interface.
func (nt NullTime) Value() (driver.Value, error) {
	if !nt.Valid {
		return nil, nil
	}
	return nt.Time, nil
}

//this function is used when JSON tries to marshal this struct.
//Without it it will be marshalled into a JSON object with fields Time and Valid.
//This is far more elegant.
func (nt NullTime) MarshalJSON() ([]byte, error) {
	if nt.Valid {
		return nt.Time.MarshalJSON()
	} else {
		return json.Marshal(nil)
	}
}

func (nt *NullTime) UnmarshalJSON(b []byte) error {
	if string(b) == "null" || string(b) == "" {
		nt.Valid = false
		return nil
	}
	v, err := time.Parse(time.RFC3339, string(b[1:len(b)-1]))
	if err != nil {
		return err
	}
	nt.Time = v
	nt.Valid = true
	return nil
}

func (nt NullTime) GetHTMLDateTime() string {
	if nt.Valid {
		return nt.Time.Format(HTMLFormDateTime)
	}
	return "N/A"
}

//This function is used to convert an HTML form value (returned from a create or edit, for instance) to a NullTime.
//It will first try parse it as a Time, if that does not work it will parse it as a Date, if that doesn't work it will parse it as a datetime
func NullTimeConverter(b string) reflect.Value {
	decodedTime := NullTime{}
	//first check if we have been given a time
	v, err := time.Parse(HTMLFormTime, b)
	if err == nil {
		v := v.AddDate(1, 0, 0) //Nasty hack so that the year is not 0000, which is valid in Go but not MsSQL
		decodedTime.Time = v
		decodedTime.Valid = true
		return reflect.ValueOf(decodedTime)
	}
	//now check if it was a date
	v, err = time.Parse(NZHTMLFormDate, b)
	if err == nil {
		decodedTime.Time = v
		decodedTime.Valid = true
		return reflect.ValueOf(decodedTime)
	}

	v, err = time.Parse(HTMLFormDate, b)
	if err == nil {
		decodedTime.Time = v
		decodedTime.Valid = true
		return reflect.ValueOf(decodedTime)
	}
	//now check if it was a datetime
	v, err = time.Parse(HTMLFormDateTime, b)
	if err == nil {
		decodedTime.Time = v
		decodedTime.Valid = true
		return reflect.ValueOf(decodedTime)
	}
	//now check if it was a date
	v, err = time.Parse(NZHTMLFormDateTime, b)
	if err == nil {
		decodedTime.Time = v
		decodedTime.Valid = true
		return reflect.ValueOf(decodedTime)
	}
	v, err = time.Parse(HTMLFormDateTime24Hr, b)
	if err == nil {
		decodedTime.Time = v
		decodedTime.Valid = true
		return reflect.ValueOf(decodedTime)
	}
	return reflect.ValueOf(decodedTime)
}

type NullString struct {
	String string
	Valid  bool // Valid is true if Time is not NULL
}

// Scan implements the Scanner interface.
func (nt *NullString) Scan(value interface{}) error {
	if value == nil {
		nt.Valid = false
		return nil
	}
	nt.String, nt.Valid = fmt.Sprintf("%s", value), true
	return nil
}

// Value implements the driver Valuer interface.
func (nt NullString) Value() (driver.Value, error) {
	if !nt.Valid {
		return nil, nil
	}
	return nt.String, nil
}

//this function is used when JSON tries to marshal this struct.
//Without it it will be marshalled into a JSON object with fields Time and Valid.
//This is far more elegant.
func (nt NullString) MarshalJSON() ([]byte, error) {
	if nt.Valid {
		return []byte(fmt.Sprintf("\"%v\"", nt.String)), nil
	} else {
		return json.Marshal(nil)
	}
}

func (nt *NullString) UnmarshalJSON(b []byte) error {
	if string(b) == "null" {
		nt.Valid = false
		return nil
	}
	nt.String = string(b[1 : len(b)-1])
	nt.Valid = true
	return nil
}

func ConvertBool(value string) reflect.Value {
	if value == "on" {
		return reflect.ValueOf(true)
	} else if v, err := strconv.ParseBool(value); err == nil {
		return reflect.ValueOf(v)
	}

	return reflect.ValueOf(false)
}

//This function is used to convert an HTML form value (returned from a create or edit, for instance) to a NullTime.
//It will first try parse it as a Time, if that does not work it will parse it as a Date.
func NullStringConverter(s string) reflect.Value {
	decodedString := NullString{}
	//If string is empty we'll make it null
	if s == "" {
		decodedString.Valid = false
	} else {
		decodedString.String = s
		decodedString.Valid = true
	}
	return reflect.ValueOf(decodedString)
}

type NullInt64 struct {
	Int64 int64
	Valid bool // Valid is true if Time is not NULL
}

// Scan implements the Scanner interface.
func (nt *NullInt64) Scan(value interface{}) error {
	if value == nil {
		nt.Valid = false
		return nil
	}
	nt.Int64, nt.Valid = value.(int64), true
	return nil
}

// Value implements the driver Valuer interface.
func (nt NullInt64) Value() (driver.Value, error) {
	if !nt.Valid {
		return nil, nil
	}
	return nt.Int64, nil
}

//this function is used when JSON tries to marshal this struct.
//Without it it will be marshalled into a JSON object with fields Time and Valid.
//This is far more elegant.
func (nt NullInt64) MarshalJSON() ([]byte, error) {
	if nt.Valid {
		return []byte(fmt.Sprintf("%v", nt.Int64)), nil
	} else {
		return json.Marshal(nil)
	}
}

func (nt *NullInt64) UnmarshalJSON(b []byte) error {
	if string(b) == "null" {
		nt.Valid = false
		return nil
	}
	var err error
	nt.Int64, err = strconv.ParseInt(string(b), 10, 64)
	if err != nil {
		return err
	}
	nt.Valid = true
	return nil
}

func NullInt64Converter(i string) reflect.Value {
	decodedInt64 := NullInt64{}
	//If string is empty we'll make it null
	if i == "" {
		decodedInt64.Valid = false
	} else {
		var err error
		decodedInt64.Int64, err = strconv.ParseInt(i, 10, 64)
		if err != nil {
			decodedInt64.Valid = false
		} else {
			decodedInt64.Valid = true
		}
	}
	return reflect.ValueOf(decodedInt64)
}

type NullFloat64 struct {
	Float64 float64
	Valid   bool // Valid is true if Time is not NULL
}

// Scan implements the Scanner interface.
func (nt *NullFloat64) Scan(value interface{}) error {
	if value == nil {
		nt.Valid = false
		return nil
	}
	nt.Float64, nt.Valid = value.(float64), true
	return nil
}

// Value implements the driver Valuer interface.
func (nt NullFloat64) Value() (driver.Value, error) {
	if !nt.Valid {
		return nil, nil
	}
	return nt.Float64, nil
}

//this function is used when JSON tries to marshal this struct.
//Without it it will be marshalled into a JSON object with fields Time and Valid.
//This is far more elegant.
func (nt NullFloat64) MarshalJSON() ([]byte, error) {
	if nt.Valid {
		return []byte(fmt.Sprintf("%v", nt.Float64)), nil
	} else {
		return json.Marshal(nil)
	}
}

func (nt *NullFloat64) UnmarshalJSON(b []byte) error {
	if string(b) == "null" {
		nt.Valid = false
		return nil
	}
	var err error
	nt.Float64, err = strconv.ParseFloat(string(b), 64)
	if err != nil {
		return err
	}
	nt.Valid = true
	return nil
}

func NullFloat64Converter(i string) reflect.Value {
	decodedFloat64 := NullFloat64{}
	//If string is empty we'll make it null
	if i == "" {
		decodedFloat64.Valid = false
	} else {
		var err error
		decodedFloat64.Float64, err = strconv.ParseFloat(i, 64)
		if err != nil {
			decodedFloat64.Valid = false
		} else {
			decodedFloat64.Valid = true
		}
	}
	return reflect.ValueOf(decodedFloat64)
}

type NullBool struct {
	Bool  bool
	Valid bool // Valid is true if Time is not NULL
}

// Scan implements the Scanner interface.
func (nt *NullBool) Scan(value interface{}) error {
	if value == nil {
		nt.Valid = false
		return nil
	}
	nt.Bool, nt.Valid = value.(bool), true
	return nil
}

// Value implements the driver Valuer interface.
func (nt NullBool) Value() (driver.Value, error) {
	if !nt.Valid {
		return nil, nil
	}
	return nt.Bool, nil
}

//this function is used when JSON tries to marshal this struct.
//Without it it will be marshalled into a JSON object with fields Time and Valid.
//This is far more elegant.
func (nt NullBool) MarshalJSON() ([]byte, error) {
	if nt.Valid {
		if nt.Bool {
			return []byte("true"), nil
		}
		return []byte("false"), nil
	} else {
		return json.Marshal(nil)
	}
}

func (nt *NullBool) UnmarshalJSON(b []byte) error {
	if string(b) == "null" {
		nt.Valid = false
		return nil
	}
	if string(b) == "true" {
		nt.Valid = true
		nt.Bool = true
		return nil
	}
	if string(b) == "false" {
		nt.Valid = true
		nt.Bool = false
	}
	return errors.New(string(b) + " is not a valid JSON bool")
}

func NullBoolConverter(i string) reflect.Value {
	decodedBool := NullBool{}
	//If string is empty we'll make it null
	if i == "" {
		decodedBool.Valid = false
	} else {
		var err error
		decodedBool.Bool, err = strconv.ParseBool(i)
		if err != nil {
			decodedBool.Valid = false
		} else {
			decodedBool.Valid = true
		}
	}
	return reflect.ValueOf(decodedBool)
}
<commit_msg>Tidy some commenting<commit_after>// this is used to give the ability to NULL times in the database
// source: https://github.com/jinzhu/gorm/issues/10

package nullables

import (
	"database/sql/driver"
	"encoding/json"
	"errors"
	"fmt"
	"reflect"
	"strconv"
	"time"
)

const (
	HTMLFormTime         = "15:04"
	HTMLFormDate         = "2006-01-02"
	HTMLFormDateTime     = "2006-01-02 3:04 PM"
	NZHTMLFormDate       = "02-01-2006"
	NZHTMLFormDateTime   = "02-01-2006 3:04 PM"
	DBTime               = "15:04:05:000"
	HTMLFormDateTime24Hr = "2006-01-02 15:04"
	DBDateTime           = "2006-01-02 15:04:05:000"
	TimeWithSeconds      = "15:04:05"
)

type NullTime struct {
	Time  time.Time
	Valid bool // Valid is true if Time is not NULL
}

// Scan implements the Scanner interface.
// Can scan from both time.Time and NullTime interfaces
func (nt *NullTime) Scan(value interface{}) error {
	if value == nil {
		nt.Valid = false
		return nil
	}

	switch iface := value.(type) {
	case time.Time:
		nt.Time, nt.Valid = iface, true
		return nil
	case NullTime:
		nt.Time, nt.Valid = iface.Time, iface.Valid
		return nil
	default:
		return nil
	}
	return nil
}

// Value implements the driver Valuer interface.
func (nt NullTime) Value() (driver.Value, error) {
	if !nt.Valid {
		return nil, nil
	}
	return nt.Time, nil
}

//this function is used when JSON tries to marshal this struct.
//Without it it will be marshalled into a JSON object with fields Time and Valid.
//This is far more elegant.
func (nt NullTime) MarshalJSON() ([]byte, error) {
	if nt.Valid {
		return nt.Time.MarshalJSON()
	} else {
		return json.Marshal(nil)
	}
}

func (nt *NullTime) UnmarshalJSON(b []byte) error {
	if string(b) == "null" || string(b) == "" {
		nt.Valid = false
		return nil
	}
	v, err := time.Parse(time.RFC3339, string(b[1:len(b)-1]))
	if err != nil {
		return err
	}
	nt.Time = v
	nt.Valid = true
	return nil
}

func (nt NullTime) GetHTMLDateTime() string {
	if nt.Valid {
		return nt.Time.Format(HTMLFormDateTime)
	}
	return "N/A"
}

//This function is used to convert an HTML form value (returned from a create or edit, for instance) to a NullTime.
//It will first try parse it as a Time, if that does not work it will parse it as a Date, if that doesn't work it will parse it as a datetime
func NullTimeConverter(b string) reflect.Value {
	decodedTime := NullTime{}
	//Try ALL the formats we think the data might be
	v, err := time.Parse(HTMLFormTime, b)
	if err == nil {
		v := v.AddDate(1, 0, 0) //Nasty hack so that the year is not 0000, which is valid in Go but not MsSQL
		decodedTime.Time = v
		decodedTime.Valid = true
		return reflect.ValueOf(decodedTime)
	}
	v, err = time.Parse(NZHTMLFormDate, b)
	if err == nil {
		decodedTime.Time = v
		decodedTime.Valid = true
		return reflect.ValueOf(decodedTime)
	}
	v, err = time.Parse(HTMLFormDate, b)
	if err == nil {
		decodedTime.Time = v
		decodedTime.Valid = true
		return reflect.ValueOf(decodedTime)
	}
	v, err = time.Parse(HTMLFormDateTime, b)
	if err == nil {
		decodedTime.Time = v
		decodedTime.Valid = true
		return reflect.ValueOf(decodedTime)
	}
	v, err = time.Parse(NZHTMLFormDateTime, b)
	if err == nil {
		decodedTime.Time = v
		decodedTime.Valid = true
		return reflect.ValueOf(decodedTime)
	}
	v, err = time.Parse(HTMLFormDateTime24Hr, b)
	if err == nil {
		decodedTime.Time = v
		decodedTime.Valid = true
		return reflect.ValueOf(decodedTime)
	}
	return reflect.ValueOf(decodedTime)
}

type NullString struct {
	String string
	Valid  bool // Valid is true if Time is not NULL
}

// Scan implements the Scanner interface.
func (nt *NullString) Scan(value interface{}) error {
	if value == nil {
		nt.Valid = false
		return nil
	}
	nt.String, nt.Valid = fmt.Sprintf("%s", value), true
	return nil
}

// Value implements the driver Valuer interface.
func (nt NullString) Value() (driver.Value, error) {
	if !nt.Valid {
		return nil, nil
	}
	return nt.String, nil
}

//this function is used when JSON tries to marshal this struct.
//Without it it will be marshalled into a JSON object with fields Time and Valid.
//This is far more elegant.
func (nt NullString) MarshalJSON() ([]byte, error) {
	if nt.Valid {
		return []byte(fmt.Sprintf("\"%v\"", nt.String)), nil
	} else {
		return json.Marshal(nil)
	}
}

func (nt *NullString) UnmarshalJSON(b []byte) error {
	if string(b) == "null" {
		nt.Valid = false
		return nil
	}
	nt.String = string(b[1 : len(b)-1])
	nt.Valid = true
	return nil
}

func ConvertBool(value string) reflect.Value {
	if value == "on" {
		return reflect.ValueOf(true)
	} else if v, err := strconv.ParseBool(value); err == nil {
		return reflect.ValueOf(v)
	}

	return reflect.ValueOf(false)
}

//This function is used to convert an HTML form value (returned from a create or edit, for instance) to a NullTime.
//It will first try parse it as a Time, if that does not work it will parse it as a Date.
func NullStringConverter(s string) reflect.Value {
	decodedString := NullString{}
	//If string is empty we'll make it null
	if s == "" {
		decodedString.Valid = false
	} else {
		decodedString.String = s
		decodedString.Valid = true
	}
	return reflect.ValueOf(decodedString)
}

type NullInt64 struct {
	Int64 int64
	Valid bool // Valid is true if Time is not NULL
}

// Scan implements the Scanner interface.
func (nt *NullInt64) Scan(value interface{}) error {
	if value == nil {
		nt.Valid = false
		return nil
	}
	nt.Int64, nt.Valid = value.(int64), true
	return nil
}

// Value implements the driver Valuer interface.
func (nt NullInt64) Value() (driver.Value, error) {
	if !nt.Valid {
		return nil, nil
	}
	return nt.Int64, nil
}

//this function is used when JSON tries to marshal this struct.
//Without it it will be marshalled into a JSON object with fields Time and Valid.
//This is far more elegant.
func (nt NullInt64) MarshalJSON() ([]byte, error) {
	if nt.Valid {
		return []byte(fmt.Sprintf("%v", nt.Int64)), nil
	} else {
		return json.Marshal(nil)
	}
}

func (nt *NullInt64) UnmarshalJSON(b []byte) error {
	if string(b) == "null" {
		nt.Valid = false
		return nil
	}
	var err error
	nt.Int64, err = strconv.ParseInt(string(b), 10, 64)
	if err != nil {
		return err
	}
	nt.Valid = true
	return nil
}

func NullInt64Converter(i string) reflect.Value {
	decodedInt64 := NullInt64{}
	//If string is empty we'll make it null
	if i == "" {
		decodedInt64.Valid = false
	} else {
		var err error
		decodedInt64.Int64, err = strconv.ParseInt(i, 10, 64)
		if err != nil {
			decodedInt64.Valid = false
		} else {
			decodedInt64.Valid = true
		}
	}
	return reflect.ValueOf(decodedInt64)
}

type NullFloat64 struct {
	Float64 float64
	Valid   bool // Valid is true if Time is not NULL
}

// Scan implements the Scanner interface.
func (nt *NullFloat64) Scan(value interface{}) error {
	if value == nil {
		nt.Valid = false
		return nil
	}
	nt.Float64, nt.Valid = value.(float64), true
	return nil
}

// Value implements the driver Valuer interface.
func (nt NullFloat64) Value() (driver.Value, error) {
	if !nt.Valid {
		return nil, nil
	}
	return nt.Float64, nil
}

//this function is used when JSON tries to marshal this struct.
//Without it it will be marshalled into a JSON object with fields Time and Valid.
//This is far more elegant.
func (nt NullFloat64) MarshalJSON() ([]byte, error) {
	if nt.Valid {
		return []byte(fmt.Sprintf("%v", nt.Float64)), nil
	} else {
		return json.Marshal(nil)
	}
}

func (nt *NullFloat64) UnmarshalJSON(b []byte) error {
	if string(b) == "null" {
		nt.Valid = false
		return nil
	}
	var err error
	nt.Float64, err = strconv.ParseFloat(string(b), 64)
	if err != nil {
		return err
	}
	nt.Valid = true
	return nil
}

func NullFloat64Converter(i string) reflect.Value {
	decodedFloat64 := NullFloat64{}
	//If string is empty we'll make it null
	if i == "" {
		decodedFloat64.Valid = false
	} else {
		var err error
		decodedFloat64.Float64, err = strconv.ParseFloat(i, 64)
		if err != nil {
			decodedFloat64.Valid = false
		} else {
			decodedFloat64.Valid = true
		}
	}
	return reflect.ValueOf(decodedFloat64)
}

type NullBool struct {
	Bool  bool
	Valid bool // Valid is true if Time is not NULL
}

// Scan implements the Scanner interface.
func (nt *NullBool) Scan(value interface{}) error {
	if value == nil {
		nt.Valid = false
		return nil
	}
	nt.Bool, nt.Valid = value.(bool), true
	return nil
}

// Value implements the driver Valuer interface.
func (nt NullBool) Value() (driver.Value, error) {
	if !nt.Valid {
		return nil, nil
	}
	return nt.Bool, nil
}

//this function is used when JSON tries to marshal this struct.
//Without it it will be marshalled into a JSON object with fields Time and Valid.
//This is far more elegant.
func (nt NullBool) MarshalJSON() ([]byte, error) {
	if nt.Valid {
		if nt.Bool {
			return []byte("true"), nil
		}
		return []byte("false"), nil
	} else {
		return json.Marshal(nil)
	}
}

func (nt *NullBool) UnmarshalJSON(b []byte) error {
	if string(b) == "null" {
		nt.Valid = false
		return nil
	}
	if string(b) == "true" {
		nt.Valid = true
		nt.Bool = true
		return nil
	}
	if string(b) == "false" {
		nt.Valid = true
		nt.Bool = false
	}
	return errors.New(string(b) + " is not a valid JSON bool")
}

func NullBoolConverter(i string) reflect.Value {
	decodedBool := NullBool{}
	//If string is empty we'll make it null
	if i == "" {
		decodedBool.Valid = false
	} else {
		var err error
		decodedBool.Bool, err = strconv.ParseBool(i)
		if err != nil {
			decodedBool.Valid = false
		} else {
			decodedBool.Valid = true
		}
	}
	return reflect.ValueOf(decodedBool)
}
<|endoftext|>


<commit_before>package blinkstick

import (
	"fmt"
	"image/color"

	"github.com/yesnault/hid"
)

// Version of Blinkstick
// One Line for this, used by release.sh script
// Keep "const Version on one line"
const Version = "0.1.5"

// vendorID blinkstick
const vendorID = 0x20a0

// productID blinkstick
const productID = 0x41e5

// usbDevice ...
type usbDevice struct {
	DeviceInfo *hid.DeviceInfo
	Device     *hid.Device
}

// Blinkstick represents a blinkstick device
type Blinkstick interface {
	List() []Blinkstick
	SetColor(color.Color) error
	Blink(color.Color, int, int) error
	GetDeviceInfo() *hid.DeviceInfo
	ListFilter(hid *hid.DeviceInfo) (bool, Blinkstick)
	getUSBDevice() *usbDevice
}

// SetColor set color
func (usbDevice *usbDevice) setColor(c color.Color, index byte) error {
	if usbDevice.Device == nil {
		if err := usbDevice.Open(); err != nil {
			return err
		}
	}
	r, g, b, _ := c.RGBA()
	d := *usbDevice.Device
	return d.WriteFeature([]byte{0x05, 0x00, index, byte(r >> 8), byte(g >> 8), byte(b >> 8)})
}

// Open open a device
func (usbDevice *usbDevice) Open() error {
	device, err := usbDevice.DeviceInfo.Open()
	if err != nil {
		return fmt.Errorf("Error while opening device: %s", err)
	}
	usbDevice.Device = &device
	return nil
}

// ListFilter is used to filter device on List
type ListFilter func(*hid.DeviceInfo) (bool, Blinkstick)

// List gets all blinkstick device
func List(opts ...ListFilter) []Blinkstick {
	out := []Blinkstick{}

	if len(opts) == 0 {
		opts = append(opts, Nano{}.ListFilter)
		opts = append(opts, Strip{}.ListFilter)
	}

	for di := range hid.Devices() {
		if di.VendorId == vendorID && di.ProductId == productID {
			//			fmt.Printf("di: %+v", di)
			for _, o := range opts {
				if toKeep, blinkstick := o(di); toKeep {
					out = append(out, blinkstick)
				}
			}

		}
	}
	return out
}
<commit_msg>feat: add flex in list<commit_after>package blinkstick

import (
	"fmt"
	"image/color"

	"github.com/yesnault/hid"
)

// Version of Blinkstick
// One Line for this, used by release.sh script
// Keep "const Version on one line"
const Version = "0.1.5"

// vendorID blinkstick
const vendorID = 0x20a0

// productID blinkstick
const productID = 0x41e5

// usbDevice ...
type usbDevice struct {
	DeviceInfo *hid.DeviceInfo
	Device     *hid.Device
}

// Blinkstick represents a blinkstick device
type Blinkstick interface {
	List() []Blinkstick
	SetColor(color.Color) error
	Blink(color.Color, int, int) error
	GetDeviceInfo() *hid.DeviceInfo
	ListFilter(hid *hid.DeviceInfo) (bool, Blinkstick)
	getUSBDevice() *usbDevice
}

// SetColor set color
func (usbDevice *usbDevice) setColor(c color.Color, index byte) error {
	if usbDevice.Device == nil {
		if err := usbDevice.Open(); err != nil {
			return err
		}
	}
	r, g, b, _ := c.RGBA()
	d := *usbDevice.Device
	return d.WriteFeature([]byte{0x05, 0x00, index, byte(r >> 8), byte(g >> 8), byte(b >> 8)})
}

// Open open a device
func (usbDevice *usbDevice) Open() error {
	device, err := usbDevice.DeviceInfo.Open()
	if err != nil {
		return fmt.Errorf("Error while opening device: %s", err)
	}
	usbDevice.Device = &device
	return nil
}

// ListFilter is used to filter device on List
type ListFilter func(*hid.DeviceInfo) (bool, Blinkstick)

// List gets all blinkstick device
func List(opts ...ListFilter) []Blinkstick {
	out := []Blinkstick{}

	if len(opts) == 0 {
		opts = append(opts, Nano{}.ListFilter)
		opts = append(opts, Flex{}.ListFilter)
		opts = append(opts, Strip{}.ListFilter)
	}

	for di := range hid.Devices() {
		if di.VendorId == vendorID && di.ProductId == productID {
			//			fmt.Printf("di: %+v", di)
			for _, o := range opts {
				if toKeep, blinkstick := o(di); toKeep {
					out = append(out, blinkstick)
				}
			}

		}
	}
	return out
}
<|endoftext|>