<commit_before>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package addons

import (
	"fmt"
	"path"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"
	"github.com/spf13/viper"

	"k8s.io/minikube/pkg/drivers/kic/oci"
	"k8s.io/minikube/pkg/kapi"
	"k8s.io/minikube/pkg/minikube/assets"
	"k8s.io/minikube/pkg/minikube/command"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/driver"
	"k8s.io/minikube/pkg/minikube/exit"
	"k8s.io/minikube/pkg/minikube/machine"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/minikube/storageclass"
	"k8s.io/minikube/pkg/util/retry"
)

// defaultStorageClassProvisioner is the name of the default storage class provisioner
const defaultStorageClassProvisioner = "standard"

// RunCallbacks runs all actions associated to an addon, but does not set it (thread-safe)
func RunCallbacks(cc *config.ClusterConfig, name string, value string) error {
	glog.Infof("Setting %s=%s in profile %q", name, value, cc.Name)
	a, valid := isAddonValid(name)
	if !valid {
		return errors.Errorf("%s is not a valid addon", name)
	}

	// Run any additional validations for this property
	if err := run(cc, name, value, a.validations); err != nil {
		return errors.Wrap(err, "running validations")
	}

	// Run any callbacks for this property
	if err := run(cc, name, value, a.callbacks); err != nil {
		return errors.Wrap(err, "running callbacks")
	}
	return nil
}

// Set sets a value in the config (not threadsafe)
func Set(cc *config.ClusterConfig, name string, value string) error {
	a, valid := isAddonValid(name)
	if !valid {
		return errors.Errorf("%s is not a valid addon", name)
	}
	return a.set(cc, name, value)
}

// SetAndSave sets a value and saves the config
func SetAndSave(profile string, name string, value string) error {
	cc, err := config.Load(profile)
	if err != nil {
		return errors.Wrap(err, "loading profile")
	}

	if err := RunCallbacks(cc, name, value); err != nil {
		return errors.Wrap(err, "run callbacks")
	}

	if err := Set(cc, name, value); err != nil {
		return errors.Wrap(err, "set")
	}

	glog.Infof("Writing out %q config to set %s=%v...", profile, name, value)
	return config.Write(profile, cc)
}

// Runs all the validation or callback functions and collects errors
func run(cc *config.ClusterConfig, name string, value string, fns []setFn) error {
	var errors []error
	for _, fn := range fns {
		err := fn(cc, name, value)
		if err != nil {
			errors = append(errors, err)
		}
	}
	if len(errors) > 0 {
		return fmt.Errorf("%v", errors)
	}
	return nil
}

// SetBool sets a bool value in the config (not threadsafe)
func SetBool(cc *config.ClusterConfig, name string, val string) error {
	b, err := strconv.ParseBool(val)
	if err != nil {
		return err
	}
	if cc.Addons == nil {
		cc.Addons = map[string]bool{}
	}
	cc.Addons[name] = b
	return nil
}

// enableOrDisableAddon updates addon status executing any commands necessary
func enableOrDisableAddon(cc *config.ClusterConfig, name string, val string) error {
	glog.Infof("Setting addon %s=%s in %q", name, val, cc.Name)
	enable, err := strconv.ParseBool(val)
	if err != nil {
		return errors.Wrapf(err, "parsing bool: %s", name)
	}
	addon := assets.Addons[name]

	// check addon status before enabling/disabling it
	if isAddonAlreadySet(cc, addon, enable) {
		glog.Warningf("addon %s should already be in state %v", name, val)
		if !enable {
			return nil
		}
	}

	// to match both ingress and ingress-dns adons
	// 	if strings.HasPrefix(name, "ingress") && enable && driver.IsKIC(cc.Driver) && runtime.GOOS != "linux" {
	// 		exit.UsageT(`Due to {{.driver_name}} networking limitations on {{.os_name}}, {{.addon_name}} addon is not supported for this driver.
	// Alternatively to use this addon you can use a vm-based driver:

	// 	'minikube start --vm=true'

	// To track the update on this work in progress feature please check:
	// https://github.com/kubernetes/minikube/issues/7332`, out.V{"driver_name": cc.Driver, "os_name": runtime.GOOS, "addon_name": name})
	// 	}

	if strings.HasPrefix(name, "istio") && enable {
		minMem := 8192
		minCPUs := 4
		if cc.Memory < minMem {
			out.WarningT("Istio needs {{.minMem}}MB of memory -- your configuration only allocates {{.memory}}MB", out.V{"minMem": minMem, "memory": cc.Memory})
		}
		if cc.CPUs < minCPUs {
			out.WarningT("Istio needs {{.minCPUs}} CPUs -- your configuration only allocates {{.cpus}} CPUs", out.V{"minCPUs": minCPUs, "cpus": cc.CPUs})
		}
	}

	// TODO(r2d4): config package should not reference API, pull this out
	api, err := machine.NewAPIClient()
	if err != nil {
		return errors.Wrap(err, "machine client")
	}
	defer api.Close()

	cp, err := config.PrimaryControlPlane(cc)
	if err != nil {
		exit.WithError("Error getting primary control plane", err)
	}

	mName := driver.MachineName(*cc, cp)
	host, err := machine.LoadHost(api, mName)
	if err != nil || !machine.IsRunning(api, mName) {
		glog.Warningf("%q is not running, setting %s=%v and skipping enablement (err=%v)", mName, addon.Name(), enable, err)
		return nil
	}

	if name == "registry" {
		if driver.NeedsPortForward(cc.Driver) {
			port, err := oci.ForwardedPort(cc.Driver, cc.Name, constants.RegistryAddonPort)
			if err != nil {
				return errors.Wrap(err, "registry port")
			}
			out.T(out.Tip, `Registry addon on with {{.driver}} uses {{.port}} please use that instead of default 5000`, out.V{"driver": cc.Driver, "port": port})
			out.T(out.Documentation, `For more information see: https://minikube.sigs.k8s.io/docs/drivers/{{.driver}}`, out.V{"driver": cc.Driver})
		}
	}

	cmd, err := machine.CommandRunner(host)
	if err != nil {
		return errors.Wrap(err, "command runner")
	}

	data := assets.GenerateTemplateData(cc.KubernetesConfig)
	return enableOrDisableAddonInternal(cc, addon, cmd, data, enable)
}

func isAddonAlreadySet(cc *config.ClusterConfig, addon *assets.Addon, enable bool) bool {
	enabled := addon.IsEnabled(cc)
	if enabled && enable {
		return true
	}

	if !enabled && !enable {
		return true
	}

	return false
}

func enableOrDisableAddonInternal(cc *config.ClusterConfig, addon *assets.Addon, cmd command.Runner, data interface{}, enable bool) error {
	deployFiles := []string{}

	for _, addon := range addon.Assets {
		var f assets.CopyableFile
		var err error
		if addon.IsTemplate() {
			f, err = addon.Evaluate(data)
			if err != nil {
				return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetSourcePath())
			}

		} else {
			f = addon
		}
		fPath := path.Join(f.GetTargetDir(), f.GetTargetName())

		if enable {
			glog.Infof("installing %s", fPath)
			if err := cmd.Copy(f); err != nil {
				return err
			}
		} else {
			glog.Infof("Removing %+v", fPath)
			defer func() {
				if err := cmd.Remove(f); err != nil {
					glog.Warningf("error removing %s; addon should still be disabled as expected", fPath)
				}
			}()
		}
		if strings.HasSuffix(fPath, ".yaml") {
			deployFiles = append(deployFiles, fPath)
		}
	}

	command := kubectlCommand(cc, deployFiles, enable)

	// Retry, because sometimes we race against an apiserver restart
	apply := func() error {
		_, err := cmd.RunCmd(command)
		if err != nil {
			glog.Warningf("apply failed, will retry: %v", err)
		}
		return err
	}

	return retry.Expo(apply, 250*time.Millisecond, 2*time.Minute)
}

// enableOrDisableStorageClasses enables or disables storage classes
func enableOrDisableStorageClasses(cc *config.ClusterConfig, name string, val string) error {
	glog.Infof("enableOrDisableStorageClasses %s=%v on %q", name, val, cc.Name)
	enable, err := strconv.ParseBool(val)
	if err != nil {
		return errors.Wrap(err, "Error parsing boolean")
	}

	class := defaultStorageClassProvisioner
	if name == "storage-provisioner-gluster" {
		class = "glusterfile"
	}

	api, err := machine.NewAPIClient()
	if err != nil {
		return errors.Wrap(err, "machine client")
	}
	defer api.Close()

	cp, err := config.PrimaryControlPlane(cc)
	if err != nil {
		return errors.Wrap(err, "getting control plane")
	}
	if !machine.IsRunning(api, driver.MachineName(*cc, cp)) {
		glog.Warningf("%q is not running, writing %s=%v to disk and skipping enablement", driver.MachineName(*cc, cp), name, val)
		return enableOrDisableAddon(cc, name, val)
	}

	storagev1, err := storageclass.GetStoragev1(cc.Name)
	if err != nil {
		return errors.Wrapf(err, "Error getting storagev1 interface %v ", err)
	}

	if enable {
		// Only StorageClass for 'name' should be marked as default
		err = storageclass.SetDefaultStorageClass(storagev1, class)
		if err != nil {
			return errors.Wrapf(err, "Error making %s the default storage class", class)
		}
	} else {
		// Unset the StorageClass as default
		err := storageclass.DisableDefaultStorageClass(storagev1, class)
		if err != nil {
			return errors.Wrapf(err, "Error disabling %s as the default storage class", class)
		}
	}

	return enableOrDisableAddon(cc, name, val)
}

func validateIngress(cc *config.ClusterConfig, name string, val string) error {
	glog.Infof("Setting addon %s=%s in %q", name, val, cc.Name)
	enable, err := strconv.ParseBool(val)
	if err != nil {
		return errors.Wrapf(err, "parsing bool: %s", name)
	}
	if name == "ingress" && enable {
		client, err := kapi.Client(viper.GetString(config.ProfileName))
		if err != nil {
			return errors.Wrapf(err, "get kube-client to validate ingress addon: %s", name)
		} else {
			err = kapi.WaitForDeploymentToStabilize(client, "kube-system", "ingress-nginx-controller", time.Minute*3)
			if err != nil {
				return errors.Wrapf(err, "Failed verifying ingress addon deployment: %s", name)
			}
		}
	}
	return nil
}

// Start enables the default addons for a profile, plus any additional
func Start(wg *sync.WaitGroup, cc *config.ClusterConfig, toEnable map[string]bool, additional []string) {
	wg.Add(1)
	defer wg.Done()

	start := time.Now()
	glog.Infof("enableAddons start: toEnable=%v, additional=%s", toEnable, additional)
	defer func() {
		glog.Infof("enableAddons completed in %s", time.Since(start))
	}()

	// Get the default values of any addons not saved to our config
	for name, a := range assets.Addons {
		defaultVal := a.IsEnabled(cc)

		_, exists := toEnable[name]
		if !exists {
			toEnable[name] = defaultVal
		}
	}

	// Apply new addons
	for _, name := range additional {
		// replace heapster as metrics-server because heapster is deprecated
		if name == "heapster" {
			name = "metrics-server"
		}
		// if the specified addon doesn't exist, skip enabling
		_, e := isAddonValid(name)
		if e {
			toEnable[name] = true
		}
	}

	toEnableList := []string{}
	for k, v := range toEnable {
		if v {
			toEnableList = append(toEnableList, k)
		}
	}
	sort.Strings(toEnableList)

	var awg sync.WaitGroup

	defer func() { // making it show after verifications( not perfect till #7613 is closed)
		out.T(out.AddonEnable, "Enabled addons: {{.addons}}", out.V{"addons": strings.Join(toEnableList, ", ")})
	}()
	for _, a := range toEnableList {
		awg.Add(1)
		go func(name string) {
			err := RunCallbacks(cc, name, "true")
			if err != nil {
				out.WarningT("Enabling '{{.name}}' returned an error: {{.error}}", out.V{"name": name, "error": err})
			}
			awg.Done()
		}(a)
	}

	// Wait until all of the addons are enabled before updating the config (not thread safe)
	awg.Wait()
	for _, a := range toEnableList {
		if err := Set(cc, a, "true"); err != nil {
			glog.Errorf("store failed: %v", err)
		}
	}
}
<commit_msg>remove unrelated changes<commit_after>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package addons

import (
	"fmt"
	"path"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/pkg/errors"
	"github.com/spf13/viper"

	"k8s.io/minikube/pkg/drivers/kic/oci"
	"k8s.io/minikube/pkg/kapi"
	"k8s.io/minikube/pkg/minikube/assets"
	"k8s.io/minikube/pkg/minikube/command"
	"k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/driver"
	"k8s.io/minikube/pkg/minikube/exit"
	"k8s.io/minikube/pkg/minikube/machine"
	"k8s.io/minikube/pkg/minikube/out"
	"k8s.io/minikube/pkg/minikube/storageclass"
	"k8s.io/minikube/pkg/util/retry"
)

// defaultStorageClassProvisioner is the name of the default storage class provisioner
const defaultStorageClassProvisioner = "standard"

// RunCallbacks runs all actions associated to an addon, but does not set it (thread-safe)
func RunCallbacks(cc *config.ClusterConfig, name string, value string) error {
	glog.Infof("Setting %s=%s in profile %q", name, value, cc.Name)
	a, valid := isAddonValid(name)
	if !valid {
		return errors.Errorf("%s is not a valid addon", name)
	}

	// Run any additional validations for this property
	if err := run(cc, name, value, a.validations); err != nil {
		return errors.Wrap(err, "running validations")
	}

	// Run any callbacks for this property
	if err := run(cc, name, value, a.callbacks); err != nil {
		return errors.Wrap(err, "running callbacks")
	}
	return nil
}

// Set sets a value in the config (not threadsafe)
func Set(cc *config.ClusterConfig, name string, value string) error {
	a, valid := isAddonValid(name)
	if !valid {
		return errors.Errorf("%s is not a valid addon", name)
	}
	return a.set(cc, name, value)
}

// SetAndSave sets a value and saves the config
func SetAndSave(profile string, name string, value string) error {
	cc, err := config.Load(profile)
	if err != nil {
		return errors.Wrap(err, "loading profile")
	}

	if err := RunCallbacks(cc, name, value); err != nil {
		return errors.Wrap(err, "run callbacks")
	}

	if err := Set(cc, name, value); err != nil {
		return errors.Wrap(err, "set")
	}

	glog.Infof("Writing out %q config to set %s=%v...", profile, name, value)
	return config.Write(profile, cc)
}

// Runs all the validation or callback functions and collects errors
func run(cc *config.ClusterConfig, name string, value string, fns []setFn) error {
	var errors []error
	for _, fn := range fns {
		err := fn(cc, name, value)
		if err != nil {
			errors = append(errors, err)
		}
	}
	if len(errors) > 0 {
		return fmt.Errorf("%v", errors)
	}
	return nil
}

// SetBool sets a bool value in the config (not threadsafe)
func SetBool(cc *config.ClusterConfig, name string, val string) error {
	b, err := strconv.ParseBool(val)
	if err != nil {
		return err
	}
	if cc.Addons == nil {
		cc.Addons = map[string]bool{}
	}
	cc.Addons[name] = b
	return nil
}

// enableOrDisableAddon updates addon status executing any commands necessary
func enableOrDisableAddon(cc *config.ClusterConfig, name string, val string) error {
	glog.Infof("Setting addon %s=%s in %q", name, val, cc.Name)
	enable, err := strconv.ParseBool(val)
	if err != nil {
		return errors.Wrapf(err, "parsing bool: %s", name)
	}
	addon := assets.Addons[name]

	// check addon status before enabling/disabling it
	if isAddonAlreadySet(cc, addon, enable) {
		glog.Warningf("addon %s should already be in state %v", name, val)
		if !enable {
			return nil
		}
	}

	//	to match both ingress and ingress-dns adons
	if strings.HasPrefix(name, "ingress") && enable && driver.IsKIC(cc.Driver) && runtime.GOOS != "linux" {
		exit.UsageT(`Due to {{.driver_name}} networking limitations on {{.os_name}}, {{.addon_name}} addon is not supported for this driver.
	Alternatively to use this addon you can use a vm-based driver:

		'minikube start --vm=true'

	To track the update on this work in progress feature please check:
	https://github.com/kubernetes/minikube/issues/7332`, out.V{"driver_name": cc.Driver, "os_name": runtime.GOOS, "addon_name": name})
	}

	if strings.HasPrefix(name, "istio") && enable {
		minMem := 8192
		minCPUs := 4
		if cc.Memory < minMem {
			out.WarningT("Istio needs {{.minMem}}MB of memory -- your configuration only allocates {{.memory}}MB", out.V{"minMem": minMem, "memory": cc.Memory})
		}
		if cc.CPUs < minCPUs {
			out.WarningT("Istio needs {{.minCPUs}} CPUs -- your configuration only allocates {{.cpus}} CPUs", out.V{"minCPUs": minCPUs, "cpus": cc.CPUs})
		}
	}

	// TODO(r2d4): config package should not reference API, pull this out
	api, err := machine.NewAPIClient()
	if err != nil {
		return errors.Wrap(err, "machine client")
	}
	defer api.Close()

	cp, err := config.PrimaryControlPlane(cc)
	if err != nil {
		exit.WithError("Error getting primary control plane", err)
	}

	mName := driver.MachineName(*cc, cp)
	host, err := machine.LoadHost(api, mName)
	if err != nil || !machine.IsRunning(api, mName) {
		glog.Warningf("%q is not running, setting %s=%v and skipping enablement (err=%v)", mName, addon.Name(), enable, err)
		return nil
	}

	if name == "registry" {
		if driver.NeedsPortForward(cc.Driver) {
			port, err := oci.ForwardedPort(cc.Driver, cc.Name, constants.RegistryAddonPort)
			if err != nil {
				return errors.Wrap(err, "registry port")
			}
			out.T(out.Tip, `Registry addon on with {{.driver}} uses {{.port}} please use that instead of default 5000`, out.V{"driver": cc.Driver, "port": port})
			out.T(out.Documentation, `For more information see: https://minikube.sigs.k8s.io/docs/drivers/{{.driver}}`, out.V{"driver": cc.Driver})
		}
	}

	cmd, err := machine.CommandRunner(host)
	if err != nil {
		return errors.Wrap(err, "command runner")
	}

	data := assets.GenerateTemplateData(cc.KubernetesConfig)
	return enableOrDisableAddonInternal(cc, addon, cmd, data, enable)
}

func isAddonAlreadySet(cc *config.ClusterConfig, addon *assets.Addon, enable bool) bool {
	enabled := addon.IsEnabled(cc)
	if enabled && enable {
		return true
	}

	if !enabled && !enable {
		return true
	}

	return false
}

func enableOrDisableAddonInternal(cc *config.ClusterConfig, addon *assets.Addon, cmd command.Runner, data interface{}, enable bool) error {
	deployFiles := []string{}

	for _, addon := range addon.Assets {
		var f assets.CopyableFile
		var err error
		if addon.IsTemplate() {
			f, err = addon.Evaluate(data)
			if err != nil {
				return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetSourcePath())
			}

		} else {
			f = addon
		}
		fPath := path.Join(f.GetTargetDir(), f.GetTargetName())

		if enable {
			glog.Infof("installing %s", fPath)
			if err := cmd.Copy(f); err != nil {
				return err
			}
		} else {
			glog.Infof("Removing %+v", fPath)
			defer func() {
				if err := cmd.Remove(f); err != nil {
					glog.Warningf("error removing %s; addon should still be disabled as expected", fPath)
				}
			}()
		}
		if strings.HasSuffix(fPath, ".yaml") {
			deployFiles = append(deployFiles, fPath)
		}
	}

	command := kubectlCommand(cc, deployFiles, enable)

	// Retry, because sometimes we race against an apiserver restart
	apply := func() error {
		_, err := cmd.RunCmd(command)
		if err != nil {
			glog.Warningf("apply failed, will retry: %v", err)
		}
		return err
	}

	return retry.Expo(apply, 250*time.Millisecond, 2*time.Minute)
}

// enableOrDisableStorageClasses enables or disables storage classes
func enableOrDisableStorageClasses(cc *config.ClusterConfig, name string, val string) error {
	glog.Infof("enableOrDisableStorageClasses %s=%v on %q", name, val, cc.Name)
	enable, err := strconv.ParseBool(val)
	if err != nil {
		return errors.Wrap(err, "Error parsing boolean")
	}

	class := defaultStorageClassProvisioner
	if name == "storage-provisioner-gluster" {
		class = "glusterfile"
	}

	api, err := machine.NewAPIClient()
	if err != nil {
		return errors.Wrap(err, "machine client")
	}
	defer api.Close()

	cp, err := config.PrimaryControlPlane(cc)
	if err != nil {
		return errors.Wrap(err, "getting control plane")
	}
	if !machine.IsRunning(api, driver.MachineName(*cc, cp)) {
		glog.Warningf("%q is not running, writing %s=%v to disk and skipping enablement", driver.MachineName(*cc, cp), name, val)
		return enableOrDisableAddon(cc, name, val)
	}

	storagev1, err := storageclass.GetStoragev1(cc.Name)
	if err != nil {
		return errors.Wrapf(err, "Error getting storagev1 interface %v ", err)
	}

	if enable {
		// Only StorageClass for 'name' should be marked as default
		err = storageclass.SetDefaultStorageClass(storagev1, class)
		if err != nil {
			return errors.Wrapf(err, "Error making %s the default storage class", class)
		}
	} else {
		// Unset the StorageClass as default
		err := storageclass.DisableDefaultStorageClass(storagev1, class)
		if err != nil {
			return errors.Wrapf(err, "Error disabling %s as the default storage class", class)
		}
	}

	return enableOrDisableAddon(cc, name, val)
}

func validateIngress(cc *config.ClusterConfig, name string, val string) error {
	glog.Infof("Setting addon %s=%s in %q", name, val, cc.Name)
	enable, err := strconv.ParseBool(val)
	if err != nil {
		return errors.Wrapf(err, "parsing bool: %s", name)
	}
	if name == "ingress" && enable {
		client, err := kapi.Client(viper.GetString(config.ProfileName))
		if err != nil {
			return errors.Wrapf(err, "get kube-client to validate ingress addon: %s", name)
		} else {
			err = kapi.WaitForDeploymentToStabilize(client, "kube-system", "ingress-nginx-controller", time.Minute*3)
			if err != nil {
				return errors.Wrapf(err, "Failed verifying ingress addon deployment: %s", name)
			}
		}
	}
	return nil
}

// Start enables the default addons for a profile, plus any additional
func Start(wg *sync.WaitGroup, cc *config.ClusterConfig, toEnable map[string]bool, additional []string) {
	wg.Add(1)
	defer wg.Done()

	start := time.Now()
	glog.Infof("enableAddons start: toEnable=%v, additional=%s", toEnable, additional)
	defer func() {
		glog.Infof("enableAddons completed in %s", time.Since(start))
	}()

	// Get the default values of any addons not saved to our config
	for name, a := range assets.Addons {
		defaultVal := a.IsEnabled(cc)

		_, exists := toEnable[name]
		if !exists {
			toEnable[name] = defaultVal
		}
	}

	// Apply new addons
	for _, name := range additional {
		// replace heapster as metrics-server because heapster is deprecated
		if name == "heapster" {
			name = "metrics-server"
		}
		// if the specified addon doesn't exist, skip enabling
		_, e := isAddonValid(name)
		if e {
			toEnable[name] = true
		}
	}

	toEnableList := []string{}
	for k, v := range toEnable {
		if v {
			toEnableList = append(toEnableList, k)
		}
	}
	sort.Strings(toEnableList)

	var awg sync.WaitGroup

	defer func() { // making it show after verifications( not perfect till #7613 is closed)
		out.T(out.AddonEnable, "Enabled addons: {{.addons}}", out.V{"addons": strings.Join(toEnableList, ", ")})
	}()
	for _, a := range toEnableList {
		awg.Add(1)
		go func(name string) {
			err := RunCallbacks(cc, name, "true")
			if err != nil {
				out.WarningT("Enabling '{{.name}}' returned an error: {{.error}}", out.V{"name": name, "error": err})
			}
			awg.Done()
		}(a)
	}

	// Wait until all of the addons are enabled before updating the config (not thread safe)
	awg.Wait()
	for _, a := range toEnableList {
		if err := Set(cc, a, "true"); err != nil {
			glog.Errorf("store failed: %v", err)
		}
	}
}
<|endoftext|>


<commit_before>package charts

import "bytes"

type ChartType string

const (
	UnknownChart ChartType = "UnknownChart"
	SimpleBar    ChartType = "SimpleBar"
	SimpleLine   ChartType = "SimpleLine"
)

type DataType string

type DataTypes []DataType

const (
	UnknownType DataType = "UnknownType"
	Text        DataType = "Text"
	Number      DataType = "Number"
	Time        DataType = "Time"
)

var charts map[string]ChartType

func sequence(types DataTypes) string {
	var seq bytes.Buffer

	for _, t := range types {
		seq.WriteString(t.String())
	}

	return seq.String()
}

func Detect(types DataTypes) ChartType {
	if chart, ok := charts[sequence(types)]; ok {
		return chart
	}

	return UnknownChart
}

func (ct ChartType) String() string {
	return string(ct)
}

func (t DataType) String() string {
	return string(t)
}

func init() {
	charts = make(map[string]ChartType)

	charts[sequence(DataTypes{Text, Number})] = SimpleBar
	charts[sequence(DataTypes{Number, Number})] = SimpleBar
	charts[sequence(DataTypes{Time, Number})] = SimpleLine
}
<commit_msg>Revert "Use string as underlining data type"<commit_after>package charts

import "bytes"

type ChartType int

const (
	UnknownChart ChartType = iota
	SimpleBar
	SimpleLine
)

type DataType int

type DataTypes []DataType

const (
	UnknownType DataType = iota
	Text
	Number
	Time
)

var charts map[string]ChartType

func sequence(types DataTypes) string {
	var seq bytes.Buffer

	for _, t := range types {
		seq.WriteString(t.String())
	}

	return seq.String()
}

func Detect(types DataTypes) ChartType {
	if chart, ok := charts[sequence(types)]; ok {
		return chart
	}

	return UnknownChart
}

func (ct ChartType) String() string {
	switch ct {
	case SimpleBar:
		return "SimpleBar"
	case SimpleLine:
		return "SimpleLine"
	}

	return "UnknownChart"
}

func (t DataType) String() string {
	switch t {
	case Text:
		return "Text"
	case Number:
		return "Number"
	case Time:
		return "Time"
	}

	return "Unknown"
}

func init() {
	charts = make(map[string]ChartType)

	charts[sequence(DataTypes{Text, Number})] = SimpleBar
	charts[sequence(DataTypes{Number, Number})] = SimpleBar
	charts[sequence(DataTypes{Time, Number})] = SimpleLine
}
<|endoftext|>


<commit_before>package main

import (
	"../nsq"
	"../util"
	"../util/notify"
	"errors"
	"log"
)

type Channel struct {
	name                string
	addClientChan       chan util.ChanReq
	removeClientChan    chan util.ChanReq
	backend             nsq.BackendQueue
	incomingMessageChan chan *nsq.Message
	memoryMsgChan       chan *nsq.Message
	inFlightMessageChan chan *nsq.Message
	ClientMessageChan   chan *nsq.Message
	requeueMessageChan  chan util.ChanReq
	finishMessageChan   chan util.ChanReq
	inFlightMessages    map[string]*nsq.Message
}

// Channel constructor
func NewChannel(channelName string, inMemSize int, dataPath string) *Channel {
	channel := &Channel{name: channelName,
		addClientChan:       make(chan util.ChanReq),
		removeClientChan:    make(chan util.ChanReq),
		backend:             nsq.NewDiskQueue(channelName, dataPath),
		incomingMessageChan: make(chan *nsq.Message, 5),
		memoryMsgChan:       make(chan *nsq.Message, inMemSize),
		inFlightMessageChan: make(chan *nsq.Message),
		ClientMessageChan:   make(chan *nsq.Message),
		requeueMessageChan:  make(chan util.ChanReq),
		finishMessageChan:   make(chan util.ChanReq),
		inFlightMessages:    make(map[string]*nsq.Message)}
	go channel.Router()
	notify.Post("new_channel", channel)
	return channel
}

// PutMessage writes to the appropriate incoming
// message channel
func (c *Channel) PutMessage(msg *nsq.Message) {
	c.incomingMessageChan <- msg
}

func (c *Channel) FinishMessage(uuidStr string) error {
	errChan := make(chan interface{})
	c.finishMessageChan <- util.ChanReq{uuidStr, errChan}
	err, _ := (<-errChan).(error)
	return err
}

func (c *Channel) RequeueMessage(uuidStr string) error {
	errChan := make(chan interface{})
	c.requeueMessageChan <- util.ChanReq{uuidStr, errChan}
	err, _ := (<-errChan).(error)
	return err
}

// Router handles the muxing of Channel messages including
// the addition of a Client to the Channel
func (c *Channel) Router() {
	go c.RequeueRouter()
	go c.MessagePump()

	exitChan := make(chan interface{})
	notify.Observe(c.name+".channel_close", exitChan)
	for {
		select {
		case msg := <-c.incomingMessageChan:
			select {
			case c.memoryMsgChan <- msg:
				log.Printf("CHANNEL(%s): wrote to memoryMsgChan", c.name)
			default:
				data, err := msg.Encode()
				if err != nil {
					log.Printf("ERROR: failed to Encode() message - %s", err.Error())
					// TODO: shrug
					continue
				}
				err = c.backend.Put(data)
				if err != nil {
					log.Printf("ERROR: t.backend.Put() - %s", err.Error())
					// TODO: requeue?
				}
				log.Printf("CHANNEL(%s): wrote to backend", c.name)
			}
		case <-exitChan:
			notify.Ignore(c.name+".channel_close", exitChan)
			return
		}
	}
}

func (c *Channel) RequeueRouter() {
	exitChan := make(chan interface{})
	notify.Observe(c.name+".channel_close", exitChan)
	for {
		select {
		case msg := <-c.inFlightMessageChan:
			c.pushInFlightMessage(msg)
			go func(msg *nsq.Message) {
				if msg.ShouldRequeue(60000) {
					err := c.RequeueMessage(util.UuidToStr(msg.Uuid))
					if err != nil {
						log.Printf("ERROR: channel(%s) - %s", c.name, err.Error())
					}
				}
			}(msg)
		case requeueReq := <-c.requeueMessageChan:
			uuidStr := requeueReq.Variable.(string)
			msg, err := c.popInFlightMessage(uuidStr)
			if err != nil {
				log.Printf("ERROR: failed to requeue message(%s) - %s", uuidStr, err.Error())
			} else {
				go func(msg *nsq.Message) {
					c.PutMessage(msg)
				}(msg)
			}
			requeueReq.RetChan <- err
		case finishReq := <-c.finishMessageChan:
			uuidStr := finishReq.Variable.(string)
			_, err := c.popInFlightMessage(uuidStr)
			if err != nil {
				log.Printf("ERROR: failed to finish message(%s) - %s", uuidStr, err.Error())
			}
			finishReq.RetChan <- err
		case <-exitChan:
			notify.Ignore(c.name+".channel_close", exitChan)
			return
		}
	}
}

func (c *Channel) pushInFlightMessage(msg *nsq.Message) {
	uuidStr := util.UuidToStr(msg.Uuid)
	c.inFlightMessages[uuidStr] = msg
}

func (c *Channel) popInFlightMessage(uuidStr string) (*nsq.Message, error) {
	msg, ok := c.inFlightMessages[uuidStr]
	if !ok {
		return nil, errors.New("UUID not in flight")
	}
	delete(c.inFlightMessages, uuidStr)
	msg.EndTimer()
	return msg, nil
}

func (c *Channel) MessagePump() {
	var msg *nsq.Message

	exitChan := make(chan interface{})
	notify.Observe(c.name+".channel_close", exitChan)
	for {
		select {
		case msg = <-c.memoryMsgChan:
		case <-c.backend.ReadReadyChan():
			buf, err := c.backend.Get()
			if err != nil {
				log.Printf("ERROR: c.backend.Get() - %s", err.Error())
				continue
			}
			msg, err = nsq.DecodeMessage(buf)
			if err != nil {
				log.Printf("ERROR: failed to decode message - %s", err.Error())
				continue
			}
		case <-exitChan:
			notify.Ignore(c.name+".channel_close", exitChan)
			return
		}

		// TODO: not sure how msg would ever be nil here
		if msg != nil {
			msg.Retries += 1
			c.inFlightMessageChan <- msg
		}

		c.ClientMessageChan <- msg
	}
}

func (c *Channel) Close() error {
	var err error

	log.Printf("CHANNEL(%s): closing", c.name)

	notify.Post(c.name+".channel_close", nil)

	err = c.backend.Close()
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>remove old go channels on Channel that were used for client tracking<commit_after>package main

import (
	"../nsq"
	"../util"
	"../util/notify"
	"errors"
	"log"
)

type Channel struct {
	name                string
	backend             nsq.BackendQueue
	incomingMessageChan chan *nsq.Message
	memoryMsgChan       chan *nsq.Message
	inFlightMessageChan chan *nsq.Message
	ClientMessageChan   chan *nsq.Message
	requeueMessageChan  chan util.ChanReq
	finishMessageChan   chan util.ChanReq
	inFlightMessages    map[string]*nsq.Message
}

// Channel constructor
func NewChannel(channelName string, inMemSize int, dataPath string) *Channel {
	channel := &Channel{name: channelName,
		backend:             nsq.NewDiskQueue(channelName, dataPath),
		incomingMessageChan: make(chan *nsq.Message, 5),
		memoryMsgChan:       make(chan *nsq.Message, inMemSize),
		inFlightMessageChan: make(chan *nsq.Message),
		ClientMessageChan:   make(chan *nsq.Message),
		requeueMessageChan:  make(chan util.ChanReq),
		finishMessageChan:   make(chan util.ChanReq),
		inFlightMessages:    make(map[string]*nsq.Message)}
	go channel.Router()
	notify.Post("new_channel", channel)
	return channel
}

// PutMessage writes to the appropriate incoming
// message channel
func (c *Channel) PutMessage(msg *nsq.Message) {
	c.incomingMessageChan <- msg
}

func (c *Channel) FinishMessage(uuidStr string) error {
	errChan := make(chan interface{})
	c.finishMessageChan <- util.ChanReq{uuidStr, errChan}
	err, _ := (<-errChan).(error)
	return err
}

func (c *Channel) RequeueMessage(uuidStr string) error {
	errChan := make(chan interface{})
	c.requeueMessageChan <- util.ChanReq{uuidStr, errChan}
	err, _ := (<-errChan).(error)
	return err
}

// Router handles the muxing of Channel messages including
// the addition of a Client to the Channel
func (c *Channel) Router() {
	go c.RequeueRouter()
	go c.MessagePump()

	exitChan := make(chan interface{})
	notify.Observe(c.name+".channel_close", exitChan)
	for {
		select {
		case msg := <-c.incomingMessageChan:
			select {
			case c.memoryMsgChan <- msg:
				log.Printf("CHANNEL(%s): wrote to memoryMsgChan", c.name)
			default:
				data, err := msg.Encode()
				if err != nil {
					log.Printf("ERROR: failed to Encode() message - %s", err.Error())
					// TODO: shrug
					continue
				}
				err = c.backend.Put(data)
				if err != nil {
					log.Printf("ERROR: t.backend.Put() - %s", err.Error())
					// TODO: requeue?
				}
				log.Printf("CHANNEL(%s): wrote to backend", c.name)
			}
		case <-exitChan:
			notify.Ignore(c.name+".channel_close", exitChan)
			return
		}
	}
}

func (c *Channel) RequeueRouter() {
	exitChan := make(chan interface{})
	notify.Observe(c.name+".channel_close", exitChan)
	for {
		select {
		case msg := <-c.inFlightMessageChan:
			c.pushInFlightMessage(msg)
			go func(msg *nsq.Message) {
				if msg.ShouldRequeue(60000) {
					err := c.RequeueMessage(util.UuidToStr(msg.Uuid))
					if err != nil {
						log.Printf("ERROR: channel(%s) - %s", c.name, err.Error())
					}
				}
			}(msg)
		case requeueReq := <-c.requeueMessageChan:
			uuidStr := requeueReq.Variable.(string)
			msg, err := c.popInFlightMessage(uuidStr)
			if err != nil {
				log.Printf("ERROR: failed to requeue message(%s) - %s", uuidStr, err.Error())
			} else {
				go func(msg *nsq.Message) {
					c.PutMessage(msg)
				}(msg)
			}
			requeueReq.RetChan <- err
		case finishReq := <-c.finishMessageChan:
			uuidStr := finishReq.Variable.(string)
			_, err := c.popInFlightMessage(uuidStr)
			if err != nil {
				log.Printf("ERROR: failed to finish message(%s) - %s", uuidStr, err.Error())
			}
			finishReq.RetChan <- err
		case <-exitChan:
			notify.Ignore(c.name+".channel_close", exitChan)
			return
		}
	}
}

func (c *Channel) pushInFlightMessage(msg *nsq.Message) {
	uuidStr := util.UuidToStr(msg.Uuid)
	c.inFlightMessages[uuidStr] = msg
}

func (c *Channel) popInFlightMessage(uuidStr string) (*nsq.Message, error) {
	msg, ok := c.inFlightMessages[uuidStr]
	if !ok {
		return nil, errors.New("UUID not in flight")
	}
	delete(c.inFlightMessages, uuidStr)
	msg.EndTimer()
	return msg, nil
}

func (c *Channel) MessagePump() {
	var msg *nsq.Message

	exitChan := make(chan interface{})
	notify.Observe(c.name+".channel_close", exitChan)
	for {
		select {
		case msg = <-c.memoryMsgChan:
		case <-c.backend.ReadReadyChan():
			buf, err := c.backend.Get()
			if err != nil {
				log.Printf("ERROR: c.backend.Get() - %s", err.Error())
				continue
			}
			msg, err = nsq.DecodeMessage(buf)
			if err != nil {
				log.Printf("ERROR: failed to decode message - %s", err.Error())
				continue
			}
		case <-exitChan:
			notify.Ignore(c.name+".channel_close", exitChan)
			return
		}

		// TODO: not sure how msg would ever be nil here
		if msg != nil {
			msg.Retries += 1
			c.inFlightMessageChan <- msg
		}

		c.ClientMessageChan <- msg
	}
}

func (c *Channel) Close() error {
	var err error

	log.Printf("CHANNEL(%s): closing", c.name)

	notify.Post(c.name+".channel_close", nil)

	err = c.backend.Close()
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package cocoa

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

// BookmarkData represents the data structure holding the bookmark information
type BookmarkData struct {
	FileSystemType string
	Path           []string
	// CNIDPath in the case of an alias file is the offset to the path element (minus header size)
	CNIDPath            []uint64
	FileCreationDate    time.Time
	FileProperties      []byte
	TypeData            []byte // from 0xf022
	ContainingFolderIDX uint32
	VolumePath          string
	VolumeIsRoot        bool
	VolumeURL           string // file://' + volPath
	VolumeName          string
	VolumeSize          int64
	VolumeCreationDate  time.Time
	VolumeUUID          string // must be uppercase
	VolumeProperties    []byte
	CreationOptions     uint32 // 512
	WasFileReference    bool   // true
	UserName            string // unknown
	CNID                uint32
	UID                 uint32 // 99
	Filename            string
}

// TargetPath returns the full path to the current target url.
func (b *BookmarkData) TargetPath() string {
	return fmt.Sprintf("%s%s", b.VolumePath, filepath.Join(b.Path...))
}

// Write converts the bookmark data into binary data and writes it to the passed writer.
// Note that the writes are buffered and written all at once.
func (b *BookmarkData) Write(w io.Writer) error {
	// buffer for the body
	buf := &bytes.Buffer{}
	// track the offset within the body so we can build the TOC
	oMap := offsetMap{}

	oMap[KBookmarkCreationOptions] = buf.Len()
	buf.Write(encodedUint32(1024))

	slashPos := buf.Len()

	if !b.VolumeIsRoot {
		// HACK, not working for well
		buf.Write([]byte{'/', 0x0, 0x0, 0x0})
		buf.Write([]byte{0x0, 0xF0, 0x0, 0x0})
		buf.Write([]byte{0x48, 0x0, 0x0, 0x0})
	}

	// write each path items one by one
	pathOffsets := make([]int, len(b.Path))
	for i, item := range b.Path {
		// track the starting offset of each item (append 4 for the body size value)
		// since we need those to create an array for the TOC
		pathOffsets[i] = 4 + buf.Len()
		// get the offset of the last item in the path
		if i == len(b.Path)-1 {
			oMap[KBookmarkFullFileName] = pathOffsets[i] - 4
		}
		buf.Write(encodedStringItem(item))
	}
	padBuf(buf)

	// offset to the start of path offsets
	// the TOC will point to here so we can find how many items are in the array
	// and access each item to rebuild the path.
	// 0x04 0x10
	oMap[KBookmarkPath] = buf.Len()
	// number of items
	binary.Write(buf, binary.LittleEndian, uint32(len(b.Path)*4))
	binary.Write(buf, binary.LittleEndian, uint32(bmk_array|bmk_st_one))
	// offset from after the header for each item
	for _, offset := range pathOffsets {
		binary.Write(buf, binary.LittleEndian, uint32(offset))
	}
	padBuf(buf)

	// each file ids for the path
	cnidOffsets := make([]int, len(b.Path))
	for i, cnid := range b.CNIDPath {
		cnidOffsets[i] = 4 + buf.Len()
		buf.Write(encodedUint64(cnid))
	}

	// 0x05 0x10
	oMap[KBookmarkCNIDPath] = buf.Len()
	binary.Write(buf, binary.LittleEndian, uint32(len(b.CNIDPath)*4))
	binary.Write(buf, binary.LittleEndian, uint32(bmk_array|bmk_st_one))
	for _, offset := range cnidOffsets {
		binary.Write(buf, binary.LittleEndian, uint32(offset))
	}
	padBuf(buf)

	// file ID 0x30 0x10
	// if b.VolumeIsRoot {
	// 	oMap[KBookmarkFileID] = buf.Len()
	// 	buf.Write(encodedUint32(b.CNID))
	// 	padBuf(buf)
	// }

	// file properties
	// 0x10 0x10
	oMap[KBookmarkFileProperties] = buf.Len()
	buf.Write(encodedBytes(b.FileProperties))
	padBuf(buf)

	// KBookmarkFileCreationDate 0x04 0x10
	oMap[KBookmarkFileCreationDate] = buf.Len()
	buf.Write(encodedTime(b.FileCreationDate))
	padBuf(buf)

	// 0x54 0x10 unknown but seems to always be 1
	// 0x55 0x10 unknown, point to the same value
	oMap[KBookmarkUnknown] = buf.Len()
	oMap[KBookmarkUnknown1] = buf.Len()
	buf.Write(encodedUint32(uint32(1)))
	padBuf(buf)

	// 0x56 0x10 bool set to true
	oMap[KBookmarkUnknown2] = buf.Len()
	buf.Write(encodedBool(true))
	padBuf(buf)

	// KBookmarkTOCPath
	if !b.VolumeIsRoot {
		oMap[KBookmarkTOCPath] = buf.Len()
		// array of something
		// nbrBytesNeeded (nbr of elements * 4 bytes)

		offsets := []uint32{
			uint32(slashPos + 52),     // '/'
			uint32(slashPos + 52 + 4), // 00F00000
			uint32(slashPos + 52 + 8), // ?
		}
		for i := 0; i < len(b.Path)-2; i++ {
			offsets = append(offsets, uint32(slashPos+52+4))
		}

		binary.Write(buf, binary.LittleEndian, uint32(42))
		binary.Write(buf, binary.LittleEndian, uint32(bmk_array|bmk_st_one))
		for _, offs := range offsets {
			binary.Write(buf, binary.LittleEndian, uint32(offs))
		}

		padBuf(buf)
	}

	// KBookmarkVolumePath 0x02 0x20
	oMap[KBookmarkVolumePath] = buf.Len()
	buf.Write(encodedStringItem(b.VolumePath))
	padBuf(buf)

	// KBookmarkVolumeURL 0x05 0x20
	oMap[KBookmarkVolumeURL] = buf.Len()
	binary.Write(buf, binary.LittleEndian, uint32(len(b.VolumeURL)))
	// only support absolute path for now
	binary.Write(buf, binary.LittleEndian, uint32(bmk_url|bmk_url_st_absolute))
	buf.Write([]byte(b.VolumeURL))
	padBuf(buf)

	// KBookmarkVolumeName 0x10 0x20
	oMap[KBookmarkVolumeName] = buf.Len()
	buf.Write(encodedStringItem(b.VolumeName))
	padBuf(buf)

	// KBookmarkVolumeUUID 0x11 0x20
	oMap[KBookmarkVolumeUUID] = buf.Len()
	buf.Write(encodedStringItem(b.VolumeUUID))
	padBuf(buf)

	// KBookmarkVolumeSize 0x12 0x20
	oMap[KBookmarkVolumeSize] = buf.Len()
	buf.Write(encodedUint64(uint64(b.VolumeSize)))
	padBuf(buf)

	// KBookmarkVolumeCreationDate 0x13 0x20
	oMap[KBookmarkVolumeCreationDate] = buf.Len()
	buf.Write(encodedTime(b.VolumeCreationDate))
	padBuf(buf)

	// KBookmarkVolumeProperties 0x20 0x20
	oMap[KBookmarkVolumeProperties] = buf.Len()
	buf.Write(encodedBytes(b.VolumeProperties))
	padBuf(buf)

	// KBookmarkVolumeIsRoot 0x30 0x20
	if b.VolumeIsRoot {
		oMap[KBookmarkVolumeIsRoot] = buf.Len()
		buf.Write(encodedBool(b.VolumeIsRoot))
		padBuf(buf)
	}

	// KBookmarkContainingFolder 0x01 0xc0
	// TODO: only for root volumes?
	if b.VolumeIsRoot {
		oMap[KBookmarkContainingFolder] = buf.Len()
		buf.Write(encodedUint32(b.ContainingFolderIDX))
		padBuf(buf)
	}

	// KBookmarkUserName 0x11 0xc0
	if b.VolumeIsRoot {
		oMap[KBookmarkUserName] = buf.Len()
		buf.Write(encodedStringItem(b.UserName))
		padBuf(buf)
	}

	// KBookmarkUID 0x12 0xc0
	if b.VolumeIsRoot {
		oMap[KBookmarkUID] = buf.Len()
		buf.Write(encodedUint32(b.UID))
		padBuf(buf)
	}

	// KBookmarkWasFileReference 0x01 0xD0
	oMap[KBookmarkWasFileReference] = buf.Len()
	buf.Write(encodedBool(b.WasFileReference))
	padBuf(buf)

	// 0xf022 byte array 0x201
	oMap[KBookmarkFileType] = buf.Len()
	b.prepareTypeData()
	buf.Write(encodedBytes(b.TypeData))
	padBuf(buf)

	// buffer the header now that we have enough data
	hbuf := bytes.NewBufferString("book")
	hbuf.Write(make([]byte, 4))
	hbuf.Write([]byte("mark"))
	hbuf.Write(make([]byte, 4))
	// size of the header
	binary.Write(hbuf, binary.LittleEndian, uint32(56))
	// size of the header
	binary.Write(hbuf, binary.LittleEndian, uint32(56))

	// convert the toc in bytes so we can calculate offsets
	toc := oMap.Bytes()

	// total size minus the header
	binary.Write(hbuf, binary.LittleEndian, 4+uint32(buf.Len()+len(toc)))
	// magic
	hbuf.Write([]byte{0x00, 0x00, 0x04, 0x10, 0x0, 0x0, 0x0, 0x0})
	// TODO: figure out those byte since they seem to set the icon
	hbuf.Write(make([]byte, 20))
	// end of header

	// offset to the TOC  (size of the body)
	binary.Write(hbuf, binary.LittleEndian, 4+uint32(buf.Len()))
	// body
	hbuf.Write(buf.Bytes())
	// toc
	hbuf.Write(toc)

	_, err := w.Write(hbuf.Bytes())
	return err
}

func (b *BookmarkData) prepareTypeData() {
	buf := &bytes.Buffer{}
	buf.Write([]byte{
		0x64, 0x6E, 0x69, 0x62, 0x00, 0x00, 0x00, 0x00,
		0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00})
	buf.Write(make([]byte, 12))
	// file extension
	ext := filepath.Ext(b.TargetPath())
	if strings.HasPrefix(ext, ".") {
		ext = ext[1:]
	}
	fmt.Println(ext)
	binary.Write(buf, binary.LittleEndian, uint32(len(ext)))
	buf.Write(make([]byte, 4))
	buf.Write([]byte(ext))
	buf.Write(make([]byte, 12))
	b.TypeData = buf.Bytes()
}

func (b *BookmarkData) String() string {
	out := fmt.Sprintf("Bookmark:\nSource Path: %s\n", filepath.Join(b.Path...))
	out += fmt.Sprintf("CNID path: %v\n", b.CNIDPath)
	out += fmt.Sprintf("CNID: %v\n", b.CNID)
	out += fmt.Sprintf("File creation date: %v\n", b.FileCreationDate)
	out += fmt.Sprintf("File Properties: %#v\n", b.FileProperties)
	return out
}

type offsetMap map[uint32]int

func (oMap offsetMap) Bytes() []byte {
	buf := &bytes.Buffer{}
	// Size of TOC in bytes, minus 8
	binary.Write(buf, binary.LittleEndian, uint32(3*4+len(oMap)*(3*4)))
	// magic number
	buf.Write([]byte{0xFE, 0xFF, 0xFF, 0xFF})
	// identifier
	binary.Write(buf, binary.LittleEndian, uint32(1))
	// Next TOC offset (or 0 if none)
	binary.Write(buf, binary.LittleEndian, uint32(0))
	// Number of entries in this TOC
	binary.Write(buf, binary.LittleEndian, uint32(len(oMap)))

	// sort keys
	keys := make([]int, len(oMap))
	i := 0
	for k := range oMap {
		keys[i] = int(k)
		i++
	}
	sort.Ints(keys)

	for _, k := range keys {
		// key
		binary.Write(buf, binary.LittleEndian, uint32(k))
		// offset
		binary.Write(buf, binary.LittleEndian, uint32(oMap[uint32(k)])+4)
		// reserved
		binary.Write(buf, binary.LittleEndian, uint32(0))
	}

	return buf.Bytes()
}
<commit_msg>trying to copy the binary order<commit_after>package cocoa

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"path/filepath"
	"sort"
	"strings"
	"time"
)

// BookmarkData represents the data structure holding the bookmark information
type BookmarkData struct {
	FileSystemType string
	Path           []string
	// CNIDPath in the case of an alias file is the offset to the path element (minus header size)
	CNIDPath            []uint64
	FileCreationDate    time.Time
	FileProperties      []byte
	TypeData            []byte // from 0xf022
	ContainingFolderIDX uint32
	VolumePath          string
	VolumeIsRoot        bool
	VolumeURL           string // file://' + volPath
	VolumeName          string
	VolumeSize          int64
	VolumeCreationDate  time.Time
	VolumeUUID          string // must be uppercase
	VolumeProperties    []byte
	CreationOptions     uint32 // 512
	WasFileReference    bool   // true
	UserName            string // unknown
	CNID                uint32
	UID                 uint32 // 99
	Filename            string
}

// TargetPath returns the full path to the current target url.
func (b *BookmarkData) TargetPath() string {
	return fmt.Sprintf("%s%s", b.VolumePath, filepath.Join(b.Path...))
}

// Write converts the bookmark data into binary data and writes it to the passed writer.
// Note that the writes are buffered and written all at once.
func (b *BookmarkData) Write(w io.Writer) error {
	// buffer for the body
	buf := &bytes.Buffer{}
	// track the offset within the body so we can build the TOC
	oMap := offsetMap{}

	oMap[KBookmarkCreationOptions] = buf.Len()
	buf.Write(encodedUint32(1024))

	slashPos := buf.Len()

	if !b.VolumeIsRoot {
		// HACK, not working for well
		buf.Write([]byte{'/', 0x0, 0x0, 0x0})
		buf.Write([]byte{0x0, 0xF0, 0x0, 0x0})
		buf.Write([]byte{0x48, 0x0, 0x0, 0x0})
	}

	// write each path items one by one
	pathOffsets := make([]int, len(b.Path))
	for i, item := range b.Path {
		// track the starting offset of each item (append 4 for the body size value)
		// since we need those to create an array for the TOC
		pathOffsets[i] = 4 + buf.Len()
		// get the offset of the last item in the path
		if i == len(b.Path)-1 {
			oMap[KBookmarkFullFileName] = pathOffsets[i] - 4
		}
		buf.Write(encodedStringItem(item))
	}
	padBuf(buf)

	// offset to the start of path offsets
	// the TOC will point to here so we can find how many items are in the array
	// and access each item to rebuild the path.
	// 0x04 0x10
	oMap[KBookmarkPath] = buf.Len()
	// number of items
	binary.Write(buf, binary.LittleEndian, uint32(len(b.Path)*4))
	binary.Write(buf, binary.LittleEndian, uint32(bmk_array|bmk_st_one))
	// offset from after the header for each item
	for _, offset := range pathOffsets {
		binary.Write(buf, binary.LittleEndian, uint32(offset))
	}
	padBuf(buf)

	// each file ids for the path
	cnidOffsets := make([]int, len(b.Path))
	for i, cnid := range b.CNIDPath {
		cnidOffsets[i] = 4 + buf.Len()
		buf.Write(encodedUint64(cnid))
	}

	// 0x05 0x10
	oMap[KBookmarkCNIDPath] = buf.Len()
	binary.Write(buf, binary.LittleEndian, uint32(len(b.CNIDPath)*4))
	binary.Write(buf, binary.LittleEndian, uint32(bmk_array|bmk_st_one))
	for _, offset := range cnidOffsets {
		binary.Write(buf, binary.LittleEndian, uint32(offset))
	}
	padBuf(buf)

	// KBookmarkFileCreationDate 0x04 0x10
	oMap[KBookmarkFileCreationDate] = buf.Len()
	buf.Write(encodedTime(b.FileCreationDate))
	padBuf(buf)

	// file ID 0x30 0x10
	// if b.VolumeIsRoot {
	// 	oMap[KBookmarkFileID] = buf.Len()
	// 	buf.Write(encodedUint32(b.CNID))
	// 	padBuf(buf)
	// }

	// file properties
	// 0x10 0x10
	oMap[KBookmarkFileProperties] = buf.Len()
	buf.Write(encodedBytes(b.FileProperties))
	padBuf(buf)

	// KBookmarkWasFileReference 0x01 0xD0
	oMap[KBookmarkWasFileReference] = buf.Len()
	buf.Write(encodedBool(b.WasFileReference))
	padBuf(buf)

	// 0x54 0x10 unknown but seems to always be 1
	// 0x55 0x10 unknown, point to the same value
	oMap[KBookmarkUnknown] = buf.Len()
	oMap[KBookmarkUnknown1] = buf.Len()
	buf.Write(encodedUint32(uint32(1)))
	padBuf(buf)

	// KBookmarkContainingFolder 0x01 0xc0
	// TODO: only for root volumes?
	if b.VolumeIsRoot {
		oMap[KBookmarkContainingFolder] = buf.Len()
		buf.Write(encodedUint64(uint64(b.ContainingFolderIDX)))
		padBuf(buf)
	}

	// 0x56 0x10 bool set to true
	oMap[KBookmarkUnknown2] = buf.Len()
	buf.Write(encodedBool(true))
	padBuf(buf)

	// KBookmarkTOCPath
	if !b.VolumeIsRoot {
		oMap[KBookmarkTOCPath] = buf.Len()
		// array of something
		// nbrBytesNeeded (nbr of elements * 4 bytes)

		offsets := []uint32{
			uint32(slashPos + 52),     // '/'
			uint32(slashPos + 52 + 4), // 00F00000
			uint32(slashPos + 52 + 8), // ?
		}
		for i := 0; i < len(b.Path)-2; i++ {
			offsets = append(offsets, uint32(slashPos+52+4))
		}

		binary.Write(buf, binary.LittleEndian, uint32(42))
		binary.Write(buf, binary.LittleEndian, uint32(bmk_array|bmk_st_one))
		for _, offs := range offsets {
			binary.Write(buf, binary.LittleEndian, uint32(offs))
		}

		padBuf(buf)
	}

	// KBookmarkVolumePath 0x02 0x20
	oMap[KBookmarkVolumePath] = buf.Len()
	buf.Write(encodedStringItem(b.VolumePath))
	padBuf(buf)

	// KBookmarkVolumeURL 0x05 0x20
	oMap[KBookmarkVolumeURL] = buf.Len()
	binary.Write(buf, binary.LittleEndian, uint32(len(b.VolumeURL)))
	// only support absolute path for now
	binary.Write(buf, binary.LittleEndian, uint32(bmk_url|bmk_url_st_absolute))
	buf.Write([]byte(b.VolumeURL))
	padBuf(buf)

	// KBookmarkVolumeName 0x10 0x20
	oMap[KBookmarkVolumeName] = buf.Len()
	buf.Write(encodedStringItem(b.VolumeName))
	padBuf(buf)

	// KBookmarkVolumeUUID 0x11 0x20
	oMap[KBookmarkVolumeUUID] = buf.Len()
	buf.Write(encodedStringItem(b.VolumeUUID))
	padBuf(buf)

	// KBookmarkVolumeSize 0x12 0x20
	oMap[KBookmarkVolumeSize] = buf.Len()
	buf.Write(encodedUint64(uint64(b.VolumeSize)))
	padBuf(buf)

	// KBookmarkVolumeCreationDate 0x13 0x20
	oMap[KBookmarkVolumeCreationDate] = buf.Len()
	buf.Write(encodedTime(b.VolumeCreationDate))
	padBuf(buf)

	// KBookmarkVolumeProperties 0x20 0x20
	oMap[KBookmarkVolumeProperties] = buf.Len()
	buf.Write(encodedBytes(b.VolumeProperties))
	padBuf(buf)

	// KBookmarkVolumeIsRoot 0x30 0x20
	if b.VolumeIsRoot {
		oMap[KBookmarkVolumeIsRoot] = buf.Len()
		buf.Write(encodedBool(b.VolumeIsRoot))
		padBuf(buf)
	}

	// KBookmarkUserName 0x11 0xc0
	if b.VolumeIsRoot {
		oMap[KBookmarkUserName] = buf.Len()
		buf.Write(encodedStringItem(b.UserName))
		padBuf(buf)
	}

	// KBookmarkUID 0x12 0xc0
	if b.VolumeIsRoot {
		oMap[KBookmarkUID] = buf.Len()
		buf.Write(encodedUint32(b.UID))
		padBuf(buf)
	}

	// 0xf022 byte array 0x201
	oMap[KBookmarkFileType] = buf.Len()
	b.prepareTypeData()
	buf.Write(encodedBytes(b.TypeData))
	padBuf(buf)

	// buffer the header now that we have enough data
	hbuf := bytes.NewBufferString("book")
	hbuf.Write(make([]byte, 4))
	hbuf.Write([]byte("mark"))
	hbuf.Write(make([]byte, 4))
	// size of the header
	binary.Write(hbuf, binary.LittleEndian, uint32(56))
	// size of the header
	binary.Write(hbuf, binary.LittleEndian, uint32(56))

	// convert the toc in bytes so we can calculate offsets
	toc := oMap.Bytes()

	// total size minus the header
	binary.Write(hbuf, binary.LittleEndian, 4+uint32(buf.Len()+len(toc)))
	// magic
	hbuf.Write([]byte{0x00, 0x00, 0x04, 0x10, 0x0, 0x0, 0x0, 0x0})
	// TODO: figure out those byte since they seem to set the icon
	hbuf.Write(make([]byte, 20))
	// end of header

	// offset to the TOC  (size of the body)
	binary.Write(hbuf, binary.LittleEndian, 4+uint32(buf.Len()))
	// body
	hbuf.Write(buf.Bytes())
	// toc
	hbuf.Write(toc)

	_, err := w.Write(hbuf.Bytes())
	return err
}

func (b *BookmarkData) prepareTypeData() {
	buf := &bytes.Buffer{}
	buf.Write([]byte{
		0x64, 0x6E, 0x69, 0x62, 0x00, 0x00, 0x00, 0x00,
		0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00})
	buf.Write(make([]byte, 12))
	// file extension
	ext := filepath.Ext(b.TargetPath())
	if strings.HasPrefix(ext, ".") {
		ext = ext[1:]
	}
	binary.Write(buf, binary.LittleEndian, uint32(len(ext)))
	buf.Write(make([]byte, 4))
	buf.Write([]byte(ext))
	buf.Write(make([]byte, 12))
	b.TypeData = buf.Bytes()
}

func (b *BookmarkData) String() string {
	out := fmt.Sprintf("Bookmark:\nSource Path: %s\n", filepath.Join(b.Path...))
	out += fmt.Sprintf("CNID path: %v\n", b.CNIDPath)
	out += fmt.Sprintf("CNID: %v\n", b.CNID)
	out += fmt.Sprintf("File creation date: %v\n", b.FileCreationDate)
	out += fmt.Sprintf("File Properties: %#v\n", b.FileProperties)
	return out
}

type offsetMap map[uint32]int

func (oMap offsetMap) Bytes() []byte {
	buf := &bytes.Buffer{}
	// Size of TOC in bytes, minus 8
	binary.Write(buf, binary.LittleEndian, uint32(3*4+len(oMap)*(3*4)))
	// magic number
	buf.Write([]byte{0xFE, 0xFF, 0xFF, 0xFF})
	// identifier
	binary.Write(buf, binary.LittleEndian, uint32(1))
	// Next TOC offset (or 0 if none)
	binary.Write(buf, binary.LittleEndian, uint32(0))
	// Number of entries in this TOC
	binary.Write(buf, binary.LittleEndian, uint32(len(oMap)))

	// sort keys
	keys := make([]int, len(oMap))
	i := 0
	for k := range oMap {
		keys[i] = int(k)
		i++
	}
	sort.Ints(keys)

	for _, k := range keys {
		// key
		binary.Write(buf, binary.LittleEndian, uint32(k))
		// offset
		binary.Write(buf, binary.LittleEndian, uint32(oMap[uint32(k)])+4)
		// reserved
		binary.Write(buf, binary.LittleEndian, uint32(0))
	}

	return buf.Bytes()
}
<|endoftext|>


<commit_before>package CloudForest

import (
	"fmt"
	"log"
	"math"
	"strconv"
)

//Keeps track of votes by trees.
//Voteing is thread safe.
type NumBallotBox struct {
	box []*RunningMean
}

//Build a new ballot box for the number of cases specified by "size".
func NewNumBallotBox(size int) *NumBallotBox {
	bb := NumBallotBox{
		make([]*RunningMean, 0, size)}
	for i := 0; i < size; i++ {
		bb.box = append(bb.box, new(RunningMean))
	}
	return &bb
}

//Vote parses the float in the string and votes for it
func (bb *NumBallotBox) Vote(casei int, pred string, weight float64) {
	v, err := strconv.ParseFloat(pred, 64)
	if err == nil {
		bb.box[casei].WeightedAdd(v, weight)
	}

}

//TallyNumerical tallies the votes for the case specified by i as
//if it is a Numerical feature. Ie it returns the mean of all votes.
func (bb *NumBallotBox) TallyNum(i int) (predicted float64) {
	predicted, _ = bb.box[i].Read()
	return
}

func (bb *NumBallotBox) Tally(i int) (predicted string) {
	mean, count := bb.box[i].Read()
	if count > 0 {
		predicted = fmt.Sprintf("%v", mean)
	} else {
		predicted = "NA"
	}
	return
}

//Tally error returns the error of the votes vs the provided feature.
//For categorical features it returns the error rate
//For numerical features it returns mean squared error.
//The provided feature must use the same index as the feature matrix
//the ballot box was constructed with.
//Missing values are ignored.
//Gini impurity is not used so this is not for use in rf implementations.
func (bb *NumBallotBox) TallyError(feature Feature) (e float64) {
	e = 0.0

	// Numerical feature. Calculate mean squared
	d := 0.0
	c := 0
	for i := 0; i < feature.Length(); i++ {
		predicted := bb.TallyNum(i)
		if !feature.IsMissing(i) && !math.IsNaN(predicted) {
			value := feature.(NumFeature).Get(i)

			d = float64(value) - predicted
			e += d * d
			c += 1
		}
	}
	if c == 0.0 {
		log.Fatal("TallyError with 0 count!")
	}
	e = e / float64(c)

	return

}

//Tally score returns the R2 score
func (bb *NumBallotBox) TallyR2Score(feature Feature) (e float64) {
	mean := 0.0
	r2 := 0.0
	total := 0
	for i := 0; i < feature.Length(); i++ {
		if !feature.IsMissing(i) {
			mean += feature.(*DenseNumFeature).Get(i)
			total++
		}
	}
	mean /= float64(total)

	for i := 0; i < feature.Length(); i++ {
		if !feature.IsMissing(i) {
			value := feature.(NumFeature).Get(i)

			d := float64(value) - mean
			r2 += d * d
		}
	}
	r2 /= float64(total)

	e = 1 - bb.TallyError(feature)/r2

	return

}
<commit_msg>Normalize error via variance<commit_after>package CloudForest

import (
	"fmt"
	"log"
	"math"
	"strconv"
)

//Keeps track of votes by trees.
//Voteing is thread safe.
type NumBallotBox struct {
	box []*RunningMean
}

//Build a new ballot box for the number of cases specified by "size".
func NewNumBallotBox(size int) *NumBallotBox {
	bb := NumBallotBox{
		make([]*RunningMean, 0, size)}
	for i := 0; i < size; i++ {
		bb.box = append(bb.box, new(RunningMean))
	}
	return &bb
}

//Vote parses the float in the string and votes for it
func (bb *NumBallotBox) Vote(casei int, pred string, weight float64) {
	v, err := strconv.ParseFloat(pred, 64)
	if err == nil {
		bb.box[casei].WeightedAdd(v, weight)
	}

}

//TallyNumerical tallies the votes for the case specified by i as
//if it is a Numerical feature. Ie it returns the mean of all votes.
func (bb *NumBallotBox) TallyNum(i int) (predicted float64) {
	predicted, _ = bb.box[i].Read()
	return
}

func (bb *NumBallotBox) Tally(i int) (predicted string) {
	mean, count := bb.box[i].Read()
	if count > 0 {
		predicted = fmt.Sprintf("%v", mean)
	} else {
		predicted = "NA"
	}
	return
}

//TallySquareError returns the error of the votes vs the provided feature.
//For categorical features it returns the error rate
//For numerical features it returns mean squared error.
//The provided feature must use the same index as the feature matrix
//the ballot box was constructed with.
//Missing values are ignored.
//Gini impurity is not used so this is not for use in rf implementations.
func (bb *NumBallotBox) TallySquaredError(feature Feature) (e float64) {
	e = 0.0

	// Numerical feature. Calculate mean squared
	d := 0.0
	c := 0
	for i := 0; i < feature.Length(); i++ {
		predicted := bb.TallyNum(i)
		if !feature.IsMissing(i) && !math.IsNaN(predicted) {
			value := feature.(NumFeature).Get(i)

			d = float64(value) - predicted
			e += d * d
			c += 1
		}
	}
	if c == 0.0 {
		log.Fatal("TallyError with 0 count!")
	}
	e = e / float64(c)

	return

}

//TallyScore returns the squared error (unexplained variance) divided by the data variance.
func (bb *NumBallotBox) TallyError(feature Feature) (e float64) {
	mean := 0.0
	r2 := 0.0
	total := 0
	for i := 0; i < feature.Length(); i++ {
		if !feature.IsMissing(i) {
			mean += feature.(*DenseNumFeature).Get(i)
			total++
		}
	}
	mean /= float64(total)

	for i := 0; i < feature.Length(); i++ {
		if !feature.IsMissing(i) {
			value := feature.(NumFeature).Get(i)

			d := float64(value) - mean
			r2 += d * d
		}
	}
	r2 /= float64(total)

	e = bb.TallySquaredError(feature) / r2

	return

}

//Tally score returns the R2 score or coefichent of determination.
func (bb *NumBallotBox) TallyR2Score(feature Feature) (e float64) {

	e = 1 - bb.TallyError(feature)

	return

}
<|endoftext|>


<commit_before>// +build linux

package shared

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"syscall"
	"unsafe"

	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/shared/units"
)

// --- pure Go functions ---

func Major(dev uint64) int {
	return int(((dev >> 8) & 0xfff) | ((dev >> 32) & (0xfffff000)))
}

func Minor(dev uint64) int {
	return int((dev & 0xff) | ((dev >> 12) & (0xffffff00)))
}

func GetFileStat(p string) (uid int, gid int, major int, minor int,
	inode uint64, nlink int, err error) {
	var stat unix.Stat_t
	err = unix.Lstat(p, &stat)
	if err != nil {
		return
	}
	uid = int(stat.Uid)
	gid = int(stat.Gid)
	inode = uint64(stat.Ino)
	nlink = int(stat.Nlink)
	major = -1
	minor = -1
	if stat.Mode&unix.S_IFBLK != 0 || stat.Mode&unix.S_IFCHR != 0 {
		major = Major(stat.Rdev)
		minor = Minor(stat.Rdev)
	}

	return
}

// FileCopy copies a file, overwriting the target if it exists.
func GetPathMode(path string) (os.FileMode, error) {
	s, err := os.Open(path)
	if err != nil {
		return os.FileMode(0000), err
	}
	defer s.Close()

	fi, err := s.Stat()
	if err != nil {
		return os.FileMode(0000), err
	}

	mode, _, _ := GetOwnerMode(fi)
	return mode, nil
}

func parseMountinfo(name string) int {
	// In case someone uses symlinks we need to look for the actual
	// mountpoint.
	actualPath, err := filepath.EvalSymlinks(name)
	if err != nil {
		return -1
	}

	f, err := os.Open("/proc/self/mountinfo")
	if err != nil {
		return -1
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := scanner.Text()
		tokens := strings.Fields(line)
		if len(tokens) < 5 {
			return -1
		}
		cleanPath := filepath.Clean(tokens[4])
		if cleanPath == actualPath {
			return 1
		}
	}

	return 0
}

func IsMountPoint(name string) bool {
	ret := parseMountinfo(name)
	if ret == 1 {
		return true
	}

	stat, err := os.Stat(name)
	if err != nil {
		return false
	}

	rootStat, err := os.Lstat(name + "/..")
	if err != nil {
		return false
	}
	// If the directory has the same device as parent, then it's not a mountpoint.
	return stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev
}

func SetSize(fd int, width int, height int) (err error) {
	var dimensions [4]uint16
	dimensions[0] = uint16(height)
	dimensions[1] = uint16(width)

	if _, _, err := unix.Syscall6(unix.SYS_IOCTL, uintptr(fd), uintptr(unix.TIOCSWINSZ), uintptr(unsafe.Pointer(&dimensions)), 0, 0, 0); err != 0 {
		return err
	}
	return nil
}

// This uses ssize_t llistxattr(const char *path, char *list, size_t size); to
// handle symbolic links (should it in the future be possible to set extended
// attributed on symlinks): If path is a symbolic link the extended attributes
// associated with the link itself are retrieved.
func llistxattr(path string, list []byte) (sz int, err error) {
	var _p0 *byte
	_p0, err = unix.BytePtrFromString(path)
	if err != nil {
		return
	}
	var _p1 unsafe.Pointer
	if len(list) > 0 {
		_p1 = unsafe.Pointer(&list[0])
	} else {
		_p1 = unsafe.Pointer(nil)
	}
	r0, _, e1 := unix.Syscall(unix.SYS_LLISTXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(list)))
	sz = int(r0)
	if e1 != 0 {
		err = e1
	}
	return
}

// GetAllXattr retrieves all extended attributes associated with a file,
// directory or symbolic link.
func GetAllXattr(path string) (xattrs map[string]string, err error) {
	e1 := fmt.Errorf("Extended attributes changed during retrieval")

	// Call llistxattr() twice: First, to determine the size of the buffer
	// we need to allocate to store the extended attributes, second, to
	// actually store the extended attributes in the buffer. Also, check if
	// the size/number of extended attributes hasn't changed between the two
	// calls.
	pre, err := llistxattr(path, nil)
	if err != nil || pre < 0 {
		return nil, err
	}
	if pre == 0 {
		return nil, nil
	}

	dest := make([]byte, pre)

	post, err := llistxattr(path, dest)
	if err != nil || post < 0 {
		return nil, err
	}
	if post != pre {
		return nil, e1
	}

	split := strings.Split(string(dest), "\x00")
	if split == nil {
		return nil, fmt.Errorf("No valid extended attribute key found")
	}
	// *listxattr functions return a list of  names  as  an unordered array
	// of null-terminated character strings (attribute names are separated
	// by null bytes ('\0')), like this: user.name1\0system.name1\0user.name2\0
	// Since we split at the '\0'-byte the last element of the slice will be
	// the empty string. We remove it:
	if split[len(split)-1] == "" {
		split = split[:len(split)-1]
	}

	xattrs = make(map[string]string, len(split))

	for _, x := range split {
		xattr := string(x)
		// Call Getxattr() twice: First, to determine the size of the
		// buffer we need to allocate to store the extended attributes,
		// second, to actually store the extended attributes in the
		// buffer. Also, check if the size of the extended attribute
		// hasn't changed between the two calls.
		pre, err = unix.Getxattr(path, xattr, nil)
		if err != nil || pre < 0 {
			return nil, err
		}

		dest = make([]byte, pre)
		post := 0
		if pre > 0 {
			post, err = unix.Getxattr(path, xattr, dest)
			if err != nil || post < 0 {
				return nil, err
			}
		}

		if post != pre {
			return nil, e1
		}

		xattrs[xattr] = string(dest)
	}

	return xattrs, nil
}

var ObjectFound = fmt.Errorf("Found requested object")

func LookupUUIDByBlockDevPath(diskDevice string) (string, error) {
	uuid := ""
	readUUID := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if (info.Mode() & os.ModeSymlink) == os.ModeSymlink {
			link, err := os.Readlink(path)
			if err != nil {
				return err
			}

			// filepath.Join() will call Clean() on the result and
			// thus resolve those ugly "../../" parts that make it
			// hard to compare the strings.
			absPath := filepath.Join("/dev/disk/by-uuid", link)
			if absPath == diskDevice {
				uuid = path
				// Will allows us to avoid needlessly travers
				// the whole directory.
				return ObjectFound
			}
		}
		return nil
	}

	err := filepath.Walk("/dev/disk/by-uuid", readUUID)
	if err != nil && err != ObjectFound {
		return "", fmt.Errorf("Failed to detect UUID: %s", err)
	}

	if uuid == "" {
		return "", fmt.Errorf("Failed to detect UUID")
	}

	lastSlash := strings.LastIndex(uuid, "/")
	return uuid[lastSlash+1:], nil
}

// Detect whether err is an errno.
func GetErrno(err error) (errno error, iserrno bool) {
	sysErr, ok := err.(*os.SyscallError)
	if ok {
		return sysErr.Err, true
	}

	pathErr, ok := err.(*os.PathError)
	if ok {
		return pathErr.Err, true
	}

	tmpErrno, ok := err.(unix.Errno)
	if ok {
		return tmpErrno, true
	}

	return nil, false
}

// Utsname returns the same info as unix.Utsname, as strings
type Utsname struct {
	Sysname    string
	Nodename   string
	Release    string
	Version    string
	Machine    string
	Domainname string
}

// Uname returns Utsname as strings
func Uname() (*Utsname, error) {
	/*
	 * Based on: https://groups.google.com/forum/#!topic/golang-nuts/Jel8Bb-YwX8
	 * there is really no better way to do this, which is
	 * unfortunate. Also, we ditch the more accepted CharsToString
	 * version in that thread, since it doesn't seem as portable,
	 * viz. github issue #206.
	 */

	uname := unix.Utsname{}
	err := unix.Uname(&uname)
	if err != nil {
		return nil, err
	}

	return &Utsname{
		Sysname:    intArrayToString(uname.Sysname),
		Nodename:   intArrayToString(uname.Nodename),
		Release:    intArrayToString(uname.Release),
		Version:    intArrayToString(uname.Version),
		Machine:    intArrayToString(uname.Machine),
		Domainname: intArrayToString(uname.Domainname),
	}, nil
}

func intArrayToString(arr interface{}) string {
	slice := reflect.ValueOf(arr)
	s := ""
	for i := 0; i < slice.Len(); i++ {
		val := slice.Index(i)
		valInt := int64(-1)

		switch val.Kind() {
		case reflect.Int:
		case reflect.Int8:
			valInt = int64(val.Int())
		case reflect.Uint:
		case reflect.Uint8:
			valInt = int64(val.Uint())
		default:
			continue
		}

		if valInt == 0 {
			break
		}

		s += string(byte(valInt))
	}

	return s
}

func Statvfs(path string) (*unix.Statfs_t, error) {
	var st unix.Statfs_t

	err := unix.Statfs(path, &st)
	if err != nil {
		return nil, err
	}

	return &st, nil
}

func DeviceTotalMemory() (int64, error) {
	// Open /proc/meminfo
	f, err := os.Open("/proc/meminfo")
	if err != nil {
		return -1, err
	}
	defer f.Close()

	// Read it line by line
	scan := bufio.NewScanner(f)
	for scan.Scan() {
		line := scan.Text()

		// We only care about MemTotal
		if !strings.HasPrefix(line, "MemTotal:") {
			continue
		}

		// Extract the before last (value) and last (unit) fields
		fields := strings.Split(line, " ")
		value := fields[len(fields)-2] + fields[len(fields)-1]

		// Feed the result to units.ParseByteSizeString to get an int value
		valueBytes, err := units.ParseByteSizeString(value)
		if err != nil {
			return -1, err
		}

		return valueBytes, nil
	}

	return -1, fmt.Errorf("Couldn't find MemTotal")
}
<commit_msg>shared: Don't open files to get their mode<commit_after>// +build linux

package shared

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"syscall"
	"unsafe"

	"golang.org/x/sys/unix"

	"github.com/lxc/lxd/shared/units"
)

// --- pure Go functions ---

func Major(dev uint64) int {
	return int(((dev >> 8) & 0xfff) | ((dev >> 32) & (0xfffff000)))
}

func Minor(dev uint64) int {
	return int((dev & 0xff) | ((dev >> 12) & (0xffffff00)))
}

func GetFileStat(p string) (uid int, gid int, major int, minor int,
	inode uint64, nlink int, err error) {
	var stat unix.Stat_t
	err = unix.Lstat(p, &stat)
	if err != nil {
		return
	}
	uid = int(stat.Uid)
	gid = int(stat.Gid)
	inode = uint64(stat.Ino)
	nlink = int(stat.Nlink)
	major = -1
	minor = -1
	if stat.Mode&unix.S_IFBLK != 0 || stat.Mode&unix.S_IFCHR != 0 {
		major = Major(stat.Rdev)
		minor = Minor(stat.Rdev)
	}

	return
}

// GetPathMode returns a os.FileMode for the provided path
func GetPathMode(path string) (os.FileMode, error) {
	fi, err := os.Stat(path)
	if err != nil {
		return os.FileMode(0000), err
	}

	mode, _, _ := GetOwnerMode(fi)
	return mode, nil
}

func parseMountinfo(name string) int {
	// In case someone uses symlinks we need to look for the actual
	// mountpoint.
	actualPath, err := filepath.EvalSymlinks(name)
	if err != nil {
		return -1
	}

	f, err := os.Open("/proc/self/mountinfo")
	if err != nil {
		return -1
	}
	defer f.Close()

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := scanner.Text()
		tokens := strings.Fields(line)
		if len(tokens) < 5 {
			return -1
		}
		cleanPath := filepath.Clean(tokens[4])
		if cleanPath == actualPath {
			return 1
		}
	}

	return 0
}

func IsMountPoint(name string) bool {
	ret := parseMountinfo(name)
	if ret == 1 {
		return true
	}

	stat, err := os.Stat(name)
	if err != nil {
		return false
	}

	rootStat, err := os.Lstat(name + "/..")
	if err != nil {
		return false
	}
	// If the directory has the same device as parent, then it's not a mountpoint.
	return stat.Sys().(*syscall.Stat_t).Dev != rootStat.Sys().(*syscall.Stat_t).Dev
}

func SetSize(fd int, width int, height int) (err error) {
	var dimensions [4]uint16
	dimensions[0] = uint16(height)
	dimensions[1] = uint16(width)

	if _, _, err := unix.Syscall6(unix.SYS_IOCTL, uintptr(fd), uintptr(unix.TIOCSWINSZ), uintptr(unsafe.Pointer(&dimensions)), 0, 0, 0); err != 0 {
		return err
	}
	return nil
}

// This uses ssize_t llistxattr(const char *path, char *list, size_t size); to
// handle symbolic links (should it in the future be possible to set extended
// attributed on symlinks): If path is a symbolic link the extended attributes
// associated with the link itself are retrieved.
func llistxattr(path string, list []byte) (sz int, err error) {
	var _p0 *byte
	_p0, err = unix.BytePtrFromString(path)
	if err != nil {
		return
	}
	var _p1 unsafe.Pointer
	if len(list) > 0 {
		_p1 = unsafe.Pointer(&list[0])
	} else {
		_p1 = unsafe.Pointer(nil)
	}
	r0, _, e1 := unix.Syscall(unix.SYS_LLISTXATTR, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(list)))
	sz = int(r0)
	if e1 != 0 {
		err = e1
	}
	return
}

// GetAllXattr retrieves all extended attributes associated with a file,
// directory or symbolic link.
func GetAllXattr(path string) (xattrs map[string]string, err error) {
	e1 := fmt.Errorf("Extended attributes changed during retrieval")

	// Call llistxattr() twice: First, to determine the size of the buffer
	// we need to allocate to store the extended attributes, second, to
	// actually store the extended attributes in the buffer. Also, check if
	// the size/number of extended attributes hasn't changed between the two
	// calls.
	pre, err := llistxattr(path, nil)
	if err != nil || pre < 0 {
		return nil, err
	}
	if pre == 0 {
		return nil, nil
	}

	dest := make([]byte, pre)

	post, err := llistxattr(path, dest)
	if err != nil || post < 0 {
		return nil, err
	}
	if post != pre {
		return nil, e1
	}

	split := strings.Split(string(dest), "\x00")
	if split == nil {
		return nil, fmt.Errorf("No valid extended attribute key found")
	}
	// *listxattr functions return a list of  names  as  an unordered array
	// of null-terminated character strings (attribute names are separated
	// by null bytes ('\0')), like this: user.name1\0system.name1\0user.name2\0
	// Since we split at the '\0'-byte the last element of the slice will be
	// the empty string. We remove it:
	if split[len(split)-1] == "" {
		split = split[:len(split)-1]
	}

	xattrs = make(map[string]string, len(split))

	for _, x := range split {
		xattr := string(x)
		// Call Getxattr() twice: First, to determine the size of the
		// buffer we need to allocate to store the extended attributes,
		// second, to actually store the extended attributes in the
		// buffer. Also, check if the size of the extended attribute
		// hasn't changed between the two calls.
		pre, err = unix.Getxattr(path, xattr, nil)
		if err != nil || pre < 0 {
			return nil, err
		}

		dest = make([]byte, pre)
		post := 0
		if pre > 0 {
			post, err = unix.Getxattr(path, xattr, dest)
			if err != nil || post < 0 {
				return nil, err
			}
		}

		if post != pre {
			return nil, e1
		}

		xattrs[xattr] = string(dest)
	}

	return xattrs, nil
}

var ObjectFound = fmt.Errorf("Found requested object")

func LookupUUIDByBlockDevPath(diskDevice string) (string, error) {
	uuid := ""
	readUUID := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if (info.Mode() & os.ModeSymlink) == os.ModeSymlink {
			link, err := os.Readlink(path)
			if err != nil {
				return err
			}

			// filepath.Join() will call Clean() on the result and
			// thus resolve those ugly "../../" parts that make it
			// hard to compare the strings.
			absPath := filepath.Join("/dev/disk/by-uuid", link)
			if absPath == diskDevice {
				uuid = path
				// Will allows us to avoid needlessly travers
				// the whole directory.
				return ObjectFound
			}
		}
		return nil
	}

	err := filepath.Walk("/dev/disk/by-uuid", readUUID)
	if err != nil && err != ObjectFound {
		return "", fmt.Errorf("Failed to detect UUID: %s", err)
	}

	if uuid == "" {
		return "", fmt.Errorf("Failed to detect UUID")
	}

	lastSlash := strings.LastIndex(uuid, "/")
	return uuid[lastSlash+1:], nil
}

// Detect whether err is an errno.
func GetErrno(err error) (errno error, iserrno bool) {
	sysErr, ok := err.(*os.SyscallError)
	if ok {
		return sysErr.Err, true
	}

	pathErr, ok := err.(*os.PathError)
	if ok {
		return pathErr.Err, true
	}

	tmpErrno, ok := err.(unix.Errno)
	if ok {
		return tmpErrno, true
	}

	return nil, false
}

// Utsname returns the same info as unix.Utsname, as strings
type Utsname struct {
	Sysname    string
	Nodename   string
	Release    string
	Version    string
	Machine    string
	Domainname string
}

// Uname returns Utsname as strings
func Uname() (*Utsname, error) {
	/*
	 * Based on: https://groups.google.com/forum/#!topic/golang-nuts/Jel8Bb-YwX8
	 * there is really no better way to do this, which is
	 * unfortunate. Also, we ditch the more accepted CharsToString
	 * version in that thread, since it doesn't seem as portable,
	 * viz. github issue #206.
	 */

	uname := unix.Utsname{}
	err := unix.Uname(&uname)
	if err != nil {
		return nil, err
	}

	return &Utsname{
		Sysname:    intArrayToString(uname.Sysname),
		Nodename:   intArrayToString(uname.Nodename),
		Release:    intArrayToString(uname.Release),
		Version:    intArrayToString(uname.Version),
		Machine:    intArrayToString(uname.Machine),
		Domainname: intArrayToString(uname.Domainname),
	}, nil
}

func intArrayToString(arr interface{}) string {
	slice := reflect.ValueOf(arr)
	s := ""
	for i := 0; i < slice.Len(); i++ {
		val := slice.Index(i)
		valInt := int64(-1)

		switch val.Kind() {
		case reflect.Int:
		case reflect.Int8:
			valInt = int64(val.Int())
		case reflect.Uint:
		case reflect.Uint8:
			valInt = int64(val.Uint())
		default:
			continue
		}

		if valInt == 0 {
			break
		}

		s += string(byte(valInt))
	}

	return s
}

func Statvfs(path string) (*unix.Statfs_t, error) {
	var st unix.Statfs_t

	err := unix.Statfs(path, &st)
	if err != nil {
		return nil, err
	}

	return &st, nil
}

func DeviceTotalMemory() (int64, error) {
	// Open /proc/meminfo
	f, err := os.Open("/proc/meminfo")
	if err != nil {
		return -1, err
	}
	defer f.Close()

	// Read it line by line
	scan := bufio.NewScanner(f)
	for scan.Scan() {
		line := scan.Text()

		// We only care about MemTotal
		if !strings.HasPrefix(line, "MemTotal:") {
			continue
		}

		// Extract the before last (value) and last (unit) fields
		fields := strings.Split(line, " ")
		value := fields[len(fields)-2] + fields[len(fields)-1]

		// Feed the result to units.ParseByteSizeString to get an int value
		valueBytes, err := units.ParseByteSizeString(value)
		if err != nil {
			return -1, err
		}

		return valueBytes, nil
	}

	return -1, fmt.Errorf("Couldn't find MemTotal")
}
<|endoftext|>


<commit_before>// Copyright 2015 The LUCI Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package lhttp

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"golang.org/x/net/context"

	"github.com/luci/luci-go/common/errors"
	"github.com/luci/luci-go/common/logging"
	"github.com/luci/luci-go/common/retry"
)

const (
	jsonContentType        = "application/json"
	jsonContentTypeForPOST = "application/json; charset=utf-8"
)

// Handler is called once or multiple times for each HTTP request that is tried.
type Handler func(*http.Response) error

// RequestGen is a generator function to create a new request. It may be called
// multiple times if an operation needs to be retried. The HTTP server is
// responsible for closing the Request body, as per http.Request Body method
// documentation.
type RequestGen func() (*http.Request, error)

// NewRequest returns a retriable request.
//
// The handler func is responsible for closing the response Body before
// returning. It should return retry.Error in case of retriable error, for
// example if a TCP connection is terminated while receiving the content.
//
// If rFn is nil, NewRequest will use a default exponential backoff strategy only
// for transient errors.
func NewRequest(ctx context.Context, c *http.Client, rFn retry.Factory, rgen RequestGen, handler Handler) func() (int, error) {
	if rFn == nil {
		rFn = retry.TransientOnly(retry.Default)
	}

	return func() (int, error) {
		status, attempts := 0, 0
		err := retry.Retry(ctx, rFn, func() error {
			attempts++
			req, err := rgen()
			if err != nil {
				return err
			}

			resp, err := c.Do(req)
			if err != nil {
				// Retry every error. This is sad when you specify an invalid hostname but
				// it's better than failing when DNS resolution is flaky.
				return errors.WrapTransient(err)
			}
			status = resp.StatusCode

			switch {
			case status == 408, status == 429, status >= 500:
				// The HTTP status code means the request should be retried.
				err = errors.WrapTransient(
					fmt.Errorf("http request failed: %s (HTTP %d)", http.StatusText(status), status))
			case status == 404 && strings.HasPrefix(req.URL.Path, "/_ah/api/"):
				// Endpoints occasionally return 404 on valid requests!
				logging.Infof(ctx, "lhttp.Do() got a Cloud Endpoints 404: %#v", resp.Header)
				err = errors.WrapTransient(
					fmt.Errorf("http request failed (endpoints): %s (HTTP %d)", http.StatusText(status), status))
			case status >= 400:
				// Any other failure code is a hard failure.
				err = fmt.Errorf("http request failed: %s (HTTP %d)", http.StatusText(status), status)
			default:
				// The handler may still return a retry.Error to indicate that the request
				// should be retried even on successful status code.
				return handler(resp)
			}

			// Drain and close the resp.Body.
			io.Copy(ioutil.Discard, resp.Body)
			resp.Body.Close()
			return err
		}, nil)
		if err != nil {
			err = fmt.Errorf("%v (attempts: %d)", err, attempts)
		}
		return status, err
	}
}

// NewRequestJSON returns a retriable request calling a JSON endpoint.
func NewRequestJSON(ctx context.Context, c *http.Client, rFn retry.Factory, url, method string, headers map[string]string, in, out interface{}) (func() (int, error), error) {
	var encoded []byte
	if in != nil {
		var err error
		if encoded, err = json.Marshal(in); err != nil {
			return nil, err
		}
	}

	return NewRequest(ctx, c, rFn, func() (*http.Request, error) {
		var body io.Reader
		if encoded != nil {
			body = bytes.NewReader(encoded)
		}

		req, err := http.NewRequest(method, url, body)
		if err != nil {
			return nil, err
		}
		if encoded != nil {
			req.Header.Set("Content-Type", jsonContentTypeForPOST)
		}
		if headers != nil {
			for k, v := range headers {
				req.Header.Add(k, v)
			}
		}
		return req, nil
	}, func(resp *http.Response) error {
		defer resp.Body.Close()
		if ct := strings.ToLower(resp.Header.Get("Content-Type")); !strings.HasPrefix(ct, jsonContentType) {
			// Non-retriable.
			return fmt.Errorf("unexpected Content-Type, expected \"%s\", got \"%s\"", jsonContentType, ct)
		}
		if out == nil {
			// The client doesn't care about the response. Still ensure the response
			// is valid json.
			out = &map[string]interface{}{}
		}
		if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
			// Retriable.
			return errors.WrapTransient(fmt.Errorf("bad response %s: %s", url, err))
		}
		return nil
	}), nil
}

// GetJSON is a shorthand. It returns the HTTP status code and error if any.
func GetJSON(ctx context.Context, rFn retry.Factory, c *http.Client, url string, out interface{}) (int, error) {
	req, err := NewRequestJSON(ctx, c, rFn, url, "GET", nil, nil, out)
	if err != nil {
		return 0, err
	}
	return req()
}

// PostJSON is a shorthand. It returns the HTTP status code and error if any.
func PostJSON(ctx context.Context, rFn retry.Factory, c *http.Client, url string, headers map[string]string, in, out interface{}) (int, error) {
	req, err := NewRequestJSON(ctx, c, rFn, url, "POST", headers, in, out)
	if err != nil {
		return 0, err
	}
	return req()
}
<commit_msg>common/lhttp: fix line wrapping of comment<commit_after>// Copyright 2015 The LUCI Authors. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

package lhttp

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"strings"

	"golang.org/x/net/context"

	"github.com/luci/luci-go/common/errors"
	"github.com/luci/luci-go/common/logging"
	"github.com/luci/luci-go/common/retry"
)

const (
	jsonContentType        = "application/json"
	jsonContentTypeForPOST = "application/json; charset=utf-8"
)

// Handler is called once or multiple times for each HTTP request that is tried.
type Handler func(*http.Response) error

// RequestGen is a generator function to create a new request. It may be called
// multiple times if an operation needs to be retried. The HTTP server is
// responsible for closing the Request body, as per http.Request Body method
// documentation.
type RequestGen func() (*http.Request, error)

// NewRequest returns a retriable request.
//
// The handler func is responsible for closing the response Body before
// returning. It should return retry.Error in case of retriable error, for
// example if a TCP connection is terminated while receiving the content.
//
// If rFn is nil, NewRequest will use a default exponential backoff strategy
// only for transient errors.
func NewRequest(ctx context.Context, c *http.Client, rFn retry.Factory, rgen RequestGen, handler Handler) func() (int, error) {
	if rFn == nil {
		rFn = retry.TransientOnly(retry.Default)
	}

	return func() (int, error) {
		status, attempts := 0, 0
		err := retry.Retry(ctx, rFn, func() error {
			attempts++
			req, err := rgen()
			if err != nil {
				return err
			}

			resp, err := c.Do(req)
			if err != nil {
				// Retry every error. This is sad when you specify an invalid hostname but
				// it's better than failing when DNS resolution is flaky.
				return errors.WrapTransient(err)
			}
			status = resp.StatusCode

			switch {
			case status == 408, status == 429, status >= 500:
				// The HTTP status code means the request should be retried.
				err = errors.WrapTransient(
					fmt.Errorf("http request failed: %s (HTTP %d)", http.StatusText(status), status))
			case status == 404 && strings.HasPrefix(req.URL.Path, "/_ah/api/"):
				// Endpoints occasionally return 404 on valid requests!
				logging.Infof(ctx, "lhttp.Do() got a Cloud Endpoints 404: %#v", resp.Header)
				err = errors.WrapTransient(
					fmt.Errorf("http request failed (endpoints): %s (HTTP %d)", http.StatusText(status), status))
			case status >= 400:
				// Any other failure code is a hard failure.
				err = fmt.Errorf("http request failed: %s (HTTP %d)", http.StatusText(status), status)
			default:
				// The handler may still return a retry.Error to indicate that the request
				// should be retried even on successful status code.
				return handler(resp)
			}

			// Drain and close the resp.Body.
			io.Copy(ioutil.Discard, resp.Body)
			resp.Body.Close()
			return err
		}, nil)
		if err != nil {
			err = fmt.Errorf("%v (attempts: %d)", err, attempts)
		}
		return status, err
	}
}

// NewRequestJSON returns a retriable request calling a JSON endpoint.
func NewRequestJSON(ctx context.Context, c *http.Client, rFn retry.Factory, url, method string, headers map[string]string, in, out interface{}) (func() (int, error), error) {
	var encoded []byte
	if in != nil {
		var err error
		if encoded, err = json.Marshal(in); err != nil {
			return nil, err
		}
	}

	return NewRequest(ctx, c, rFn, func() (*http.Request, error) {
		var body io.Reader
		if encoded != nil {
			body = bytes.NewReader(encoded)
		}

		req, err := http.NewRequest(method, url, body)
		if err != nil {
			return nil, err
		}
		if encoded != nil {
			req.Header.Set("Content-Type", jsonContentTypeForPOST)
		}
		if headers != nil {
			for k, v := range headers {
				req.Header.Add(k, v)
			}
		}
		return req, nil
	}, func(resp *http.Response) error {
		defer resp.Body.Close()
		if ct := strings.ToLower(resp.Header.Get("Content-Type")); !strings.HasPrefix(ct, jsonContentType) {
			// Non-retriable.
			return fmt.Errorf("unexpected Content-Type, expected \"%s\", got \"%s\"", jsonContentType, ct)
		}
		if out == nil {
			// The client doesn't care about the response. Still ensure the response
			// is valid json.
			out = &map[string]interface{}{}
		}
		if err := json.NewDecoder(resp.Body).Decode(out); err != nil {
			// Retriable.
			return errors.WrapTransient(fmt.Errorf("bad response %s: %s", url, err))
		}
		return nil
	}), nil
}

// GetJSON is a shorthand. It returns the HTTP status code and error if any.
func GetJSON(ctx context.Context, rFn retry.Factory, c *http.Client, url string, out interface{}) (int, error) {
	req, err := NewRequestJSON(ctx, c, rFn, url, "GET", nil, nil, out)
	if err != nil {
		return 0, err
	}
	return req()
}

// PostJSON is a shorthand. It returns the HTTP status code and error if any.
func PostJSON(ctx context.Context, rFn retry.Factory, c *http.Client, url string, headers map[string]string, in, out interface{}) (int, error) {
	req, err := NewRequestJSON(ctx, c, rFn, url, "POST", headers, in, out)
	if err != nil {
		return 0, err
	}
	return req()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/url"
)

const (
	DESC int = iota
	ASC
)

type Item interface {
	PaginationValue(key string) interface{}
}

type Interface interface {
	Equal(order string, i, j int) bool
	Value(order string, i int) interface{}
	Len() int
}

type Page struct {
	items []Item
}

func (p *Page) Equal(order string, i, j int) bool {
	return p.Value(order, i) == p.Value(order, j)
}

func (p *Page) Value(order string, i int) interface{} {
	return p.items[i].PaginationValue(order)
}

func (p *Page) Len() int {
	return len(p.items)
}

type Config struct {
	pageSize  int
	order     string
	direction int
}

type Cursor struct {
	Value  interface{}
	Offset int
	Order  string
}

type Pagination struct {
	Cursor
	config Config
}

func (p *Pagination) max(items Interface) int {
	if items.Len() <= p.config.pageSize {
		return items.Len() - 1
	} else {
		return p.config.pageSize - 1
	}
}

func (p *Pagination) equalCount(items Interface, order string, max int) int {
	c := 0
	for i := 0; i < p.config.pageSize; i++ {
		if items.Equal(order, i, max) {
			c += 1
		}
	}
	return c
}

func NewPaginationFromUrl(u *url.URL, c Config) Pagination {
	return Pagination{config: c}
}

func (p *Pagination) next(items Interface) *Pagination {
	max := p.max(items)
	value := items.Value(p.Order, max)
	offset := p.equalCount(items, p.Order, max)

	if offset == p.config.pageSize && value == p.Value {
		offset += p.Offset
	}

	return &Pagination{Cursor{value, offset, p.Order}, p.config}
}

type Comment struct {
	text       string
	created_at int
	updated_at int
}

func (c *Comment) PaginationValue(key string) interface{} {
	switch {
	case key == "created_at":
		return c.created_at
	case key == "updated_at":
		return c.updated_at
	default:
		return nil
	}
}

func main() {
	pagination := Pagination{config: Config{pageSize: 2, order: "created_at"}}

	items := &Page{[]Item{
		&Comment{"a", 0, 4},
		&Comment{"b", 1, 4},
		&Comment{"c", 2, 5},
		&Comment{"d", 3, 5},
	}}
	next := pagination.next(items)

	fmt.Printf("next value: %i, offset: %i\n", next.Value, next.Offset)
}
<commit_msg>Fix bug that ignored the first item of next page<commit_after>package main

import (
	"fmt"
	"net/url"
)

const (
	DESC int = iota
	ASC
)

type Item interface {
	PaginationValue(key string) interface{}
}

type Interface interface {
	Equal(order string, i, j int) bool
	Value(order string, i int) interface{}
	Len() int
}

type Page struct {
	items []Item
}

func (p *Page) Equal(order string, i, j int) bool {
	return p.Value(order, i) == p.Value(order, j)
}

func (p *Page) Value(order string, i int) interface{} {
	return p.items[i].PaginationValue(order)
}

func (p *Page) Len() int {
	return len(p.items)
}

type Config struct {
	pageSize  int
	order     string
	direction int
}

type Cursor struct {
	Value  interface{}
	Offset int
	Order  string
}

type Pagination struct {
	Cursor
	config Config
}

func (p *Pagination) max(items Interface) int {
	if items.Len() <= p.config.pageSize {
		return items.Len() - 1
	} else {
		return p.config.pageSize
	}
}

func (p *Pagination) equalCount(items Interface, order string, max int) int {
	c := 0
	for i := 0; i < p.config.pageSize; i++ {
		if items.Equal(order, i, max) {
			c += 1
		}
	}
	return c
}

func NewPaginationFromUrl(u *url.URL, c Config) Pagination {
	return Pagination{config: c}
}

func (p *Pagination) next(items Interface) *Pagination {
	max := p.max(items)
	value := items.Value(p.Order, max)
	offset := p.equalCount(items, p.Order, max)

	if offset == p.config.pageSize && value == p.Value {
		offset += p.Offset
	}

	return &Pagination{Cursor{value, offset, p.Order}, p.config}
}

type Comment struct {
	text       string
	created_at int
	updated_at int
}

func (c *Comment) PaginationValue(key string) interface{} {
	switch {
	case key == "created_at":
		return c.created_at
	case key == "updated_at":
		return c.updated_at
	default:
		return nil
	}
}

func main() {
	pagination := Pagination{config: Config{pageSize: 2, order: "created_at"}}

	items := &Page{[]Item{
		&Comment{"a", 0, 4},
		&Comment{"b", 1, 4},
		&Comment{"c", 2, 5},
		&Comment{"d", 3, 5},
	}}
	next := pagination.next(items)

	fmt.Printf("next value: %i, offset: %i\n", next.Value, next.Offset)
}
<|endoftext|>