<commit_before>package cloudca

import (
	"fmt"
	"github.com/cloud-ca/go-cloudca"
	"github.com/cloud-ca/go-cloudca/api"
	"github.com/cloud-ca/go-cloudca/services/cloudca"
	"github.com/hashicorp/terraform/helper/schema"
	"strings"
)

func resourceCloudcaVolume() *schema.Resource {
	return &schema.Resource{
		Create: resourceCloudcaVolumeCreate,
		Read:   resourceCloudcaVolumeRead,
		Update: resourceCloudcaVolumeUpdate,
		Delete: resourceCloudcaVolumeDelete,

		Schema: map[string]*schema.Schema{
			"service_code": &schema.Schema{
				Type:        schema.TypeString,
				Required:    true,
				ForceNew:    true,
				Description: "A cloudca service code",
			},
			"environment_name": &schema.Schema{
				Type:        schema.TypeString,
				Required:    true,
				ForceNew:    true,
				Description: "Name of environment where port forwarding rule should be created",
			},
			"name": &schema.Schema{
				Type:        schema.TypeString,
				Required:    true,
				ForceNew:    true,
				Description: "The name of the volume to be created",
			},
			"zone_name": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				ForceNew:    true,
				Description: "The zone into which the volume will be create",
			},
			"storage_tier": &schema.Schema{
				Type:        schema.TypeString,
				Required:    true,
				ForceNew:    true,
				Description: "The storage tier name",
			},
			"size_in_gb": &schema.Schema{
				Type:        schema.TypeInt,
				Required:    true,
				ForceNew:    true,
				Description: "The size of the disk volume in gigabytes",
			},
			"instance_id": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The id of the instance to which the volume should be attached",
			},
		},
	}
}

func resourceCloudcaVolumeCreate(d *schema.ResourceData, meta interface{}) error {
	ccaClient := meta.(*cca.CcaClient)
	resources, _ := ccaClient.GetResources(d.Get("service_code").(string), d.Get("environment_name").(string))
	ccaResources := resources.(cloudca.Resources)

	storageTier := d.Get("storage_tier").(string)
	size := d.Get("size_in_gb").(int)
	diskOfferingId, err := retrieveDiskOfferingId(&ccaResources, storageTier, size)
	if err != nil {
		return err
	}
	volumeToCreate := cloudca.Volume{
		Name:           d.Get("name").(string),
		DiskOfferingId: diskOfferingId,
	}

	if zoneName, ok := d.GetOk("zone_name"); ok {
		if zoneId, err := retrieveZoneId(&ccaResources, zoneName.(string)); err != nil {
			return err
		} else {
			volumeToCreate.ZoneId = zoneId
		}
	}

	newVolume, err := ccaResources.Volumes.Create(volumeToCreate)
	if err != nil {
		return err
	}
	if instanceId, ok := d.GetOk("instance_id"); ok {
		err = ccaResources.Volumes.AttachToInstance(newVolume, instanceId.(string))
		if err != nil {
			return err
		}
	}
	d.SetId(newVolume.Id)
	return resourceCloudcaVolumeRead(d, meta)
}

func resourceCloudcaVolumeRead(d *schema.ResourceData, meta interface{}) error {
	ccaClient := meta.(*cca.CcaClient)
	resources, _ := ccaClient.GetResources(d.Get("service_code").(string), d.Get("environment_name").(string))
	ccaResources := resources.(cloudca.Resources)

	volume, err := ccaResources.Volumes.Get(d.Id())
	if err != nil {
		return handleVolumeNotFoundError(err, d)
	}
	d.Set("name", volume.Name)
	d.Set("zone_name", volume.ZoneName)
	d.Set("storage_tier", volume.StorageTier)
	d.Set("size_in_gb", volume.Size)
	d.Set("instance_id", volume.InstanceId)
	return nil
}

func resourceCloudcaVolumeUpdate(d *schema.ResourceData, meta interface{}) error {
	ccaClient := meta.(*cca.CcaClient)
	resources, _ := ccaClient.GetResources(d.Get("service_code").(string), d.Get("environment_name").(string))
	ccaResources := resources.(cloudca.Resources)

	d.Partial(true)
	if d.HasChange("instance_id") {
		oldInstanceId, newInstanceId := d.GetChange("instance_id")
		volume := &cloudca.Volume{
			Id: d.Id(),
		}
		if oldInstanceId != "" {
			err := ccaResources.Volumes.DetachFromInstance(volume)
			if err != nil {
				return fmt.Errorf("%s %s", oldInstanceId, newInstanceId)
			}
		}
		if newInstanceId != "" {
			err := ccaResources.Volumes.AttachToInstance(volume, newInstanceId.(string))
			if err != nil {
				return err
			}
		}
		d.SetPartial("instance_id")
	}
	d.Partial(false)
	return resourceCloudcaVolumeRead(d, meta)
}

func resourceCloudcaVolumeDelete(d *schema.ResourceData, meta interface{}) error {
	ccaClient := meta.(*cca.CcaClient)
	resources, _ := ccaClient.GetResources(d.Get("service_code").(string), d.Get("environment_name").(string))
	ccaResources := resources.(cloudca.Resources)
	if instanceId, ok := d.GetOk("instance_id"); ok && instanceId != "" {
		volume := &cloudca.Volume{
			Id: d.Id(),
		}
		err := ccaResources.Volumes.DetachFromInstance(volume)
		if err != nil {
			return err
		}
	}
	if err := ccaResources.Volumes.Delete(d.Id()); err != nil {
		return handleVolumeNotFoundError(err, d)
	}
	return nil
}

func retrieveDiskOfferingId(ccaResources *cloudca.Resources, storageTier string, size int) (id string, err error) {
	diskOfferings, err := ccaResources.DiskOfferings.List()
	if err != nil {
		return "", err
	}
	for _, diskOffering := range diskOfferings {
		if strings.EqualFold(diskOffering.StorageTier, storageTier) && diskOffering.GbSize == size {
			return diskOffering.Id, nil
		}
	}
	return "", fmt.Errorf("No valid disk offering's were found with storage tier: %s and size: %d", storageTier, size)
}

func retrieveZoneId(ccaResources *cloudca.Resources, zoneName string) (zoneId string, nerr error) {
	zones, err := ccaResources.Zones.List()
	if err != nil {
		return "", err
	}
	for _, zone := range zones {
		if strings.EqualFold(zone.Name, zoneName) {
			return zone.Id, nil
		}
	}
	return "", fmt.Errorf("Zone with name %s could not be found", zoneName)
}

func handleVolumeNotFoundError(err error, d *schema.ResourceData) error {
	if ccaError, ok := err.(api.CcaErrorResponse); ok {
		if ccaError.StatusCode == 404 {
			fmt.Errorf("Volume with id='%s' was not found", d.Id())
			d.SetId("")
			return nil
		}
	}
	return err
}
<commit_msg>Use cloudca.Volume.GbSize instead of .Size<commit_after>package cloudca

import (
	"fmt"
	"github.com/cloud-ca/go-cloudca"
	"github.com/cloud-ca/go-cloudca/api"
	"github.com/cloud-ca/go-cloudca/services/cloudca"
	"github.com/hashicorp/terraform/helper/schema"
	"strings"
)

func resourceCloudcaVolume() *schema.Resource {
	return &schema.Resource{
		Create: resourceCloudcaVolumeCreate,
		Read:   resourceCloudcaVolumeRead,
		Update: resourceCloudcaVolumeUpdate,
		Delete: resourceCloudcaVolumeDelete,

		Schema: map[string]*schema.Schema{
			"service_code": &schema.Schema{
				Type:        schema.TypeString,
				Required:    true,
				ForceNew:    true,
				Description: "A cloudca service code",
			},
			"environment_name": &schema.Schema{
				Type:        schema.TypeString,
				Required:    true,
				ForceNew:    true,
				Description: "Name of environment where port forwarding rule should be created",
			},
			"name": &schema.Schema{
				Type:        schema.TypeString,
				Required:    true,
				ForceNew:    true,
				Description: "The name of the volume to be created",
			},
			"zone_name": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				ForceNew:    true,
				Description: "The zone into which the volume will be create",
			},
			"storage_tier": &schema.Schema{
				Type:        schema.TypeString,
				Required:    true,
				ForceNew:    true,
				Description: "The storage tier name",
			},
			"size_in_gb": &schema.Schema{
				Type:        schema.TypeInt,
				Required:    true,
				ForceNew:    true,
				Description: "The size of the disk volume in gigabytes",
			},
			"instance_id": &schema.Schema{
				Type:        schema.TypeString,
				Optional:    true,
				Description: "The id of the instance to which the volume should be attached",
			},
		},
	}
}

func resourceCloudcaVolumeCreate(d *schema.ResourceData, meta interface{}) error {
	ccaClient := meta.(*cca.CcaClient)
	resources, _ := ccaClient.GetResources(d.Get("service_code").(string), d.Get("environment_name").(string))
	ccaResources := resources.(cloudca.Resources)

	storageTier := d.Get("storage_tier").(string)
	size := d.Get("size_in_gb").(int)
	diskOfferingId, err := retrieveDiskOfferingId(&ccaResources, storageTier, size)
	if err != nil {
		return err
	}
	volumeToCreate := cloudca.Volume{
		Name:           d.Get("name").(string),
		DiskOfferingId: diskOfferingId,
	}

	if zoneName, ok := d.GetOk("zone_name"); ok {
		if zoneId, err := retrieveZoneId(&ccaResources, zoneName.(string)); err != nil {
			return err
		} else {
			volumeToCreate.ZoneId = zoneId
		}
	}

	newVolume, err := ccaResources.Volumes.Create(volumeToCreate)
	if err != nil {
		return err
	}
	if instanceId, ok := d.GetOk("instance_id"); ok {
		err = ccaResources.Volumes.AttachToInstance(newVolume, instanceId.(string))
		if err != nil {
			return err
		}
	}
	d.SetId(newVolume.Id)
	return resourceCloudcaVolumeRead(d, meta)
}

func resourceCloudcaVolumeRead(d *schema.ResourceData, meta interface{}) error {
	ccaClient := meta.(*cca.CcaClient)
	resources, _ := ccaClient.GetResources(d.Get("service_code").(string), d.Get("environment_name").(string))
	ccaResources := resources.(cloudca.Resources)

	volume, err := ccaResources.Volumes.Get(d.Id())
	if err != nil {
		return handleVolumeNotFoundError(err, d)
	}
	d.Set("name", volume.Name)
	d.Set("zone_name", volume.ZoneName)
	d.Set("storage_tier", volume.StorageTier)
	d.Set("size_in_gb", volume.GbSize)
	d.Set("instance_id", volume.InstanceId)
	return nil
}

func resourceCloudcaVolumeUpdate(d *schema.ResourceData, meta interface{}) error {
	ccaClient := meta.(*cca.CcaClient)
	resources, _ := ccaClient.GetResources(d.Get("service_code").(string), d.Get("environment_name").(string))
	ccaResources := resources.(cloudca.Resources)

	d.Partial(true)
	if d.HasChange("instance_id") {
		oldInstanceId, newInstanceId := d.GetChange("instance_id")
		volume := &cloudca.Volume{
			Id: d.Id(),
		}
		if oldInstanceId != "" {
			err := ccaResources.Volumes.DetachFromInstance(volume)
			if err != nil {
				return fmt.Errorf("%s %s", oldInstanceId, newInstanceId)
			}
		}
		if newInstanceId != "" {
			err := ccaResources.Volumes.AttachToInstance(volume, newInstanceId.(string))
			if err != nil {
				return err
			}
		}
		d.SetPartial("instance_id")
	}
	d.Partial(false)
	return resourceCloudcaVolumeRead(d, meta)
}

func resourceCloudcaVolumeDelete(d *schema.ResourceData, meta interface{}) error {
	ccaClient := meta.(*cca.CcaClient)
	resources, _ := ccaClient.GetResources(d.Get("service_code").(string), d.Get("environment_name").(string))
	ccaResources := resources.(cloudca.Resources)
	if instanceId, ok := d.GetOk("instance_id"); ok && instanceId != "" {
		volume := &cloudca.Volume{
			Id: d.Id(),
		}
		err := ccaResources.Volumes.DetachFromInstance(volume)
		if err != nil {
			return err
		}
	}
	if err := ccaResources.Volumes.Delete(d.Id()); err != nil {
		return handleVolumeNotFoundError(err, d)
	}
	return nil
}

func retrieveDiskOfferingId(ccaResources *cloudca.Resources, storageTier string, size int) (id string, err error) {
	diskOfferings, err := ccaResources.DiskOfferings.List()
	if err != nil {
		return "", err
	}
	for _, diskOffering := range diskOfferings {
		if strings.EqualFold(diskOffering.StorageTier, storageTier) && diskOffering.GbSize == size {
			return diskOffering.Id, nil
		}
	}
	return "", fmt.Errorf("No valid disk offering's were found with storage tier: %s and size: %d", storageTier, size)
}

func retrieveZoneId(ccaResources *cloudca.Resources, zoneName string) (zoneId string, nerr error) {
	zones, err := ccaResources.Zones.List()
	if err != nil {
		return "", err
	}
	for _, zone := range zones {
		if strings.EqualFold(zone.Name, zoneName) {
			return zone.Id, nil
		}
	}
	return "", fmt.Errorf("Zone with name %s could not be found", zoneName)
}

func handleVolumeNotFoundError(err error, d *schema.ResourceData) error {
	if ccaError, ok := err.(api.CcaErrorResponse); ok {
		if ccaError.StatusCode == 404 {
			fmt.Errorf("Volume with id='%s' was not found", d.Id())
			d.SetId("")
			return nil
		}
	}
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/julienmoumne/hotshell/cmd/testutil"
	. "gopkg.in/check.v1"
	"testing"
)

func TestBuilder(t *testing.T) { TestingT(t) }

type TestHsMan struct{}

var _ = Suite(&TestHsMan{})

func (s *TestHsMan) TestMan(c *C) {
	driver := testutil.Driver{Main: main}
	actualStdout, _, err := driver.Run()
	c.Check(err, IsNil)
	c.Check(actualStdout, Equals, expectedMan)
}

var expectedMan = `.TH hs 1 "11 May 2017"
.SH NAME
hs \- Interactive single keystroke menus for the shell
.SH SYNOPSIS
\fBhs\fP [-f <arg>...] [Options]
.SH DESCRIPTION
Hotshell is a command-line application to efficiently recall and share commands.
.SH OPTIONS
.TP
\fB\fB\-\-default\fR\fP
Load the default menu to get to know Hotshell
.TP
\fB\fB\-f\fR, \fB\-\-file\fR \fIFILE|DIR\fR\fP
Specify an alternate definition file (defaults: ./hs.js, ~/.hs/hs.js)
.TP
\fB\fB\-\-chdir\fR\fP
Set the working directory to the location of the menu definition
.TP
\fB\fB\-\-generate-demo\fR\fP
Generate an interactive HTML demo to stdout
.TP
\fB\fB\-\-generate-doc\fR\fP
Generate a markdown documentation of the menu to stdout
.TP
\fB\fB\-v\fR, \fB\-\-version\fR\fP
Print version information and quit
`
<commit_msg>test: make hs-man test independent of the current time<commit_after>package main

import (
	"github.com/julienmoumne/hotshell/cmd/testutil"
	. "gopkg.in/check.v1"
	"testing"
	"fmt"
	"time"
)

func TestBuilder(t *testing.T) { TestingT(t) }

type TestHsMan struct{}

var _ = Suite(&TestHsMan{})

func (s *TestHsMan) TestMan(c *C) {
	driver := testutil.Driver{Main: main}
	actualStdout, _, err := driver.Run()
	c.Check(err, IsNil)
	c.Check(actualStdout, Equals, expectedMan)
}

var expectedMan = fmt.Sprintf(`.TH hs 1 "%s"
.SH NAME
hs \- Interactive single keystroke menus for the shell
.SH SYNOPSIS
\fBhs\fP [-f <arg>...] [Options]
.SH DESCRIPTION
Hotshell is a command-line application to efficiently recall and share commands.
.SH OPTIONS
.TP
\fB\fB\-\-default\fR\fP
Load the default menu to get to know Hotshell
.TP
\fB\fB\-f\fR, \fB\-\-file\fR \fIFILE|DIR\fR\fP
Specify an alternate definition file (defaults: ./hs.js, ~/.hs/hs.js)
.TP
\fB\fB\-\-chdir\fR\fP
Set the working directory to the location of the menu definition
.TP
\fB\fB\-\-generate-demo\fR\fP
Generate an interactive HTML demo to stdout
.TP
\fB\fB\-\-generate-doc\fR\fP
Generate a markdown documentation of the menu to stdout
.TP
\fB\fB\-v\fR, \fB\-\-version\fR\fP
Print version information and quit
`, time.Now().Format("02 January 2006"))
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"os"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	pkgConfig "k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/console"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/exit"
	pkgutil "k8s.io/minikube/pkg/util"
)

// ProfileCmd represents the profile command
var ProfileCmd = &cobra.Command{
	Use:   "profile [MINIKUBE_PROFILE_NAME].  You can return to the default minikube profile by running `minikube profile default`",
	Short: "Profile gets or sets the current minikube profile",
	Long:  "profile sets the current minikube profile, or gets the current profile if no arguments are provided.  This is used to run and manage multiple minikube instance.  You can return to the default minikube profile by running `minikube profile default`",
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) == 0 {
			profile := viper.GetString(pkgConfig.MachineProfile)
			console.OutLn("%s", profile)
			os.Exit(0)
		}

		if len(args) > 1 {
			exit.Usage("usage: minikube profile [MINIKUBE_PROFILE_NAME]")
		}

		profile := args[0]
		if profile == "default" {
			profile = "minikube"
		}
		err := Set(pkgConfig.MachineProfile, profile)
		if err != nil {
			exit.WithError("Setting profile failed", err)
		}
		cc, err := pkgConfig.Load()
		if err != nil && !os.IsNotExist(err) {
			console.ErrLn("Error loading profile config: %v", err)
		}
		// don't change context is user has started the minikube with --keep-context
		if !cc.MachineConfig.KeepContext {
			err = pkgutil.SetCurrentContext(constants.KubeconfigPath, profile)
			console.ErrLn("error while setting kubectl current context :  %v", err)
		}

		console.Success("minikube profile was successfully set to %s", profile)
	},
}
<commit_msg>notify user of skipping<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"os"

	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	pkgConfig "k8s.io/minikube/pkg/minikube/config"
	"k8s.io/minikube/pkg/minikube/console"
	"k8s.io/minikube/pkg/minikube/constants"
	"k8s.io/minikube/pkg/minikube/exit"
	pkgutil "k8s.io/minikube/pkg/util"
)

// ProfileCmd represents the profile command
var ProfileCmd = &cobra.Command{
	Use:   "profile [MINIKUBE_PROFILE_NAME].  You can return to the default minikube profile by running `minikube profile default`",
	Short: "Profile gets or sets the current minikube profile",
	Long:  "profile sets the current minikube profile, or gets the current profile if no arguments are provided.  This is used to run and manage multiple minikube instance.  You can return to the default minikube profile by running `minikube profile default`",
	Run: func(cmd *cobra.Command, args []string) {
		if len(args) == 0 {
			profile := viper.GetString(pkgConfig.MachineProfile)
			console.OutLn("%s", profile)
			os.Exit(0)
		}

		if len(args) > 1 {
			exit.Usage("usage: minikube profile [MINIKUBE_PROFILE_NAME]")
		}

		profile := args[0]
		if profile == "default" {
			profile = "minikube"
		}
		err := Set(pkgConfig.MachineProfile, profile)
		if err != nil {
			exit.WithError("Setting profile failed", err)
		}
		cc, err := pkgConfig.Load()
		if err != nil && !os.IsNotExist(err) {
			console.ErrLn("Error loading profile config: %v", err)
		}
		if cc.MachineConfig.KeepContext {
			console.Success("Skipped switching kubectl context for %s , because --keep-context", profile)
			console.Success("To connect to this cluster, use: kubectl --context=%s", profile)
		} else {
			err := pkgutil.SetCurrentContext(constants.KubeconfigPath, profile)
			if err != nil {
				console.ErrLn("error while setting kubectl current context :  %v", err)
			}
		}
		console.Success("minikube profile was successfully set to %s", profile)
	},
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"os/exec"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/labels"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Autoscaling", func() {
	f := NewFramework("autoscaling")
	var nodeCount int
	var coresPerNode int
	var memCapacityMb int

	BeforeEach(func() {
		SkipUnlessProviderIs("gce")

		nodes, err := f.Client.Nodes().List(labels.Everything(), fields.Everything())
		expectNoError(err)
		nodeCount = len(nodes.Items)
		Expect(nodeCount).NotTo(BeZero())
		cpu := nodes.Items[0].Status.Capacity[api.ResourceCPU]
		mem := nodes.Items[0].Status.Capacity[api.ResourceMemory]
		coresPerNode = int((&cpu).MilliValue() / 1000)
		memCapacityMb = int((&mem).Value() / 1024 / 1024)
	})

	AfterEach(func() {
		cleanUpAutoscaler()
	})

	It("[Skipped][Autoscaling Suite] should scale cluster size based on cpu utilization", func() {
		setUpAutoscaler("cpu/node_utilization", 0.4, nodeCount, nodeCount+1)

		// Consume 50% CPU
		millicoresPerReplica := 500
		rc := NewStaticResourceConsumer("cpu-utilization", nodeCount*coresPerNode, millicoresPerReplica*nodeCount*coresPerNode, 0, int64(millicoresPerReplica), 100, f)
		expectNoError(waitForClusterSize(f.Client, nodeCount+1, 20*time.Minute))

		rc.CleanUp()
		expectNoError(waitForClusterSize(f.Client, nodeCount, 20*time.Minute))
	})

	It("[Skipped] should scale cluster size based on cpu reservation", func() {
		setUpAutoscaler("cpu/node_reservation", 0.5, nodeCount, nodeCount+1)

		ReserveCpu(f, "cpu-reservation", 600*nodeCount*coresPerNode)
		expectNoError(waitForClusterSize(f.Client, nodeCount+1, 20*time.Minute))

		expectNoError(DeleteRC(f.Client, f.Namespace.Name, "cpu-reservation"))
		expectNoError(waitForClusterSize(f.Client, nodeCount, 20*time.Minute))
	})

	It("[Skipped][Autoscaling Suite] should scale cluster size based on memory utilization", func() {
		setUpAutoscaler("memory/node_utilization", 0.6, nodeCount, nodeCount+1)

		// Consume 60% of total memory capacity
		megabytesPerReplica := int(memCapacityMb * 6 / 10 / coresPerNode)
		rc := NewStaticResourceConsumer("mem-utilization", nodeCount*coresPerNode, 0, megabytesPerReplica*nodeCount*coresPerNode, 100, int64(megabytesPerReplica+100), f)
		expectNoError(waitForClusterSize(f.Client, nodeCount+1, 20*time.Minute))

		rc.CleanUp()
		expectNoError(waitForClusterSize(f.Client, nodeCount, 20*time.Minute))
	})

	It("[Skipped] should scale cluster size based on memory reservation", func() {
		setUpAutoscaler("memory/node_reservation", 0.5, nodeCount, nodeCount+1)

		ReserveMemory(f, "memory-reservation", nodeCount*memCapacityMb*6/10)
		expectNoError(waitForClusterSize(f.Client, nodeCount+1, 20*time.Minute))

		expectNoError(DeleteRC(f.Client, f.Namespace.Name, "memory-reservation"))
		expectNoError(waitForClusterSize(f.Client, nodeCount, 20*time.Minute))
	})
})

func setUpAutoscaler(metric string, target float64, min, max int) {
	// TODO integrate with kube-up.sh script once it will support autoscaler setup.
	By("Setting up autoscaler to scale based on " + metric)
	out, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "set-autoscaling",
		testContext.CloudConfig.NodeInstanceGroup,
		"--project="+testContext.CloudConfig.ProjectID,
		"--zone="+testContext.CloudConfig.Zone,
		"--custom-metric-utilization=metric=custom.cloudmonitoring.googleapis.com/kubernetes.io/"+metric+fmt.Sprintf(",utilization-target=%v", target)+",utilization-target-type=GAUGE",
		fmt.Sprintf("--min-num-replicas=%v", min),
		fmt.Sprintf("--max-num-replicas=%v", max),
	).CombinedOutput()
	expectNoError(err, "Output: "+string(out))
}

func cleanUpAutoscaler() {
	By("Removing autoscaler")
	out, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "stop-autoscaling",
		testContext.CloudConfig.NodeInstanceGroup,
		"--project="+testContext.CloudConfig.ProjectID,
		"--zone="+testContext.CloudConfig.Zone,
	).CombinedOutput()
	expectNoError(err, "Output: "+string(out))
}

func ReserveCpu(f *Framework, id string, millicores int) {
	By(fmt.Sprintf("Running RC which reserves %v millicores", millicores))
	config := &RCConfig{
		Client:     f.Client,
		Name:       id,
		Namespace:  f.Namespace.Name,
		Timeout:    10 * time.Minute,
		Image:      "gcr.io/google_containers/pause",
		Replicas:   millicores / 100,
		CpuRequest: 100,
	}
	expectNoError(RunRC(*config))
}

func ReserveMemory(f *Framework, id string, megabytes int) {
	By(fmt.Sprintf("Running RC which reserves %v MB of memory", megabytes))
	config := &RCConfig{
		Client:     f.Client,
		Name:       id,
		Namespace:  f.Namespace.Name,
		Timeout:    10 * time.Minute,
		Image:      "gcr.io/google_containers/pause",
		Replicas:   megabytes / 500,
		MemRequest: 500 * 1024 * 1024,
	}
	expectNoError(RunRC(*config))
}
<commit_msg>Fixed flaky cluster size autoscaling e2e.<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"os/exec"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/labels"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	scaleUpTimeout   = 20 * time.Minute
	scaleDownTimeout = 30 * time.Minute
)

var _ = Describe("Autoscaling", func() {
	f := NewFramework("autoscaling")
	var nodeCount int
	var coresPerNode int
	var memCapacityMb int

	BeforeEach(func() {
		SkipUnlessProviderIs("gce")

		nodes, err := f.Client.Nodes().List(labels.Everything(), fields.Everything())
		expectNoError(err)
		nodeCount = len(nodes.Items)
		Expect(nodeCount).NotTo(BeZero())
		cpu := nodes.Items[0].Status.Capacity[api.ResourceCPU]
		mem := nodes.Items[0].Status.Capacity[api.ResourceMemory]
		coresPerNode = int((&cpu).MilliValue() / 1000)
		memCapacityMb = int((&mem).Value() / 1024 / 1024)
	})

	AfterEach(func() {
		cleanUpAutoscaler()
	})

	It("[Skipped][Autoscaling Suite] should scale cluster size based on cpu utilization", func() {
		setUpAutoscaler("cpu/node_utilization", 0.4, nodeCount, nodeCount+1)

		// Consume 50% CPU
		millicoresPerReplica := 500
		rc := NewStaticResourceConsumer("cpu-utilization", nodeCount*coresPerNode, millicoresPerReplica*nodeCount*coresPerNode, 0, int64(millicoresPerReplica), 100, f)
		expectNoError(waitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout))

		rc.CleanUp()
		expectNoError(waitForClusterSize(f.Client, nodeCount, scaleDownTimeout))
	})

	It("[Skipped] should scale cluster size based on cpu reservation", func() {
		setUpAutoscaler("cpu/node_reservation", 0.5, nodeCount, nodeCount+1)

		ReserveCpu(f, "cpu-reservation", 600*nodeCount*coresPerNode)
		expectNoError(waitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout))

		expectNoError(DeleteRC(f.Client, f.Namespace.Name, "cpu-reservation"))
		expectNoError(waitForClusterSize(f.Client, nodeCount, scaleDownTimeout))
	})

	It("[Skipped][Autoscaling Suite] should scale cluster size based on memory utilization", func() {
		setUpAutoscaler("memory/node_utilization", 0.6, nodeCount, nodeCount+1)

		// Consume 60% of total memory capacity
		megabytesPerReplica := int(memCapacityMb * 6 / 10 / coresPerNode)
		rc := NewStaticResourceConsumer("mem-utilization", nodeCount*coresPerNode, 0, megabytesPerReplica*nodeCount*coresPerNode, 100, int64(megabytesPerReplica+100), f)
		expectNoError(waitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout))

		rc.CleanUp()
		expectNoError(waitForClusterSize(f.Client, nodeCount, scaleDownTimeout))
	})

	It("[Skipped] should scale cluster size based on memory reservation", func() {
		setUpAutoscaler("memory/node_reservation", 0.5, nodeCount, nodeCount+1)

		ReserveMemory(f, "memory-reservation", nodeCount*memCapacityMb*6/10)
		expectNoError(waitForClusterSize(f.Client, nodeCount+1, scaleUpTimeout))

		expectNoError(DeleteRC(f.Client, f.Namespace.Name, "memory-reservation"))
		expectNoError(waitForClusterSize(f.Client, nodeCount, scaleDownTimeout))
	})
})

func setUpAutoscaler(metric string, target float64, min, max int) {
	// TODO integrate with kube-up.sh script once it will support autoscaler setup.
	By("Setting up autoscaler to scale based on " + metric)
	out, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "set-autoscaling",
		testContext.CloudConfig.NodeInstanceGroup,
		"--project="+testContext.CloudConfig.ProjectID,
		"--zone="+testContext.CloudConfig.Zone,
		"--custom-metric-utilization=metric=custom.cloudmonitoring.googleapis.com/kubernetes.io/"+metric+fmt.Sprintf(",utilization-target=%v", target)+",utilization-target-type=GAUGE",
		fmt.Sprintf("--min-num-replicas=%v", min),
		fmt.Sprintf("--max-num-replicas=%v", max),
	).CombinedOutput()
	expectNoError(err, "Output: "+string(out))
}

func cleanUpAutoscaler() {
	By("Removing autoscaler")
	out, err := exec.Command("gcloud", "compute", "instance-groups", "managed", "stop-autoscaling",
		testContext.CloudConfig.NodeInstanceGroup,
		"--project="+testContext.CloudConfig.ProjectID,
		"--zone="+testContext.CloudConfig.Zone,
	).CombinedOutput()
	expectNoError(err, "Output: "+string(out))
}

func ReserveCpu(f *Framework, id string, millicores int) {
	By(fmt.Sprintf("Running RC which reserves %v millicores", millicores))
	config := &RCConfig{
		Client:     f.Client,
		Name:       id,
		Namespace:  f.Namespace.Name,
		Timeout:    10 * time.Minute,
		Image:      "gcr.io/google_containers/pause",
		Replicas:   millicores / 100,
		CpuRequest: 100,
	}
	expectNoError(RunRC(*config))
}

func ReserveMemory(f *Framework, id string, megabytes int) {
	By(fmt.Sprintf("Running RC which reserves %v MB of memory", megabytes))
	config := &RCConfig{
		Client:     f.Client,
		Name:       id,
		Namespace:  f.Namespace.Name,
		Timeout:    10 * time.Minute,
		Image:      "gcr.io/google_containers/pause",
		Replicas:   megabytes / 500,
		MemRequest: 500 * 1024 * 1024,
	}
	expectNoError(RunRC(*config))
}
<|endoftext|>


<commit_before>package raspberry

import (
	"github.com/remogatto/egl"
	"github.com/remogatto/egl/platform"
	"log"
	"runtime/debug"
	"unsafe"
)

var (
	DefaultConfigAttributes = []int32{
		egl.RED_SIZE, 8,
		egl.GREEN_SIZE, 8,
		egl.BLUE_SIZE, 8,
		egl.ALPHA_SIZE, 8,
		egl.DEPTH_SIZE, 8,
		egl.SURFACE_TYPE, egl.WINDOW_BIT,
		egl.NONE,
	}
	DefaultContextAttributes = []int32{
		egl.CONTEXT_CLIENT_VERSION, 2,
		egl.NONE,
	}
	dstRect, srcRect            egl.VCRect
	NativeWindow                egl.EGLDispmanxWindow
	screen_width, screen_height uint32
)

func assert(cond bool) {
	if !cond {
		debug.PrintStack()
		panic("Assertion failed!")
	}
}

func Initialize(configAttr, contextAttr []int32) {
	platform.Display = egl.GetDisplay(egl.DEFAULT_DISPLAY)
	if ok := egl.Initialize(platform.Display, nil, nil); !ok {
		egl.LogError(egl.GetError())
	}
	if ok := egl.ChooseConfig(platform.Display, configAttr, &platform.Config, 1, &platform.NumConfig); !ok {
		egl.LogError(egl.GetError())
	}
	if ok := egl.GetConfigAttrib(platform.Display, platform.Config, egl.NATIVE_VISUAL_ID, &platform.VisualId); !ok {
		egl.LogError(egl.GetError())
	}
	egl.BindAPI(egl.OPENGL_ES_API)
	platform.Context = egl.CreateContext(platform.Display, platform.Config, egl.NO_CONTEXT, &contextAttr[0])
	screen_width, screen_height = egl.GraphicsGetDisplaySize(0)
	log.Printf("Display size W: %d H: %d\n", screen_width, screen_height)

	dstRect.X = 0
	dstRect.Y = 0
	dstRect.Width = int32(screen_width)
	dstRect.Height = int32(screen_height)

	srcRect.X = 0
	srcRect.Y = 0
	srcRect.Width = int32(screen_width << 16)
	srcRect.Height = int32(screen_height << 16)

	dispman_display := egl.VCDispmanxDisplayOpen(0 /* LCD */)
	dispman_update := egl.VCDispmanxUpdateStart(0)

	dispman_element := egl.VCDispmanxElementAdd(
		dispman_update,
		dispman_display,
		0, /*layer */
		&dstRect,
		0, /*src */
		&srcRect,
		egl.DISPMANX_PROTECTION_NONE,
		nil, /*alpha */
		nil, /*clamp */
		0 /*transform */)

	NativeWindow.Element = dispman_element
	NativeWindow.Width = int(screen_width)
	NativeWindow.Height = int(screen_height)
	egl.VCDispmanxUpdateSubmitSync(dispman_update)

	platform.Surface = egl.CreateWindowSurface(
		platform.Display,
		platform.Config,
		egl.NativeWindowType(unsafe.Pointer(&NativeWindow)),
		nil)
	assert(platform.Surface != egl.NO_SURFACE)

	// connect the context to the surface
	result := egl.MakeCurrent(platform.Display, platform.Surface, platform.Surface, platform.Context)
	assert(result)

	var val int32
	if ok := egl.QuerySurface(platform.Display, &val, egl.WIDTH, platform.Surface); !ok {
		egl.LogError(egl.GetError())
	}

	if ok := egl.QuerySurface(platform.Display, &val, egl.HEIGHT, platform.Surface); !ok {
		egl.LogError(egl.GetError())
	}
	if ok := egl.GetConfigAttrib(platform.Display, platform.Config, egl.SURFACE_TYPE, &val); !ok {
		egl.LogError(egl.GetError())
	}
}
<commit_msg>Fix support for RPI (first try)<commit_after>package raspberry

import (
	"github.com/remogatto/egl"
	"github.com/remogatto/egl/platform"
	"log"
	"runtime/debug"
	"unsafe"
)

var (
	DefaultConfigAttributes = []int32{
		egl.RED_SIZE, 8,
		egl.GREEN_SIZE, 8,
		egl.BLUE_SIZE, 8,
		egl.ALPHA_SIZE, 8,
		egl.DEPTH_SIZE, 8,
		egl.SURFACE_TYPE, egl.WINDOW_BIT,
		egl.NONE,
	}
	DefaultContextAttributes = []int32{
		egl.CONTEXT_CLIENT_VERSION, 2,
		egl.NONE,
	}
)

func Initialize(configAttr, contextAttr []int32) *platform.EGLState {
	var (
		config           egl.Config
		numConfig        int32
		visualId         int32
		width, height    int32
		dstRect, srcRect egl.VCRect
		nativeWindow     egl.EGLDispmanxWindow
	)
	display := egl.GetDisplay(egl.DEFAULT_DISPLAY)
	if ok := egl.Initialize(egl.Display, nil, nil); !ok {
		egl.LogError(egl.GetError())
	}
	if ok := egl.ChooseConfig(display, configAttr, &config, 1, &numConfig); !ok {
		egl.LogError(egl.GetError())
	}
	if ok := egl.GetConfigAttrib(display, config, egl.NATIVE_VISUAL_ID, &visualId); !ok {
		egl.LogError(egl.GetError())
	}
	egl.BindAPI(egl.OPENGL_ES_API)
	context := egl.CreateContext(display, config, egl.NO_CONTEXT, &contextAttr[0])

	width, height = egl.GraphicsGetDisplaySize(0)

	dstRect.X = 0
	dstRect.Y = 0
	dstRect.Width = int32(width)
	dstRect.Height = int32(height)

	srcRect.X = 0
	srcRect.Y = 0
	srcRect.Width = int32(width << 16)
	srcRect.Height = int32(height << 16)

	dispman_display := egl.VCDispmanxDisplayOpen(0)
	dispman_update := egl.VCDispmanxUpdateStart(0)
	dispman_element := egl.VCDispmanxElementAdd(
		dispman_update,
		dispman_display,
		0, /*layer */
		&dstRect,
		0, /*src */
		&srcRect,
		egl.DISPMANX_PROTECTION_NONE,
		nil, /*alpha */
		nil, /*clamp */
		0 /*transform */)

	nativeWindow.Element = dispman_element
	nativeWindow.Width = int(width)
	nativeWindow.Height = int(height)
	egl.VCDispmanxUpdateSubmitSync(dispman_update)

	surface := egl.CreateWindowSurface(
		display,
		config,
		egl.NativeWindowType(unsafe.Pointer(&nativeWindow)),
		nil)

	if surface == egl.NO_SURFACE {
		panic("Error in creating EGL surface")
	}

	return &platform.EGLState{
		Display:           display,
		Config:            config,
		Context:           context,
		Surface:           surface,
		NumConfig:         numConfig,
		VisualId:          visualId,
		ContextAttributes: contextAttr,
		ConfigAttributes:  configAttr,
		SurfaceWidth:      int(width),
		SurfaceHeight:     int(height),
	}
}
<|endoftext|>


<commit_before>package gpio

import (
	"time"

	"github.com/hybridgroup/gobot"
)

var _ gobot.Driver = (*GroveTouchDriver)(nil)
var _ gobot.Driver = (*GroveSoundSensorDriver)(nil)
var _ gobot.Driver = (*GroveButtonDriver)(nil)
var _ gobot.Driver = (*GroveBuzzerDriver)(nil)
var _ gobot.Driver = (*GroveLightSensorDriver)(nil)
var _ gobot.Driver = (*GroveLedDriver)(nil)
var _ gobot.Driver = (*GroveRotaryDriver)(nil)
var _ gobot.Driver = (*GroveRelayDriver)(nil)

type GroveRelayDriver struct {
	*RelayDriver
}

type GroveRotaryDriver struct {
	*AnalogSensorDriver
}

type GroveLedDriver struct {
	*LedDriver
}

type GroveLightSensorDriver struct {
	*AnalogSensorDriver
}

type GroveBuzzerDriver struct {
	*BuzzerDriver
}

type GroveButtonDriver struct {
	*ButtonDriver
}

type GroveSoundSensorDriver struct {
	*AnalogSensorDriver
}

type GroveTouchDriver struct {
	*ButtonDriver
}

func NewGroveTouchDriver(a DigitalReader, name string, pin string, v ...time.Duration) *GroveButtonDriver {
	return &GroveTouchDriver{
		ButtonDriver: NewButtonDriver(a, name, pin, v...),
	}
}

func NewGroveButtonDriver(a DigitalReader, name string, pin string, v ...time.Duration) *GroveButtonDriver {
	return &GroveButtonDriver{
		ButtonDriver: NewButtonDriver(a, name, pin, v...),
	}
}

func NewGroveBuzzerDriver(a DigitalWriter, name string, pin string) *GroveBuzzerDriver {
	return &GroveBuzzerDriver{
		BuzzerDriver: NewBuzzerDriver(a, name, pin),
	}
}

func NewGroveLedDriver(a DigitalWriter, name string, pin string) *GroveLedDriver {
	return &GroveLedDriver{
		LedDriver: NewLedDriver(a, name, pin),
	}
}

func NewGroveRelayDriver(a DigitalWriter, name string, pin string) *GroveRelayDriver {
	return &GroveRelayDriver{
		RelayDriver: NewRelayDriver(a, name, pin),
	}
}

func NewGroveRotaryDriver(a AnalogReader, name string, pin string, v ...time.Duration) *GroveRotaryDriver {
	return &GroveRotaryDriver{
		AnalogSensorDriver: NewAnalogSensorDriver(a, name, pin, v...),
	}
}

func NewGroveLightSensorDriver(a AnalogReader, name string, pin string, v ...time.Duration) *GroveLightSensorDriver {
	return &GroveLightSensorDriver{
		AnalogSensorDriver: NewAnalogSensorDriver(a, name, pin, v...),
	}
}

func NewGroveSoundSensorDriver(a AnalogReader, name string, pin string, v ...time.Duration) *GroveSoundSensorDriver {
	return &GroveSoundSensorDriver{
		AnalogSensorDriver: NewAnalogSensorDriver(a, name, pin, v...),
	}
}
<commit_msg>Fix incorrect return type for NewGroveTouchDriver<commit_after>package gpio

import (
	"time"

	"github.com/hybridgroup/gobot"
)

var _ gobot.Driver = (*GroveTouchDriver)(nil)
var _ gobot.Driver = (*GroveSoundSensorDriver)(nil)
var _ gobot.Driver = (*GroveButtonDriver)(nil)
var _ gobot.Driver = (*GroveBuzzerDriver)(nil)
var _ gobot.Driver = (*GroveLightSensorDriver)(nil)
var _ gobot.Driver = (*GroveLedDriver)(nil)
var _ gobot.Driver = (*GroveRotaryDriver)(nil)
var _ gobot.Driver = (*GroveRelayDriver)(nil)

type GroveRelayDriver struct {
	*RelayDriver
}

type GroveRotaryDriver struct {
	*AnalogSensorDriver
}

type GroveLedDriver struct {
	*LedDriver
}

type GroveLightSensorDriver struct {
	*AnalogSensorDriver
}

type GroveBuzzerDriver struct {
	*BuzzerDriver
}

type GroveButtonDriver struct {
	*ButtonDriver
}

type GroveSoundSensorDriver struct {
	*AnalogSensorDriver
}

type GroveTouchDriver struct {
	*ButtonDriver
}

func NewGroveTouchDriver(a DigitalReader, name string, pin string, v ...time.Duration) *GroveTouchDriver {
	return &GroveTouchDriver{
		ButtonDriver: NewButtonDriver(a, name, pin, v...),
	}
}

func NewGroveButtonDriver(a DigitalReader, name string, pin string, v ...time.Duration) *GroveButtonDriver {
	return &GroveButtonDriver{
		ButtonDriver: NewButtonDriver(a, name, pin, v...),
	}
}

func NewGroveBuzzerDriver(a DigitalWriter, name string, pin string) *GroveBuzzerDriver {
	return &GroveBuzzerDriver{
		BuzzerDriver: NewBuzzerDriver(a, name, pin),
	}
}

func NewGroveLedDriver(a DigitalWriter, name string, pin string) *GroveLedDriver {
	return &GroveLedDriver{
		LedDriver: NewLedDriver(a, name, pin),
	}
}

func NewGroveRelayDriver(a DigitalWriter, name string, pin string) *GroveRelayDriver {
	return &GroveRelayDriver{
		RelayDriver: NewRelayDriver(a, name, pin),
	}
}

func NewGroveRotaryDriver(a AnalogReader, name string, pin string, v ...time.Duration) *GroveRotaryDriver {
	return &GroveRotaryDriver{
		AnalogSensorDriver: NewAnalogSensorDriver(a, name, pin, v...),
	}
}

func NewGroveLightSensorDriver(a AnalogReader, name string, pin string, v ...time.Duration) *GroveLightSensorDriver {
	return &GroveLightSensorDriver{
		AnalogSensorDriver: NewAnalogSensorDriver(a, name, pin, v...),
	}
}

func NewGroveSoundSensorDriver(a AnalogReader, name string, pin string, v ...time.Duration) *GroveSoundSensorDriver {
	return &GroveSoundSensorDriver{
		AnalogSensorDriver: NewAnalogSensorDriver(a, name, pin, v...),
	}
}
<|endoftext|>


<commit_before>/*
Package structor contains interface and default implementation of EL
(expression language) evaluator which evaluates EL expressions within tags of
the struct, using optional additional struct as an extra context.

Basic idea is to use simple expression language within Go struct tags to
compute struct fields based on other fields or provided additional context.

Due usage of reflection and EL interpretation, this package is hardly suitable
for tasks, requiring high performance, but rather intended to be used during
application setup or in cases where high performance is not an ultimate goal.

See tests for examples of usage with xpath, regexp, goquery, encryption,
reading from files, shell invocation, etc.
*/
package structor

import (
	"fmt"
	"reflect"
	"unsafe"

	multierror "github.com/hashicorp/go-multierror"

	"github.com/nikolay-turpitko/structor/el"
	"github.com/nikolay-turpitko/structor/funcs/use"
	"github.com/nikolay-turpitko/structor/scanner"
)

// Evaluator is an interface of evaluator, which gets structure and extra
// context as input, iterates over `s`'s fields and evaluate expression tag on
// every field.
type Evaluator interface {
	Eval(s, extra interface{}) error
}

// Interpreters is a map of tag names to el.Interpreters.  Used to register
// different interpreters for different tag names.
//
// Only first tag name on the struct field is currently recognized and
// processed. So, only one EL expression per structure field, but different
// fields of the same structure can be processed by different interpreters.
type Interpreters map[string]el.Interpreter

// WholeTag constant can be used as tag name in the Interpreters to indicate
// that whole tag value should be passed to the interpreter.
//
// Registering interpreter to the whole tag value conflicts with any other
// usage of the struct's tag, but can be convenient to simplify complex EL
// expressions with quotes (regexp, for example).
//
// WholeTag interpreter is probed after all other registered interpreters.
const WholeTag = ""

// NewEvaluatorWithOptions returns Evaluator with desired settings.
//
// Only first tag with EL will be recognized and used (only one
// expression per struct field). Different fields of the same struct can be
// processed using different EL interpreters.
//
//  scanner - is a scanner implementation to be used to scan tags.
//  interpreters - is a map of registered tag names to EL interpreters.
//  options - is an Options structure.
func NewEvaluatorWithOptions(
	scanner scanner.Scanner,
	interpreters Interpreters,
	options Options) Evaluator {
	if len(interpreters) == 0 {
		panic("no interpreters registered")
	}
	return &evaluator{scanner, interpreters, options}
}

// NewEvaluator returns Evaluator with desired settings.
// It invokes NewEvaluatorWithOptions with default options.
func NewEvaluator(
	scanner scanner.Scanner,
	interpreters Interpreters) Evaluator {
	return NewEvaluatorWithOptions(scanner, interpreters, Options{})
}

// NewDefaultEvaluator returns default Evaluator implementation. Default
// implementation uses tag "eval" for expressions and EL interpreter, based on
// `"text/template"`.
//
//  funcs - custom functions, available for interpreter;
func NewDefaultEvaluator(funcs use.FuncMap) Evaluator {
	return NewEvaluator(
		scanner.Default,
		Interpreters{
			"eval": &el.DefaultInterpreter{Funcs: funcs},
		})
}

// NewNonmutatingEvaluator creates Evaluator implementation which does not
// change original structure (does not save evaluated results) itself.  Though,
// interpreters can change structures' fields as a side effect.  In this case
// Evaluator can be used as a visitor of fields with tags for which it have
// registered interpreters.  It will invoke registered interpreter for field
// with corresponded tag.  Interpreter can then manipulate it's own state or
// el.Context.  For example, it can store processing results into context's
// Extra field.
//
// See NewEvaluator() for additional information.
func NewNonmutatingEvaluator(
	scanner scanner.Scanner,
	interpreters Interpreters) Evaluator {
	return NewEvaluatorWithOptions(scanner, interpreters, Options{NonMutating: true})
}

type evaluator struct {
	scanner      scanner.Scanner
	interpreters Interpreters
	options      Options
}

// Options is an options to create Evaluator.
type Options struct {

	// NonMutating creates non-mutating Evaluator, see NewNonmutatingEvaluator.
	NonMutating bool

	// EvalEmptyTags causes Evaluator to invoke Interpreter for fields with
	// empty tags.
	EvalEmptyTags bool
}

func (ev evaluator) Eval(s, extra interface{}) error {
	v := reflect.ValueOf(s)
	t := v.Type()
	k := t.Kind()
	for v.IsValid() && (k == reflect.Interface || k == reflect.Ptr) {
		v = v.Elem()
		if v.IsValid() {
			t = v.Type()
			k = t.Kind()
		}
	}
	if k != reflect.Struct || !v.CanSet() {
		return fmt.Errorf("structor: %T: not a settable struct", s)
	}
	return multierror.Prefix(
		ev.eval(
			"",
			nil,
			reflect.ValueOf(s),
			&el.Context{
				Struct:   s,
				Extra:    extra,
				EvalExpr: ev.evalExpr,
				LongName: fmt.Sprintf("%T", s),
			}), "structor:")
}

func (ev evaluator) evalExpr(
	intrprName, expr string,
	ctx *el.Context) (interface{}, error) {
	intrpr, ok := ev.interpreters[intrprName]
	if !ok {
		return nil, fmt.Errorf("unknown interpreter: %s", intrprName)
	}
	return intrpr.Execute(expr, ctx)
}

func (ev evaluator) eval(
	expr string,
	interpreter el.Interpreter,
	v reflect.Value,
	ctx *el.Context) (err error) {
	// Note: only errors returned by recursive call should not be prefixed.
	defer func() {
		if r := recover(); r != nil {
			var ok bool
			if err, ok = r.(error); !ok {
				err = multierror.Prefix(
					fmt.Errorf("%v", r),
					fmt.Sprintf("<<%s>>", ctx.LongName))
			}
		}
	}()
	if !v.IsValid() {
		return nil
	}
	t := v.Type()
	k := t.Kind()
	if v.CanAddr() && !v.CanSet() {
		// https://stackoverflow.com/a/43918797/2063744
		v = reflect.NewAt(t, unsafe.Pointer(v.UnsafeAddr())).Elem()
	}
	elV, elT, elK := v, t, k
	for elV.IsValid() && (elK == reflect.Interface || elK == reflect.Ptr) {
		elV = v.Elem()
		if elV.IsValid() {
			if elV.CanAddr() && !elV.CanSet() {
				// https://stackoverflow.com/a/43918797/2063744
				elV = reflect.NewAt(elT, unsafe.Pointer(elV.UnsafeAddr())).Elem()
			}
			elT = elV.Type()
			elK = elT.Kind()
		}
	}
	var merr *multierror.Error
	var ctxSub interface{}
	if (expr != "" || ev.options.EvalEmptyTags) && interpreter != nil {
		ctx.Val = nil
		if elV.IsValid() {
			ctx.Val = elV.Interface()
		}
		if result, err := interpreter.Execute(expr, ctx); err != nil {
			merr = multierror.Append(
				merr,
				multierror.Prefix(err, fmt.Sprintf("<<%s>>", ctx.LongName)))
		} else {
			ctxSub = result
			if !ev.options.NonMutating {
				if result == nil {
					if v.IsValid() {
						v.Set(reflect.Zero(t))
					}
				} else {
					vnv := reflect.ValueOf(result)
					if vnv.Type().ConvertibleTo(t) || elK != reflect.Struct {
						// Try to convert, it may give a panic with suitable
						// message.
						v.Set(vnv.Convert(t))
					}
				}
			}
		}
	}
	switch elK {
	case reflect.Slice, reflect.Array:
		prevLongName := ctx.LongName
		for i, l := 0, elV.Len(); i < l; i++ {
			v := elV.Index(i)
			ctx.Name = v.Type().Name()
			ctx.LongName = fmt.Sprintf("%s[%d]", ctx.LongName, i)
			ctx.Tags = nil
			err := ev.eval("", nil, v, ctx)
			ctx.LongName = prevLongName
			merr = multierror.Append(merr, err)
		}
	case reflect.Struct:
		ctx.Sub = ctxSub
		prevLongName := ctx.LongName
		for i, l := 0, elV.NumField(); i < l; i++ {
			tf := elT.Field(i)
			tags, err := ev.scanner.Tags(tf.Tag)
			if err != nil {
				merr = multierror.Append(
					merr,
					multierror.Prefix(err, fmt.Sprintf("<<%s>>", ctx.LongName)))
				break
			}
			expr := ""
			var interpreter el.Interpreter
			for k, t := range tags {
				if i, ok := ev.interpreters[k]; ok {
					delete(tags, k)
					expr, interpreter = t, i
				}
			}
			if i, ok := ev.interpreters[WholeTag]; ok {
				delete(tags, WholeTag)
				expr, interpreter = string(tf.Tag), i
			}
			ctx.Name = tf.Name
			ctx.LongName = fmt.Sprintf("%s.%s", ctx.LongName, tf.Name)
			ctx.Tags = tags
			v := elV.Field(i)
			err = ev.eval(expr, interpreter, v, ctx)
			ctx.LongName = prevLongName
			merr = multierror.Append(merr, err)
		}
	}
	return merr.ErrorOrNil()
}

// AddressableCopy returns a pointer to the addressable copy of the struct.
func AddressableCopy(s interface{}) interface{} {
	v := reflect.Indirect(reflect.ValueOf(s))
	s2 := reflect.New(v.Type())
	s2.Elem().Set(v)
	return s2.Interface()
}
<commit_msg>Use whole tag interpreter only if other was not found<commit_after>/*
Package structor contains interface and default implementation of EL
(expression language) evaluator which evaluates EL expressions within tags of
the struct, using optional additional struct as an extra context.

Basic idea is to use simple expression language within Go struct tags to
compute struct fields based on other fields or provided additional context.

Due usage of reflection and EL interpretation, this package is hardly suitable
for tasks, requiring high performance, but rather intended to be used during
application setup or in cases where high performance is not an ultimate goal.

See tests for examples of usage with xpath, regexp, goquery, encryption,
reading from files, shell invocation, etc.
*/
package structor

import (
	"fmt"
	"reflect"
	"unsafe"

	multierror "github.com/hashicorp/go-multierror"

	"github.com/nikolay-turpitko/structor/el"
	"github.com/nikolay-turpitko/structor/funcs/use"
	"github.com/nikolay-turpitko/structor/scanner"
)

// Evaluator is an interface of evaluator, which gets structure and extra
// context as input, iterates over `s`'s fields and evaluate expression tag on
// every field.
type Evaluator interface {
	Eval(s, extra interface{}) error
}

// Interpreters is a map of tag names to el.Interpreters.  Used to register
// different interpreters for different tag names.
//
// Only first tag name on the struct field is currently recognized and
// processed. So, only one EL expression per structure field, but different
// fields of the same structure can be processed by different interpreters.
type Interpreters map[string]el.Interpreter

// WholeTag constant can be used as tag name in the Interpreters to indicate
// that whole tag value should be passed to the interpreter.
//
// Registering interpreter to the whole tag value conflicts with any other
// usage of the struct's tag, but can be convenient to simplify complex EL
// expressions with quotes (regexp, for example).
//
// WholeTag interpreter is probed after all other registered interpreters.
const WholeTag = ""

// NewEvaluatorWithOptions returns Evaluator with desired settings.
//
// Only first tag with EL will be recognized and used (only one
// expression per struct field). Different fields of the same struct can be
// processed using different EL interpreters.
//
//  scanner - is a scanner implementation to be used to scan tags.
//  interpreters - is a map of registered tag names to EL interpreters.
//  options - is an Options structure.
func NewEvaluatorWithOptions(
	scanner scanner.Scanner,
	interpreters Interpreters,
	options Options) Evaluator {
	if len(interpreters) == 0 {
		panic("no interpreters registered")
	}
	return &evaluator{scanner, interpreters, options}
}

// NewEvaluator returns Evaluator with desired settings.
// It invokes NewEvaluatorWithOptions with default options.
func NewEvaluator(
	scanner scanner.Scanner,
	interpreters Interpreters) Evaluator {
	return NewEvaluatorWithOptions(scanner, interpreters, Options{})
}

// NewDefaultEvaluator returns default Evaluator implementation. Default
// implementation uses tag "eval" for expressions and EL interpreter, based on
// `"text/template"`.
//
//  funcs - custom functions, available for interpreter;
func NewDefaultEvaluator(funcs use.FuncMap) Evaluator {
	return NewEvaluator(
		scanner.Default,
		Interpreters{
			"eval": &el.DefaultInterpreter{Funcs: funcs},
		})
}

// NewNonmutatingEvaluator creates Evaluator implementation which does not
// change original structure (does not save evaluated results) itself.  Though,
// interpreters can change structures' fields as a side effect.  In this case
// Evaluator can be used as a visitor of fields with tags for which it have
// registered interpreters.  It will invoke registered interpreter for field
// with corresponded tag.  Interpreter can then manipulate it's own state or
// el.Context.  For example, it can store processing results into context's
// Extra field.
//
// See NewEvaluator() for additional information.
func NewNonmutatingEvaluator(
	scanner scanner.Scanner,
	interpreters Interpreters) Evaluator {
	return NewEvaluatorWithOptions(scanner, interpreters, Options{NonMutating: true})
}

type evaluator struct {
	scanner      scanner.Scanner
	interpreters Interpreters
	options      Options
}

// Options is an options to create Evaluator.
type Options struct {

	// NonMutating creates non-mutating Evaluator, see NewNonmutatingEvaluator.
	NonMutating bool

	// EvalEmptyTags causes Evaluator to invoke Interpreter for fields with
	// empty tags.
	EvalEmptyTags bool
}

func (ev evaluator) Eval(s, extra interface{}) error {
	v := reflect.ValueOf(s)
	t := v.Type()
	k := t.Kind()
	for v.IsValid() && (k == reflect.Interface || k == reflect.Ptr) {
		v = v.Elem()
		if v.IsValid() {
			t = v.Type()
			k = t.Kind()
		}
	}
	if k != reflect.Struct || !v.CanSet() {
		return fmt.Errorf("structor: %T: not a settable struct", s)
	}
	return multierror.Prefix(
		ev.eval(
			"",
			nil,
			reflect.ValueOf(s),
			&el.Context{
				Struct:   s,
				Extra:    extra,
				EvalExpr: ev.evalExpr,
				LongName: fmt.Sprintf("%T", s),
			}), "structor:")
}

func (ev evaluator) evalExpr(
	intrprName, expr string,
	ctx *el.Context) (interface{}, error) {
	intrpr, ok := ev.interpreters[intrprName]
	if !ok {
		return nil, fmt.Errorf("unknown interpreter: %s", intrprName)
	}
	return intrpr.Execute(expr, ctx)
}

func (ev evaluator) eval(
	expr string,
	interpreter el.Interpreter,
	v reflect.Value,
	ctx *el.Context) (err error) {
	// Note: only errors returned by recursive call should not be prefixed.
	defer func() {
		if r := recover(); r != nil {
			var ok bool
			if err, ok = r.(error); !ok {
				err = multierror.Prefix(
					fmt.Errorf("%v", r),
					fmt.Sprintf("<<%s>>", ctx.LongName))
			}
		}
	}()
	if !v.IsValid() {
		return nil
	}
	t := v.Type()
	k := t.Kind()
	if v.CanAddr() && !v.CanSet() {
		// https://stackoverflow.com/a/43918797/2063744
		v = reflect.NewAt(t, unsafe.Pointer(v.UnsafeAddr())).Elem()
	}
	elV, elT, elK := v, t, k
	for elV.IsValid() && (elK == reflect.Interface || elK == reflect.Ptr) {
		elV = v.Elem()
		if elV.IsValid() {
			if elV.CanAddr() && !elV.CanSet() {
				// https://stackoverflow.com/a/43918797/2063744
				elV = reflect.NewAt(elT, unsafe.Pointer(elV.UnsafeAddr())).Elem()
			}
			elT = elV.Type()
			elK = elT.Kind()
		}
	}
	var merr *multierror.Error
	var ctxSub interface{}
	if (expr != "" || ev.options.EvalEmptyTags) && interpreter != nil {
		ctx.Val = nil
		if elV.IsValid() {
			ctx.Val = elV.Interface()
		}
		if result, err := interpreter.Execute(expr, ctx); err != nil {
			merr = multierror.Append(
				merr,
				multierror.Prefix(err, fmt.Sprintf("<<%s>>", ctx.LongName)))
		} else {
			ctxSub = result
			if !ev.options.NonMutating {
				if result == nil {
					if v.IsValid() {
						v.Set(reflect.Zero(t))
					}
				} else {
					vnv := reflect.ValueOf(result)
					if vnv.Type().ConvertibleTo(t) || elK != reflect.Struct {
						// Try to convert, it may give a panic with suitable
						// message.
						v.Set(vnv.Convert(t))
					}
				}
			}
		}
	}
	switch elK {
	case reflect.Slice, reflect.Array:
		prevLongName := ctx.LongName
		for i, l := 0, elV.Len(); i < l; i++ {
			v := elV.Index(i)
			ctx.Name = v.Type().Name()
			ctx.LongName = fmt.Sprintf("%s[%d]", ctx.LongName, i)
			ctx.Tags = nil
			err := ev.eval("", nil, v, ctx)
			ctx.LongName = prevLongName
			merr = multierror.Append(merr, err)
		}
	case reflect.Struct:
		ctx.Sub = ctxSub
		prevLongName := ctx.LongName
		for i, l := 0, elV.NumField(); i < l; i++ {
			tf := elT.Field(i)
			tags, err := ev.scanner.Tags(tf.Tag)
			if err != nil {
				merr = multierror.Append(
					merr,
					multierror.Prefix(err, fmt.Sprintf("<<%s>>", ctx.LongName)))
				break
			}
			expr := ""
			var interpreter el.Interpreter
			for k, t := range tags {
				if i, ok := ev.interpreters[k]; ok {
					delete(tags, k)
					expr, interpreter = t, i
				}
			}
			if i, ok := ev.interpreters[WholeTag]; ok && interpreter == nil {
				delete(tags, WholeTag)
				expr, interpreter = string(tf.Tag), i
			}
			ctx.Name = tf.Name
			ctx.LongName = fmt.Sprintf("%s.%s", ctx.LongName, tf.Name)
			ctx.Tags = tags
			v := elV.Field(i)
			err = ev.eval(expr, interpreter, v, ctx)
			ctx.LongName = prevLongName
			merr = multierror.Append(merr, err)
		}
	}
	return merr.ErrorOrNil()
}

// AddressableCopy returns a pointer to the addressable copy of the struct.
func AddressableCopy(s interface{}) interface{} {
	v := reflect.Indirect(reflect.ValueOf(s))
	s2 := reflect.New(v.Type())
	s2.Elem().Set(v)
	return s2.Interface()
}
<|endoftext|>


<commit_before>package coreunix

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	gopath "path"

	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
	syncds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/sync"
	bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
	key "github.com/ipfs/go-ipfs/blocks/key"
	bserv "github.com/ipfs/go-ipfs/blockservice"
	"github.com/ipfs/go-ipfs/exchange/offline"
	importer "github.com/ipfs/go-ipfs/importer"
	"github.com/ipfs/go-ipfs/importer/chunk"
	mfs "github.com/ipfs/go-ipfs/mfs"
	"github.com/ipfs/go-ipfs/pin"
	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"

	bs "github.com/ipfs/go-ipfs/blocks/blockstore"
	"github.com/ipfs/go-ipfs/commands/files"
	core "github.com/ipfs/go-ipfs/core"
	dag "github.com/ipfs/go-ipfs/merkledag"
	unixfs "github.com/ipfs/go-ipfs/unixfs"
	logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log"
)

var log = logging.Logger("coreunix")

var folderData = unixfs.FolderPBData()

// how many bytes of progress to wait before sending a progress update message
const progressReaderIncrement = 1024 * 256

type Link struct {
	Name, Hash string
	Size       uint64
}

type Object struct {
	Hash  string
	Links []Link
}

type hiddenFileError struct {
	fileName string
}

func (e *hiddenFileError) Error() string {
	return fmt.Sprintf("%s is a hidden file", e.fileName)
}

type ignoreFileError struct {
	fileName string
}

func (e *ignoreFileError) Error() string {
	return fmt.Sprintf("%s is an ignored file", e.fileName)
}

type AddedObject struct {
	Name  string
	Hash  string `json:",omitempty"`
	Bytes int64  `json:",omitempty"`
}

func NewAdder(ctx context.Context, n *core.IpfsNode, out chan interface{}) (*Adder, error) {
	mr, err := mfs.NewRoot(ctx, n.DAG, newDirNode(), nil)
	if err != nil {
		return nil, err
	}

	return &Adder{
		mr:       mr,
		ctx:      ctx,
		node:     n,
		out:      out,
		Progress: false,
		Hidden:   true,
		Pin:      true,
		Trickle:  false,
		Wrap:     false,
		Chunker:  "",
	}, nil
}

// Internal structure for holding the switches passed to the `add` call
type Adder struct {
	ctx      context.Context
	node     *core.IpfsNode
	out      chan interface{}
	Progress bool
	Hidden   bool
	Pin      bool
	Trickle  bool
	Silent   bool
	Wrap     bool
	Chunker  string
	root     *dag.Node
	mr       *mfs.Root
	unlocker bs.Unlocker
	tempRoot key.Key
}

// Perform the actual add & pin locally, outputting results to reader
func (adder Adder) add(reader io.Reader) (*dag.Node, error) {
	chnk, err := chunk.FromString(reader, adder.Chunker)
	if err != nil {
		return nil, err
	}

	if adder.Trickle {
		return importer.BuildTrickleDagFromReader(
			adder.node.DAG,
			chnk,
		)
	}
	return importer.BuildDagFromReader(
		adder.node.DAG,
		chnk,
	)
}

func (adder *Adder) RootNode() (*dag.Node, error) {
	// for memoizing
	if adder.root != nil {
		return adder.root, nil
	}

	root, err := adder.mr.GetValue().GetNode()
	if err != nil {
		return nil, err
	}

	// if not wrapping, AND one root file, use that hash as root.
	if !adder.Wrap && len(root.Links) == 1 {
		root, err = root.Links[0].GetNode(adder.ctx, adder.node.DAG)
		if err != nil {
			return nil, err
		}
	}

	adder.root = root
	return root, err
}

func (adder *Adder) PinRoot() error {
	root, err := adder.RootNode()
	if err != nil {
		return err
	}
	if !adder.Pin {
		return nil
	}

	rnk, err := adder.node.DAG.Add(root)
	if err != nil {
		return err
	}

	if adder.tempRoot != "" {
		err := adder.node.Pinning.Unpin(adder.ctx, adder.tempRoot, true)
		if err != nil {
			return err
		}
		adder.tempRoot = rnk
	}

	adder.node.Pinning.PinWithMode(rnk, pin.Recursive)
	return adder.node.Pinning.Flush()
}

func (adder *Adder) Finalize() (*dag.Node, error) {
	root := adder.mr.GetValue()

	// cant just call adder.RootNode() here as we need the name for printing
	rootNode, err := root.GetNode()
	if err != nil {
		return nil, err
	}

	var name string
	if !adder.Wrap {
		name = rootNode.Links[0].Name

		dir, ok := adder.mr.GetValue().(*mfs.Directory)
		if !ok {
			return nil, fmt.Errorf("root is not a directory")
		}

		root, err = dir.Child(name)
		if err != nil {
			return nil, err
		}
	}

	err = adder.outputDirs(name, root)
	if err != nil {
		return nil, err
	}

	err = adder.mr.Close()
	if err != nil {
		return nil, err
	}

	rootNode, err = root.GetNode()
	if err != nil {
		return nil, err
	}
	return rootNode, nil
}

func (adder *Adder) outputDirs(path string, fs mfs.FSNode) error {
	nd, err := fs.GetNode()
	if err != nil {
		return err
	}

	if !bytes.Equal(nd.Data, folderData) || fs.Type() != mfs.TDir {
		return nil
	}

	dir, ok := fs.(*mfs.Directory)
	if !ok {
		return fmt.Errorf("received FSNode of type TDir that was not a Directory")
	}

	for _, name := range dir.ListNames() {
		child, err := dir.Child(name)
		if err != nil {
			return err
		}

		err = adder.outputDirs(gopath.Join(path, name), child)
		if err != nil {
			return err
		}
	}

	return outputDagnode(adder.out, path, nd)
}

// Add builds a merkledag from the a reader, pinning all objects to the local
// datastore. Returns a key representing the root node.
func Add(n *core.IpfsNode, r io.Reader) (string, error) {
	defer n.Blockstore.PinLock().Unlock()

	fileAdder, err := NewAdder(n.Context(), n, nil)
	if err != nil {
		return "", err
	}

	node, err := fileAdder.add(r)
	if err != nil {
		return "", err
	}
	k, err := node.Key()
	if err != nil {
		return "", err
	}

	return k.String(), nil
}

// AddR recursively adds files in |path|.
func AddR(n *core.IpfsNode, root string) (key string, err error) {
	n.Blockstore.PinLock().Unlock()

	stat, err := os.Lstat(root)
	if err != nil {
		return "", err
	}

	f, err := files.NewSerialFile(root, root, false, stat)
	if err != nil {
		return "", err
	}
	defer f.Close()

	fileAdder, err := NewAdder(n.Context(), n, nil)
	if err != nil {
		return "", err
	}

	err = fileAdder.addFile(f)
	if err != nil {
		return "", err
	}

	nd, err := fileAdder.Finalize()
	if err != nil {
		return "", err
	}

	k, err := nd.Key()
	if err != nil {
		return "", err
	}

	return k.String(), nil
}

// AddWrapped adds data from a reader, and wraps it with a directory object
// to preserve the filename.
// Returns the path of the added file ("<dir hash>/filename"), the DAG node of
// the directory, and and error if any.
func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.Node, error) {
	file := files.NewReaderFile(filename, filename, ioutil.NopCloser(r), nil)
	fileAdder, err := NewAdder(n.Context(), n, nil)
	if err != nil {
		return "", nil, err
	}
	fileAdder.Wrap = true

	defer n.Blockstore.PinLock().Unlock()

	err = fileAdder.addFile(file)
	if err != nil {
		return "", nil, err
	}

	dagnode, err := fileAdder.Finalize()
	if err != nil {
		return "", nil, err
	}

	k, err := dagnode.Key()
	if err != nil {
		return "", nil, err
	}

	return gopath.Join(k.String(), filename), dagnode, nil
}

func (adder *Adder) addNode(node *dag.Node, path string) error {
	// patch it into the root
	if path == "" {
		key, err := node.Key()
		if err != nil {
			return err
		}

		path = key.B58String()
	}

	dir := gopath.Dir(path)
	if dir != "." {
		if err := mfs.Mkdir(adder.mr, dir, true, false); err != nil {
			return err
		}
	}

	if err := mfs.PutNode(adder.mr, path, node); err != nil {
		return err
	}

	if !adder.Silent {
		return outputDagnode(adder.out, path, node)
	}
	return nil
}

// Add the given file while respecting the adder.
func (adder *Adder) AddFile(file files.File) error {
	adder.unlocker = adder.node.Blockstore.PinLock()
	defer func() {
		adder.unlocker.Unlock()
	}()

	return adder.addFile(file)
}

func (adder *Adder) addFile(file files.File) error {
	err := adder.maybePauseForGC()
	if err != nil {
		return err
	}

	if file.IsDirectory() {
		return adder.addDir(file)
	}

	// case for symlink
	if s, ok := file.(*files.Symlink); ok {
		sdata, err := unixfs.SymlinkData(s.Target)
		if err != nil {
			return err
		}

		dagnode := &dag.Node{Data: sdata}
		_, err = adder.node.DAG.Add(dagnode)
		if err != nil {
			return err
		}

		return adder.addNode(dagnode, s.FileName())
	}

	// case for regular file
	// if the progress flag was specified, wrap the file so that we can send
	// progress updates to the client (over the output channel)
	var reader io.Reader = file
	if adder.Progress {
		reader = &progressReader{file: file, out: adder.out}
	}

	dagnode, err := adder.add(reader)
	if err != nil {
		return err
	}

	// patch it into the root
	return adder.addNode(dagnode, file.FileName())
}

func (adder *Adder) addDir(dir files.File) error {
	log.Infof("adding directory: %s", dir.FileName())

	err := mfs.Mkdir(adder.mr, dir.FileName(), true, false)
	if err != nil {
		return err
	}

	for {
		file, err := dir.NextFile()
		if err != nil && err != io.EOF {
			return err
		}
		if file == nil {
			break
		}

		// Skip hidden files when adding recursively, unless Hidden is enabled.
		if files.IsHidden(file) && !adder.Hidden {
			log.Infof("%s is hidden, skipping", file.FileName())
			continue
		}
		err = adder.addFile(file)
		if err != nil {
			return err
		}
	}

	return nil
}

func (adder *Adder) maybePauseForGC() error {
	if adder.node.Blockstore.GCRequested() {
		err := adder.PinRoot()
		if err != nil {
			return err
		}

		adder.unlocker.Unlock()
		adder.unlocker = adder.node.Blockstore.PinLock()
	}
	return nil
}

// outputDagnode sends dagnode info over the output channel
func outputDagnode(out chan interface{}, name string, dn *dag.Node) error {
	if out == nil {
		return nil
	}

	o, err := getOutput(dn)
	if err != nil {
		return err
	}

	out <- &AddedObject{
		Hash: o.Hash,
		Name: name,
	}

	return nil
}

func NewMemoryDagService() dag.DAGService {
	// build mem-datastore for editor's intermediary nodes
	bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
	bsrv := bserv.New(bs, offline.Exchange(bs))
	return dag.NewDAGService(bsrv)
}

// TODO: generalize this to more than unix-fs nodes.
func newDirNode() *dag.Node {
	return &dag.Node{Data: unixfs.FolderPBData()}
}

// from core/commands/object.go
func getOutput(dagnode *dag.Node) (*Object, error) {
	key, err := dagnode.Key()
	if err != nil {
		return nil, err
	}

	output := &Object{
		Hash:  key.B58String(),
		Links: make([]Link, len(dagnode.Links)),
	}

	for i, link := range dagnode.Links {
		output.Links[i] = Link{
			Name: link.Name,
			//Hash: link.Hash.B58String(),
			Size: link.Size,
		}
	}

	return output, nil
}

type progressReader struct {
	file         files.File
	out          chan interface{}
	bytes        int64
	lastProgress int64
}

func (i *progressReader) Read(p []byte) (int, error) {
	n, err := i.file.Read(p)

	i.bytes += int64(n)
	if i.bytes-i.lastProgress >= progressReaderIncrement || err == io.EOF {
		i.lastProgress = i.bytes
		i.out <- &AddedObject{
			Name:  i.file.FileName(),
			Bytes: i.bytes,
		}
	}

	return n, err
}
<commit_msg>redundency-be-gone<commit_after>package coreunix

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	gopath "path"

	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
	syncds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/sync"
	bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
	key "github.com/ipfs/go-ipfs/blocks/key"
	bserv "github.com/ipfs/go-ipfs/blockservice"
	"github.com/ipfs/go-ipfs/exchange/offline"
	importer "github.com/ipfs/go-ipfs/importer"
	"github.com/ipfs/go-ipfs/importer/chunk"
	mfs "github.com/ipfs/go-ipfs/mfs"
	"github.com/ipfs/go-ipfs/pin"
	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"

	bs "github.com/ipfs/go-ipfs/blocks/blockstore"
	"github.com/ipfs/go-ipfs/commands/files"
	core "github.com/ipfs/go-ipfs/core"
	dag "github.com/ipfs/go-ipfs/merkledag"
	unixfs "github.com/ipfs/go-ipfs/unixfs"
	logging "gx/ipfs/Qmazh5oNUVsDZTs2g59rq8aYQqwpss8tcUWQzor5sCCEuH/go-log"
)

var log = logging.Logger("coreunix")

var folderData = unixfs.FolderPBData()

// how many bytes of progress to wait before sending a progress update message
const progressReaderIncrement = 1024 * 256

type Link struct {
	Name, Hash string
	Size       uint64
}

type Object struct {
	Hash  string
	Links []Link
}

type hiddenFileError struct {
	fileName string
}

func (e *hiddenFileError) Error() string {
	return fmt.Sprintf("%s is a hidden file", e.fileName)
}

type ignoreFileError struct {
	fileName string
}

func (e *ignoreFileError) Error() string {
	return fmt.Sprintf("%s is an ignored file", e.fileName)
}

type AddedObject struct {
	Name  string
	Hash  string `json:",omitempty"`
	Bytes int64  `json:",omitempty"`
}

func NewAdder(ctx context.Context, n *core.IpfsNode, out chan interface{}) (*Adder, error) {
	mr, err := mfs.NewRoot(ctx, n.DAG, newDirNode(), nil)
	if err != nil {
		return nil, err
	}

	return &Adder{
		mr:       mr,
		ctx:      ctx,
		node:     n,
		out:      out,
		Progress: false,
		Hidden:   true,
		Pin:      true,
		Trickle:  false,
		Wrap:     false,
		Chunker:  "",
	}, nil
}

// Internal structure for holding the switches passed to the `add` call
type Adder struct {
	ctx      context.Context
	node     *core.IpfsNode
	out      chan interface{}
	Progress bool
	Hidden   bool
	Pin      bool
	Trickle  bool
	Silent   bool
	Wrap     bool
	Chunker  string
	root     *dag.Node
	mr       *mfs.Root
	unlocker bs.Unlocker
	tempRoot key.Key
}

// Perform the actual add & pin locally, outputting results to reader
func (adder Adder) add(reader io.Reader) (*dag.Node, error) {
	chnk, err := chunk.FromString(reader, adder.Chunker)
	if err != nil {
		return nil, err
	}

	if adder.Trickle {
		return importer.BuildTrickleDagFromReader(
			adder.node.DAG,
			chnk,
		)
	}
	return importer.BuildDagFromReader(
		adder.node.DAG,
		chnk,
	)
}

func (adder *Adder) RootNode() (*dag.Node, error) {
	// for memoizing
	if adder.root != nil {
		return adder.root, nil
	}

	root, err := adder.mr.GetValue().GetNode()
	if err != nil {
		return nil, err
	}

	// if not wrapping, AND one root file, use that hash as root.
	if !adder.Wrap && len(root.Links) == 1 {
		root, err = root.Links[0].GetNode(adder.ctx, adder.node.DAG)
		if err != nil {
			return nil, err
		}
	}

	adder.root = root
	return root, err
}

func (adder *Adder) PinRoot() error {
	root, err := adder.RootNode()
	if err != nil {
		return err
	}
	if !adder.Pin {
		return nil
	}

	rnk, err := adder.node.DAG.Add(root)
	if err != nil {
		return err
	}

	if adder.tempRoot != "" {
		err := adder.node.Pinning.Unpin(adder.ctx, adder.tempRoot, true)
		if err != nil {
			return err
		}
		adder.tempRoot = rnk
	}

	adder.node.Pinning.PinWithMode(rnk, pin.Recursive)
	return adder.node.Pinning.Flush()
}

func (adder *Adder) Finalize() (*dag.Node, error) {
	root := adder.mr.GetValue()

	// cant just call adder.RootNode() here as we need the name for printing
	rootNode, err := root.GetNode()
	if err != nil {
		return nil, err
	}

	var name string
	if !adder.Wrap {
		name = rootNode.Links[0].Name

		dir, ok := adder.mr.GetValue().(*mfs.Directory)
		if !ok {
			return nil, fmt.Errorf("root is not a directory")
		}

		root, err = dir.Child(name)
		if err != nil {
			return nil, err
		}
	}

	err = adder.outputDirs(name, root)
	if err != nil {
		return nil, err
	}

	err = adder.mr.Close()
	if err != nil {
		return nil, err
	}

	return root.GetNode()
}

func (adder *Adder) outputDirs(path string, fs mfs.FSNode) error {
	nd, err := fs.GetNode()
	if err != nil {
		return err
	}

	if !bytes.Equal(nd.Data, folderData) || fs.Type() != mfs.TDir {
		return nil
	}

	dir, ok := fs.(*mfs.Directory)
	if !ok {
		return fmt.Errorf("received FSNode of type TDir that was not a Directory")
	}

	for _, name := range dir.ListNames() {
		child, err := dir.Child(name)
		if err != nil {
			return err
		}

		err = adder.outputDirs(gopath.Join(path, name), child)
		if err != nil {
			return err
		}
	}

	return outputDagnode(adder.out, path, nd)
}

// Add builds a merkledag from the a reader, pinning all objects to the local
// datastore. Returns a key representing the root node.
func Add(n *core.IpfsNode, r io.Reader) (string, error) {
	defer n.Blockstore.PinLock().Unlock()

	fileAdder, err := NewAdder(n.Context(), n, nil)
	if err != nil {
		return "", err
	}

	node, err := fileAdder.add(r)
	if err != nil {
		return "", err
	}
	k, err := node.Key()
	if err != nil {
		return "", err
	}

	return k.String(), nil
}

// AddR recursively adds files in |path|.
func AddR(n *core.IpfsNode, root string) (key string, err error) {
	n.Blockstore.PinLock().Unlock()

	stat, err := os.Lstat(root)
	if err != nil {
		return "", err
	}

	f, err := files.NewSerialFile(root, root, false, stat)
	if err != nil {
		return "", err
	}
	defer f.Close()

	fileAdder, err := NewAdder(n.Context(), n, nil)
	if err != nil {
		return "", err
	}

	err = fileAdder.addFile(f)
	if err != nil {
		return "", err
	}

	nd, err := fileAdder.Finalize()
	if err != nil {
		return "", err
	}

	k, err := nd.Key()
	if err != nil {
		return "", err
	}

	return k.String(), nil
}

// AddWrapped adds data from a reader, and wraps it with a directory object
// to preserve the filename.
// Returns the path of the added file ("<dir hash>/filename"), the DAG node of
// the directory, and and error if any.
func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.Node, error) {
	file := files.NewReaderFile(filename, filename, ioutil.NopCloser(r), nil)
	fileAdder, err := NewAdder(n.Context(), n, nil)
	if err != nil {
		return "", nil, err
	}
	fileAdder.Wrap = true

	defer n.Blockstore.PinLock().Unlock()

	err = fileAdder.addFile(file)
	if err != nil {
		return "", nil, err
	}

	dagnode, err := fileAdder.Finalize()
	if err != nil {
		return "", nil, err
	}

	k, err := dagnode.Key()
	if err != nil {
		return "", nil, err
	}

	return gopath.Join(k.String(), filename), dagnode, nil
}

func (adder *Adder) addNode(node *dag.Node, path string) error {
	// patch it into the root
	if path == "" {
		key, err := node.Key()
		if err != nil {
			return err
		}

		path = key.B58String()
	}

	dir := gopath.Dir(path)
	if dir != "." {
		if err := mfs.Mkdir(adder.mr, dir, true, false); err != nil {
			return err
		}
	}

	if err := mfs.PutNode(adder.mr, path, node); err != nil {
		return err
	}

	if !adder.Silent {
		return outputDagnode(adder.out, path, node)
	}
	return nil
}

// Add the given file while respecting the adder.
func (adder *Adder) AddFile(file files.File) error {
	adder.unlocker = adder.node.Blockstore.PinLock()
	defer func() {
		adder.unlocker.Unlock()
	}()

	return adder.addFile(file)
}

func (adder *Adder) addFile(file files.File) error {
	err := adder.maybePauseForGC()
	if err != nil {
		return err
	}

	if file.IsDirectory() {
		return adder.addDir(file)
	}

	// case for symlink
	if s, ok := file.(*files.Symlink); ok {
		sdata, err := unixfs.SymlinkData(s.Target)
		if err != nil {
			return err
		}

		dagnode := &dag.Node{Data: sdata}
		_, err = adder.node.DAG.Add(dagnode)
		if err != nil {
			return err
		}

		return adder.addNode(dagnode, s.FileName())
	}

	// case for regular file
	// if the progress flag was specified, wrap the file so that we can send
	// progress updates to the client (over the output channel)
	var reader io.Reader = file
	if adder.Progress {
		reader = &progressReader{file: file, out: adder.out}
	}

	dagnode, err := adder.add(reader)
	if err != nil {
		return err
	}

	// patch it into the root
	return adder.addNode(dagnode, file.FileName())
}

func (adder *Adder) addDir(dir files.File) error {
	log.Infof("adding directory: %s", dir.FileName())

	err := mfs.Mkdir(adder.mr, dir.FileName(), true, false)
	if err != nil {
		return err
	}

	for {
		file, err := dir.NextFile()
		if err != nil && err != io.EOF {
			return err
		}
		if file == nil {
			break
		}

		// Skip hidden files when adding recursively, unless Hidden is enabled.
		if files.IsHidden(file) && !adder.Hidden {
			log.Infof("%s is hidden, skipping", file.FileName())
			continue
		}
		err = adder.addFile(file)
		if err != nil {
			return err
		}
	}

	return nil
}

func (adder *Adder) maybePauseForGC() error {
	if adder.node.Blockstore.GCRequested() {
		err := adder.PinRoot()
		if err != nil {
			return err
		}

		adder.unlocker.Unlock()
		adder.unlocker = adder.node.Blockstore.PinLock()
	}
	return nil
}

// outputDagnode sends dagnode info over the output channel
func outputDagnode(out chan interface{}, name string, dn *dag.Node) error {
	if out == nil {
		return nil
	}

	o, err := getOutput(dn)
	if err != nil {
		return err
	}

	out <- &AddedObject{
		Hash: o.Hash,
		Name: name,
	}

	return nil
}

func NewMemoryDagService() dag.DAGService {
	// build mem-datastore for editor's intermediary nodes
	bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
	bsrv := bserv.New(bs, offline.Exchange(bs))
	return dag.NewDAGService(bsrv)
}

// TODO: generalize this to more than unix-fs nodes.
func newDirNode() *dag.Node {
	return &dag.Node{Data: unixfs.FolderPBData()}
}

// from core/commands/object.go
func getOutput(dagnode *dag.Node) (*Object, error) {
	key, err := dagnode.Key()
	if err != nil {
		return nil, err
	}

	output := &Object{
		Hash:  key.B58String(),
		Links: make([]Link, len(dagnode.Links)),
	}

	for i, link := range dagnode.Links {
		output.Links[i] = Link{
			Name: link.Name,
			//Hash: link.Hash.B58String(),
			Size: link.Size,
		}
	}

	return output, nil
}

type progressReader struct {
	file         files.File
	out          chan interface{}
	bytes        int64
	lastProgress int64
}

func (i *progressReader) Read(p []byte) (int, error) {
	n, err := i.file.Read(p)

	i.bytes += int64(n)
	if i.bytes-i.lastProgress >= progressReaderIncrement || err == io.EOF {
		i.lastProgress = i.bytes
		i.out <- &AddedObject{
			Name:  i.file.FileName(),
			Bytes: i.bytes,
		}
	}

	return n, err
}
<|endoftext|>


<commit_before>package util

// util/IntsRef.java

/* An empty integer array for convenience */
var EMPTY_INTS = []int{}

/*
Represents []int, as a slice (offset + length) into an existing []int.
The ints member should never be nil; use EMPTY_INTS if necessary.

Go's native slice is always preferrable unless the reference pointer
need to remain unchanged, in which case, this class is more useful.
*/
type IntsRef struct {
	// The contents of teh IntsRef. Should never be nil.
	Ints []int
	// Offset of first valid integer.
	Offset int
	// Length of used ints.
	Length int
}

func NewEmptyIntsRef() *IntsRef {
	return &IntsRef{Ints: EMPTY_INTS}
}

func (a *IntsRef) At(i int) int {
	return a.Ints[a.Offset+i]
}

func (a *IntsRef) Value() []int {
	return a.Ints[a.Offset : a.Offset+a.Length]
}

/* Signed int order comparison */
func (a *IntsRef) Less(other *IntsRef) bool {
	if a == other {
		return false
	}

	aInts := a.Ints
	aUpto := a.Offset
	bInts := other.Ints
	bUpto := other.Offset

	var aStop int
	if a.Length < other.Length {
		aStop = aUpto + a.Length
	} else {
		aStop = aUpto + other.Length
	}

	for aUpto < aStop {
		aInt := aInts[aUpto]
		aUpto++
		bInt := bInts[bUpto]
		bUpto++
		if aInt > bInt {
			return false
		} else if aInt < bInt {
			return true
		}
	}

	// one is a prefix of the other, or, they are equal:
	return a.Length < other.Length
}

func (a *IntsRef) CopyInts(other *IntsRef) {
	if len(a.Ints)-a.Offset < other.Length {
		a.Ints = make([]int, other.Length)
		a.Offset = 0
	}
	copy(a.Ints, other.Ints[other.Offset:other.Offset+other.Length])
	a.Length = other.Length
}

/*
Used to grow the reference slice.

In general this should not be used as it does not take the offset into account.
*/
func (a *IntsRef) Grow(newLength int) {
	assert(a.Offset == 0)
	if len(a.Ints) < newLength {
		a.Ints = GrowIntSlice(a.Ints, newLength)
	}
}

// util/IntsRefBuilder.java

type IntsRefBuilder struct {
	ref *IntsRef
}

func NewIntsRefBuilder() *IntsRefBuilder {
	return &IntsRefBuilder{
		ref: NewEmptyIntsRef(),
	}
}

func (a *IntsRefBuilder) Length() int {
	return a.ref.Length
}

func (a *IntsRefBuilder) Clear() {
	a.ref.Length = 0
}

func (a *IntsRefBuilder) At(offset int) int {
	return a.ref.Ints[offset]
}

func (a *IntsRefBuilder) Append(i int) {
	a.Grow(a.ref.Length + 1)
	a.ref.Ints[a.ref.Length] = i
	a.ref.Length++
}

func (a *IntsRefBuilder) Grow(newLength int) {
	a.ref.Ints = GrowIntSlice(a.ref.Ints, newLength)
}

func (a *IntsRefBuilder) CopyIntSlice([]int) {
	panic("niy")
}

func (a *IntsRefBuilder) CopyInts(ints *IntsRef) {
	a.CopyIntSlice(ints.Ints[ints.Offset : ints.Offset+ints.Length])
}

func (a *IntsRefBuilder) Get() *IntsRef {
	assert2(a.ref.Offset == 0, "Modifying the offset of the returned ref is illegal")
	return a.ref
}
<commit_msg>implement IntsRefBuilder.copyIntSlice()<commit_after>package util

// util/IntsRef.java

/* An empty integer array for convenience */
var EMPTY_INTS = []int{}

/*
Represents []int, as a slice (offset + length) into an existing []int.
The ints member should never be nil; use EMPTY_INTS if necessary.

Go's native slice is always preferrable unless the reference pointer
need to remain unchanged, in which case, this class is more useful.
*/
type IntsRef struct {
	// The contents of teh IntsRef. Should never be nil.
	Ints []int
	// Offset of first valid integer.
	Offset int
	// Length of used ints.
	Length int
}

func NewEmptyIntsRef() *IntsRef {
	return &IntsRef{Ints: EMPTY_INTS}
}

func (a *IntsRef) At(i int) int {
	return a.Ints[a.Offset+i]
}

func (a *IntsRef) Value() []int {
	return a.Ints[a.Offset : a.Offset+a.Length]
}

/* Signed int order comparison */
func (a *IntsRef) Less(other *IntsRef) bool {
	if a == other {
		return false
	}

	aInts := a.Ints
	aUpto := a.Offset
	bInts := other.Ints
	bUpto := other.Offset

	var aStop int
	if a.Length < other.Length {
		aStop = aUpto + a.Length
	} else {
		aStop = aUpto + other.Length
	}

	for aUpto < aStop {
		aInt := aInts[aUpto]
		aUpto++
		bInt := bInts[bUpto]
		bUpto++
		if aInt > bInt {
			return false
		} else if aInt < bInt {
			return true
		}
	}

	// one is a prefix of the other, or, they are equal:
	return a.Length < other.Length
}

func (a *IntsRef) CopyInts(other *IntsRef) {
	if len(a.Ints)-a.Offset < other.Length {
		a.Ints = make([]int, other.Length)
		a.Offset = 0
	}
	copy(a.Ints, other.Ints[other.Offset:other.Offset+other.Length])
	a.Length = other.Length
}

/*
Used to grow the reference slice.

In general this should not be used as it does not take the offset into account.
*/
func (a *IntsRef) Grow(newLength int) {
	assert(a.Offset == 0)
	if len(a.Ints) < newLength {
		a.Ints = GrowIntSlice(a.Ints, newLength)
	}
}

// util/IntsRefBuilder.java

type IntsRefBuilder struct {
	ref *IntsRef
}

func NewIntsRefBuilder() *IntsRefBuilder {
	return &IntsRefBuilder{
		ref: NewEmptyIntsRef(),
	}
}

func (a *IntsRefBuilder) Length() int {
	return a.ref.Length
}

func (a *IntsRefBuilder) Clear() {
	a.ref.Length = 0
}

func (a *IntsRefBuilder) At(offset int) int {
	return a.ref.Ints[offset]
}

func (a *IntsRefBuilder) Append(i int) {
	a.Grow(a.ref.Length + 1)
	a.ref.Ints[a.ref.Length] = i
	a.ref.Length++
}

func (a *IntsRefBuilder) Grow(newLength int) {
	a.ref.Ints = GrowIntSlice(a.ref.Ints, newLength)
}

func (a *IntsRefBuilder) CopyIntSlice(other []int) {
	a.Grow(len(other))
	copy(a.ref.Ints, other)
	a.ref.Length = len(other)
}

func (a *IntsRefBuilder) CopyInts(ints *IntsRef) {
	a.CopyIntSlice(ints.Ints[ints.Offset : ints.Offset+ints.Length])
}

func (a *IntsRefBuilder) Get() *IntsRef {
	assert2(a.ref.Offset == 0, "Modifying the offset of the returned ref is illegal")
	return a.ref
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/cio"
	"github.com/containerd/containerd/containers"
	"github.com/containerd/containerd/namespaces"
	"github.com/containerd/containerd/oci"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"
)

const imageName = "docker.io/library/alpine:latest"

func main() {
	// morr power!
	runtime.GOMAXPROCS(runtime.NumCPU())

	app := cli.NewApp()
	app.Name = "containerd-stress"
	app.Description = "stress test a containerd daemon"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "debug",
			Usage: "set debug output in the logs",
		},
		cli.StringFlag{
			Name:  "address,a",
			Value: "/run/containerd/containerd.sock",
			Usage: "path to the containerd socket",
		},
		cli.IntFlag{
			Name:  "concurrent,c",
			Value: 1,
			Usage: "set the concurrency of the stress test",
		},
		cli.DurationFlag{
			Name:  "duration,d",
			Value: 1 * time.Minute,
			Usage: "set the duration of the stress test",
		},
	}
	app.Before = func(context *cli.Context) error {
		if context.GlobalBool("debug") {
			logrus.SetLevel(logrus.DebugLevel)
		}
		return nil
	}
	app.Action = func(context *cli.Context) error {
		config := config{
			Address:     context.GlobalString("address"),
			Duration:    context.GlobalDuration("duration"),
			Concurrency: context.GlobalInt("concurrent"),
		}
		return test(config)
	}
	if err := app.Run(os.Args); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

type config struct {
	Concurrency int
	Duration    time.Duration
	Address     string
}

func (c config) newClient() (*containerd.Client, error) {
	return containerd.New(c.Address)
}

func test(c config) error {
	var (
		wg  sync.WaitGroup
		ctx = namespaces.WithNamespace(context.Background(), "stress")
	)

	client, err := c.newClient()
	if err != nil {
		return err
	}
	defer client.Close()
	if err := cleanup(ctx, client); err != nil {
		return err
	}
	logrus.Infof("pulling %s", imageName)
	image, err := client.Pull(ctx, imageName, containerd.WithPullUnpack)
	if err != nil {
		return err
	}
	logrus.Info("generating spec from image")
	tctx, cancel := context.WithTimeout(ctx, c.Duration)
	go func() {
		s := make(chan os.Signal, 1)
		signal.Notify(s, syscall.SIGTERM, syscall.SIGINT)
		<-s
		cancel()
	}()

	var (
		workers []*worker
		start   = time.Now()
	)
	logrus.Info("starting stress test run...")
	for i := 0; i < c.Concurrency; i++ {
		wg.Add(1)
		spec, err := oci.GenerateSpec(ctx, client,
			&containers.Container{},
			oci.WithImageConfig(image),
			oci.WithProcessArgs("true"))
		if err != nil {
			return err
		}
		w := &worker{
			id:     i,
			wg:     &wg,
			spec:   spec,
			image:  image,
			client: client,
		}
		workers = append(workers, w)
		go w.run(ctx, tctx)
	}
	wg.Wait()

	var (
		total    int
		failures int
		end      = time.Now().Sub(start).Seconds()
	)
	logrus.Infof("ending test run in %0.3f seconds", end)
	for _, w := range workers {
		total += w.count
		failures += w.failures
	}
	logrus.WithField("failures", failures).Infof(
		"create/start/delete %d containers in %0.3f seconds (%0.3f c/sec) or (%0.3f sec/c)",
		total,
		end,
		float64(total)/end,
		end/float64(total),
	)
	return nil
}

type worker struct {
	id          int
	wg          *sync.WaitGroup
	count       int
	failures    int
	waitContext context.Context

	client *containerd.Client
	image  containerd.Image
	spec   *specs.Spec
}

func (w *worker) run(ctx, tctx context.Context) {
	defer func() {
		w.wg.Done()
		logrus.Infof("worker %d finished", w.id)
	}()
	wctx, cancel := context.WithCancel(ctx)
	w.waitContext = wctx
	go func() {
		<-tctx.Done()
		cancel()
	}()
	for {
		select {
		case <-tctx.Done():
			return
		default:
		}

		w.count++
		id := w.getID()
		logrus.Debugf("starting container %s", id)
		if err := w.runContainer(ctx, id); err != nil {
			if err != context.DeadlineExceeded ||
				!strings.Contains(err.Error(), context.DeadlineExceeded.Error()) {
				w.failures++
				logrus.WithError(err).Errorf("running container %s", id)

			}
		}
	}
}

func (w *worker) runContainer(ctx context.Context, id string) error {
	// fix up cgroups path for a default config
	w.spec.Linux.CgroupsPath = filepath.Join("/", "stress", id)
	c, err := w.client.NewContainer(ctx, id,
		containerd.WithSpec(w.spec),
		containerd.WithNewSnapshot(id, w.image),
	)
	if err != nil {
		return err
	}
	defer c.Delete(ctx, containerd.WithSnapshotCleanup)

	task, err := c.NewTask(ctx, cio.NullIO)
	if err != nil {
		return err
	}
	defer task.Delete(ctx, containerd.WithProcessKill)

	statusC, err := task.Wait(ctx)
	if err != nil {
		return err
	}

	if err := task.Start(ctx); err != nil {
		return err
	}
	status := <-statusC
	_, _, err = status.Result()
	if err != nil {
		if err == context.DeadlineExceeded || err == context.Canceled {
			return nil
		}
		w.failures++
	}
	return nil
}

func (w *worker) getID() string {
	return fmt.Sprintf("%d-%d", w.id, w.count)
}

// cleanup cleans up any containers in the "stress" namespace before the test run
func cleanup(ctx context.Context, client *containerd.Client) error {
	containers, err := client.Containers(ctx)
	if err != nil {
		return err
	}
	for _, c := range containers {
		task, err := c.Task(ctx, nil)
		if err == nil {
			task.Delete(ctx, containerd.WithProcessKill)
		}
		if err := c.Delete(ctx, containerd.WithSnapshotCleanup); err != nil {
			if derr := c.Delete(ctx); derr == nil {
				continue
			}
			return err
		}
	}
	return nil
}
<commit_msg>Add exec support to stress test tool<commit_after>package main

import (
	"context"
	"fmt"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/cio"
	"github.com/containerd/containerd/containers"
	"github.com/containerd/containerd/namespaces"
	"github.com/containerd/containerd/oci"
	specs "github.com/opencontainers/runtime-spec/specs-go"
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"
)

const imageName = "docker.io/library/alpine:latest"

func main() {
	// morr power!
	runtime.GOMAXPROCS(runtime.NumCPU())

	app := cli.NewApp()
	app.Name = "containerd-stress"
	app.Description = "stress test a containerd daemon"
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "debug",
			Usage: "set debug output in the logs",
		},
		cli.StringFlag{
			Name:  "address,a",
			Value: "/run/containerd/containerd.sock",
			Usage: "path to the containerd socket",
		},
		cli.IntFlag{
			Name:  "concurrent,c",
			Value: 1,
			Usage: "set the concurrency of the stress test",
		},
		cli.DurationFlag{
			Name:  "duration,d",
			Value: 1 * time.Minute,
			Usage: "set the duration of the stress test",
		},
		cli.BoolFlag{
			Name:  "exec",
			Usage: "add execs to the stress tests",
		},
	}
	app.Before = func(context *cli.Context) error {
		if context.GlobalBool("debug") {
			logrus.SetLevel(logrus.DebugLevel)
		}
		return nil
	}
	app.Action = func(context *cli.Context) error {
		config := config{
			Address:     context.GlobalString("address"),
			Duration:    context.GlobalDuration("duration"),
			Concurrency: context.GlobalInt("concurrent"),
			Exec:        context.GlobalBool("exec"),
		}
		return test(config)
	}
	if err := app.Run(os.Args); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
}

type config struct {
	Concurrency int
	Duration    time.Duration
	Address     string
	Exec        bool
}

func (c config) newClient() (*containerd.Client, error) {
	return containerd.New(c.Address)
}

func test(c config) error {
	var (
		wg  sync.WaitGroup
		ctx = namespaces.WithNamespace(context.Background(), "stress")
	)

	client, err := c.newClient()
	if err != nil {
		return err
	}
	defer client.Close()
	if err := cleanup(ctx, client); err != nil {
		return err
	}
	logrus.Infof("pulling %s", imageName)
	image, err := client.Pull(ctx, imageName, containerd.WithPullUnpack)
	if err != nil {
		return err
	}
	logrus.Info("generating spec from image")
	tctx, cancel := context.WithTimeout(ctx, c.Duration)
	go func() {
		s := make(chan os.Signal, 1)
		signal.Notify(s, syscall.SIGTERM, syscall.SIGINT)
		<-s
		cancel()
	}()

	var (
		workers []*worker
		start   = time.Now()
	)
	logrus.Info("starting stress test run...")
	args := oci.WithProcessArgs("true")
	if c.Exec {
		args = oci.WithProcessArgs("sleep", "10")
	}
	for i := 0; i < c.Concurrency; i++ {
		wg.Add(1)
		spec, err := oci.GenerateSpec(ctx, client,
			&containers.Container{},
			oci.WithImageConfig(image),
			args,
		)
		if err != nil {
			return err
		}
		w := &worker{
			id:     i,
			wg:     &wg,
			spec:   spec,
			image:  image,
			client: client,
			doExec: c.Exec,
		}
		workers = append(workers, w)
		go w.run(ctx, tctx)
	}
	wg.Wait()

	var (
		total    int
		failures int
		end      = time.Now().Sub(start).Seconds()
	)
	logrus.Infof("ending test run in %0.3f seconds", end)
	for _, w := range workers {
		total += w.count
		failures += w.failures
	}
	logrus.WithField("failures", failures).Infof(
		"create/start/delete %d containers in %0.3f seconds (%0.3f c/sec) or (%0.3f sec/c)",
		total,
		end,
		float64(total)/end,
		end/float64(total),
	)
	return nil
}

type worker struct {
	id       int
	wg       *sync.WaitGroup
	count    int
	failures int

	client *containerd.Client
	image  containerd.Image
	spec   *specs.Spec
	doExec bool
}

func (w *worker) run(ctx, tctx context.Context) {
	defer func() {
		w.wg.Done()
		logrus.Infof("worker %d finished", w.id)
	}()
	for {
		select {
		case <-tctx.Done():
			return
		default:
		}

		w.count++
		id := w.getID()
		logrus.Debugf("starting container %s", id)
		if err := w.runContainer(ctx, id); err != nil {
			if err != context.DeadlineExceeded ||
				!strings.Contains(err.Error(), context.DeadlineExceeded.Error()) {
				w.failures++
				logrus.WithError(err).Errorf("running container %s", id)

			}
		}
	}
}

func (w *worker) runContainer(ctx context.Context, id string) error {
	// fix up cgroups path for a default config
	w.spec.Linux.CgroupsPath = filepath.Join("/", "stress", id)
	c, err := w.client.NewContainer(ctx, id,
		containerd.WithSpec(w.spec),
		containerd.WithNewSnapshot(id, w.image),
	)
	if err != nil {
		return err
	}
	defer c.Delete(ctx, containerd.WithSnapshotCleanup)

	task, err := c.NewTask(ctx, cio.NullIO)
	if err != nil {
		return err
	}
	defer task.Delete(ctx, containerd.WithProcessKill)

	statusC, err := task.Wait(ctx)
	if err != nil {
		return err
	}
	if err := task.Start(ctx); err != nil {
		return err
	}
	if w.doExec {
		for i := 0; i < 256; i++ {
			if err := w.exec(ctx, i, task); err != nil {
				w.failures++
				logrus.WithError(err).Error("exec failure")
			}
		}
	}
	if err := task.Kill(ctx, syscall.SIGKILL); err != nil {
		return err
	}
	status := <-statusC
	_, _, err = status.Result()
	if err != nil {
		if err == context.DeadlineExceeded || err == context.Canceled {
			return nil
		}
		w.failures++
	}
	return nil
}

func (w *worker) exec(ctx context.Context, i int, t containerd.Task) error {
	pSpec := *w.spec.Process
	pSpec.Args = []string{"true"}
	process, err := t.Exec(ctx, strconv.Itoa(i), &pSpec, cio.NullIO)
	if err != nil {
		return err
	}
	defer process.Delete(ctx)
	status, err := process.Wait(ctx)
	if err != nil {
		return err
	}
	if err := process.Start(ctx); err != nil {
		return err
	}
	<-status
	return nil
}

func (w *worker) getID() string {
	return fmt.Sprintf("%d-%d", w.id, w.count)
}

// cleanup cleans up any containers in the "stress" namespace before the test run
func cleanup(ctx context.Context, client *containerd.Client) error {
	containers, err := client.Containers(ctx)
	if err != nil {
		return err
	}
	for _, c := range containers {
		task, err := c.Task(ctx, nil)
		if err == nil {
			task.Delete(ctx, containerd.WithProcessKill)
		}
		if err := c.Delete(ctx, containerd.WithSnapshotCleanup); err != nil {
			if derr := c.Delete(ctx); derr == nil {
				continue
			}
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha512"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/objectcache"
	"io"
	"os"
	"path"
)

func convertToObject(pathname, objectsDir string) error {
	file, err := os.Open(pathname)
	if err != nil {
		return err
	}
	defer file.Close()
	hasher := sha512.New()
	_, err = io.Copy(hasher, file)
	if err != nil {
		return err
	}
	var hashVal hash.Hash
	copy(hashVal[:], hasher.Sum(nil))
	objPathname := path.Join(objectsDir, objectcache.HashToFilename(hashVal))
	if err = os.MkdirAll(path.Dir(objPathname), 0755); err != nil {
		return err
	}
	return os.Rename(pathname, objPathname)
}
<commit_msg>Strip append-only and immutable attributes in fs2objectcache.<commit_after>package main

import (
	"crypto/sha512"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/objectcache"
	"io"
	"os"
	"os/exec"
	"path"
)

func convertToObject(pathname, objectsDir string) error {
	file, err := os.Open(pathname)
	if err != nil {
		return err
	}
	defer file.Close()
	hasher := sha512.New()
	_, err = io.Copy(hasher, file)
	if err != nil {
		return err
	}
	var hashVal hash.Hash
	copy(hashVal[:], hasher.Sum(nil))
	objPathname := path.Join(objectsDir, objectcache.HashToFilename(hashVal))
	if err = os.MkdirAll(path.Dir(objPathname), 0755); err != nil {
		return err
	}
	err = os.Rename(pathname, objPathname)
	if err == nil {
		return nil
	}
	if os.IsPermission(err) {
		// Blindly attempt to remove immutable attribute.
		cmd := exec.Command("chattr", "-ai", pathname)
		cmd.Run()
	}
	return os.Rename(pathname, objPathname)
}
<|endoftext|>