<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path"
	"syscall"

	"github.com/golang/glog"
	"github.com/kubernetes-incubator/external-storage/lib/controller"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/strategicpatch"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"

	"github.com/dustin/go-humanize"
	"github.com/virtuozzo/goploop-cli"
	"github.com/virtuozzo/ploop-flexvol/vstorage"
)

const (
	parentProvisionerAnn = "vzFSParentProvisioner"
	vzShareAnn           = "vzShare"
)

type vzFSProvisioner struct {
	// Kubernetes Client. Use to retrieve secrets with Virtuozzo Storage credentials
	client kubernetes.Interface
}

func newVzFSProvisioner(client kubernetes.Interface) controller.Provisioner {
	return &vzFSProvisioner{
		client: client,
	}
}

var _ controller.Provisioner = &vzFSProvisioner{}

const provisionerDir = "/export/virtuozzo-provisioner/"
const mountDir = provisionerDir + "mnt/"

func prepareVstorage(options map[string]string, clusterName string, clusterPassword string) error {
	mount := mountDir + clusterName
	mounted, _ := vstorage.IsVstorage(mount)
	if mounted {
		return nil
	}

	if err := os.MkdirAll(mount, 0755); err != nil {
		return err
	}

	v := vstorage.Vstorage{clusterName}
	p, _ := v.Mountpoint()
	if p != "" {
		return syscall.Mount(p, mount, "", syscall.MS_BIND, "")
	}

	if err := v.Auth(clusterPassword); err != nil {
		return err
	}
	if err := v.Mount(mount); err != nil {
		return err
	}

	return nil
}

func createPloop(mount string, options map[string]string) error {
	var (
		volumePath, volumeID, size string
	)

	for k, v := range options {
		switch k {
		case "volumePath":
			volumePath = v
		case "volumeID":
			volumeID = v
		case "size":
			size = v
		case "vzsReplicas":
		case "vzsFailureDomain":
		case "vzsEncoding":
		case "vzsTier":
		case "kubernetes.io/readwrite":
		case "kubernetes.io/fsType":
		default:
		}
	}

	if volumePath == "" {
		return fmt.Errorf("volumePath isn't specified")
	}

	if volumeID == "" {
		return fmt.Errorf("volumeID isn't specified")
	}

	if size == "" {
		return fmt.Errorf("size isn't specified")
	}

	// get a human readable size from the map
	bytes, _ := humanize.ParseBytes(size)

	// ploop driver takes kilobytes, so convert it
	volumeSize := bytes / 1024

	// create ploop deltas path
	if err := os.MkdirAll(path.Join(mount, options["deltasPath"]), 0755); err != nil {
		return err
	}

	ploopPath := path.Join(mount, options["volumePath"], options["volumeID"])
	deltaPath := path.Join(mount, options["deltasPath"], options["volumeID"])
	// Create the ploop volume
	_, err := ploop.PloopVolumeCreate(ploopPath, volumeSize, deltaPath)
	if err != nil {
		return err
	}

	for k, v := range options {
		attr := ""
		switch k {
		case "vzsReplicas":
			attr = "replicas"
		case "vzsTier":
			attr = "tier"
		case "vzsEncoding":
			attr = "encoding"
		case "vzsFailureDomain":
			attr = "failure-domain"
		}
		if attr != "" {
			cmd := "vstorage"
			args := []string{"set-attr", "-R", ploopPath,
				fmt.Sprintf("%s=%s", attr, v)}
			err = exec.Command(cmd, args...).Run()
		}

		if err != nil {
			os.RemoveAll(ploopPath)
			return fmt.Errorf("Unable to set %s to %s: %v", attr, v, err)
		}
	}

	return nil
}

func (p *vzFSProvisioner) patchSecret(oldSecret, newSecret *v1.Secret) error {
	oldData, err := json.Marshal(oldSecret)
	if err != nil {
		glog.Errorf("failed to marshal secret %s: %v", newSecret.Name, err)
		return err
	}
	newData, err := json.Marshal(newSecret)
	if err != nil {
		glog.Errorf("failed to marshal secret patch %s: %v", newSecret.Name, err)
		return err
	}

	patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Secret{})
	if err != nil {
		glog.Errorf("failed to create patch for secret %s: %v", newSecret.Name, err)
		return err
	}
	glog.Infof("Secret %s patch: %s", newSecret.Name, string(patchBytes))

	_, err = p.client.Core().Secrets(newSecret.ObjectMeta.Namespace).Patch(newSecret.Name, types.StrategicMergePatchType, patchBytes)
	return err
}

// Provision creates a storage asset and returns a PV object representing it.
func (p *vzFSProvisioner) Provision(options controller.VolumeOptions) (*v1.PersistentVolume, error) {
	modes := options.PVC.Spec.AccessModes
	if len(modes) == 0 {
		// if AccessModes field is absent, ReadWriteOnce is used by default
		modes = append(modes, v1.ReadWriteOnce)
	} else {
		if len(modes) != 1 || modes[0] != v1.ReadWriteOnce {
			return nil, fmt.Errorf("Virtuozzo flexvolume provisioner supports only ReadWriteOnce access mode")
		}
	}
	capacity := options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	bytes := capacity.Value()

	if options.PVC.Spec.Selector != nil {
		return nil, fmt.Errorf("claim Selector is not supported")
	}
	share := fmt.Sprintf("kubernetes-dynamic-pvc-%s", options.PVC.UID)

	glog.Infof("Add %s %s", share, humanize.Bytes(uint64(bytes)))

	storageClassOptions := map[string]string{}
	for k, v := range options.Parameters {
		storageClassOptions[k] = v
	}

	storageClassOptions["volumeID"] = share
	storageClassOptions["size"] = fmt.Sprintf("%d", bytes)
	secretName := storageClassOptions["secretName"]
	delete(storageClassOptions, "secretName")

	secret, err := p.client.Core().Secrets(options.PVC.Namespace).Get(secretName, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}

	name := string(secret.Data["clusterName"][:len(secret.Data["clusterName"])])
	password := string(secret.Data["clusterPassword"][:len(secret.Data["clusterPassword"])])
	if err := prepareVstorage(storageClassOptions, name, password); err != nil {
		return nil, err
	}

	if err := createPloop(mountDir+name, storageClassOptions); err != nil {
		return nil, err
	}

	u := uuid.NewUUID()
	storageClassOptions["clusterName"] = name
	pv := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: options.PVName,
			UID:  u,
			Annotations: map[string]string{
				parentProvisionerAnn: *provisionerID,
				vzShareAnn:           share,
			},
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy,
			AccessModes:                   modes,
			Capacity: v1.ResourceList{
				v1.ResourceName(v1.ResourceStorage): options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)],
			},
			PersistentVolumeSource: v1.PersistentVolumeSource{
				FlexVolume: &v1.FlexVolumeSource{
					Driver:    "virtuozzo/ploop",
					SecretRef: &v1.LocalObjectReference{Name: secretName},
					Options:   storageClassOptions,
				},
			},
		},
	}

	newSecret := *secret
	finalizer := fmt.Sprintf("virtuozzo.com/%s-pv", u)
	idx := -1
	for i, f := range newSecret.Finalizers {
		if f == finalizer {
			idx = i
			break
		}
	}
	if idx == -1 {
		newSecret.Finalizers = append(newSecret.Finalizers, finalizer)
		if err = p.patchSecret(secret, &newSecret); err != nil {
			glog.Errorf("Failed to update finalizers in secret: %s", secretName)
			return nil, err
		}
	}
	glog.Infof("successfully created virtuozzo storage share: %s", share)
	return pv, nil
}

// Delete removes the storage asset that was created by Provision represented
// by the given PV.
func (p *vzFSProvisioner) Delete(volume *v1.PersistentVolume) error {
	ann, ok := volume.Annotations[parentProvisionerAnn]
	if !ok {
		return errors.New("Parent provisioner name annotation not found on PV")
	}
	if ann != *provisionerID {
		return &controller.IgnoredError{"parent provisioner name annotation on PV does not match ours"}
	}
	share, ok := volume.Annotations[vzShareAnn]
	if !ok {
		return errors.New("vz share annotation not found on PV")
	}

	secretName := volume.Spec.PersistentVolumeSource.FlexVolume.SecretRef.Name
	options := volume.Spec.PersistentVolumeSource.FlexVolume.Options

	secret, err := p.client.Core().Secrets(volume.Spec.ClaimRef.Namespace).Get(secretName, metav1.GetOptions{})
	if err != nil {
		return err
	}

	name := string(secret.Data["clusterName"][:len(secret.Data["clusterName"])])
	password := string(secret.Data["clusterPassword"][:len(secret.Data["clusterPassword"])])
	mount := mountDir + name
	if err := prepareVstorage(options, name, password); err != nil {
		return err
	}

	ploopPath := path.Join(mount, options["volumePath"], options["volumeID"])
	vol, err := ploop.PloopVolumeOpen(ploopPath)
	if err != nil {
		return err
	}
	glog.Infof("Delete: %s", ploopPath)
	if err = vol.Delete(); err != nil {
		return err
	}

	newSecret := *secret
	finalizer := fmt.Sprintf("virtuozzo.com/%s-pv", volume.UID)
	idx := -1
	for i, f := range newSecret.Finalizers {
		if f == finalizer {
			idx = i
			break
		}
	}
	if idx != -1 {
		newSecret.Finalizers = append(newSecret.Finalizers[:idx], newSecret.Finalizers[idx+1:]...)
		if err = p.patchSecret(secret, &newSecret); err != nil {
			glog.Errorf("Failed to update finalizers in secret: %s", secretName)
			return err
		}
	}

	glog.Infof("successfully delete virtuozzo storage share: %s", share)

	return nil
}

var (
	master          = flag.String("master", "", "Master URL")
	kubeconfig      = flag.String("kubeconfig", "", "Absolute path to the kubeconfig")
	provisionerID   = flag.String("id", "", "Unique provisioner id")
	provisionerName = flag.String("name", "virtuozzo.com/virtuozzo-storage", "Unique provisioner name")
)

func main() {
	flag.Parse()
	flag.Set("logtostderr", "true")
	if *provisionerID == "" {
		glog.Fatalf("You should provide unique provisioner name!")
	}

	var config *rest.Config
	var err error
	if *master != "" || *kubeconfig != "" {
		config, err = clientcmd.BuildConfigFromFlags(*master, *kubeconfig)
	} else {
		config, err = rest.InClusterConfig()
	}

	if err != nil {
		glog.Fatalf("Failed to create config: %v", err)
	}
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		glog.Fatalf("Failed to create client: %v", err)
	}

	// The controller needs to know what the server version is because out-of-tree
	// provisioners aren't officially supported until 1.5
	serverVersion, err := clientset.Discovery().ServerVersion()
	if err != nil {
		glog.Fatalf("Error getting server version: %v", err)
	}

	// Create the provisioner: it implements the Provisioner interface expected by
	// the controller
	vzFSProvisioner := newVzFSProvisioner(clientset)

	// Start the provision controller which will dynamically provision Virtuozzo Storage PVs
	pc := controller.NewProvisionController(clientset,
		*provisionerName,
		vzFSProvisioner,
		serverVersion.GitVersion,
	)

	pc.Run(wait.NeverStop)
}
<commit_msg>In case of Provision error, ploop volume should be cleaned up<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path"
	"syscall"

	"github.com/golang/glog"
	"github.com/kubernetes-incubator/external-storage/lib/controller"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/strategicpatch"
	"k8s.io/apimachinery/pkg/util/uuid"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/pkg/api/v1"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"

	"github.com/dustin/go-humanize"
	"github.com/virtuozzo/goploop-cli"
	"github.com/virtuozzo/ploop-flexvol/vstorage"
)

const (
	parentProvisionerAnn = "vzFSParentProvisioner"
	vzShareAnn           = "vzShare"
)

type vzFSProvisioner struct {
	// Kubernetes Client. Use to retrieve secrets with Virtuozzo Storage credentials
	client kubernetes.Interface
}

func newVzFSProvisioner(client kubernetes.Interface) controller.Provisioner {
	return &vzFSProvisioner{
		client: client,
	}
}

var _ controller.Provisioner = &vzFSProvisioner{}

const provisionerDir = "/export/virtuozzo-provisioner/"
const mountDir = provisionerDir + "mnt/"

func prepareVstorage(options map[string]string, clusterName string, clusterPassword string) error {
	mount := mountDir + clusterName
	mounted, _ := vstorage.IsVstorage(mount)
	if mounted {
		return nil
	}

	if err := os.MkdirAll(mount, 0755); err != nil {
		return err
	}

	v := vstorage.Vstorage{clusterName}
	p, _ := v.Mountpoint()
	if p != "" {
		return syscall.Mount(p, mount, "", syscall.MS_BIND, "")
	}

	if err := v.Auth(clusterPassword); err != nil {
		return err
	}
	if err := v.Mount(mount); err != nil {
		return err
	}

	return nil
}

func createPloop(mount string, options map[string]string) error {
	var (
		volumePath, volumeID, size string
	)

	for k, v := range options {
		switch k {
		case "volumePath":
			volumePath = v
		case "volumeID":
			volumeID = v
		case "size":
			size = v
		case "vzsReplicas":
		case "vzsFailureDomain":
		case "vzsEncoding":
		case "vzsTier":
		case "kubernetes.io/readwrite":
		case "kubernetes.io/fsType":
		default:
		}
	}

	if volumePath == "" {
		return fmt.Errorf("volumePath isn't specified")
	}

	if volumeID == "" {
		return fmt.Errorf("volumeID isn't specified")
	}

	if size == "" {
		return fmt.Errorf("size isn't specified")
	}

	// get a human readable size from the map
	bytes, _ := humanize.ParseBytes(size)

	// ploop driver takes kilobytes, so convert it
	volumeSize := bytes / 1024

	// create ploop deltas path
	if err := os.MkdirAll(path.Join(mount, options["deltasPath"]), 0755); err != nil {
		return err
	}

	ploopPath := path.Join(mount, options["volumePath"], options["volumeID"])
	deltaPath := path.Join(mount, options["deltasPath"], options["volumeID"])
	// Create the ploop volume
	_, err := ploop.PloopVolumeCreate(ploopPath, volumeSize, deltaPath)
	if err != nil {
		return err
	}

	for k, v := range options {
		attr := ""
		switch k {
		case "vzsReplicas":
			attr = "replicas"
		case "vzsTier":
			attr = "tier"
		case "vzsEncoding":
			attr = "encoding"
		case "vzsFailureDomain":
			attr = "failure-domain"
		}
		if attr != "" {
			cmd := "vstorage"
			args := []string{"set-attr", "-R", ploopPath,
				fmt.Sprintf("%s=%s", attr, v)}
			err = exec.Command(cmd, args...).Run()
		}

		if err != nil {
			os.RemoveAll(ploopPath)
			return fmt.Errorf("Unable to set %s to %s: %v", attr, v, err)
		}
	}

	return nil
}

func (p *vzFSProvisioner) patchSecret(oldSecret, newSecret *v1.Secret) error {
	oldData, err := json.Marshal(oldSecret)
	if err != nil {
		glog.Errorf("failed to marshal secret %s: %v", newSecret.Name, err)
		return err
	}
	newData, err := json.Marshal(newSecret)
	if err != nil {
		glog.Errorf("failed to marshal secret patch %s: %v", newSecret.Name, err)
		return err
	}

	patchBytes, err := strategicpatch.CreateTwoWayMergePatch(oldData, newData, v1.Secret{})
	if err != nil {
		glog.Errorf("failed to create patch for secret %s: %v", newSecret.Name, err)
		return err
	}
	glog.Infof("Secret %s patch: %s", newSecret.Name, string(patchBytes))

	_, err = p.client.Core().Secrets(newSecret.ObjectMeta.Namespace).Patch(newSecret.Name, types.StrategicMergePatchType, patchBytes)
	return err
}

func removePloop(mount string, options map[string]string) error {
	ploopPath := path.Join(mount, options["volumePath"], options["volumeID"])
	vol, err := ploop.PloopVolumeOpen(ploopPath)
	if err != nil {
		return err
	}
	glog.Infof("Delete: %s", ploopPath)
	return vol.Delete()
}

// Provision creates a storage asset and returns a PV object representing it.
func (p *vzFSProvisioner) Provision(options controller.VolumeOptions) (*v1.PersistentVolume, error) {
	modes := options.PVC.Spec.AccessModes
	if len(modes) == 0 {
		// if AccessModes field is absent, ReadWriteOnce is used by default
		modes = append(modes, v1.ReadWriteOnce)
	} else {
		if len(modes) != 1 || modes[0] != v1.ReadWriteOnce {
			return nil, fmt.Errorf("Virtuozzo flexvolume provisioner supports only ReadWriteOnce access mode")
		}
	}
	capacity := options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)]
	bytes := capacity.Value()

	if options.PVC.Spec.Selector != nil {
		return nil, fmt.Errorf("claim Selector is not supported")
	}
	share := fmt.Sprintf("kubernetes-dynamic-pvc-%s", options.PVC.UID)

	glog.Infof("Add %s %s", share, humanize.Bytes(uint64(bytes)))

	storageClassOptions := map[string]string{}
	for k, v := range options.Parameters {
		storageClassOptions[k] = v
	}

	storageClassOptions["volumeID"] = share
	storageClassOptions["size"] = fmt.Sprintf("%d", bytes)
	secretName := storageClassOptions["secretName"]
	delete(storageClassOptions, "secretName")

	secret, err := p.client.Core().Secrets(options.PVC.Namespace).Get(secretName, metav1.GetOptions{})
	if err != nil {
		return nil, err
	}

	name := string(secret.Data["clusterName"][:len(secret.Data["clusterName"])])
	password := string(secret.Data["clusterPassword"][:len(secret.Data["clusterPassword"])])
	if err := prepareVstorage(storageClassOptions, name, password); err != nil {
		return nil, err
	}

	if err := createPloop(mountDir+name, storageClassOptions); err != nil {
		return nil, err
	}

	u := uuid.NewUUID()
	storageClassOptions["clusterName"] = name
	pv := &v1.PersistentVolume{
		ObjectMeta: metav1.ObjectMeta{
			Name: options.PVName,
			UID:  u,
			Annotations: map[string]string{
				parentProvisionerAnn: *provisionerID,
				vzShareAnn:           share,
			},
		},
		Spec: v1.PersistentVolumeSpec{
			PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy,
			AccessModes:                   modes,
			Capacity: v1.ResourceList{
				v1.ResourceName(v1.ResourceStorage): options.PVC.Spec.Resources.Requests[v1.ResourceName(v1.ResourceStorage)],
			},
			PersistentVolumeSource: v1.PersistentVolumeSource{
				FlexVolume: &v1.FlexVolumeSource{
					Driver:    "virtuozzo/ploop",
					SecretRef: &v1.LocalObjectReference{Name: secretName},
					Options:   storageClassOptions,
				},
			},
		},
	}

	newSecret := *secret
	finalizer := fmt.Sprintf("virtuozzo.com/%s-pv", u)
	idx := -1
	for i, f := range newSecret.Finalizers {
		if f == finalizer {
			idx = i
			break
		}
	}
	if idx == -1 {
		newSecret.Finalizers = append(newSecret.Finalizers, finalizer)
		if err = p.patchSecret(secret, &newSecret); err != nil {
			glog.Errorf("Failed to update finalizers in secret: %s", secretName)
			if e := removePloop(mountDir+name, storageClassOptions); e != nil {
				err = fmt.Errorf("Add finalizer error: %v; cleanup ploop-volume error: %v", err, e)
			}
			return nil, err
		}
	}
	glog.Infof("successfully created virtuozzo storage share: %s", share)
	return pv, nil
}

// Delete removes the storage asset that was created by Provision represented
// by the given PV.
func (p *vzFSProvisioner) Delete(volume *v1.PersistentVolume) error {
	ann, ok := volume.Annotations[parentProvisionerAnn]
	if !ok {
		return errors.New("Parent provisioner name annotation not found on PV")
	}
	if ann != *provisionerID {
		return &controller.IgnoredError{"parent provisioner name annotation on PV does not match ours"}
	}
	share, ok := volume.Annotations[vzShareAnn]
	if !ok {
		return errors.New("vz share annotation not found on PV")
	}

	secretName := volume.Spec.PersistentVolumeSource.FlexVolume.SecretRef.Name
	options := volume.Spec.PersistentVolumeSource.FlexVolume.Options

	secret, err := p.client.Core().Secrets(volume.Spec.ClaimRef.Namespace).Get(secretName, metav1.GetOptions{})
	if err != nil {
		return err
	}

	name := string(secret.Data["clusterName"][:len(secret.Data["clusterName"])])
	password := string(secret.Data["clusterPassword"][:len(secret.Data["clusterPassword"])])
	mount := mountDir + name
	if err := prepareVstorage(options, name, password); err != nil {
		return err
	}

	if err = removePloop(mount, options); err != nil {
		return err
	}

	newSecret := *secret
	finalizer := fmt.Sprintf("virtuozzo.com/%s-pv", volume.UID)
	idx := -1
	for i, f := range newSecret.Finalizers {
		if f == finalizer {
			idx = i
			break
		}
	}
	if idx != -1 {
		newSecret.Finalizers = append(newSecret.Finalizers[:idx], newSecret.Finalizers[idx+1:]...)
		if err = p.patchSecret(secret, &newSecret); err != nil {
			glog.Errorf("Failed to update finalizers in secret: %s", secretName)
			return err
		}
	}

	glog.Infof("successfully delete virtuozzo storage share: %s", share)

	return nil
}

var (
	master          = flag.String("master", "", "Master URL")
	kubeconfig      = flag.String("kubeconfig", "", "Absolute path to the kubeconfig")
	provisionerID   = flag.String("id", "", "Unique provisioner id")
	provisionerName = flag.String("name", "virtuozzo.com/virtuozzo-storage", "Unique provisioner name")
)

func main() {
	flag.Parse()
	flag.Set("logtostderr", "true")
	if *provisionerID == "" {
		glog.Fatalf("You should provide unique provisioner name!")
	}

	var config *rest.Config
	var err error
	if *master != "" || *kubeconfig != "" {
		config, err = clientcmd.BuildConfigFromFlags(*master, *kubeconfig)
	} else {
		config, err = rest.InClusterConfig()
	}

	if err != nil {
		glog.Fatalf("Failed to create config: %v", err)
	}
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		glog.Fatalf("Failed to create client: %v", err)
	}

	// The controller needs to know what the server version is because out-of-tree
	// provisioners aren't officially supported until 1.5
	serverVersion, err := clientset.Discovery().ServerVersion()
	if err != nil {
		glog.Fatalf("Error getting server version: %v", err)
	}

	// Create the provisioner: it implements the Provisioner interface expected by
	// the controller
	vzFSProvisioner := newVzFSProvisioner(clientset)

	// Start the provision controller which will dynamically provision Virtuozzo Storage PVs
	pc := controller.NewProvisionController(clientset,
		*provisionerName,
		vzFSProvisioner,
		serverVersion.GitVersion,
	)

	pc.Run(wait.NeverStop)
}
<|endoftext|>


<commit_before>package bsw

var VERSION = "2.0.0"
<commit_msg>Changes version to const<commit_after>package bsw

const VERSION = "2.0.0"
<|endoftext|>


<commit_before>// Copyright 2017 The Kubernetes Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package jwe

import (
	"crypto/rand"
	"crypto/rsa"
	"log"
	"sync"

	authApi "github.com/kubernetes/dashboard/src/app/backend/auth/api"
	syncApi "github.com/kubernetes/dashboard/src/app/backend/sync/api"
	jose "gopkg.in/square/go-jose.v2"
	"k8s.io/api/core/v1"
	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
	metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/watch"
)

// Entries held by resource used to synchronize encryption key data.
const (
	holderMapKeyEntry  = "priv"
	holderMapCertEntry = "pub"
)

// KeyHolder is responsible for generating, storing and synchronizing encryption key used for token
// generation/decryption.
type KeyHolder interface {
	// Returns encrypter instance that can be used to encrypt data.
	Encrypter() jose.Encrypter
	// Returns encryption key that can be used to decrypt data.
	Key() *rsa.PrivateKey
	// Forces refresh of encryption key synchronized with kubernetes resource (secret).
	Refresh()
}

// Implements KeyHolder interface
type rsaKeyHolder struct {
	// 256-byte random RSA key pair. Synced with a key saved in a secret.
	key          *rsa.PrivateKey
	synchronizer syncApi.Synchronizer
	mux          sync.Mutex
}

// Encrypter implements key holder interface. See KeyHolder for more information.
// Used encryption algorithms:
//    - Content encryption: AES-GCM (256)
//    - Key management: RSA-OAEP-SHA256
func (self *rsaKeyHolder) Encrypter() jose.Encrypter {
	publicKey := &self.Key().PublicKey
	encrypter, err := jose.NewEncrypter(jose.A256GCM, jose.Recipient{Algorithm: jose.RSA_OAEP_256, Key: publicKey}, nil)
	if err != nil {
		panic(err)
	}

	return encrypter
}

// Key implements key holder interface. See KeyHolder for more information.
func (self *rsaKeyHolder) Key() *rsa.PrivateKey {
	self.mux.Lock()
	defer self.mux.Unlock()
	return self.key
}

// Refresh implements key holder interface. See KeyHolder for more information.
func (self *rsaKeyHolder) Refresh() {
	self.synchronizer.Refresh()
	self.update(self.synchronizer.Get())
}

// Handler function executed by synchronizer used to store encryption key. It is called whenever watched object
// is created or updated.
func (self *rsaKeyHolder) update(obj runtime.Object) {
	self.mux.Lock()
	defer self.mux.Unlock()
	secret := obj.(*v1.Secret)
	priv, err := ParseRSAKey(string(secret.Data[holderMapKeyEntry]), string(secret.Data[holderMapCertEntry]))
	if err != nil {
		// Secret was probably tampered with. Delete it and let it be recreated from local copy.
		err := self.synchronizer.Delete()
		if err != nil {
			panic(err)
		}

		return
	}

	self.key = priv
}

// Handler function executed by synchronizer used to store encryption key. It is called whenever watched object
// is gets deleted. It is then recreated based on local key.
func (self *rsaKeyHolder) recreate(obj runtime.Object) {
	secret := obj.(*v1.Secret)
	log.Printf("Synchronized secret %s has been deleted. Recreating.", secret.Name)
	self.synchronizer.Create(self.getEncryptionKeyHolder())
}

func (self *rsaKeyHolder) init() {
	self.initEncryptionKey()

	// Register event handlers
	self.synchronizer.RegisterActionHandler(self.update, watch.Added, watch.Modified)
	self.synchronizer.RegisterActionHandler(self.recreate, watch.Deleted)

	// Try to init key from synchronized object
	if obj := self.synchronizer.Get(); obj != nil {
		log.Print("Initializing JWE encryption key from synchronized object")
		self.update(obj)
		return
	}

	// Try to save generated key in a secret
	log.Printf("Storing encryption key in a secret")
	err := self.synchronizer.Create(self.getEncryptionKeyHolder())
	if err != nil && !k8sErrors.IsAlreadyExists(err) {
		panic(err)
	}
}

func (self *rsaKeyHolder) getEncryptionKeyHolder() runtime.Object {
	priv, pub := ExportRSAKeyOrDie(self.Key())
	return &v1.Secret{
		ObjectMeta: metaV1.ObjectMeta{
			Namespace: authApi.EncryptionKeyHolderNamespace,
			Name:      authApi.EncryptionKeyHolderName,
		},

		Data: map[string][]byte{
			holderMapKeyEntry:  []byte(priv),
			holderMapCertEntry: []byte(pub),
		},
	}
}

// Generates encryption key used to encrypt token payload.
func (self *rsaKeyHolder) initEncryptionKey() {
	log.Print("Generating JWE encryption key")
	self.mux.Lock()
	defer self.mux.Unlock()

	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		panic(err)
	}

	self.key = privateKey
}

// NewRSAKeyHolder creates new KeyHolder instance.
func NewRSAKeyHolder(synchronizer syncApi.Synchronizer) KeyHolder {
	holder := &rsaKeyHolder{
		synchronizer: synchronizer,
	}

	holder.init()
	return holder
}
<commit_msg>Update secret instead of deleting and recreating (#2667)<commit_after>// Copyright 2017 The Kubernetes Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package jwe

import (
	"crypto/rand"
	"crypto/rsa"
	"log"
	"sync"

	authApi "github.com/kubernetes/dashboard/src/app/backend/auth/api"
	syncApi "github.com/kubernetes/dashboard/src/app/backend/sync/api"
	jose "gopkg.in/square/go-jose.v2"
	"k8s.io/api/core/v1"
	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
	metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/watch"
)

// Entries held by resource used to synchronize encryption key data.
const (
	holderMapKeyEntry  = "priv"
	holderMapCertEntry = "pub"
)

// KeyHolder is responsible for generating, storing and synchronizing encryption key used for token
// generation/decryption.
type KeyHolder interface {
	// Returns encrypter instance that can be used to encrypt data.
	Encrypter() jose.Encrypter
	// Returns encryption key that can be used to decrypt data.
	Key() *rsa.PrivateKey
	// Forces refresh of encryption key synchronized with kubernetes resource (secret).
	Refresh()
}

// Implements KeyHolder interface
type rsaKeyHolder struct {
	// 256-byte random RSA key pair. Synced with a key saved in a secret.
	key          *rsa.PrivateKey
	synchronizer syncApi.Synchronizer
	mux          sync.Mutex
}

// Encrypter implements key holder interface. See KeyHolder for more information.
// Used encryption algorithms:
//    - Content encryption: AES-GCM (256)
//    - Key management: RSA-OAEP-SHA256
func (self *rsaKeyHolder) Encrypter() jose.Encrypter {
	publicKey := &self.Key().PublicKey
	encrypter, err := jose.NewEncrypter(jose.A256GCM, jose.Recipient{Algorithm: jose.RSA_OAEP_256, Key: publicKey}, nil)
	if err != nil {
		panic(err)
	}

	return encrypter
}

// Key implements key holder interface. See KeyHolder for more information.
func (self *rsaKeyHolder) Key() *rsa.PrivateKey {
	self.mux.Lock()
	defer self.mux.Unlock()
	return self.key
}

// Refresh implements key holder interface. See KeyHolder for more information.
func (self *rsaKeyHolder) Refresh() {
	self.synchronizer.Refresh()
	self.update(self.synchronizer.Get())
}

// Handler function executed by synchronizer used to store encryption key. It is called whenever watched object
// is created or updated.
func (self *rsaKeyHolder) update(obj runtime.Object) {
	self.mux.Lock()
	defer self.mux.Unlock()
	secret := obj.(*v1.Secret)
	priv, err := ParseRSAKey(string(secret.Data[holderMapKeyEntry]), string(secret.Data[holderMapCertEntry]))
	if err != nil {
		// Secret was probably tampered with. Update it based on local key.
		err := self.synchronizer.Update(self.getEncryptionKeyHolder())
		if err != nil {
			panic(err)
		}

		return
	}

	self.key = priv
}

// Handler function executed by synchronizer used to store encryption key. It is called whenever watched object
// is gets deleted. It is then recreated based on local key.
func (self *rsaKeyHolder) recreate(obj runtime.Object) {
	secret := obj.(*v1.Secret)
	log.Printf("Synchronized secret %s has been deleted. Recreating.", secret.Name)
	self.synchronizer.Create(self.getEncryptionKeyHolder())
}

func (self *rsaKeyHolder) init() {
	self.initEncryptionKey()

	// Register event handlers
	self.synchronizer.RegisterActionHandler(self.update, watch.Added, watch.Modified)
	self.synchronizer.RegisterActionHandler(self.recreate, watch.Deleted)

	// Try to init key from synchronized object
	if obj := self.synchronizer.Get(); obj != nil {
		log.Print("Initializing JWE encryption key from synchronized object")
		self.update(obj)
		return
	}

	// Try to save generated key in a secret
	log.Printf("Storing encryption key in a secret")
	err := self.synchronizer.Create(self.getEncryptionKeyHolder())
	if err != nil && !k8sErrors.IsAlreadyExists(err) {
		panic(err)
	}
}

func (self *rsaKeyHolder) getEncryptionKeyHolder() runtime.Object {
	priv, pub := ExportRSAKeyOrDie(self.Key())
	return &v1.Secret{
		ObjectMeta: metaV1.ObjectMeta{
			Namespace: authApi.EncryptionKeyHolderNamespace,
			Name:      authApi.EncryptionKeyHolderName,
		},

		Data: map[string][]byte{
			holderMapKeyEntry:  []byte(priv),
			holderMapCertEntry: []byte(pub),
		},
	}
}

// Generates encryption key used to encrypt token payload.
func (self *rsaKeyHolder) initEncryptionKey() {
	log.Print("Generating JWE encryption key")
	self.mux.Lock()
	defer self.mux.Unlock()

	privateKey, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		panic(err)
	}

	self.key = privateKey
}

// NewRSAKeyHolder creates new KeyHolder instance.
func NewRSAKeyHolder(synchronizer syncApi.Synchronizer) KeyHolder {
	holder := &rsaKeyHolder{
		synchronizer: synchronizer,
	}

	holder.init()
	return holder
}
<|endoftext|>


<commit_before>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package modload

import (
	"context"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path"
	"path/filepath"
	"strings"

	"cmd/go/internal/cfg"
	"cmd/go/internal/fsys"
	"cmd/go/internal/imports"
	"cmd/go/internal/modindex"
	"cmd/go/internal/search"

	"golang.org/x/mod/module"
)

type stdFilter int8

const (
	omitStd = stdFilter(iota)
	includeStd
)

// matchPackages is like m.MatchPackages, but uses a local variable (rather than
// a global) for tags, can include or exclude packages in the standard library,
// and is restricted to the given list of modules.
func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) {
	m.Pkgs = []string{}

	isMatch := func(string) bool { return true }
	treeCanMatch := func(string) bool { return true }
	if !m.IsMeta() {
		isMatch = search.MatchPattern(m.Pattern())
		treeCanMatch = search.TreeCanMatchPattern(m.Pattern())
	}

	have := map[string]bool{
		"builtin": true, // ignore pseudo-package that exists only for documentation
	}
	if !cfg.BuildContext.CgoEnabled {
		have["runtime/cgo"] = true // ignore during walk
	}

	type pruning int8
	const (
		pruneVendor = pruning(1 << iota)
		pruneGoMod
	)

	walkPkgs := func(root, importPathRoot string, prune pruning) {
		root = filepath.Clean(root)
		err := fsys.Walk(root, func(path string, fi fs.FileInfo, err error) error {
			if err != nil {
				m.AddError(err)
				return nil
			}

			want := true
			elem := ""

			// Don't use GOROOT/src but do walk down into it.
			if path == root {
				if importPathRoot == "" {
					return nil
				}
			} else {
				// Avoid .foo, _foo, and testdata subdirectory trees.
				_, elem = filepath.Split(path)
				if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" {
					want = false
				}
			}

			name := importPathRoot + filepath.ToSlash(path[len(root):])
			if importPathRoot == "" {
				name = name[1:] // cut leading slash
			}
			if !treeCanMatch(name) {
				want = false
			}

			if !fi.IsDir() {
				if fi.Mode()&fs.ModeSymlink != 0 && want && strings.Contains(m.Pattern(), "...") {
					if target, err := fsys.Stat(path); err == nil && target.IsDir() {
						fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path)
					}
				}
				return nil
			}

			if !want {
				return filepath.SkipDir
			}
			// Stop at module boundaries.
			if (prune&pruneGoMod != 0) && path != root {
				if fi, err := os.Stat(filepath.Join(path, "go.mod")); err == nil && !fi.IsDir() {
					return filepath.SkipDir
				}
			}

			if !have[name] {
				have[name] = true
				if isMatch(name) {
					if _, _, err := scanDir(root, path, tags); err != imports.ErrNoGo {
						m.Pkgs = append(m.Pkgs, name)
					}
				}
			}

			if elem == "vendor" && (prune&pruneVendor != 0) {
				return filepath.SkipDir
			}
			return nil
		})
		if err != nil {
			m.AddError(err)
		}
	}

	if filter == includeStd {
		walkPkgs(cfg.GOROOTsrc, "", pruneGoMod)
		if treeCanMatch("cmd") {
			walkPkgs(filepath.Join(cfg.GOROOTsrc, "cmd"), "cmd", pruneGoMod)
		}
	}

	if cfg.BuildMod == "vendor" {
		mod := MainModules.mustGetSingleMainModule()
		if modRoot := MainModules.ModRoot(mod); modRoot != "" {
			walkPkgs(modRoot, MainModules.PathPrefix(mod), pruneGoMod|pruneVendor)
			walkPkgs(filepath.Join(modRoot, "vendor"), "", pruneVendor)
		}
		return
	}

	for _, mod := range modules {
		if !treeCanMatch(mod.Path) {
			continue
		}

		var (
			root, modPrefix string
			isLocal         bool
		)
		if MainModules.Contains(mod.Path) {
			if MainModules.ModRoot(mod) == "" {
				continue // If there is no main module, we can't search in it.
			}
			root = MainModules.ModRoot(mod)
			modPrefix = MainModules.PathPrefix(mod)
			isLocal = true
		} else {
			var err error
			const needSum = true
			root, isLocal, err = fetch(ctx, mod, needSum)
			if err != nil {
				m.AddError(err)
				continue
			}
			modPrefix = mod.Path
		}
		if mi, err := modindex.Get(root); err == nil {
			walkFromIndex(ctx, m, tags, root, mi, have, modPrefix)
			continue
		} else if !errors.Is(err, modindex.ErrNotIndexed) {
			m.AddError(err)
		}

		prune := pruneVendor
		if isLocal {
			prune |= pruneGoMod
		}
		walkPkgs(root, modPrefix, prune)
	}

	return
}

// walkFromIndex matches packages in a module using the module index. modroot
// is the module's root directory on disk, index is the ModuleIndex for the
// module, and importPathRoot is the module's path prefix.
func walkFromIndex(ctx context.Context, m *search.Match, tags map[string]bool, modroot string, index *modindex.ModuleIndex, have map[string]bool, importPathRoot string) {
	isMatch := func(string) bool { return true }
	treeCanMatch := func(string) bool { return true }
	if !m.IsMeta() {
		isMatch = search.MatchPattern(m.Pattern())
		treeCanMatch = search.TreeCanMatchPattern(m.Pattern())
	}
loopPackages:
	for _, reldir := range index.Packages() {
		// Avoid .foo, _foo, and testdata subdirectory trees.
		p := reldir
		for {
			elem, rest, found := strings.Cut(p, string(filepath.Separator))
			if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" {
				continue loopPackages
			}
			if found && elem == "vendor" {
				// Ignore this path if it contains the element "vendor" anywhere
				// except for the last element (packages named vendor are allowed
				// for historical reasons). Note that found is true when this
				// isn't the last path element.
				continue loopPackages
			}
			if !found {
				// Didn't find the separator, so we're considering the last element.
				break
			}
			p = rest
		}

		// Don't use GOROOT/src.
		if reldir == "" && importPathRoot == "" {
			continue
		}

		name := path.Join(importPathRoot, filepath.ToSlash(reldir))
		if !treeCanMatch(name) {
			continue
		}

		if !have[name] {
			have[name] = true
			if isMatch(name) {
				if _, _, err := index.ScanDir(reldir, tags); err != imports.ErrNoGo {
					m.Pkgs = append(m.Pkgs, name)
				}
			}
		}
	}
}

// MatchInModule identifies the packages matching the given pattern within the
// given module version, which does not need to be in the build list or module
// requirement graph.
//
// If m is the zero module.Version, MatchInModule matches the pattern
// against the standard library (std and cmd) in GOROOT/src.
func MatchInModule(ctx context.Context, pattern string, m module.Version, tags map[string]bool) *search.Match {
	match := search.NewMatch(pattern)
	if m == (module.Version{}) {
		matchPackages(ctx, match, tags, includeStd, nil)
	}

	LoadModFile(ctx) // Sets Target, needed by fetch and matchPackages.

	if !match.IsLiteral() {
		matchPackages(ctx, match, tags, omitStd, []module.Version{m})
		return match
	}

	const needSum = true
	root, isLocal, err := fetch(ctx, m, needSum)
	if err != nil {
		match.Errs = []error{err}
		return match
	}

	dir, haveGoFiles, err := dirInModule(pattern, m.Path, root, isLocal)
	if err != nil {
		match.Errs = []error{err}
		return match
	}
	if haveGoFiles {
		if _, _, err := scanDir(root, dir, tags); err != imports.ErrNoGo {
			// ErrNoGo indicates that the directory is not actually a Go package,
			// perhaps due to the tags in use. Any other non-nil error indicates a
			// problem with one or more of the Go source files, but such an error does
			// not stop the package from existing, so it has no impact on matching.
			match.Pkgs = []string{pattern}
		}
	}
	return match
}
<commit_msg>cmd/go: parallelize matchPackages work in each module<commit_after>// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package modload

import (
	"context"
	"errors"
	"fmt"
	"io/fs"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"

	"cmd/go/internal/cfg"
	"cmd/go/internal/fsys"
	"cmd/go/internal/imports"
	"cmd/go/internal/modindex"
	"cmd/go/internal/par"
	"cmd/go/internal/search"

	"golang.org/x/mod/module"
)

type stdFilter int8

const (
	omitStd = stdFilter(iota)
	includeStd
)

// matchPackages is like m.MatchPackages, but uses a local variable (rather than
// a global) for tags, can include or exclude packages in the standard library,
// and is restricted to the given list of modules.
func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) {
	m.Pkgs = []string{}

	isMatch := func(string) bool { return true }
	treeCanMatch := func(string) bool { return true }
	if !m.IsMeta() {
		isMatch = search.MatchPattern(m.Pattern())
		treeCanMatch = search.TreeCanMatchPattern(m.Pattern())
	}

	var mu sync.Mutex
	have := map[string]bool{
		"builtin": true, // ignore pseudo-package that exists only for documentation
	}
	addPkg := func(p string) {
		mu.Lock()
		m.Pkgs = append(m.Pkgs, p)
		mu.Unlock()
	}
	if !cfg.BuildContext.CgoEnabled {
		have["runtime/cgo"] = true // ignore during walk
	}

	type pruning int8
	const (
		pruneVendor = pruning(1 << iota)
		pruneGoMod
	)

	q := par.NewQueue(runtime.GOMAXPROCS(0))

	walkPkgs := func(root, importPathRoot string, prune pruning) {
		root = filepath.Clean(root)
		err := fsys.Walk(root, func(path string, fi fs.FileInfo, err error) error {
			if err != nil {
				m.AddError(err)
				return nil
			}

			want := true
			elem := ""

			// Don't use GOROOT/src but do walk down into it.
			if path == root {
				if importPathRoot == "" {
					return nil
				}
			} else {
				// Avoid .foo, _foo, and testdata subdirectory trees.
				_, elem = filepath.Split(path)
				if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" {
					want = false
				}
			}

			name := importPathRoot + filepath.ToSlash(path[len(root):])
			if importPathRoot == "" {
				name = name[1:] // cut leading slash
			}
			if !treeCanMatch(name) {
				want = false
			}

			if !fi.IsDir() {
				if fi.Mode()&fs.ModeSymlink != 0 && want && strings.Contains(m.Pattern(), "...") {
					if target, err := fsys.Stat(path); err == nil && target.IsDir() {
						fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", path)
					}
				}
				return nil
			}

			if !want {
				return filepath.SkipDir
			}
			// Stop at module boundaries.
			if (prune&pruneGoMod != 0) && path != root {
				if fi, err := os.Stat(filepath.Join(path, "go.mod")); err == nil && !fi.IsDir() {
					return filepath.SkipDir
				}
			}

			if !have[name] {
				have[name] = true
				if isMatch(name) {
					q.Add(func() {
						if _, _, err := scanDir(root, path, tags); err != imports.ErrNoGo {
							addPkg(name)
						}
					})
				}
			}

			if elem == "vendor" && (prune&pruneVendor != 0) {
				return filepath.SkipDir
			}
			return nil
		})
		if err != nil {
			m.AddError(err)
		}
	}

	// Wait for all in-flight operations to complete before returning.
	defer func() {
		<-q.Idle()
		sort.Strings(m.Pkgs) // sort everything we added for determinism
	}()

	if filter == includeStd {
		walkPkgs(cfg.GOROOTsrc, "", pruneGoMod)
		if treeCanMatch("cmd") {
			walkPkgs(filepath.Join(cfg.GOROOTsrc, "cmd"), "cmd", pruneGoMod)
		}
	}

	if cfg.BuildMod == "vendor" {
		mod := MainModules.mustGetSingleMainModule()
		if modRoot := MainModules.ModRoot(mod); modRoot != "" {
			walkPkgs(modRoot, MainModules.PathPrefix(mod), pruneGoMod|pruneVendor)
			walkPkgs(filepath.Join(modRoot, "vendor"), "", pruneVendor)
		}
		return
	}

	for _, mod := range modules {
		if !treeCanMatch(mod.Path) {
			continue
		}

		var (
			root, modPrefix string
			isLocal         bool
		)
		if MainModules.Contains(mod.Path) {
			if MainModules.ModRoot(mod) == "" {
				continue // If there is no main module, we can't search in it.
			}
			root = MainModules.ModRoot(mod)
			modPrefix = MainModules.PathPrefix(mod)
			isLocal = true
		} else {
			var err error
			const needSum = true
			root, isLocal, err = fetch(ctx, mod, needSum)
			if err != nil {
				m.AddError(err)
				continue
			}
			modPrefix = mod.Path
		}
		if mi, err := modindex.Get(root); err == nil {
			walkFromIndex(mi, modPrefix, isMatch, treeCanMatch, tags, have, addPkg)
			continue
		} else if !errors.Is(err, modindex.ErrNotIndexed) {
			m.AddError(err)
		}

		prune := pruneVendor
		if isLocal {
			prune |= pruneGoMod
		}
		walkPkgs(root, modPrefix, prune)
	}

	return
}

// walkFromIndex matches packages in a module using the module index. modroot
// is the module's root directory on disk, index is the ModuleIndex for the
// module, and importPathRoot is the module's path prefix.
func walkFromIndex(index *modindex.ModuleIndex, importPathRoot string, isMatch, treeCanMatch func(string) bool, tags, have map[string]bool, addPkg func(string)) {
loopPackages:
	for _, reldir := range index.Packages() {
		// Avoid .foo, _foo, and testdata subdirectory trees.
		p := reldir
		for {
			elem, rest, found := strings.Cut(p, string(filepath.Separator))
			if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" {
				continue loopPackages
			}
			if found && elem == "vendor" {
				// Ignore this path if it contains the element "vendor" anywhere
				// except for the last element (packages named vendor are allowed
				// for historical reasons). Note that found is true when this
				// isn't the last path element.
				continue loopPackages
			}
			if !found {
				// Didn't find the separator, so we're considering the last element.
				break
			}
			p = rest
		}

		// Don't use GOROOT/src.
		if reldir == "" && importPathRoot == "" {
			continue
		}

		name := path.Join(importPathRoot, filepath.ToSlash(reldir))
		if !treeCanMatch(name) {
			continue
		}

		if !have[name] {
			have[name] = true
			if isMatch(name) {
				if _, _, err := index.ScanDir(reldir, tags); err != imports.ErrNoGo {
					addPkg(name)
				}
			}
		}
	}
}

// MatchInModule identifies the packages matching the given pattern within the
// given module version, which does not need to be in the build list or module
// requirement graph.
//
// If m is the zero module.Version, MatchInModule matches the pattern
// against the standard library (std and cmd) in GOROOT/src.
func MatchInModule(ctx context.Context, pattern string, m module.Version, tags map[string]bool) *search.Match {
	match := search.NewMatch(pattern)
	if m == (module.Version{}) {
		matchPackages(ctx, match, tags, includeStd, nil)
	}

	LoadModFile(ctx) // Sets Target, needed by fetch and matchPackages.

	if !match.IsLiteral() {
		matchPackages(ctx, match, tags, omitStd, []module.Version{m})
		return match
	}

	const needSum = true
	root, isLocal, err := fetch(ctx, m, needSum)
	if err != nil {
		match.Errs = []error{err}
		return match
	}

	dir, haveGoFiles, err := dirInModule(pattern, m.Path, root, isLocal)
	if err != nil {
		match.Errs = []error{err}
		return match
	}
	if haveGoFiles {
		if _, _, err := scanDir(root, dir, tags); err != imports.ErrNoGo {
			// ErrNoGo indicates that the directory is not actually a Go package,
			// perhaps due to the tags in use. Any other non-nil error indicates a
			// problem with one or more of the Go source files, but such an error does
			// not stop the package from existing, so it has no impact on matching.
			match.Pkgs = []string{pattern}
		}
	}
	return match
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sparc64

import "cmd/internal/obj"

// General purpose registers, kept in the low bits of Prog.Reg.
const (
	// integer
	REG_R0 = obj.RBaseSPARC64 + iota
	REG_R1
	REG_R2
	REG_R3
	REG_R4
	REG_R5
	REG_R6
	REG_R7
	REG_R8
	REG_R9
	REG_R10
	REG_R11
	REG_R12
	REG_R13
	REG_R14
	REG_R15
	REG_R16
	REG_R17
	REG_R18
	REG_R19
	REG_R20
	REG_R21
	REG_R22
	REG_R23
	REG_R24
	REG_R25
	REG_R26
	REG_R27
	REG_R28
	REG_R29
	REG_R30
	REG_R31

	// single-precision floating point
	REG_F0
	REG_F1
	REG_F2
	REG_F3
	REG_F4
	REG_F5
	REG_F6
	REG_F7
	REG_F8
	REG_F9
	REG_F10
	REG_F11
	REG_F12
	REG_F13
	REG_F14
	REG_F15
	REG_F16
	REG_F17
	REG_F18
	REG_F19
	REG_F20
	REG_F21
	REG_F22
	REG_F23
	REG_F24
	REG_F25
	REG_F26
	REG_F27
	REG_F28
	REG_F29
	REG_F30
	REG_F31

	// double-precision floating point; the first half is aliased to
	// single-precision registers, that is: Dn is aliased to Fn, Fn+1,
	// where n ≤ 30.
	REG_D0
	REG_D32
	REG_D2
	REG_D34
	REG_D4
	REG_D36
	REG_D6
	REG_D38
	REG_D8
	REG_D40
	REG_D10
	REG_D42
	REG_D12
	REG_D44
	REG_D14
	REG_D46
	REG_D16
	REG_D48
	REG_D18
	REG_D50
	REG_D20
	REG_D52
	REG_D22
	REG_D54
	REG_D24
	REG_D56
	REG_D26
	REG_D58
	REG_D28
	REG_D60
	REG_D30
	REG_D62
)

const (
	// floating-point condition-code registers
	REG_FCC0 = REG_R0 + 256 + iota
	REG_FCC1
	REG_FCC2
	REG_FCC3
)

const (
	// integer condition-code flags
	REG_ICC = REG_R0 + 384
	REG_XCC = REG_R0 + 384 + 2
)

const (
	REG_SPECIAL = REG_R0 + 512

	REG_CCR  = REG_SPECIAL + 2
	REG_TICK = REG_SPECIAL + 4
	REG_RPC  = REG_SPECIAL + 5

	REG_LAST = REG_R0 + 1024
)

// Register assignments:
const (
	REG_ZR   = REG_R0
	REG_RSP  = REG_R14
	REG_LR   = REG_R15
	REG_GOT  = REG_R23
	REG_TMP  = REG_R26
	REG_RT1  = REG_R27
	REG_RT2  = REG_R28
	REG_CTXT = REG_R29
	REG_RFP  = REG_R30
	REG_G    = REG_R31
	REG_FTMP = REG_F0
	REG_DTMP = REG_D0
)

const (
	REG_MIN = REG_R1
	REG_MAX = REG_R25
)

const (
	StackAlign            = 8      // ABI says 16
	StackBias             = 0x7ff  // craziness
	WindowSaveAreaSize    = 16 * 8 // only slots for RFP and PLR used
	ArgumentsSaveAreaSize = 0      // unused, normally 6 * 8
	MinStackFrameSize     = WindowSaveAreaSize + ArgumentsSaveAreaSize
)

const (
	BIG = 1<<12 - 1 // magnitude of smallest negative immediate
)

// Prog.mark
const (
	FOLL = 1 << iota
	LABEL
	LEAF
)

const (
	ClassUnknown = iota

	ClassReg        // R1..R31
	ClassFloatReg   // F0..F31
	ClassDoubleReg  // D0..D62
	ClassCond       // ICC, XCC
	ClassFloatCond  // FCC0..FCC3
	ClassSpecialReg // TICK, CCR, etc

	ClassZero       // $0 or ZR
	ClassConst5     // unsigned 5-bit constant
	ClassConst6     // unsigned 6-bit constant
	ClassConst13    // signed 13-bit constant
	ClassConst31_   // signed 32-bit constant, negative
	ClassConst31    // signed 32-bit constant, positive or zero
	ClassConst32    // 32-bit constant
	ClassConst      // 64-bit constant
	ClassFloatConst // floating-point constant

	ClassRegReg     // $(Rn+Rm) or $(Rn)(Rm*1)
	ClassRegConst13 // $n(R), n is 13-bit signed
	ClassRegConst   // $n(R), n large

	ClassIndirRegReg // (Rn+Rm) or (Rn)(Rm*1)
	ClassIndir0      // (R)
	ClassIndir13     // n(R), n is 13-bit signed
	ClassIndir       // n(R), n large

	ClassShortBranch // n(PC) branch target, n is 21-bit signed, mod 4

	ClassAddr // $sym(SB)
	ClassMem  // sym(SB)

	ClassTextSize
	ClassNone
)

var cnames = []string{
	ClassUnknown:     "ClassUnknown",
	ClassReg:         "ClassReg",
	ClassFloatReg:    "ClassFloatReg",
	ClassDoubleReg:   "ClassDoubleReg",
	ClassCond:        "ClassCond",
	ClassFloatCond:   "ClassFloatCond",
	ClassSpecialReg:  "ClassSpecialReg",
	ClassZero:        "ClassZero",
	ClassConst5:      "ClassConst5",
	ClassConst6:      "ClassConst6",
	ClassConst13:     "ClassConst13",
	ClassConst31_:    "ClassConst31-",
	ClassConst31:     "ClassConst31+",
	ClassConst32:     "ClassConst32",
	ClassConst:       "ClassConst",
	ClassFloatConst:  "ClassFloatConst",
	ClassRegReg:      "ClassRegReg",
	ClassRegConst13:  "ClassRegConst13",
	ClassRegConst:    "ClassRegConst",
	ClassIndirRegReg: "ClassIndirRegReg",
	ClassIndir0:      "ClassIndir0",
	ClassIndir13:     "ClassIndir13",
	ClassIndir:       "ClassIndir",
	ClassShortBranch: "ClassShortBranch",
	ClassAddr:        "ClassAddr",
	ClassMem:         "ClassMem",
	ClassTextSize:    "ClassTextSize",
	ClassNone:        "ClassNone",
}

//go:generate go run ../stringer.go -i $GOFILE -o anames.go -p sparc64

const (
	AADD = obj.ABaseSPARC64 + obj.A_ARCHSPECIFIC + iota
	AADDCC
	AADDC
	AADDCCC
	AAND
	AANDCC
	AANDN
	AANDNCC

	// These are the two-operand SPARCv9 32-, and 64-bit, branch
	// on integer condition codes with prediction (BPcc), not the
	// single-operand SPARCv8 32-bit branch on integer condition
	// codes (Bicc).
	ABN
	ABNE
	ABE
	ABG
	ABLE
	ABGE
	ABL
	ABGU
	ABLEU
	ABCC
	ABCS
	ABPOS
	ABNEG
	ABVC
	ABVS

	ABRZ
	ABRLEZ
	ABRLZ
	ABRNZ
	ABRGZ
	ABRGEZ
	ACASW
	ACASD
	AFABSS
	AFABSD
	AFADDS
	AFADDD
	AFBA
	AFBN
	AFBU
	AFBG
	AFBUG
	AFBL
	AFBUL
	AFBLG
	AFBNE
	AFBE
	AFBUE
	AFBGE
	AFBUGE
	AFBLE
	AFBULE
	AFBO
	AFCMPS
	AFCMPD
	AFDIVS
	AFDIVD
	AFITOS
	AFITOD
	AFLUSH
	AFMOVS // the SPARC64 instruction, and alias for loads and stores
	AFMOVD // the SPARC64 instruction, and alias for loads and stores
	AFMULS
	AFMULD
	AFSMULD
	AFNEGS
	AFNEGD
	AFSQRTS
	AFSQRTD
	AFSTOX
	AFDTOX
	AFSTOI
	AFDTOI
	AFSTOD
	AFDTOS
	AFSUBS
	AFSUBD
	AFXTOS
	AFXTOD
	AJMPL
	ALDSB
	ALDSH
	ALDSW
	ALDUB
	ALDUH
	ALDUW
	ALDD
	ALDSF
	ALDDF
	AMEMBAR
	AMULD
	ASDIVD
	AUDIVD
	AOR
	AORCC
	AORN
	AORNCC
	ARD
	ASETHI
	ASLLW
	ASRLW
	ASRAW
	ASLLD
	ASRLD
	ASRAD
	ASTB
	ASTH
	ASTW
	ASTD
	ASTSF
	ASTDF
	ASUB
	ASUBCC
	ASUBC
	ASUBCCC
	ATA
	AXOR
	AXORCC
	AXNOR
	AXNORCC

	// Pseudo-instructions, aliases to SPARC64 instructions and
	// synthetic instructions.
	ACMP // SUBCC R1, R2, ZR
	ANEG
	AMOVUB
	AMOVB
	AMOVUH
	AMOVH
	AMOVUW
	AMOVW
	AMOVD // also the SPARC64 synthetic instruction
	ARNOP // SETHI $0, ZR

	// These are aliases to two-operand SPARCv9 32-, and 64-bit,
	// branch on integer condition codes with prediction (BPcc),
	// with ICC implied.
	ABNW
	ABNEW
	ABEW
	ABGW
	ABLEW
	ABGEW
	ABLW
	ABGUW
	ABLEUW
	ABCCW
	ABCSW
	ABPOSW
	ABNEGW
	ABVCW
	ABVSW

	// These are aliases to two-operand SPARCv9 32-, and 64-bit,
	// branch on integer condition codes with prediction (BPcc),
	// with XCC implied.
	ABND
	ABNED
	ABED
	ABGD
	ABLED
	ABGED
	ABLD
	ABGUD
	ABLEUD
	ABCCD
	ABCSD
	ABPOSD
	ABNEGD
	ABVCD
	ABVSD

	AWORD
	ADWORD

	ALAST
)
<commit_msg>cmd/internal/obj/sparc64: REG_MIN is REG_R0, not REG_R1<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sparc64

import "cmd/internal/obj"

// General purpose registers, kept in the low bits of Prog.Reg.
const (
	// integer
	REG_R0 = obj.RBaseSPARC64 + iota
	REG_R1
	REG_R2
	REG_R3
	REG_R4
	REG_R5
	REG_R6
	REG_R7
	REG_R8
	REG_R9
	REG_R10
	REG_R11
	REG_R12
	REG_R13
	REG_R14
	REG_R15
	REG_R16
	REG_R17
	REG_R18
	REG_R19
	REG_R20
	REG_R21
	REG_R22
	REG_R23
	REG_R24
	REG_R25
	REG_R26
	REG_R27
	REG_R28
	REG_R29
	REG_R30
	REG_R31

	// single-precision floating point
	REG_F0
	REG_F1
	REG_F2
	REG_F3
	REG_F4
	REG_F5
	REG_F6
	REG_F7
	REG_F8
	REG_F9
	REG_F10
	REG_F11
	REG_F12
	REG_F13
	REG_F14
	REG_F15
	REG_F16
	REG_F17
	REG_F18
	REG_F19
	REG_F20
	REG_F21
	REG_F22
	REG_F23
	REG_F24
	REG_F25
	REG_F26
	REG_F27
	REG_F28
	REG_F29
	REG_F30
	REG_F31

	// double-precision floating point; the first half is aliased to
	// single-precision registers, that is: Dn is aliased to Fn, Fn+1,
	// where n ≤ 30.
	REG_D0
	REG_D32
	REG_D2
	REG_D34
	REG_D4
	REG_D36
	REG_D6
	REG_D38
	REG_D8
	REG_D40
	REG_D10
	REG_D42
	REG_D12
	REG_D44
	REG_D14
	REG_D46
	REG_D16
	REG_D48
	REG_D18
	REG_D50
	REG_D20
	REG_D52
	REG_D22
	REG_D54
	REG_D24
	REG_D56
	REG_D26
	REG_D58
	REG_D28
	REG_D60
	REG_D30
	REG_D62
)

const (
	// floating-point condition-code registers
	REG_FCC0 = REG_R0 + 256 + iota
	REG_FCC1
	REG_FCC2
	REG_FCC3
)

const (
	// integer condition-code flags
	REG_ICC = REG_R0 + 384
	REG_XCC = REG_R0 + 384 + 2
)

const (
	REG_SPECIAL = REG_R0 + 512

	REG_CCR  = REG_SPECIAL + 2
	REG_TICK = REG_SPECIAL + 4
	REG_RPC  = REG_SPECIAL + 5

	REG_LAST = REG_R0 + 1024
)

// Register assignments:
const (
	REG_ZR   = REG_R0
	REG_RSP  = REG_R14
	REG_LR   = REG_R15
	REG_GOT  = REG_R23
	REG_TMP  = REG_R26
	REG_RT1  = REG_R27
	REG_RT2  = REG_R28
	REG_CTXT = REG_R29
	REG_RFP  = REG_R30
	REG_G    = REG_R31
	REG_FTMP = REG_F0
	REG_DTMP = REG_D0
)

const (
	REG_MIN = REG_R0
	REG_MAX = REG_R25
)

const (
	StackAlign            = 8      // ABI says 16
	StackBias             = 0x7ff  // craziness
	WindowSaveAreaSize    = 16 * 8 // only slots for RFP and PLR used
	ArgumentsSaveAreaSize = 0      // unused, normally 6 * 8
	MinStackFrameSize     = WindowSaveAreaSize + ArgumentsSaveAreaSize
)

const (
	BIG = 1<<12 - 1 // magnitude of smallest negative immediate
)

// Prog.mark
const (
	FOLL = 1 << iota
	LABEL
	LEAF
)

const (
	ClassUnknown = iota

	ClassReg        // R1..R31
	ClassFloatReg   // F0..F31
	ClassDoubleReg  // D0..D62
	ClassCond       // ICC, XCC
	ClassFloatCond  // FCC0..FCC3
	ClassSpecialReg // TICK, CCR, etc

	ClassZero       // $0 or ZR
	ClassConst5     // unsigned 5-bit constant
	ClassConst6     // unsigned 6-bit constant
	ClassConst13    // signed 13-bit constant
	ClassConst31_   // signed 32-bit constant, negative
	ClassConst31    // signed 32-bit constant, positive or zero
	ClassConst32    // 32-bit constant
	ClassConst      // 64-bit constant
	ClassFloatConst // floating-point constant

	ClassRegReg     // $(Rn+Rm) or $(Rn)(Rm*1)
	ClassRegConst13 // $n(R), n is 13-bit signed
	ClassRegConst   // $n(R), n large

	ClassIndirRegReg // (Rn+Rm) or (Rn)(Rm*1)
	ClassIndir0      // (R)
	ClassIndir13     // n(R), n is 13-bit signed
	ClassIndir       // n(R), n large

	ClassShortBranch // n(PC) branch target, n is 21-bit signed, mod 4

	ClassAddr // $sym(SB)
	ClassMem  // sym(SB)

	ClassTextSize
	ClassNone
)

var cnames = []string{
	ClassUnknown:     "ClassUnknown",
	ClassReg:         "ClassReg",
	ClassFloatReg:    "ClassFloatReg",
	ClassDoubleReg:   "ClassDoubleReg",
	ClassCond:        "ClassCond",
	ClassFloatCond:   "ClassFloatCond",
	ClassSpecialReg:  "ClassSpecialReg",
	ClassZero:        "ClassZero",
	ClassConst5:      "ClassConst5",
	ClassConst6:      "ClassConst6",
	ClassConst13:     "ClassConst13",
	ClassConst31_:    "ClassConst31-",
	ClassConst31:     "ClassConst31+",
	ClassConst32:     "ClassConst32",
	ClassConst:       "ClassConst",
	ClassFloatConst:  "ClassFloatConst",
	ClassRegReg:      "ClassRegReg",
	ClassRegConst13:  "ClassRegConst13",
	ClassRegConst:    "ClassRegConst",
	ClassIndirRegReg: "ClassIndirRegReg",
	ClassIndir0:      "ClassIndir0",
	ClassIndir13:     "ClassIndir13",
	ClassIndir:       "ClassIndir",
	ClassShortBranch: "ClassShortBranch",
	ClassAddr:        "ClassAddr",
	ClassMem:         "ClassMem",
	ClassTextSize:    "ClassTextSize",
	ClassNone:        "ClassNone",
}

//go:generate go run ../stringer.go -i $GOFILE -o anames.go -p sparc64

const (
	AADD = obj.ABaseSPARC64 + obj.A_ARCHSPECIFIC + iota
	AADDCC
	AADDC
	AADDCCC
	AAND
	AANDCC
	AANDN
	AANDNCC

	// These are the two-operand SPARCv9 32-, and 64-bit, branch
	// on integer condition codes with prediction (BPcc), not the
	// single-operand SPARCv8 32-bit branch on integer condition
	// codes (Bicc).
	ABN
	ABNE
	ABE
	ABG
	ABLE
	ABGE
	ABL
	ABGU
	ABLEU
	ABCC
	ABCS
	ABPOS
	ABNEG
	ABVC
	ABVS

	ABRZ
	ABRLEZ
	ABRLZ
	ABRNZ
	ABRGZ
	ABRGEZ
	ACASW
	ACASD
	AFABSS
	AFABSD
	AFADDS
	AFADDD
	AFBA
	AFBN
	AFBU
	AFBG
	AFBUG
	AFBL
	AFBUL
	AFBLG
	AFBNE
	AFBE
	AFBUE
	AFBGE
	AFBUGE
	AFBLE
	AFBULE
	AFBO
	AFCMPS
	AFCMPD
	AFDIVS
	AFDIVD
	AFITOS
	AFITOD
	AFLUSH
	AFMOVS // the SPARC64 instruction, and alias for loads and stores
	AFMOVD // the SPARC64 instruction, and alias for loads and stores
	AFMULS
	AFMULD
	AFSMULD
	AFNEGS
	AFNEGD
	AFSQRTS
	AFSQRTD
	AFSTOX
	AFDTOX
	AFSTOI
	AFDTOI
	AFSTOD
	AFDTOS
	AFSUBS
	AFSUBD
	AFXTOS
	AFXTOD
	AJMPL
	ALDSB
	ALDSH
	ALDSW
	ALDUB
	ALDUH
	ALDUW
	ALDD
	ALDSF
	ALDDF
	AMEMBAR
	AMULD
	ASDIVD
	AUDIVD
	AOR
	AORCC
	AORN
	AORNCC
	ARD
	ASETHI
	ASLLW
	ASRLW
	ASRAW
	ASLLD
	ASRLD
	ASRAD
	ASTB
	ASTH
	ASTW
	ASTD
	ASTSF
	ASTDF
	ASUB
	ASUBCC
	ASUBC
	ASUBCCC
	ATA
	AXOR
	AXORCC
	AXNOR
	AXNORCC

	// Pseudo-instructions, aliases to SPARC64 instructions and
	// synthetic instructions.
	ACMP // SUBCC R1, R2, ZR
	ANEG
	AMOVUB
	AMOVB
	AMOVUH
	AMOVH
	AMOVUW
	AMOVW
	AMOVD // also the SPARC64 synthetic instruction
	ARNOP // SETHI $0, ZR

	// These are aliases to two-operand SPARCv9 32-, and 64-bit,
	// branch on integer condition codes with prediction (BPcc),
	// with ICC implied.
	ABNW
	ABNEW
	ABEW
	ABGW
	ABLEW
	ABGEW
	ABLW
	ABGUW
	ABLEUW
	ABCCW
	ABCSW
	ABPOSW
	ABNEGW
	ABVCW
	ABVSW

	// These are aliases to two-operand SPARCv9 32-, and 64-bit,
	// branch on integer condition codes with prediction (BPcc),
	// with XCC implied.
	ABND
	ABNED
	ABED
	ABGD
	ABLED
	ABGED
	ABLD
	ABGUD
	ABLEUD
	ABCCD
	ABCSD
	ABPOSD
	ABNEGD
	ABVCD
	ABVSD

	AWORD
	ADWORD

	ALAST
)
<|endoftext|>


<commit_before>package gitclient

import (
	"bufio"
	"bytes"
	"cred-alert/mimetype"
	"cred-alert/scanners"
	"cred-alert/scanners/filescanner"
	"cred-alert/sniff"
	"errors"
	"fmt"
	"strings"

	"code.cloudfoundry.org/lager"

	git "gopkg.in/libgit2/git2go.v24"
)

const defaultRemoteName = "origin"

var ErrInterrupted = errors.New("interrupted")

type client struct {
	cloneOptions *git.CloneOptions
}

//go:generate counterfeiter . Client

type Client interface {
	BranchTargets(string) (map[string]string, error)
	Clone(string, string) (*git.Repository, error)
	GetParents(*git.Repository, *git.Oid) ([]*git.Oid, error)
	Fetch(string) (map[string][]*git.Oid, error)
	HardReset(string, *git.Oid) error
	Diff(repositoryPath string, a, b *git.Oid) (string, error)
	BranchCredentialCounts(lager.Logger, string, sniff.Sniffer, git.BranchType) (map[string]uint, error)
}

func New(privateKeyPath, publicKeyPath string) *client {
	credentialsCallback := newCredentialsCallback(privateKeyPath, publicKeyPath)
	return &client{
		cloneOptions: &git.CloneOptions{
			FetchOptions: &git.FetchOptions{
				UpdateFetchhead: true,
				RemoteCallbacks: git.RemoteCallbacks{
					CredentialsCallback:      credentialsCallback,
					CertificateCheckCallback: certificateCheckCallback,
				},
			},
		},
	}
}

func (c *client) BranchTargets(repositoryPath string) (map[string]string, error) {
	repo, err := git.OpenRepository(repositoryPath)
	if err != nil {
		return nil, err
	}
	defer repo.Free()

	it, err := repo.NewBranchIterator(git.BranchAll)
	if err != nil {
		return nil, err
	}

	var branch *git.Branch
	branches := map[string]string{}
	for {
		branch, _, err = it.Next()
		if err != nil {
			break
		}

		branchName, err := branch.Name()
		if err != nil {
			break
		}

		target := branch.Target()
		if target == nil { // origin/HEAD has no target
			continue
		}

		branches[branchName] = branch.Target().String()
	}

	if branch != nil {
		branch.Free()
	}

	return branches, nil
}

func (c *client) Clone(sshURL, dest string) (*git.Repository, error) {
	return git.Clone(sshURL, dest, c.cloneOptions)
}

func (c *client) GetParents(repo *git.Repository, child *git.Oid) ([]*git.Oid, error) {
	object, err := repo.Lookup(child)
	if err != nil {
		return nil, err
	}
	defer object.Free()

	commit, err := object.AsCommit()
	if err != nil {
		return nil, err
	}
	defer commit.Free()

	var parents []*git.Oid
	var i uint
	for i = 0; i < commit.ParentCount(); i++ {
		parents = append(parents, commit.ParentId(i))
	}

	return parents, nil
}

func (c *client) Fetch(repositoryPath string) (map[string][]*git.Oid, error) {
	repo, err := git.OpenRepository(repositoryPath)
	if err != nil {
		return nil, err
	}
	defer repo.Free()

	remote, err := repo.Remotes.Lookup(defaultRemoteName)
	if err != nil {
		return nil, err
	}
	defer remote.Free()

	changes := map[string][]*git.Oid{}
	updateTipsCallback := func(refname string, a *git.Oid, b *git.Oid) git.ErrorCode {
		changes[refname] = []*git.Oid{a, b}
		return 0
	}

	// bleh
	c.cloneOptions.FetchOptions.RemoteCallbacks.UpdateTipsCallback = updateTipsCallback

	var msg string
	err = remote.Fetch([]string{}, c.cloneOptions.FetchOptions, msg)
	if err != nil {
		return nil, err
	}

	return changes, nil
}

func (c *client) HardReset(repositoryPath string, oid *git.Oid) error {
	repo, err := git.OpenRepository(repositoryPath)
	if err != nil {
		return err
	}
	defer repo.Free()

	object, err := repo.Lookup(oid)
	if err != nil {
		return err
	}
	defer object.Free()

	commit, err := object.AsCommit()
	if err != nil {
		return err
	}
	defer commit.Free()

	return repo.ResetToCommit(commit, git.ResetHard, &git.CheckoutOpts{
		Strategy: git.CheckoutForce,
	})
}

func newCredentialsCallback(privateKeyPath, publicKeyPath string) git.CredentialsCallback {
	return func(url string, username string, allowedTypes git.CredType) (git.ErrorCode, *git.Cred) {
		passphrase := ""
		ret, cred := git.NewCredSshKey(username, publicKeyPath, privateKeyPath, passphrase)
		if ret != 0 {
			fmt.Printf("ret: %d\n", ret)
		}
		return git.ErrorCode(ret), &cred
	}
}

func certificateCheckCallback(cert *git.Certificate, valid bool, hostname string) git.ErrorCode {
	// should return an error code if the cert isn't valid
	return git.ErrorCode(0)
}

func (c *client) Diff(repositoryPath string, parent, child *git.Oid) (string, error) {
	repo, err := git.OpenRepository(repositoryPath)
	if err != nil {
		return "", err
	}
	defer repo.Free()

	var aTree *git.Tree
	if parent != nil {
		var err error
		aTree, err = objectToTree(repo, parent)
		if err != nil {
			return "", err
		}
		defer aTree.Free()
	}

	bTree, err := objectToTree(repo, child)
	if err != nil {
		return "", err
	}
	defer bTree.Free()

	options, err := git.DefaultDiffOptions()
	if err != nil {
		return "", err
	}

	diff, err := repo.DiffTreeToTree(aTree, bTree, &options)
	if err != nil {
		return "", err
	}
	defer diff.Free()

	numDeltas, err := diff.NumDeltas()
	if err != nil {
		return "", err
	}

	var results []string
	for i := 0; i < numDeltas; i++ {
		patch, err := diff.Patch(i)
		if err != nil {
			return "", err
		}
		patchString, err := patch.String()
		if err != nil {
			return "", err
		}
		patch.Free()

		results = append(results, patchString)
	}

	return strings.Join(results, "\n"), nil
}

func (c *client) BranchCredentialCounts(
	logger lager.Logger,
	repositoryPath string,
	sniffer sniff.Sniffer,
	branchType git.BranchType,
) (map[string]uint, error) {
	repo, err := git.OpenRepository(repositoryPath)
	if err != nil {
		return nil, err
	}
	defer repo.Free()

	it, err := repo.NewBranchIterator(branchType)
	if err != nil {
		return nil, err
	}
	defer it.Free()

	var branch *git.Branch
	var target *git.Oid
	var commit *git.Commit
	var tree *git.Tree
	var branchName string
	var blob *git.Blob

	entryCounts := make(map[git.Oid]uint)
	branchCounts := make(map[string]uint)

	for {
		branch, _, err = it.Next()
		if err != nil {
			break
		}

		target = branch.Target()
		if target == nil {
			continue
		}

		commit, err = repo.LookupCommit(target)
		if err != nil {
			return nil, err
		}

		tree, err = commit.Tree()
		if err != nil {
			return nil, err
		}

		branchName, err = branch.Name()
		if err != nil {
			return nil, err
		}

		err = tree.Walk(func(root string, entry *git.TreeEntry) int {
			if entry.Type == git.ObjectBlob {
				if count, ok := entryCounts[*entry.Id]; ok {
					if count > 0 {
						branchCounts[branchName] += count
					}
					return 0
				}

				blob, err = repo.LookupBlob(entry.Id)
				if err != nil {
					return -1
				}

				var count uint
				r := bufio.NewReader(bytes.NewReader(blob.Contents()))
				mime := mimetype.Mimetype(logger, r)
				if mime == "" || strings.HasPrefix(mime, "text") {
					sniffer.Sniff(
						logger,
						filescanner.New(r, entry.Name),
						func(lager.Logger, scanners.Violation) error {
							count++
							return nil
						},
					)
				}

				entryCounts[*entry.Id] = count
				branchCounts[branchName] += count
			}

			return 0
		})

		if err != nil {
			return nil, err
		}
	}

	if blob != nil {
		blob.Free()
	}

	if tree != nil {
		tree.Free()
	}

	if commit != nil {
		commit.Free()
	}

	if branch != nil {
		branch.Free()
	}

	return branchCounts, nil
}

func objectToTree(repo *git.Repository, oid *git.Oid) (*git.Tree, error) {
	object, err := repo.Lookup(oid)
	if err != nil {
		return nil, err
	}
	defer object.Free()

	commit, err := object.AsCommit()
	if err != nil {
		return nil, err
	}
	defer commit.Free()

	tree, err := commit.Tree()
	if err != nil {
		return nil, err
	}

	return tree, nil
}
<commit_msg>Copy Oids that we get from the UpdateTipsCallback after a fetch<commit_after>package gitclient

import (
	"bufio"
	"bytes"
	"cred-alert/mimetype"
	"cred-alert/scanners"
	"cred-alert/scanners/filescanner"
	"cred-alert/sniff"
	"errors"
	"fmt"
	"strings"

	"code.cloudfoundry.org/lager"

	git "gopkg.in/libgit2/git2go.v24"
)

const defaultRemoteName = "origin"

var ErrInterrupted = errors.New("interrupted")

type client struct {
	cloneOptions *git.CloneOptions
}

//go:generate counterfeiter . Client

type Client interface {
	BranchTargets(string) (map[string]string, error)
	Clone(string, string) (*git.Repository, error)
	GetParents(*git.Repository, *git.Oid) ([]*git.Oid, error)
	Fetch(string) (map[string][]*git.Oid, error)
	HardReset(string, *git.Oid) error
	Diff(repositoryPath string, a, b *git.Oid) (string, error)
	BranchCredentialCounts(lager.Logger, string, sniff.Sniffer, git.BranchType) (map[string]uint, error)
}

func New(privateKeyPath, publicKeyPath string) *client {
	credentialsCallback := newCredentialsCallback(privateKeyPath, publicKeyPath)
	return &client{
		cloneOptions: &git.CloneOptions{
			FetchOptions: &git.FetchOptions{
				UpdateFetchhead: true,
				RemoteCallbacks: git.RemoteCallbacks{
					CredentialsCallback:      credentialsCallback,
					CertificateCheckCallback: certificateCheckCallback,
				},
			},
		},
	}
}

func (c *client) BranchTargets(repositoryPath string) (map[string]string, error) {
	repo, err := git.OpenRepository(repositoryPath)
	if err != nil {
		return nil, err
	}
	defer repo.Free()

	it, err := repo.NewBranchIterator(git.BranchAll)
	if err != nil {
		return nil, err
	}

	var branch *git.Branch
	branches := map[string]string{}
	for {
		branch, _, err = it.Next()
		if err != nil {
			break
		}

		branchName, err := branch.Name()
		if err != nil {
			break
		}

		target := branch.Target()
		if target == nil { // origin/HEAD has no target
			continue
		}

		branches[branchName] = branch.Target().String()
	}

	if branch != nil {
		branch.Free()
	}

	return branches, nil
}

func (c *client) Clone(sshURL, dest string) (*git.Repository, error) {
	return git.Clone(sshURL, dest, c.cloneOptions)
}

func (c *client) GetParents(repo *git.Repository, child *git.Oid) ([]*git.Oid, error) {
	object, err := repo.Lookup(child)
	if err != nil {
		return nil, err
	}
	defer object.Free()

	commit, err := object.AsCommit()
	if err != nil {
		return nil, err
	}
	defer commit.Free()

	var parents []*git.Oid
	var i uint
	for i = 0; i < commit.ParentCount(); i++ {
		parents = append(parents, commit.ParentId(i))
	}

	return parents, nil
}

func (c *client) Fetch(repositoryPath string) (map[string][]*git.Oid, error) {
	repo, err := git.OpenRepository(repositoryPath)
	if err != nil {
		return nil, err
	}
	defer repo.Free()

	remote, err := repo.Remotes.Lookup(defaultRemoteName)
	if err != nil {
		return nil, err
	}
	defer remote.Free()

	changes := map[string][]*git.Oid{}
	updateTipsCallback := func(refname string, a *git.Oid, b *git.Oid) git.ErrorCode {
		changes[refname] = []*git.Oid{a.Copy(), b.Copy()}
		return 0
	}

	// bleh
	c.cloneOptions.FetchOptions.RemoteCallbacks.UpdateTipsCallback = updateTipsCallback

	var msg string
	err = remote.Fetch([]string{}, c.cloneOptions.FetchOptions, msg)
	if err != nil {
		return nil, err
	}

	return changes, nil
}

func (c *client) HardReset(repositoryPath string, oid *git.Oid) error {
	repo, err := git.OpenRepository(repositoryPath)
	if err != nil {
		return err
	}
	defer repo.Free()

	object, err := repo.Lookup(oid)
	if err != nil {
		return err
	}
	defer object.Free()

	commit, err := object.AsCommit()
	if err != nil {
		return err
	}
	defer commit.Free()

	return repo.ResetToCommit(commit, git.ResetHard, &git.CheckoutOpts{
		Strategy: git.CheckoutForce,
	})
}

func newCredentialsCallback(privateKeyPath, publicKeyPath string) git.CredentialsCallback {
	return func(url string, username string, allowedTypes git.CredType) (git.ErrorCode, *git.Cred) {
		passphrase := ""
		ret, cred := git.NewCredSshKey(username, publicKeyPath, privateKeyPath, passphrase)
		if ret != 0 {
			fmt.Printf("ret: %d\n", ret)
		}
		return git.ErrorCode(ret), &cred
	}
}

func certificateCheckCallback(cert *git.Certificate, valid bool, hostname string) git.ErrorCode {
	// should return an error code if the cert isn't valid
	return git.ErrorCode(0)
}

func (c *client) Diff(repositoryPath string, parent, child *git.Oid) (string, error) {
	repo, err := git.OpenRepository(repositoryPath)
	if err != nil {
		return "", err
	}
	defer repo.Free()

	var aTree *git.Tree
	if parent != nil {
		var err error
		aTree, err = objectToTree(repo, parent)
		if err != nil {
			return "", err
		}
		defer aTree.Free()
	}

	bTree, err := objectToTree(repo, child)
	if err != nil {
		return "", err
	}
	defer bTree.Free()

	options, err := git.DefaultDiffOptions()
	if err != nil {
		return "", err
	}

	diff, err := repo.DiffTreeToTree(aTree, bTree, &options)
	if err != nil {
		return "", err
	}
	defer diff.Free()

	numDeltas, err := diff.NumDeltas()
	if err != nil {
		return "", err
	}

	var results []string
	for i := 0; i < numDeltas; i++ {
		patch, err := diff.Patch(i)
		if err != nil {
			return "", err
		}
		patchString, err := patch.String()
		if err != nil {
			return "", err
		}
		patch.Free()

		results = append(results, patchString)
	}

	return strings.Join(results, "\n"), nil
}

func (c *client) BranchCredentialCounts(
	logger lager.Logger,
	repositoryPath string,
	sniffer sniff.Sniffer,
	branchType git.BranchType,
) (map[string]uint, error) {
	repo, err := git.OpenRepository(repositoryPath)
	if err != nil {
		return nil, err
	}
	defer repo.Free()

	it, err := repo.NewBranchIterator(branchType)
	if err != nil {
		return nil, err
	}
	defer it.Free()

	var branch *git.Branch
	var target *git.Oid
	var commit *git.Commit
	var tree *git.Tree
	var branchName string
	var blob *git.Blob

	entryCounts := make(map[git.Oid]uint)
	branchCounts := make(map[string]uint)

	for {
		branch, _, err = it.Next()
		if err != nil {
			break
		}

		target = branch.Target()
		if target == nil {
			continue
		}

		commit, err = repo.LookupCommit(target)
		if err != nil {
			return nil, err
		}

		tree, err = commit.Tree()
		if err != nil {
			return nil, err
		}

		branchName, err = branch.Name()
		if err != nil {
			return nil, err
		}

		err = tree.Walk(func(root string, entry *git.TreeEntry) int {
			if entry.Type == git.ObjectBlob {
				if count, ok := entryCounts[*entry.Id]; ok {
					if count > 0 {
						branchCounts[branchName] += count
					}
					return 0
				}

				blob, err = repo.LookupBlob(entry.Id)
				if err != nil {
					return -1
				}

				var count uint
				r := bufio.NewReader(bytes.NewReader(blob.Contents()))
				mime := mimetype.Mimetype(logger, r)
				if mime == "" || strings.HasPrefix(mime, "text") {
					sniffer.Sniff(
						logger,
						filescanner.New(r, entry.Name),
						func(lager.Logger, scanners.Violation) error {
							count++
							return nil
						},
					)
				}

				entryCounts[*entry.Id] = count
				branchCounts[branchName] += count
			}

			return 0
		})

		if err != nil {
			return nil, err
		}
	}

	if blob != nil {
		blob.Free()
	}

	if tree != nil {
		tree.Free()
	}

	if commit != nil {
		commit.Free()
	}

	if branch != nil {
		branch.Free()
	}

	return branchCounts, nil
}

func objectToTree(repo *git.Repository, oid *git.Oid) (*git.Tree, error) {
	object, err := repo.Lookup(oid)
	if err != nil {
		return nil, err
	}
	defer object.Free()

	commit, err := object.AsCommit()
	if err != nil {
		return nil, err
	}
	defer commit.Free()

	tree, err := commit.Tree()
	if err != nil {
		return nil, err
	}

	return tree, nil
}
<|endoftext|>


<commit_before>package uploads

import (
	"os"

	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/app/flow"
	"github.com/materials-commons/mcstore/pkg/db/dai"
	"github.com/materials-commons/mcstore/pkg/db/schema"
)

type UploadRequest struct {
	*flow.Request
	Owner string
}

type UploadService interface {
	Upload(req *UploadRequest) error
}

type uploadService struct {
	tracker *uploadTracker
	files   dai.Files
	uploads dai.Uploads
}

func NewUploadService() *uploadService {
	return &uploadService{}
}

func (s *uploadService) Upload(req *UploadRequest) error {
	dir := s.requestDir(req.Request)

	if s.allBlocksUploaded(req.UploadID(), req.FlowTotalChunks) {
		s.assemble(req, dir)
		return nil
	}

	if err := s.Write(dir, req.Request); err != nil {
		return err
	}

	id := req.UploadID()
	s.tracker.increment(id)
	return nil
}

func (s *uploadService) allBlocksUploaded(id string, totalChunks int32) bool {
	count := s.tracker.count(id)
	return count == totalChunks
}

func (s *uploadService) assemble(req *UploadRequest, dir string) {
	file, err := s.createFile(req)
	if err != nil {
		// log
		return
	}

	dest, err := os.Create(app.MCDir.FilePath(file.ID))
	if err != nil {
		// log
		return
	}

	chunkSupplier := newDirChunkSupplier(dir)
	if err := assembleRequest(chunkSupplier, dest); err != nil {
		// log
		return
	}

	finisher := newFinisher(nil)
	if err := finisher.finish(req, file.ID, req.DirectoryID); err != nil {
		// log
		return
	}

	s.tracker.clear(req.UploadID())
}

func (s *uploadService) createFile(req *UploadRequest) (*schema.File, error) {
	upload, err := s.uploads.ByID(req.FlowIdentifier)
	if err != nil {
		return nil, err
	}
	file := schema.NewFile(req.FlowFileName, req.Owner)

	f, err := s.files.Insert(&file, upload.DirectoryID, upload.ProjectID)
	if err != nil {
		return nil, err
	}
	return f, nil
}

func (s *uploadService) requestDir(req *flow.Request) string {
	requestPath := &mcdirRequestPath{}
	return requestPath.Dir(req)
}

func (s *uploadService) Write(dest string, req *flow.Request) error {
	writer := &fileRequestWriter{}
	return writer.Write(dest, req)
}
<commit_msg>Test syste. Clean up on errors. Add extra logging.<commit_after>package uploads

import (
	"io"
	"os"

	"github.com/materials-commons/mcstore/pkg/app"
	"github.com/materials-commons/mcstore/pkg/app/flow"
	"github.com/materials-commons/mcstore/pkg/db"
	"github.com/materials-commons/mcstore/pkg/db/dai"
	"github.com/materials-commons/mcstore/pkg/db/schema"
)

type UploadRequest struct {
	*flow.Request
	Owner string
}

type UploadService interface {
	Upload(req *UploadRequest) error
}

type uploadService struct {
	tracker *uploadTracker
	files   dai.Files
	uploads dai.Uploads
}

func NewUploadService() *uploadService {
	session := db.RSessionMust()
	return &uploadService{
		tracker: newUploadTracker(),
		files:   dai.NewRFiles(session),
		uploads: dai.NewRUploads(session),
	}
}

func (s *uploadService) Upload(req *UploadRequest) error {
	dir := s.requestDir(req.Request)

	if err := s.Write(dir, req.Request); err != nil {
		return err
	}

	id := req.UploadID()
	s.tracker.increment(id)

	if s.allBlocksUploaded(id, req.FlowTotalChunks) {
		if file, err := s.assemble(req, dir); err != nil {
			if file != nil {
				err2 := s.cleanup(req, file.ID)
				app.Log.Errorf("Assembly failed for uploaded file: attempted cleanup of database entry returned: %s", err2)
			}
			return err
		}
	}

	return nil
}

func (s *uploadService) allBlocksUploaded(id string, totalChunks int32) bool {
	count := s.tracker.count(id)
	return count == totalChunks
}

func (s *uploadService) assemble(req *UploadRequest, dir string) (*schema.File, error) {
	// Create file entry in database
	file, err := s.createFile(req)
	if err != nil {
		app.Log.Errorf("Assembly failed for request %s, couldn't create file in database: %s", req.FlowIdentifier, err)
		return nil, err
	}

	// Create on disk entry to write chunks to
	dest, err := s.createDest(file.ID)
	if err != nil {
		app.Log.Errorf("Assembly failed for request %s, couldn't create file on disk: %s", req.FlowIdentifier, err)
		return file, err
	}

	// Assemble the chunks
	chunkSupplier := newDirChunkSupplier(dir)
	if err := assembleRequest(chunkSupplier, dest); err != nil {
		app.Log.Errorf("Assembly failed for request %s, couldn't assemble request: %s", req.FlowIdentifier, err)
		return file, err
	}

	// Finish updating the file state.
	finisher := newFinisher(s.files)
	if err := finisher.finish(req, file.ID, req.DirectoryID); err != nil {
		app.Log.Errorf("Assembly failed for request %s, couldn't finish request: %s", req.FlowIdentifier, err)
		return file, err
	}

	app.Log.Infof("successfully upload fileID", file.ID)
	// Remove upload request.
	s.tracker.clear(req.UploadID())
	s.uploads.Delete(req.UploadID())
	return nil, nil
}

func (s *uploadService) createFile(req *UploadRequest) (*schema.File, error) {
	upload, err := s.uploads.ByID(req.FlowIdentifier)
	if err != nil {
		return nil, err
	}
	file := schema.NewFile(upload.File.Name, req.Owner)

	f, err := s.files.Insert(&file, upload.DirectoryID, upload.ProjectID)
	app.Log.Infof("Created file %s, in %s %s\n", f.ID, upload.DirectoryID, upload.ProjectID)
	if err != nil {
		return nil, err
	}
	return f, nil
}

// createDest creates the destination file and ensures that the directory
// path is also created.
func (s *uploadService) createDest(fileID string) (io.Writer, error) {
	dir := app.MCDir.FileDir(fileID)
	if err := os.MkdirAll(dir, 0700); err != nil {
		return nil, err
	}
	return os.Create(app.MCDir.FilePath(fileID))
}

func (s *uploadService) requestDir(req *flow.Request) string {
	requestPath := &mcdirRequestPath{}
	return requestPath.Dir(req)
}

func (s *uploadService) Write(dest string, req *flow.Request) error {
	writer := &fileRequestWriter{}
	return writer.Write(dest, req)
}

func (s *uploadService) cleanup(req *UploadRequest, fileID string) error {
	upload, err := s.uploads.ByID(req.FlowIdentifier)
	if err != nil {
		return err
	}
	_, err = s.files.Delete(fileID, upload.DirectoryID, upload.ProjectID)
	return err
}
<|endoftext|>


<commit_before>package slack

type ChannelResource struct {
	Id   string
	Name string
}

type ChannelResponse struct {
	Response
	Channel ChannelResource
}

type Response struct {
	Ok bool
}

type RTMStartResponse struct {
	Ok    bool
	Error string
	Url   string
	Self  *SelfResource
	Team  *TeamResource
	Users []UserResource
}

type SelfResource struct {
	Id   string `json:"id"`
	Name string `json:"name"`
}

type TeamResource struct {
	Id          string `json:"id"`
	Name        string `json:"name"`
	EmailDomain string `json:"email_domain"`
	Domain      string `json:"domain"`
}

type UserResource struct {
	Id   string `json:"id"`
	Name string `json:"name"`
}
<commit_msg>slack/schema.go: ChatPostMessageResponse<commit_after>package slack

type ChannelResource struct {
	Id   string
	Name string
}

type ChannelResponse struct {
	Response
	Channel ChannelResource
}

type ChatPostMessageResponse struct {
	Response
	Channel string
	Message MessageResource
}

type MessageResource struct {
}

type Response struct {
	Ok    bool
	Error string
}

type RTMStartResponse struct {
	Response
	Url   string
	Self  *SelfResource
	Team  *TeamResource
	Users []UserResource
}

type SelfResource struct {
	Id   string `json:"id"`
	Name string `json:"name"`
}

type TeamResource struct {
	Id          string `json:"id"`
	Name        string `json:"name"`
	EmailDomain string `json:"email_domain"`
	Domain      string `json:"domain"`
}

type UserResource struct {
	Id   string `json:"id"`
	Name string `json:"name"`
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the LGPLv3, see LICENCE file for details.

// This file is a simplyfied copy of github.com/juju/testing/mgo.go

package elasticsearch

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"syscall"
	"testing"

	"github.com/juju/loggo"
	jujutesting "github.com/juju/testing"

	gc "launchpad.net/gocheck"
)

type ElasticSearchInstance struct {
	Dir      string
	HttpPort int
	Server   *exec.Cmd
	exited   <-chan struct{}
}

var (
	ElasticSearchServer = &ElasticSearchInstance{}
	logger              = loggo.GetLogger("juju.testing.elasticsearchsuite")
)

const (
	maxStartAttempts = 5

	//elasticsearch exits with code 143 on SIGTERM
	elasticSearchSigTermErrCode = 143
)

func ElasticSearchTestPackage(t *testing.T) {
	if err := ElasticSearchServer.Start(); err != nil {
		t.Fatal(err)
	}
	defer ElasticSearchServer.Destroy()
	gc.TestingT(t)
}

func (es *ElasticSearchInstance) kill(sig syscall.Signal) {
	es.Server.Process.Signal(sig)
	<-es.exited
	es.Server = nil
	es.exited = nil
}

func (es *ElasticSearchInstance) run() error {
	if es.Server != nil {
		panic("elasticsearch is already running")
	}
	configFile, err := es.writeConfig()
	if err != nil {
		return err
	}
	server := exec.Command("elasticsearch", "--config="+configFile)
	out, err := server.StdoutPipe()
	if err != nil {
		return err
	}
	exited := make(chan struct{})
	started := make(chan error)
	listening := make(chan error, 1)
	go func() {
		err := <-started
		if err != nil {
			close(listening)
			close(exited)
			return
		}
		var buf bytes.Buffer
		prefix := fmt.Sprintf("inet[/127.0.0.1:%v", es.HttpPort)
		if readUntilMatching(prefix, io.TeeReader(out, &buf), regexp.MustCompile("node.*started")) {
			listening <- nil
		} else {
			err := fmt.Errorf("elasticsearch failed to listen on port %v using config %v", es.HttpPort, configFile)
			if strings.Contains(buf.String(), "Address alrady in use") {
				err = addrAlreadyInUseError{err}
			}
			listening <- err
		}
		lines := readLastLines(prefix, io.MultiReader(&buf, out), 20)
		err = server.Wait()
		exitErr, _ := err.(*exec.ExitError)
		defer close(exited)
		if exitErr != nil {
			exitCode := exitErr.Sys().(syscall.WaitStatus).ExitStatus()
			if exitCode == elasticSearchSigTermErrCode {
				return
			}
		}
		if err != nil || exitErr != nil && exitErr.Exited() {
			logger.Errorf("elasticsearch has exited without being killed")
			for _, line := range lines {
				logger.Errorf("elasticsearch: %s", line)
			}
		}
	}()
	es.exited = exited
	err = server.Start()
	started <- err
	if err != nil {
		return err
	}
	err = <-listening
	close(listening)
	if err != nil {
		return err
	}
	es.Server = server
	return nil
}

func (es *ElasticSearchInstance) writeConfig() (string, error) {
	if es.Dir == "" {
		return "", fmt.Errorf("directory not set")
	}
	file, err := os.Create(es.Dir + "/elasticsearch.yml")
	if err != nil {
		return "", err
	}
	defer file.Close()
	fmt.Fprintf(file, "cluster.name: elasticsearch_testing\n")
	fmt.Fprintf(file, "path.data: %s/data\n", es.Dir)
	fmt.Fprintf(file, "path.logs: %s/log/\n", es.Dir)
	fmt.Fprintf(file, "network.host: 127.0.0.1\n")
	fmt.Fprintf(file, "http.port: %d\n", es.HttpPort)
	return file.Name(), nil
}

func (es *ElasticSearchInstance) Destroy() {
	if es.Server != nil {
		term := syscall.SIGTERM
		logger.Debugf("killing elasticsearch pid %d in %s on port %d with %s", es.Server.Process.Pid, es.Dir, es.HttpPort, term)
		es.kill(term)
		os.RemoveAll(es.Dir)
		es.Dir = ""
		return
	}
	logger.Errorf("Destroy called when elasticsearch was not started")
}
func (es *ElasticSearchInstance) Start() error {
	dir, err := ioutil.TempDir("", "test-es")
	if err != nil {
		return err
	}
	es.Dir = dir
	logger.Debugf("starting elasticsearch in ", es.Dir)
	for i := 0; i < maxStartAttempts; i++ {
		es.HttpPort = jujutesting.FindTCPPort()
		err = es.run()
		switch err.(type) {
		case addrAlreadyInUseError:
			logger.Debugf("failed to start elasticssearch: %v, trying another port", err)
			continue
		case nil:
			logger.Debugf("started elasticsearch pid %d in %s on port %d", es.Server.Process.Pid, es.Dir, es.HttpPort)
		default:
			es.HttpPort = 0
			os.RemoveAll(es.Dir)
			es.Dir = ""
			logger.Warningf("failed to start elasticsearch %v", err)
		}
		break
	}
	return err
}

func (es *ElasticSearchInstance) dropAll(db *Database) error {
	//for index in curl 'localhost:9200/_cat/indices?v'
	// delete index
	names, err := db.CatIndices()
	if err != nil {
		return err
	}
	for _, name := range names {
		db.DeleteIndex(name)
	}
	return nil
}

type ElasticSearchSuite struct {
	*ElasticSearchInstance
	db *Database
}

func (s *ElasticSearchSuite) SetUpSuite(c *gc.C) {
	s.ElasticSearchInstance = ElasticSearchServer
	s.db = &Database{"127.0.0.1", ElasticSearchServer.HttpPort}
}

func (s *ElasticSearchSuite) TearDownSuite(c *gc.C) {
}

func (s *ElasticSearchSuite) SetUpTest(c *gc.C) {
	s.dropAll(s.db)
}

func (s *ElasticSearchSuite) TearDownTest(c *gc.C) {
}

func readLastLines(prefix string, r io.Reader, n int) []string {
	sc := bufio.NewScanner(r)
	lines := make([]string, n)
	i := 0
	for sc.Scan() {
		if line := strings.TrimRight(sc.Text(), "\n"); line != "" {
			logger.Tracef("%s: %s", prefix, line)
			lines[i%n] = line
			i++
		}
	}
	if err := sc.Err(); err != nil {
		panic(err)
	}
	final := make([]string, 0, n+1)
	if i > n {
		final = append(final, fmt.Sprintf("[%d lines omitted]", i-n))
	}
	for j := 0; j < n; j++ {
		if line := lines[(j+i)%n]; line != "" {
			final = append(final, line)
		}
	}
	return final
}
func readUntilMatching(prefix string, r io.Reader, re *regexp.Regexp) bool {
	sc := bufio.NewScanner(r)
	for sc.Scan() {
		line := sc.Text()
		logger.Tracef("%s: %s", prefix, line)
		if re.MatchString(line) {
			return true
		}
	}
	return false
}

type addrAlreadyInUseError struct {
	error
}
<commit_msg>fix typo alrady->already<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the LGPLv3, see LICENCE file for details.

// This file is a simplyfied copy of github.com/juju/testing/mgo.go

package elasticsearch

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"syscall"
	"testing"

	"github.com/juju/loggo"
	jujutesting "github.com/juju/testing"

	gc "launchpad.net/gocheck"
)

type ElasticSearchInstance struct {
	Dir      string
	HttpPort int
	Server   *exec.Cmd
	exited   <-chan struct{}
}

var (
	ElasticSearchServer = &ElasticSearchInstance{}
	logger              = loggo.GetLogger("juju.testing.elasticsearchsuite")
)

const (
	maxStartAttempts = 5

	//elasticsearch exits with code 143 on SIGTERM
	elasticSearchSigTermErrCode = 143
)

func ElasticSearchTestPackage(t *testing.T) {
	if err := ElasticSearchServer.Start(); err != nil {
		t.Fatal(err)
	}
	defer ElasticSearchServer.Destroy()
	gc.TestingT(t)
}

func (es *ElasticSearchInstance) kill(sig syscall.Signal) {
	es.Server.Process.Signal(sig)
	<-es.exited
	es.Server = nil
	es.exited = nil
}

func (es *ElasticSearchInstance) run() error {
	if es.Server != nil {
		panic("elasticsearch is already running")
	}
	configFile, err := es.writeConfig()
	if err != nil {
		return err
	}
	server := exec.Command("elasticsearch", "--config="+configFile)
	out, err := server.StdoutPipe()
	if err != nil {
		return err
	}
	exited := make(chan struct{})
	started := make(chan error)
	listening := make(chan error, 1)
	go func() {
		err := <-started
		if err != nil {
			close(listening)
			close(exited)
			return
		}
		var buf bytes.Buffer
		prefix := fmt.Sprintf("inet[/127.0.0.1:%v", es.HttpPort)
		if readUntilMatching(prefix, io.TeeReader(out, &buf), regexp.MustCompile("node.*started")) {
			listening <- nil
		} else {
			err := fmt.Errorf("elasticsearch failed to listen on port %v using config %v", es.HttpPort, configFile)
			if strings.Contains(buf.String(), "Address already in use") {
				err = addrAlreadyInUseError{err}
			}
			listening <- err
		}
		lines := readLastLines(prefix, io.MultiReader(&buf, out), 20)
		err = server.Wait()
		exitErr, _ := err.(*exec.ExitError)
		defer close(exited)
		if exitErr != nil {
			exitCode := exitErr.Sys().(syscall.WaitStatus).ExitStatus()
			if exitCode == elasticSearchSigTermErrCode {
				return
			}
		}
		if err != nil || exitErr != nil && exitErr.Exited() {
			logger.Errorf("elasticsearch has exited without being killed")
			for _, line := range lines {
				logger.Errorf("elasticsearch: %s", line)
			}
		}
	}()
	es.exited = exited
	err = server.Start()
	started <- err
	if err != nil {
		return err
	}
	err = <-listening
	close(listening)
	if err != nil {
		return err
	}
	es.Server = server
	return nil
}

func (es *ElasticSearchInstance) writeConfig() (string, error) {
	if es.Dir == "" {
		return "", fmt.Errorf("directory not set")
	}
	file, err := os.Create(es.Dir + "/elasticsearch.yml")
	if err != nil {
		return "", err
	}
	defer file.Close()
	fmt.Fprintf(file, "cluster.name: elasticsearch_testing\n")
	fmt.Fprintf(file, "path.data: %s/data\n", es.Dir)
	fmt.Fprintf(file, "path.logs: %s/log/\n", es.Dir)
	fmt.Fprintf(file, "network.host: 127.0.0.1\n")
	fmt.Fprintf(file, "http.port: %d\n", es.HttpPort)
	return file.Name(), nil
}

func (es *ElasticSearchInstance) Destroy() {
	if es.Server != nil {
		term := syscall.SIGTERM
		logger.Debugf("killing elasticsearch pid %d in %s on port %d with %s", es.Server.Process.Pid, es.Dir, es.HttpPort, term)
		es.kill(term)
		os.RemoveAll(es.Dir)
		es.Dir = ""
		return
	}
	logger.Errorf("Destroy called when elasticsearch was not started")
}
func (es *ElasticSearchInstance) Start() error {
	dir, err := ioutil.TempDir("", "test-es")
	if err != nil {
		return err
	}
	es.Dir = dir
	logger.Debugf("starting elasticsearch in ", es.Dir)
	for i := 0; i < maxStartAttempts; i++ {
		es.HttpPort = jujutesting.FindTCPPort()
		err = es.run()
		switch err.(type) {
		case addrAlreadyInUseError:
			logger.Debugf("failed to start elasticssearch: %v, trying another port", err)
			continue
		case nil:
			logger.Debugf("started elasticsearch pid %d in %s on port %d", es.Server.Process.Pid, es.Dir, es.HttpPort)
		default:
			es.HttpPort = 0
			os.RemoveAll(es.Dir)
			es.Dir = ""
			logger.Warningf("failed to start elasticsearch %v", err)
		}
		break
	}
	return err
}

func (es *ElasticSearchInstance) dropAll(db *Database) error {
	//for index in curl 'localhost:9200/_cat/indices?v'
	// delete index
	names, err := db.CatIndices()
	if err != nil {
		return err
	}
	for _, name := range names {
		db.DeleteIndex(name)
	}
	return nil
}

type ElasticSearchSuite struct {
	*ElasticSearchInstance
	db *Database
}

func (s *ElasticSearchSuite) SetUpSuite(c *gc.C) {
	s.ElasticSearchInstance = ElasticSearchServer
	s.db = &Database{"127.0.0.1", ElasticSearchServer.HttpPort}
}

func (s *ElasticSearchSuite) TearDownSuite(c *gc.C) {
}

func (s *ElasticSearchSuite) SetUpTest(c *gc.C) {
	s.dropAll(s.db)
}

func (s *ElasticSearchSuite) TearDownTest(c *gc.C) {
}

func readLastLines(prefix string, r io.Reader, n int) []string {
	sc := bufio.NewScanner(r)
	lines := make([]string, n)
	i := 0
	for sc.Scan() {
		if line := strings.TrimRight(sc.Text(), "\n"); line != "" {
			logger.Tracef("%s: %s", prefix, line)
			lines[i%n] = line
			i++
		}
	}
	if err := sc.Err(); err != nil {
		panic(err)
	}
	final := make([]string, 0, n+1)
	if i > n {
		final = append(final, fmt.Sprintf("[%d lines omitted]", i-n))
	}
	for j := 0; j < n; j++ {
		if line := lines[(j+i)%n]; line != "" {
			final = append(final, line)
		}
	}
	return final
}
func readUntilMatching(prefix string, r io.Reader, re *regexp.Regexp) bool {
	sc := bufio.NewScanner(r)
	for sc.Scan() {
		line := sc.Text()
		logger.Tracef("%s: %s", prefix, line)
		if re.MatchString(line) {
			return true
		}
	}
	return false
}

type addrAlreadyInUseError struct {
	error
}
<|endoftext|>