<commit_before>package google

import (
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
	"google.golang.org/api/compute/v1"
)

func resourceComputeProjectMetadataItem() *schema.Resource {
	return &schema.Resource{
		Create: resourceComputeProjectMetadataItemCreate,
		Read:   resourceComputeProjectMetadataItemRead,
		Update: resourceComputeProjectMetadataItemUpdate,
		Delete: resourceComputeProjectMetadataItemDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"key": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"value": {
				Type:     schema.TypeString,
				Required: true,
			},
			"project": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
		},

		Timeouts: &schema.ResourceTimeout{
			Create: schema.DefaultTimeout(5 * time.Minute),
			Update: schema.DefaultTimeout(5 * time.Minute),
			Delete: schema.DefaultTimeout(5 * time.Minute),
		},
	}
}

func resourceComputeProjectMetadataItemCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	projectID, err := getProject(d, config)
	if err != nil {
		return err
	}

	key := d.Get("key").(string)
	val := d.Get("value").(string)

	err = updateComputeCommonInstanceMetadata(config, projectID, key, &val, int(d.Timeout(schema.TimeoutCreate).Minutes()))
	if err != nil {
		return err
	}

	d.SetId(key)

	return nil
}

func resourceComputeProjectMetadataItemRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	projectID, err := getProject(d, config)
	if err != nil {
		return err
	}

	log.Printf("[DEBUG] Loading project metadata: %s", projectID)
	project, err := config.clientCompute.Projects.Get(projectID).Do()
	if err != nil {
		return fmt.Errorf("Error loading project '%s': %s", projectID, err)
	}

	md := flattenMetadata(project.CommonInstanceMetadata)
	val, ok := md[d.Id()]
	if !ok {
		// Resource no longer exists
		d.SetId("")
		return nil
	}

	d.Set("project", projectID)
	d.Set("key", d.Id())
	d.Set("value", val)

	return nil
}

func resourceComputeProjectMetadataItemUpdate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	projectID, err := getProject(d, config)
	if err != nil {
		return err
	}

	if d.HasChange("value") {
		key := d.Get("key").(string)
		_, n := d.GetChange("value")
		new := n.(string)

		err = updateComputeCommonInstanceMetadata(config, projectID, key, &new, int(d.Timeout(schema.TimeoutUpdate).Minutes()))
		if err != nil {
			return err
		}
	}
	return nil
}

func resourceComputeProjectMetadataItemDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	projectID, err := getProject(d, config)
	if err != nil {
		return err
	}

	key := d.Get("key").(string)

	err = updateComputeCommonInstanceMetadata(config, projectID, key, nil, int(d.Timeout(schema.TimeoutDelete).Minutes()))
	if err != nil {
		return err
	}

	d.SetId("")
	return nil
}

func updateComputeCommonInstanceMetadata(config *Config, projectID string, key string, afterVal *string, timeout int) error {
	updateMD := func() error {
		log.Printf("[DEBUG] Loading project metadata: %s", projectID)
		project, err := config.clientCompute.Projects.Get(projectID).Do()
		if err != nil {
			return fmt.Errorf("Error loading project '%s': %s", projectID, err)
		}

		md := flattenMetadata(project.CommonInstanceMetadata)

		val, ok := md[key]

		if !ok {
			if afterVal == nil {
				// Asked to set no value and we didn't find one - we're done
				return nil
			}
		} else {
			if afterVal != nil && *afterVal == val {
				// Asked to set a value and it's already set - we're done.
				return nil
			}
		}

		if afterVal == nil {
			delete(md, key)
		} else {
			md[key] = *afterVal
		}

		// Attempt to write the new value now
		op, err := config.clientCompute.Projects.SetCommonInstanceMetadata(
			projectID,
			&compute.Metadata{
				Fingerprint: project.CommonInstanceMetadata.Fingerprint,
				Items:       expandComputeMetadata(md),
			},
		).Do()

		if err != nil {
			return err
		}

		log.Printf("[DEBUG] SetCommonInstanceMetadata: %d (%s)", op.Id, op.SelfLink)

		return computeOperationWaitTime(config, op, project.Name, "SetCommonInstanceMetadata", timeout)
	}

	return MetadataRetryWrapper(updateMD)
}
<commit_msg>Update timeouts - our unit tests are failing on these resources. (#4995)<commit_after>package google

import (
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform-plugin-sdk/helper/schema"
	"google.golang.org/api/compute/v1"
)

func resourceComputeProjectMetadataItem() *schema.Resource {
	return &schema.Resource{
		Create: resourceComputeProjectMetadataItemCreate,
		Read:   resourceComputeProjectMetadataItemRead,
		Update: resourceComputeProjectMetadataItemUpdate,
		Delete: resourceComputeProjectMetadataItemDelete,
		Importer: &schema.ResourceImporter{
			State: schema.ImportStatePassthrough,
		},

		Schema: map[string]*schema.Schema{
			"key": {
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"value": {
				Type:     schema.TypeString,
				Required: true,
			},
			"project": {
				Type:     schema.TypeString,
				Optional: true,
				Computed: true,
				ForceNew: true,
			},
		},

		Timeouts: &schema.ResourceTimeout{
			Create: schema.DefaultTimeout(7 * time.Minute),
			Update: schema.DefaultTimeout(7 * time.Minute),
			Delete: schema.DefaultTimeout(7 * time.Minute),
		},
	}
}

func resourceComputeProjectMetadataItemCreate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	projectID, err := getProject(d, config)
	if err != nil {
		return err
	}

	key := d.Get("key").(string)
	val := d.Get("value").(string)

	err = updateComputeCommonInstanceMetadata(config, projectID, key, &val, int(d.Timeout(schema.TimeoutCreate).Minutes()))
	if err != nil {
		return err
	}

	d.SetId(key)

	return nil
}

func resourceComputeProjectMetadataItemRead(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	projectID, err := getProject(d, config)
	if err != nil {
		return err
	}

	log.Printf("[DEBUG] Loading project metadata: %s", projectID)
	project, err := config.clientCompute.Projects.Get(projectID).Do()
	if err != nil {
		return fmt.Errorf("Error loading project '%s': %s", projectID, err)
	}

	md := flattenMetadata(project.CommonInstanceMetadata)
	val, ok := md[d.Id()]
	if !ok {
		// Resource no longer exists
		d.SetId("")
		return nil
	}

	d.Set("project", projectID)
	d.Set("key", d.Id())
	d.Set("value", val)

	return nil
}

func resourceComputeProjectMetadataItemUpdate(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	projectID, err := getProject(d, config)
	if err != nil {
		return err
	}

	if d.HasChange("value") {
		key := d.Get("key").(string)
		_, n := d.GetChange("value")
		new := n.(string)

		err = updateComputeCommonInstanceMetadata(config, projectID, key, &new, int(d.Timeout(schema.TimeoutUpdate).Minutes()))
		if err != nil {
			return err
		}
	}
	return nil
}

func resourceComputeProjectMetadataItemDelete(d *schema.ResourceData, meta interface{}) error {
	config := meta.(*Config)

	projectID, err := getProject(d, config)
	if err != nil {
		return err
	}

	key := d.Get("key").(string)

	err = updateComputeCommonInstanceMetadata(config, projectID, key, nil, int(d.Timeout(schema.TimeoutDelete).Minutes()))
	if err != nil {
		return err
	}

	d.SetId("")
	return nil
}

func updateComputeCommonInstanceMetadata(config *Config, projectID string, key string, afterVal *string, timeout int) error {
	updateMD := func() error {
		log.Printf("[DEBUG] Loading project metadata: %s", projectID)
		project, err := config.clientCompute.Projects.Get(projectID).Do()
		if err != nil {
			return fmt.Errorf("Error loading project '%s': %s", projectID, err)
		}

		md := flattenMetadata(project.CommonInstanceMetadata)

		val, ok := md[key]

		if !ok {
			if afterVal == nil {
				// Asked to set no value and we didn't find one - we're done
				return nil
			}
		} else {
			if afterVal != nil && *afterVal == val {
				// Asked to set a value and it's already set - we're done.
				return nil
			}
		}

		if afterVal == nil {
			delete(md, key)
		} else {
			md[key] = *afterVal
		}

		// Attempt to write the new value now
		op, err := config.clientCompute.Projects.SetCommonInstanceMetadata(
			projectID,
			&compute.Metadata{
				Fingerprint: project.CommonInstanceMetadata.Fingerprint,
				Items:       expandComputeMetadata(md),
			},
		).Do()

		if err != nil {
			return err
		}

		log.Printf("[DEBUG] SetCommonInstanceMetadata: %d (%s)", op.Id, op.SelfLink)

		return computeOperationWaitTime(config, op, project.Name, "SetCommonInstanceMetadata", timeout)
	}

	return MetadataRetryWrapper(updateMD)
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backend

import (
	"context"
	"time"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"

	"go.chromium.org/luci/resultdb/internal/tasks"
)

type invocationProcessor struct {
	bqExporter
}

// permanentInvocationTaskErrTag set in an error indicates that the err is not
// resolvable by retry.
var permanentInvocationTaskErrTag = errors.BoolTag{
	Key: errors.NewTagKey("permanent failure to process invocation task"),
}

// runTask leases, runs and deletes a task.
// May return tasks.ErrConflict.
// Updates taskAttemptMetric and taskDurationMetric.
func (b *backend) runTask(ctx context.Context, taskType tasks.Type, id string) (err error) {
	leaseDuration := 10 * time.Minute
	switch taskType {
	case tasks.TryFinalizeInvocation:
		leaseDuration = time.Minute
	}
	if b.ForceLeaseDuration > 0 {
		leaseDuration = b.ForceLeaseDuration
	}

	// Update metrics.
	startTime := clock.Now(ctx)
	taskStatus := transientFailure
	defer func() {
		// Send metrics to tsmon.
		typeStr := string(taskType)

		duration := float64(time.Since(startTime).Milliseconds())
		taskDurationMetric.Add(ctx, duration, typeStr, taskStatus)

		taskAttemptMetric.Add(ctx, 1, typeStr, taskStatus)
	}()

	invID, payload, err := tasks.Lease(ctx, taskType, id, leaseDuration)
	switch {
	case err == tasks.ErrConflict:
		logging.Warningf(ctx, "Conflict while trying to lease the task")
		// It's possible another worker has leased the task, and it's fine, skip.
		return nil
	case err != nil:
		return err
	}

	switch taskType {
	case tasks.BQExport:
		err = b.exportResultsToBigQuery(ctx, invID, payload)
	case tasks.TryFinalizeInvocation:
		err = tryFinalizeInvocation(ctx, invID)
	default:
		err = errors.Reason("unexpected task type %q", taskType).Err()
	}
	if err != nil {
		if permanentInvocationTaskErrTag.In(err) {
			taskStatus = permanentFailure
			logging.Errorf(ctx, "permanent failure to process the task: %s", err)
			return tasks.Delete(ctx, taskType, id)
		}
		return err
	}

	// Invocation task is done, delete the row from spanner.
	if err := tasks.Delete(ctx, taskType, id); err != nil {
		return err
	}

	taskStatus = success
	return nil
}

// runInvocationTasks gets invocation tasks and dispatches them to workers.
func (b *backend) runInvocationTasks(ctx context.Context, taskType tasks.Type) {
	mCtx, cancel := context.WithCancel(ctx)
	defer cancel()
	go b.cron(ctx, time.Minute, func(ctx context.Context) error {
		recordOldestTaskMetric(mCtx, taskType)
		return nil
	})

	workers := b.taskWorkers
	if workers == 0 {
		workers = 1
	}
	b.cronGroup(ctx, workers, time.Second, func(ctx context.Context, replica int) error {
		ids, err := tasks.Sample(ctx, taskType, time.Now(), 100)
		if err != nil {
			return errors.Annotate(err, "failed to query invocation tasks").Err()
		}

		var processed []string
		defer func() {
			if len(processed) > 0 {
				logging.Infof(ctx, "processed tasks: %q", processed)
			}
		}()

		for _, id := range ids {
			ctx := logging.SetField(ctx, "task_id", id)
			switch err := b.runTask(ctx, taskType, id); {
			case err == tasks.ErrConflict:
				logging.Warningf(ctx, "Conflict while trying to lease the task")
			case err != nil:
				return errors.Annotate(err, "failed to process task %s", id).Err()
			default:
				processed = append(processed, id)
			}
		}
		return nil
	})
}
<commit_msg>[resultdb] Reduce task sampling<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backend

import (
	"context"
	"time"

	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"

	"go.chromium.org/luci/resultdb/internal/tasks"
)

type invocationProcessor struct {
	bqExporter
}

// permanentInvocationTaskErrTag set in an error indicates that the err is not
// resolvable by retry.
var permanentInvocationTaskErrTag = errors.BoolTag{
	Key: errors.NewTagKey("permanent failure to process invocation task"),
}

// runTask leases, runs and deletes a task.
// May return tasks.ErrConflict.
// Updates taskAttemptMetric and taskDurationMetric.
func (b *backend) runTask(ctx context.Context, taskType tasks.Type, id string) (err error) {
	leaseDuration := 10 * time.Minute
	switch taskType {
	case tasks.TryFinalizeInvocation:
		leaseDuration = time.Minute
	}
	if b.ForceLeaseDuration > 0 {
		leaseDuration = b.ForceLeaseDuration
	}

	// Update metrics.
	startTime := clock.Now(ctx)
	taskStatus := transientFailure
	defer func() {
		// Send metrics to tsmon.
		typeStr := string(taskType)

		duration := float64(time.Since(startTime).Milliseconds())
		taskDurationMetric.Add(ctx, duration, typeStr, taskStatus)

		taskAttemptMetric.Add(ctx, 1, typeStr, taskStatus)
	}()

	invID, payload, err := tasks.Lease(ctx, taskType, id, leaseDuration)
	switch {
	case err == tasks.ErrConflict:
		logging.Warningf(ctx, "Conflict while trying to lease the task")
		// It's possible another worker has leased the task, and it's fine, skip.
		return nil
	case err != nil:
		return err
	}

	switch taskType {
	case tasks.BQExport:
		err = b.exportResultsToBigQuery(ctx, invID, payload)
	case tasks.TryFinalizeInvocation:
		err = tryFinalizeInvocation(ctx, invID)
	default:
		err = errors.Reason("unexpected task type %q", taskType).Err()
	}
	if err != nil {
		if permanentInvocationTaskErrTag.In(err) {
			taskStatus = permanentFailure
			logging.Errorf(ctx, "permanent failure to process the task: %s", err)
			return tasks.Delete(ctx, taskType, id)
		}
		return err
	}

	// Invocation task is done, delete the row from spanner.
	if err := tasks.Delete(ctx, taskType, id); err != nil {
		return err
	}

	taskStatus = success
	return nil
}

// runInvocationTasks gets invocation tasks and dispatches them to workers.
func (b *backend) runInvocationTasks(ctx context.Context, taskType tasks.Type) {
	mCtx, cancel := context.WithCancel(ctx)
	defer cancel()
	go b.cron(ctx, time.Minute, func(ctx context.Context) error {
		recordOldestTaskMetric(mCtx, taskType)
		return nil
	})

	workers := b.taskWorkers
	if workers == 0 {
		workers = 1
	}
	b.cronGroup(ctx, workers, time.Second, func(ctx context.Context, replica int) error {
		ids, err := tasks.Sample(ctx, taskType, time.Now(), 100)
		if err != nil {
			return errors.Annotate(err, "failed to query invocation tasks").Err()
		}

		for _, id := range ids {
			ctx := logging.SetField(ctx, "task_id", id)
			switch err := b.runTask(ctx, taskType, id); {

			case err == tasks.ErrConflict:
				logging.Warningf(ctx, "Conflict while trying to lease the task")

			case err != nil:
				// Do not bail on other task ids, otherwise we sample tasks too often
				// which is expensive.
				logging.Errorf(ctx, "Task failed: %s", err)

			default:
				logging.Infof(ctx, "Task succeeded")
			}
		}
		return nil
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package k8sclient

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/golang/glog"
	"github.com/kubernetes-incubator/cluster-proportional-vertical-autoscaler/pkg/version"

	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

// K8sClient - Wraps all needed client functionalities for autoscaler
type K8sClient interface {
	// GetClusterSize counts schedulable nodes and cores in the cluster
	GetClusterSize() (*ClusterSize, error)
	// UpdateResources updates the resource needs for the containers in the target
	UpdateResources(resources map[string]apiv1.ResourceRequirements) error
}

// k8sClient - Wraps all Kubernetes API client functionality.
type k8sClient struct {
	target        *targetSpec
	clientset     kubernetes.Interface
	clusterStatus *ClusterSize
	dryRun        bool
}

// NewK8sClient gives a k8sClient with the given dependencies.
func NewK8sClient(namespace, target, kubeconfig string, dryRun bool) (K8sClient, error) {
	var config *rest.Config
	var err error
	if kubeconfig != "" {
		config, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
	} else {
		config, err = rest.InClusterConfig()
	}
	if err != nil {
		return nil, err
	}
	config.UserAgent = userAgent()
	// Use protobufs for communication with apiserver.
	config.ContentType = "application/vnd.kubernetes.protobuf"
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		return nil, err
	}

	tgt, err := makeTarget(clientset, target, namespace)
	if err != nil {
		return nil, err
	}

	return &k8sClient{
		clientset: clientset,
		target:    tgt,
		dryRun:    dryRun,
	}, nil
}

func userAgent() string {
	command := ""
	if len(os.Args) > 0 && len(os.Args[0]) > 0 {
		command = filepath.Base(os.Args[0])
	}
	if len(command) == 0 {
		command = "cpvpa"
	}
	return command + "/" + version.VERSION
}

func makeTarget(client kubernetes.Interface, target, namespace string) (*targetSpec, error) {
	splits := strings.Split(target, "/")
	if len(splits) != 2 {
		return nil, fmt.Errorf("target format error: %v", target)
	}
	kind := splits[0]
	name := splits[1]

	kind, groupVersions, err := discoverAPI(client, kind)
	if err != nil {
		return nil, err
	}

	tgt, err := newTargetSpec(kind, groupVersions, namespace, name)
	if err != nil {
		return nil, err
	}

	glog.V(4).Infof("Discovered target %s in %v", target, tgt.GroupVersion)
	return tgt, nil
}

func discoverAPI(client kubernetes.Interface, kindArg string) (kind string, groupVersions map[string]bool, err error) {
	var plural string
	switch strings.ToLower(kindArg) {
	case "deployment":
		kind = "Deployment"
		plural = "deployments"
	case "daemonset":
		kind = "DaemonSet"
		plural = "daemonsets"
	case "replicaset":
		kind = "ReplicaSet"
		plural = "replicasets"
	default:
		return "", nil, fmt.Errorf("unknown kind %q", kindArg)
	}

	resourceLists, err := client.Discovery().ServerPreferredNamespacedResources()
	if err != nil {
		return "", nil, fmt.Errorf("failed to discover apigroup for kind %q: %v", kind, err)
	}

	groupVersions = map[string]bool{}
	for _, resourceList := range resourceLists {
		for _, res := range resourceList.APIResources {
			if res.Name == plural {
				kind = res.Kind
				groupVersions[resourceList.GroupVersion] = true
			}
		}
	}

	return kind, groupVersions, nil
}

// targetSpec stores the scalable target resource.
type targetSpec struct {
	Kind         string
	GroupVersion string
	Namespace    string
	Name         string
	patcher      patchFunc
}

// Captures the namespace and name to patch, and calls the best
// resource-specific patch method.
type patchFunc func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error

func newTargetSpec(kind string, groupVersions map[string]bool, namespace, name string) (*targetSpec, error) {
	groupVer, patcher, err := findPatcher(kind, groupVersions)
	if err != nil {
		return nil, err
	}

	return &targetSpec{
		Kind:         kind,
		GroupVersion: groupVer,
		Namespace:    namespace,
		Name:         name,
		patcher:      patcher,
	}, nil
}

func (tgt *targetSpec) Patch(client kubernetes.Interface, pt types.PatchType, data []byte) error {
	return tgt.patcher(client, tgt.Namespace, tgt.Name, pt, data)
}

// findPatcher returns a groupVersion string and a patch function for the
// specified kind.  This is needed because, at least in theory, the schema of a
// resource could change dramatically, and we should use statically versioned
// types everywhere.  In practice, it's unlikely that the bits we care about
// would change (since we PATCH).  Alas, there's not a great way to dynamically
// use whatever is "latest".  The fallout of this is that we will need to update
// this program when new API group-versions are introduced.
func findPatcher(kind string, groupVersions map[string]bool) (string, patchFunc, error) {
	switch strings.ToLower(kind) {
	case "deployment":
		return findDeploymentPatcher(groupVersions)
	case "daemonset":
		return findDaemonSetPatcher(groupVersions)
	case "replicaset":
		return findReplicaSetPatcher(groupVersions)
	}
	// This should not happen, we already validated it.
	return "", nil, fmt.Errorf("unknown target kind: %s", kind)
}

func findDeploymentPatcher(groupVersions map[string]bool) (string, patchFunc, error) {
	// Find the best API to use - newest API first.
	if groupVersions["apps/v1"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.AppsV1().Deployments(namespace).Patch(name, pt, data)
			return err
		}
		return "apps/v1", patchFunc(fn), nil
	}
	if groupVersions["apps/v1beta2"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.AppsV1beta2().Deployments(namespace).Patch(name, pt, data)
			return err
		}
		return "apps/v1beta2", patchFunc(fn), nil
	}
	if groupVersions["apps/v1beta1"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.AppsV1beta1().Deployments(namespace).Patch(name, pt, data)
			return err
		}
		return "apps/v1beta1", patchFunc(fn), nil
	}
	if groupVersions["extensions/v1beta1"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.ExtensionsV1beta1().Deployments(namespace).Patch(name, pt, data)
			return err
		}
		return "extensions/v1beta1", patchFunc(fn), nil
	}
	return "", nil, fmt.Errorf("no supported API group for target: %v", groupVersions)
}

func findDaemonSetPatcher(groupVersions map[string]bool) (string, patchFunc, error) {
	// Find the best API to use - newest API first.
	if groupVersions["apps/v1"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.AppsV1().DaemonSets(namespace).Patch(name, pt, data)
			return err
		}
		return "apps/v1", patchFunc(fn), nil
	}
	if groupVersions["apps/v1beta2"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.AppsV1beta2().DaemonSets(namespace).Patch(name, pt, data)
			return err
		}
		return "apps/v1beta2", patchFunc(fn), nil
	}
	if groupVersions["extensions/v1beta1"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.ExtensionsV1beta1().DaemonSets(namespace).Patch(name, pt, data)
			return err
		}
		return "extensions/v1beta1", patchFunc(fn), nil
	}
	return "", nil, fmt.Errorf("no supported API group for target: %v", groupVersions)
}

func findReplicaSetPatcher(groupVersions map[string]bool) (string, patchFunc, error) {
	// Find the best API to use - newest API first.
	if groupVersions["apps/v1"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.AppsV1().ReplicaSets(namespace).Patch(name, pt, data)
			return err
		}
		return "apps/v1", patchFunc(fn), nil
	}
	if groupVersions["apps/v1beta2"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.AppsV1beta2().ReplicaSets(namespace).Patch(name, pt, data)
			return err
		}
		return "apps/v1beta2", patchFunc(fn), nil
	}
	if groupVersions["extensions/v1beta1"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.ExtensionsV1beta1().ReplicaSets(namespace).Patch(name, pt, data)
			return err
		}
		return "extensions/v1beta1", patchFunc(fn), nil
	}
	return "", nil, fmt.Errorf("no supported API group for target: %v", groupVersions)
}

// ClusterSize defines the cluster status.
type ClusterSize struct {
	Nodes int
	Cores int
}

func (k *k8sClient) GetClusterSize() (clusterStatus *ClusterSize, err error) {
	opt := metav1.ListOptions{Watch: false}

	nodes, err := k.clientset.CoreV1().Nodes().List(opt)
	if err != nil || nodes == nil {
		return nil, err
	}
	clusterStatus = &ClusterSize{}
	clusterStatus.Nodes = len(nodes.Items)
	var tc resource.Quantity
	// All nodes are considered, even those that are marked as unshedulable,
	// this includes the master.
	for _, node := range nodes.Items {
		tc.Add(node.Status.Capacity[apiv1.ResourceCPU])
	}

	tcInt64, tcOk := tc.AsInt64()
	if !tcOk {
		return nil, fmt.Errorf("unable to compute integer values of cores in the cluster")
	}
	clusterStatus.Cores = int(tcInt64)
	k.clusterStatus = clusterStatus
	return clusterStatus, nil
}

func (k *k8sClient) UpdateResources(resources map[string]apiv1.ResourceRequirements) error {
	ctrs := []interface{}{}
	for ctrName, res := range resources {
		ctrs = append(ctrs, map[string]interface{}{
			"name":      ctrName,
			"resources": res,
		})
	}
	patch := map[string]interface{}{
		"apiVersion": fmt.Sprintf("%s", k.target.GroupVersion),
		"kind":       k.target.Kind,
		"metadata": map[string]interface{}{
			"name": k.target.Name,
		},
		"spec": map[string]interface{}{
			"template": map[string]interface{}{
				"spec": map[string]interface{}{
					"containers": ctrs,
				},
			},
		},
	}

	jb, err := json.Marshal(patch)
	if err != nil {
		return fmt.Errorf("can't marshal patch to JSON: %v", err)
	}

	if k.dryRun {
		glog.Infof("Performing dry-run, no updates will take affect.")
		return nil
	}
	if err := k.target.Patch(k.clientset, types.StrategicMergePatchType, jb); err != nil {
		return fmt.Errorf("patch failed: %v", err)
	}

	return nil
}
<commit_msg>Prevent target kind to be an unsupported one<commit_after>/*
Copyright 2016 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package k8sclient

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/golang/glog"
	"github.com/kubernetes-incubator/cluster-proportional-vertical-autoscaler/pkg/version"

	apiv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

// K8sClient - Wraps all needed client functionalities for autoscaler
type K8sClient interface {
	// GetClusterSize counts schedulable nodes and cores in the cluster
	GetClusterSize() (*ClusterSize, error)
	// UpdateResources updates the resource needs for the containers in the target
	UpdateResources(resources map[string]apiv1.ResourceRequirements) error
}

// k8sClient - Wraps all Kubernetes API client functionality.
type k8sClient struct {
	target        *targetSpec
	clientset     kubernetes.Interface
	clusterStatus *ClusterSize
	dryRun        bool
}

// NewK8sClient gives a k8sClient with the given dependencies.
func NewK8sClient(namespace, target, kubeconfig string, dryRun bool) (K8sClient, error) {
	var config *rest.Config
	var err error
	if kubeconfig != "" {
		config, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
	} else {
		config, err = rest.InClusterConfig()
	}
	if err != nil {
		return nil, err
	}
	config.UserAgent = userAgent()
	// Use protobufs for communication with apiserver.
	config.ContentType = "application/vnd.kubernetes.protobuf"
	clientset, err := kubernetes.NewForConfig(config)
	if err != nil {
		return nil, err
	}

	tgt, err := makeTarget(clientset, target, namespace)
	if err != nil {
		return nil, err
	}

	return &k8sClient{
		clientset: clientset,
		target:    tgt,
		dryRun:    dryRun,
	}, nil
}

func userAgent() string {
	command := ""
	if len(os.Args) > 0 && len(os.Args[0]) > 0 {
		command = filepath.Base(os.Args[0])
	}
	if len(command) == 0 {
		command = "cpvpa"
	}
	return command + "/" + version.VERSION
}

func makeTarget(client kubernetes.Interface, target, namespace string) (*targetSpec, error) {
	splits := strings.Split(target, "/")
	if len(splits) != 2 {
		return nil, fmt.Errorf("target format error: %v", target)
	}
	kind := splits[0]
	name := splits[1]

	kind, groupVersions, err := discoverAPI(client, kind)
	if err != nil {
		return nil, err
	}

	tgt, err := newTargetSpec(kind, groupVersions, namespace, name)
	if err != nil {
		return nil, err
	}

	glog.V(4).Infof("Discovered target %s in %v", target, tgt.GroupVersion)
	return tgt, nil
}

func discoverAPI(client kubernetes.Interface, kindArg string) (kind string, groupVersions map[string]bool, err error) {
	var plural string
	switch strings.ToLower(kindArg) {
	case "deployment":
		kind = "Deployment"
		plural = "deployments"
	case "daemonset":
		kind = "DaemonSet"
		plural = "daemonsets"
	case "replicaset":
		kind = "ReplicaSet"
		plural = "replicasets"
	default:
		return "", nil, fmt.Errorf("unknown kind %q", kindArg)
	}

	resourceLists, err := client.Discovery().ServerPreferredNamespacedResources()
	if err != nil {
		return "", nil, fmt.Errorf("failed to discover apigroup for kind %q: %v", kind, err)
	}

	groupVersions = map[string]bool{}
	for _, resourceList := range resourceLists {
		for _, res := range resourceList.APIResources {
			if res.Name == plural && res.Kind == kind {
				groupVersions[resourceList.GroupVersion] = true
			}
		}
	}

	return kind, groupVersions, nil
}

// targetSpec stores the scalable target resource.
type targetSpec struct {
	Kind         string
	GroupVersion string
	Namespace    string
	Name         string
	patcher      patchFunc
}

// Captures the namespace and name to patch, and calls the best
// resource-specific patch method.
type patchFunc func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error

func newTargetSpec(kind string, groupVersions map[string]bool, namespace, name string) (*targetSpec, error) {
	groupVer, patcher, err := findPatcher(kind, groupVersions)
	if err != nil {
		return nil, err
	}

	return &targetSpec{
		Kind:         kind,
		GroupVersion: groupVer,
		Namespace:    namespace,
		Name:         name,
		patcher:      patcher,
	}, nil
}

func (tgt *targetSpec) Patch(client kubernetes.Interface, pt types.PatchType, data []byte) error {
	return tgt.patcher(client, tgt.Namespace, tgt.Name, pt, data)
}

// findPatcher returns a groupVersion string and a patch function for the
// specified kind.  This is needed because, at least in theory, the schema of a
// resource could change dramatically, and we should use statically versioned
// types everywhere.  In practice, it's unlikely that the bits we care about
// would change (since we PATCH).  Alas, there's not a great way to dynamically
// use whatever is "latest".  The fallout of this is that we will need to update
// this program when new API group-versions are introduced.
func findPatcher(kind string, groupVersions map[string]bool) (string, patchFunc, error) {
	switch strings.ToLower(kind) {
	case "deployment":
		return findDeploymentPatcher(groupVersions)
	case "daemonset":
		return findDaemonSetPatcher(groupVersions)
	case "replicaset":
		return findReplicaSetPatcher(groupVersions)
	}
	// This should not happen, we already validated it.
	return "", nil, fmt.Errorf("unknown target kind: %s", kind)
}

func findDeploymentPatcher(groupVersions map[string]bool) (string, patchFunc, error) {
	// Find the best API to use - newest API first.
	if groupVersions["apps/v1"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.AppsV1().Deployments(namespace).Patch(name, pt, data)
			return err
		}
		return "apps/v1", patchFunc(fn), nil
	}
	if groupVersions["apps/v1beta2"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.AppsV1beta2().Deployments(namespace).Patch(name, pt, data)
			return err
		}
		return "apps/v1beta2", patchFunc(fn), nil
	}
	if groupVersions["apps/v1beta1"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.AppsV1beta1().Deployments(namespace).Patch(name, pt, data)
			return err
		}
		return "apps/v1beta1", patchFunc(fn), nil
	}
	if groupVersions["extensions/v1beta1"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.ExtensionsV1beta1().Deployments(namespace).Patch(name, pt, data)
			return err
		}
		return "extensions/v1beta1", patchFunc(fn), nil
	}
	return "", nil, fmt.Errorf("no supported API group for target: %v", groupVersions)
}

func findDaemonSetPatcher(groupVersions map[string]bool) (string, patchFunc, error) {
	// Find the best API to use - newest API first.
	if groupVersions["apps/v1"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.AppsV1().DaemonSets(namespace).Patch(name, pt, data)
			return err
		}
		return "apps/v1", patchFunc(fn), nil
	}
	if groupVersions["apps/v1beta2"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.AppsV1beta2().DaemonSets(namespace).Patch(name, pt, data)
			return err
		}
		return "apps/v1beta2", patchFunc(fn), nil
	}
	if groupVersions["extensions/v1beta1"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.ExtensionsV1beta1().DaemonSets(namespace).Patch(name, pt, data)
			return err
		}
		return "extensions/v1beta1", patchFunc(fn), nil
	}
	return "", nil, fmt.Errorf("no supported API group for target: %v", groupVersions)
}

func findReplicaSetPatcher(groupVersions map[string]bool) (string, patchFunc, error) {
	// Find the best API to use - newest API first.
	if groupVersions["apps/v1"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.AppsV1().ReplicaSets(namespace).Patch(name, pt, data)
			return err
		}
		return "apps/v1", patchFunc(fn), nil
	}
	if groupVersions["apps/v1beta2"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.AppsV1beta2().ReplicaSets(namespace).Patch(name, pt, data)
			return err
		}
		return "apps/v1beta2", patchFunc(fn), nil
	}
	if groupVersions["extensions/v1beta1"] {
		fn := func(client kubernetes.Interface, namespace, name string, pt types.PatchType, data []byte) error {
			_, err := client.ExtensionsV1beta1().ReplicaSets(namespace).Patch(name, pt, data)
			return err
		}
		return "extensions/v1beta1", patchFunc(fn), nil
	}
	return "", nil, fmt.Errorf("no supported API group for target: %v", groupVersions)
}

// ClusterSize defines the cluster status.
type ClusterSize struct {
	Nodes int
	Cores int
}

func (k *k8sClient) GetClusterSize() (clusterStatus *ClusterSize, err error) {
	opt := metav1.ListOptions{Watch: false}

	nodes, err := k.clientset.CoreV1().Nodes().List(opt)
	if err != nil || nodes == nil {
		return nil, err
	}
	clusterStatus = &ClusterSize{}
	clusterStatus.Nodes = len(nodes.Items)
	var tc resource.Quantity
	// All nodes are considered, even those that are marked as unshedulable,
	// this includes the master.
	for _, node := range nodes.Items {
		tc.Add(node.Status.Capacity[apiv1.ResourceCPU])
	}

	tcInt64, tcOk := tc.AsInt64()
	if !tcOk {
		return nil, fmt.Errorf("unable to compute integer values of cores in the cluster")
	}
	clusterStatus.Cores = int(tcInt64)
	k.clusterStatus = clusterStatus
	return clusterStatus, nil
}

func (k *k8sClient) UpdateResources(resources map[string]apiv1.ResourceRequirements) error {
	ctrs := []interface{}{}
	for ctrName, res := range resources {
		ctrs = append(ctrs, map[string]interface{}{
			"name":      ctrName,
			"resources": res,
		})
	}
	patch := map[string]interface{}{
		"apiVersion": fmt.Sprintf("%s", k.target.GroupVersion),
		"kind":       k.target.Kind,
		"metadata": map[string]interface{}{
			"name": k.target.Name,
		},
		"spec": map[string]interface{}{
			"template": map[string]interface{}{
				"spec": map[string]interface{}{
					"containers": ctrs,
				},
			},
		},
	}

	jb, err := json.Marshal(patch)
	if err != nil {
		return fmt.Errorf("can't marshal patch to JSON: %v", err)
	}

	if k.dryRun {
		glog.Infof("Performing dry-run, no updates will take affect.")
		return nil
	}
	if err := k.target.Patch(k.clientset, types.StrategicMergePatchType, jb); err != nil {
		return fmt.Errorf("patch failed: %v", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package servicing

import (
	"github.com/go-kit/kit/log"
	"github.com/prometheus/client_golang/prometheus"
	"k8s.io/apimachinery/pkg/labels"

	"github.com/sapcc/kubernikus/pkg/controller/config"
	"github.com/sapcc/kubernikus/pkg/controller/servicing/coreos"
	"github.com/sapcc/kubernikus/pkg/controller/servicing/flatcar"
	kubernikus_lister "github.com/sapcc/kubernikus/pkg/generated/listers/kubernikus/v1"
)

type servicingNodesCollector struct {
	klusters    kubernikus_lister.KlusterLister
	nodeListers *NodeListerFactory
	updating    *prometheus.Desc
	waiting     *prometheus.Desc
	kubelet     *prometheus.Desc
	proxy       *prometheus.Desc
	osimage     *prometheus.Desc
	logger      log.Logger
}

// RegisterServicingNodesCollector does what the method name sais
func RegisterServicingNodesCollector(logger log.Logger, factories config.Factories) {
	collector := &servicingNodesCollector{
		updating: prometheus.NewDesc(
			"kubernikus_servicing_nodes_state_updating",
			"Amount of nodes per servicing action/state",
			[]string{"kluster_id", "state"},
			prometheus.Labels{},
		),
		waiting: prometheus.NewDesc(
			"kubernikus_servicing_nodes_state_waiting",
			"Amount of nodes that are waiting for updates per state",
			[]string{"kluster_id", "state"},
			prometheus.Labels{},
		),
		kubelet: prometheus.NewDesc(
			"kubernikus_servicing_nodes_version_kubelet",
			"Update Status of Nodes per Kluster",
			[]string{"kluster_id", "version"},
			prometheus.Labels{},
		),
		proxy: prometheus.NewDesc(
			"kubernikus_servicing_nodes_version_proxy",
			"Update Status of Nodes per Kluster",
			[]string{"kluster_id", "version"},
			prometheus.Labels{},
		),
		osimage: prometheus.NewDesc(
			"kubernikus_servicing_nodes_version_osimage",
			"Update Status of Nodes per Kluster",
			[]string{"kluster_id", "version"},
			prometheus.Labels{},
		),
		klusters: factories.Kubernikus.Kubernikus().V1().Klusters().Lister(),
		nodeListers: &NodeListerFactory{
			Logger:          logger,
			NodeObservatory: factories.NodesObservatory.NodeInformer(),
			CoreOSVersion:   &coreos.Version{},
			CoreOSRelease:   &coreos.Release{},
			FlatcarVersion:  &flatcar.Version{},
			FlatcarRelease:  &flatcar.Release{},
		},
		logger: logger,
	}

	prometheus.MustRegister(collector)
}

//Each and every collector must implement the Describe function.
//It essentially writes all descriptors to the prometheus desc channel.
func (c *servicingNodesCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- c.updating
	ch <- c.waiting
	ch <- c.kubelet
	ch <- c.proxy
	ch <- c.osimage
}

//Collect implements required collect function for all promehteus collectors
func (c *servicingNodesCollector) Collect(ch chan<- prometheus.Metric) {
	klusters, err := c.klusters.List(labels.Everything())
	if err != nil {
		c.logger.Log("msg", "Failed to list klusters", "err", err)
		return
	}

	for _, kluster := range klusters {
		nodes, err := c.nodeListers.Make(kluster)
		if err != nil {
			c.logger.Log("msg", "Failed to list nodes", "kluster", kluster.Name, "err", err)
			continue
		}

		updatingStarted := float64(len(nodes.Updating()))
		updatingFailed := float64(len(nodes.Failed()))
		updatingSuccessful := float64(len(nodes.Successful()))

		waitingReboot := float64(len(nodes.Reboot()))
		waitingReplace := float64(len(nodes.Replace()))
		waitingUptodate := float64(len(nodes.All())) - waitingReboot - waitingReplace

		kubeletVersions := map[string]int{}
		proxyVersions := map[string]int{}
		osVersions := map[string]int{}

		for _, node := range nodes.All() {
			kubeletVersions[node.Status.NodeInfo.KubeletVersion]++
			proxyVersions[node.Status.NodeInfo.KubeProxyVersion]++

			osVersion, err := flatcar.ExractVersion(node)
			if err != nil {
				if osVersion, err = coreos.ExractVersion(node); err != nil {
					continue
				}
			}
			osVersions[osVersion.String()]++
		}

		ch <- prometheus.MustNewConstMetric(c.updating, prometheus.GaugeValue, updatingStarted, kluster.GetName(), "started")
		ch <- prometheus.MustNewConstMetric(c.updating, prometheus.GaugeValue, updatingFailed, kluster.GetName(), "failed")
		ch <- prometheus.MustNewConstMetric(c.updating, prometheus.GaugeValue, updatingSuccessful, kluster.GetName(), "successful")
		ch <- prometheus.MustNewConstMetric(c.waiting, prometheus.GaugeValue, waitingReboot, kluster.GetName(), "reboot")
		ch <- prometheus.MustNewConstMetric(c.waiting, prometheus.GaugeValue, waitingReplace, kluster.GetName(), "replace")
		ch <- prometheus.MustNewConstMetric(c.waiting, prometheus.GaugeValue, waitingUptodate, kluster.GetName(), "uptodate")

		for version, count := range kubeletVersions {
			ch <- prometheus.MustNewConstMetric(c.kubelet, prometheus.GaugeValue, float64(count), kluster.GetName(), version)
		}

		for version, count := range proxyVersions {
			ch <- prometheus.MustNewConstMetric(c.proxy, prometheus.GaugeValue, float64(count), kluster.GetName(), version)
		}

		for version, count := range osVersions {
			ch <- prometheus.MustNewConstMetric(c.osimage, prometheus.GaugeValue, float64(count), kluster.GetName(), version)
		}
	}
}
<commit_msg>servicing controller: skip over kluster in Running state when collecting metrics<commit_after>package servicing

import (
	"github.com/go-kit/kit/log"
	"github.com/prometheus/client_golang/prometheus"
	"k8s.io/apimachinery/pkg/labels"

	"github.com/sapcc/kubernikus/pkg/api/models"
	"github.com/sapcc/kubernikus/pkg/controller/config"
	"github.com/sapcc/kubernikus/pkg/controller/servicing/coreos"
	"github.com/sapcc/kubernikus/pkg/controller/servicing/flatcar"
	kubernikus_lister "github.com/sapcc/kubernikus/pkg/generated/listers/kubernikus/v1"
)

type servicingNodesCollector struct {
	klusters    kubernikus_lister.KlusterLister
	nodeListers *NodeListerFactory
	updating    *prometheus.Desc
	waiting     *prometheus.Desc
	kubelet     *prometheus.Desc
	proxy       *prometheus.Desc
	osimage     *prometheus.Desc
	logger      log.Logger
}

// RegisterServicingNodesCollector does what the method name sais
func RegisterServicingNodesCollector(logger log.Logger, factories config.Factories) {
	collector := &servicingNodesCollector{
		updating: prometheus.NewDesc(
			"kubernikus_servicing_nodes_state_updating",
			"Amount of nodes per servicing action/state",
			[]string{"kluster_id", "state"},
			prometheus.Labels{},
		),
		waiting: prometheus.NewDesc(
			"kubernikus_servicing_nodes_state_waiting",
			"Amount of nodes that are waiting for updates per state",
			[]string{"kluster_id", "state"},
			prometheus.Labels{},
		),
		kubelet: prometheus.NewDesc(
			"kubernikus_servicing_nodes_version_kubelet",
			"Update Status of Nodes per Kluster",
			[]string{"kluster_id", "version"},
			prometheus.Labels{},
		),
		proxy: prometheus.NewDesc(
			"kubernikus_servicing_nodes_version_proxy",
			"Update Status of Nodes per Kluster",
			[]string{"kluster_id", "version"},
			prometheus.Labels{},
		),
		osimage: prometheus.NewDesc(
			"kubernikus_servicing_nodes_version_osimage",
			"Update Status of Nodes per Kluster",
			[]string{"kluster_id", "version"},
			prometheus.Labels{},
		),
		klusters: factories.Kubernikus.Kubernikus().V1().Klusters().Lister(),
		nodeListers: &NodeListerFactory{
			Logger:          logger,
			NodeObservatory: factories.NodesObservatory.NodeInformer(),
			CoreOSVersion:   &coreos.Version{},
			CoreOSRelease:   &coreos.Release{},
			FlatcarVersion:  &flatcar.Version{},
			FlatcarRelease:  &flatcar.Release{},
		},
		logger: logger,
	}

	prometheus.MustRegister(collector)
}

//Each and every collector must implement the Describe function.
//It essentially writes all descriptors to the prometheus desc channel.
func (c *servicingNodesCollector) Describe(ch chan<- *prometheus.Desc) {
	ch <- c.updating
	ch <- c.waiting
	ch <- c.kubelet
	ch <- c.proxy
	ch <- c.osimage
}

//Collect implements required collect function for all promehteus collectors
func (c *servicingNodesCollector) Collect(ch chan<- prometheus.Metric) {
	klusters, err := c.klusters.List(labels.Everything())
	if err != nil {
		c.logger.Log("msg", "Failed to list klusters", "err", err)
		return
	}

	for _, kluster := range klusters {
		if klusters.Status.Phase != models.KlusterPhaseRunning {
			continue
		}
		nodes, err := c.nodeListers.Make(kluster)
		if err != nil {
			c.logger.Log("msg", "Failed to list nodes", "kluster", kluster.Name, "err", err)
			continue
		}

		updatingStarted := float64(len(nodes.Updating()))
		updatingFailed := float64(len(nodes.Failed()))
		updatingSuccessful := float64(len(nodes.Successful()))

		waitingReboot := float64(len(nodes.Reboot()))
		waitingReplace := float64(len(nodes.Replace()))
		waitingUptodate := float64(len(nodes.All())) - waitingReboot - waitingReplace

		kubeletVersions := map[string]int{}
		proxyVersions := map[string]int{}
		osVersions := map[string]int{}

		for _, node := range nodes.All() {
			kubeletVersions[node.Status.NodeInfo.KubeletVersion]++
			proxyVersions[node.Status.NodeInfo.KubeProxyVersion]++

			osVersion, err := flatcar.ExractVersion(node)
			if err != nil {
				if osVersion, err = coreos.ExractVersion(node); err != nil {
					continue
				}
			}
			osVersions[osVersion.String()]++
		}

		ch <- prometheus.MustNewConstMetric(c.updating, prometheus.GaugeValue, updatingStarted, kluster.GetName(), "started")
		ch <- prometheus.MustNewConstMetric(c.updating, prometheus.GaugeValue, updatingFailed, kluster.GetName(), "failed")
		ch <- prometheus.MustNewConstMetric(c.updating, prometheus.GaugeValue, updatingSuccessful, kluster.GetName(), "successful")
		ch <- prometheus.MustNewConstMetric(c.waiting, prometheus.GaugeValue, waitingReboot, kluster.GetName(), "reboot")
		ch <- prometheus.MustNewConstMetric(c.waiting, prometheus.GaugeValue, waitingReplace, kluster.GetName(), "replace")
		ch <- prometheus.MustNewConstMetric(c.waiting, prometheus.GaugeValue, waitingUptodate, kluster.GetName(), "uptodate")

		for version, count := range kubeletVersions {
			ch <- prometheus.MustNewConstMetric(c.kubelet, prometheus.GaugeValue, float64(count), kluster.GetName(), version)
		}

		for version, count := range proxyVersions {
			ch <- prometheus.MustNewConstMetric(c.proxy, prometheus.GaugeValue, float64(count), kluster.GetName(), version)
		}

		for version, count := range osVersions {
			ch <- prometheus.MustNewConstMetric(c.osimage, prometheus.GaugeValue, float64(count), kluster.GetName(), version)
		}
	}
}
<|endoftext|>


<commit_before>package repo_handle

import (
	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/config"
	"github.com/go-git/go-git/v5/plumbing"
	"github.com/go-git/go-git/v5/plumbing/object"
)

// Handle is a solution to get away from the worktree when working with the git repository,
// caching the necessary data from the worktree during initialization,
// and then working exclusively with git objects.
type Handle interface {
	Repository() Repository
	Submodule(submodulePath string) (SubmoduleHandle, error)
	Submodules() []SubmoduleHandle
}

type SubmoduleHandle interface {
	Handle
	Config() *config.Submodule
	Status() *git.SubmoduleStatus
}

type Repository interface {
	CommitObject(h plumbing.Hash) (*object.Commit, error)
	BlobObject(h plumbing.Hash) (*object.Blob, error)
}

func NewHandle(repository *git.Repository) (Handle, error) {
	h := newHandle(repository)

	submoduleHandleList, err := getSubmoduleHandleList(repository)
	if err != nil {
		return nil, err
	}

	h.submoduleHandleList = submoduleHandleList

	return h, nil
}

func getSubmoduleHandleList(parentRepository *git.Repository) ([]SubmoduleHandle, error) {
	var list []SubmoduleHandle

	w, err := parentRepository.Worktree()
	if err != nil {
		return nil, err
	}

	ss, err := w.Submodules()
	if err != nil {
		return nil, err
	}

	for _, s := range ss {
		submoduleRepository, err := s.Repository()
		if err != nil {
			return nil, err
		}

		submoduleStatus, err := s.Status()
		if err != nil {
			return nil, err
		}

		handle, err := NewHandle(submoduleRepository)
		if err != nil {
			return nil, err
		}

		submoduleHandle := newRepositorySubmoduleHandle(handle, s.Config(), submoduleStatus)
		list = append(list, submoduleHandle)
	}

	return list, nil
}
<commit_msg>[logging] Repo handle: detailed error<commit_after>package repo_handle

import (
	"fmt"

	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/config"
	"github.com/go-git/go-git/v5/plumbing"
	"github.com/go-git/go-git/v5/plumbing/object"
)

// Handle is a solution to get away from the worktree when working with the git repository,
// caching the necessary data from the worktree during initialization,
// and then working exclusively with git objects.
type Handle interface {
	Repository() Repository
	Submodule(submodulePath string) (SubmoduleHandle, error)
	Submodules() []SubmoduleHandle
}

type SubmoduleHandle interface {
	Handle
	Config() *config.Submodule
	Status() *git.SubmoduleStatus
}

type Repository interface {
	CommitObject(h plumbing.Hash) (*object.Commit, error)
	BlobObject(h plumbing.Hash) (*object.Blob, error)
}

func NewHandle(repository *git.Repository) (Handle, error) {
	h := newHandle(repository)

	submoduleHandleList, err := getSubmoduleHandleList(repository)
	if err != nil {
		return nil, err
	}

	h.submoduleHandleList = submoduleHandleList

	return h, nil
}

func getSubmoduleHandleList(parentRepository *git.Repository) ([]SubmoduleHandle, error) {
	var list []SubmoduleHandle

	w, err := parentRepository.Worktree()
	if err != nil {
		return nil, err
	}

	ss, err := w.Submodules()
	if err != nil {
		return nil, err
	}

	for _, s := range ss {
		submoduleRepository, err := s.Repository()
		if err != nil {
			return nil, fmt.Errorf("unable to get submodule %q repository: %s", s.Config().Path, err)
		}

		submoduleStatus, err := s.Status()
		if err != nil {
			return nil, fmt.Errorf("unable to get submodule %q status: %s", s.Config().Path, err)
		}

		handle, err := NewHandle(submoduleRepository)
		if err != nil {
			return nil, err
		}

		submoduleHandle := newRepositorySubmoduleHandle(handle, s.Config(), submoduleStatus)
		list = append(list, submoduleHandle)
	}

	return list, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package activator

import (
	"fmt"
)

const (
	// Name is the name of the component.
	Name = "activator"
	// K8sServiceName is the name of the activator Kubernetes service.
	K8sServiceName = "activator-service"
	// RevisionHeaderName is the header key for revision name.
	RevisionHeaderName string = "knative-serving-revision"
	// RevisionHeaderNamespace is the header key for revision's namespace.
	RevisionHeaderNamespace string = "knative-serving-namespace"
)

// RevisionID is the combination of namespace and revision name
type RevisionID struct {
	Namespace string
	Name      string
}

// String returns the namespaced name of the RevisionID.
func (rev RevisionID) String() string {
	return fmt.Sprintf("%s/%s", rev.Namespace, rev.Name)
}
<commit_msg>The string in the constant is removed (#4219)<commit_after>/*
Copyright 2018 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package activator

import (
	"fmt"
)

const (
	// Name is the name of the component.
	Name = "activator"
	// K8sServiceName is the name of the activator Kubernetes service.
	K8sServiceName = "activator-service"
	// RevisionHeaderName is the header key for revision name.
	RevisionHeaderName = "knative-serving-revision"
	// RevisionHeaderNamespace is the header key for revision's namespace.
	RevisionHeaderNamespace = "knative-serving-namespace"
)

// RevisionID is the combination of namespace and revision name
type RevisionID struct {
	Namespace string
	Name      string
}

// String returns the namespaced name of the RevisionID.
func (rev RevisionID) String() string {
	return fmt.Sprintf("%s/%s", rev.Namespace, rev.Name)
}
<|endoftext|>


<commit_before>package tunnel

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/base64"
	"fmt"
	"net"
	"net/http"
	"sync"
	"time"

	"github.com/gorilla/websocket"
	"github.com/rancher/k3s/pkg/daemons/config"
	"github.com/rancher/remotedialer"
	"github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/transport"
)

var (
	ports = map[string]bool{
		"10250": true,
		"10010": true,
	}
)

func getAddresses(endpoint *v1.Endpoints) []string {
	serverAddresses := []string{}
	if endpoint == nil {
		return serverAddresses
	}
	for _, subset := range endpoint.Subsets {
		var port string
		if len(subset.Ports) > 0 {
			port = fmt.Sprint(subset.Ports[0].Port)
		}
		for _, address := range subset.Addresses {
			serverAddress := address.IP
			if port != "" {
				serverAddress += ":" + port
			}
			serverAddresses = append(serverAddresses, serverAddress)
		}
	}
	return serverAddresses
}

func Setup(config *config.Node) error {
	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode)
	if err != nil {
		return err
	}

	transportConfig, err := restConfig.TransportConfig()
	if err != nil {
		return err
	}

	client, err := kubernetes.NewForConfig(restConfig)
	if err != nil {
		return err
	}

	addresses := []string{config.ServerAddress}

	endpoint, _ := client.CoreV1().Endpoints("default").Get("kubernetes", metav1.GetOptions{})
	if endpoint != nil {
		addresses = getAddresses(endpoint)
	}

	disconnect := map[string]context.CancelFunc{}

	wg := &sync.WaitGroup{}
	for _, address := range addresses {
		if _, ok := disconnect[address]; !ok {
			disconnect[address] = connect(wg, address, config, transportConfig)
		}
	}
	wg.Wait()

	go func() {
	connect:
		for {
			watch, err := client.CoreV1().Endpoints("default").Watch(metav1.ListOptions{
				FieldSelector: fields.Set{"metadata.name": "kubernetes"}.String(),
			})
			if err != nil {
				logrus.Errorf("Unable to watch for endpoints: %v", err)
				time.Sleep(5 * time.Second)
				continue connect
			}
		watching:
			for {
				select {
				case ev, ok := <-watch.ResultChan():
					if !ok {
						logrus.Error("endpoint watch channel closed")
						continue connect
					}
					endpoint, ok := ev.Object.(*v1.Endpoints)
					if !ok {
						logrus.Error("could not case event object to endpoint")
						continue watching
					}

					validEndpoint := map[string]bool{}
					var addresses = getAddresses(endpoint)
					for _, address := range addresses {
						validEndpoint[address] = true
						if _, ok := disconnect[address]; !ok {
							disconnect[address] = connect(wg, address, config, transportConfig)
						}
					}

					for address, cancel := range disconnect {
						if !validEndpoint[address] {
							cancel()
							delete(disconnect, address)
						}
					}
				}
			}
		}
	}()

	return nil
}

func connect(waitGroup *sync.WaitGroup, address string, config *config.Node, transportConfig *transport.Config) context.CancelFunc {
	wsURL := fmt.Sprintf("wss://%s/v1-k3s/connect", address)
	headers := map[string][]string{
		"X-K3s-NodeName": {config.AgentConfig.NodeName},
	}
	ws := &websocket.Dialer{}

	if len(config.CACerts) > 0 {
		pool := x509.NewCertPool()
		pool.AppendCertsFromPEM(config.CACerts)
		ws.TLSClientConfig = &tls.Config{
			RootCAs: pool,
		}
	}

	if transportConfig.Username != "" {
		auth := transportConfig.Username + ":" + transportConfig.Password
		auth = base64.StdEncoding.EncodeToString([]byte(auth))
		headers["Authorization"] = []string{"Basic " + auth}
	}

	once := sync.Once{}
	if waitGroup != nil {
		waitGroup.Add(1)
	}

	ctx, cancel := context.WithCancel(context.Background())

	go func() {
		for {
			remotedialer.ClientConnect(ctx, wsURL, http.Header(headers), ws, func(proto, address string) bool {
				host, port, err := net.SplitHostPort(address)
				return err == nil && proto == "tcp" && ports[port] && host == "127.0.0.1"
			}, func(_ context.Context) error {
				if waitGroup != nil {
					once.Do(waitGroup.Done)
				}
				return nil
			})

			if ctx.Err() != nil {
				logrus.Infof("Stopping tunnel to %s", wsURL)
				return
			}
		}
	}()

	return cancel
}
<commit_msg>Cleanup remotedialer tunnel logs<commit_after>package tunnel

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/base64"
	"fmt"
	"net"
	"net/http"
	"sync"
	"time"

	"github.com/gorilla/websocket"
	"github.com/rancher/k3s/pkg/daemons/config"
	"github.com/rancher/remotedialer"
	"github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/fields"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	"k8s.io/client-go/transport"
)

var (
	ports = map[string]bool{
		"10250": true,
		"10010": true,
	}
)

func getAddresses(endpoint *v1.Endpoints) []string {
	serverAddresses := []string{}
	if endpoint == nil {
		return serverAddresses
	}
	for _, subset := range endpoint.Subsets {
		var port string
		if len(subset.Ports) > 0 {
			port = fmt.Sprint(subset.Ports[0].Port)
		}
		for _, address := range subset.Addresses {
			serverAddress := address.IP
			if port != "" {
				serverAddress += ":" + port
			}
			serverAddresses = append(serverAddresses, serverAddress)
		}
	}
	return serverAddresses
}

func Setup(config *config.Node) error {
	restConfig, err := clientcmd.BuildConfigFromFlags("", config.AgentConfig.KubeConfigNode)
	if err != nil {
		return err
	}

	transportConfig, err := restConfig.TransportConfig()
	if err != nil {
		return err
	}

	client, err := kubernetes.NewForConfig(restConfig)
	if err != nil {
		return err
	}

	addresses := []string{config.ServerAddress}

	endpoint, _ := client.CoreV1().Endpoints("default").Get("kubernetes", metav1.GetOptions{})
	if endpoint != nil {
		addresses = getAddresses(endpoint)
	}

	disconnect := map[string]context.CancelFunc{}

	wg := &sync.WaitGroup{}
	for _, address := range addresses {
		if _, ok := disconnect[address]; !ok {
			disconnect[address] = connect(wg, address, config, transportConfig)
		}
	}
	wg.Wait()

	go func() {
	connect:
		for {
			watch, err := client.CoreV1().Endpoints("default").Watch(metav1.ListOptions{
				FieldSelector: fields.Set{"metadata.name": "kubernetes"}.String(),
			})
			if err != nil {
				logrus.Errorf("Unable to watch for tunnel endpoints: %v", err)
				time.Sleep(5 * time.Second)
				continue connect
			}
		watching:
			for {
				select {
				case ev, ok := <-watch.ResultChan():
					if !ok {
						logrus.Error("Tunnel endpoint watch channel closed")
						continue connect
					}
					endpoint, ok := ev.Object.(*v1.Endpoints)
					if !ok {
						logrus.Error("Tunnel could not case event object to endpoint")
						continue watching
					}

					var addresses = getAddresses(endpoint)
					logrus.Infof("Tunnel endpoint watch event: %v", addresses)

					validEndpoint := map[string]bool{}

					for _, address := range addresses {
						validEndpoint[address] = true
						if _, ok := disconnect[address]; !ok {
							disconnect[address] = connect(nil, address, config, transportConfig)
						}
					}

					for address, cancel := range disconnect {
						if !validEndpoint[address] {
							cancel()
							delete(disconnect, address)
						}
					}
				}
			}
		}
	}()

	return nil
}

func connect(waitGroup *sync.WaitGroup, address string, config *config.Node, transportConfig *transport.Config) context.CancelFunc {
	wsURL := fmt.Sprintf("wss://%s/v1-k3s/connect", address)
	headers := map[string][]string{
		"X-K3s-NodeName": {config.AgentConfig.NodeName},
	}
	ws := &websocket.Dialer{}

	if len(config.CACerts) > 0 {
		pool := x509.NewCertPool()
		pool.AppendCertsFromPEM(config.CACerts)
		ws.TLSClientConfig = &tls.Config{
			RootCAs: pool,
		}
	}

	if transportConfig.Username != "" {
		auth := transportConfig.Username + ":" + transportConfig.Password
		auth = base64.StdEncoding.EncodeToString([]byte(auth))
		headers["Authorization"] = []string{"Basic " + auth}
	}

	once := sync.Once{}
	if waitGroup != nil {
		waitGroup.Add(1)
	}

	ctx, cancel := context.WithCancel(context.Background())

	go func() {
		for {
			remotedialer.ClientConnect(ctx, wsURL, http.Header(headers), ws, func(proto, address string) bool {
				host, port, err := net.SplitHostPort(address)
				return err == nil && proto == "tcp" && ports[port] && host == "127.0.0.1"
			}, func(_ context.Context) error {
				if waitGroup != nil {
					once.Do(waitGroup.Done)
				}
				return nil
			})

			if ctx.Err() != nil {
				logrus.Infof("Stopped tunnel to %s", wsURL)
				return
			}
		}
	}()

	return cancel
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package blobserver

import (
	"errors"
	"fmt"
	"net/http"
	"sync"

	"camlistore.org/pkg/jsonconfig"
)

var ErrHandlerTypeNotFound = errors.New("requested handler type not loaded")

type FindHandlerByTyper interface {
	// FindHandlerByType finds a handler by its handlerType and
	// returns its prefix and handler if it's loaded.  If it's not
	// loaded, the error will be ErrHandlerTypeNotFound.
	//
	// This is used by handlers to find siblings (such as the "ui" type handler)
	// which might have more knowledge about the configuration for discovery, etc.
	//
	// Note that if this is called during handler construction
	// time, only the prefix may be returned with a nil handler
	// and nil err.  Unlike GetHandler and GetStorage, this does
	// not cause the prefix to load immediately. At runtime (after
	// construction of all handlers), then prefix and handler will
	// both be non-nil when err is nil.
	FindHandlerByType(handlerType string) (prefix string, handler interface{}, err error)

	// AllHandlers returns a map from prefix to handler type, and
	// a map from prefix to handler.
	AllHandlers() (map[string]string, map[string]interface{})
}

type Loader interface {
	FindHandlerByTyper

	// MyPrefix returns the prefix of the handler currently being constructed,
	// with both leading and trailing slashes (e.g. "/ui/").
	MyPrefix() string

	// BaseURL returns the server's base URL, without trailing slash, and not including
	// the prefix (as returned by MyPrefix).
	BaseURL() string

	// GetHandlerType returns the handler's configured type, but does
	// not force it to start being loaded yet.
	GetHandlerType(prefix string) string // returns "" if unknown

	// GetHandler returns either a Storage or an http.Handler.
	// It forces the handler to be loaded and returns an error if
	// a cycle is created.
	GetHandler(prefix string) (interface{}, error)

	// GetStorage is like GetHandler but requires that the Handler be
	// a storage Handler.
	GetStorage(prefix string) (Storage, error)
}

// HandlerIniter is an optional interface which can be implemented
// by Storage or http.Handlers (from StorageConstructor or HandlerConstructor)
// to be called once all the handlers have been created.
type HandlerIniter interface {
	InitHandler(FindHandlerByTyper) error
}

// A StorageConstructor returns a Storage implementation from a Loader
// environment and a configuration.
type StorageConstructor func(Loader, jsonconfig.Obj) (Storage, error)

// A HandlerConstructor returns an http.Handler from a Loader
// environment and a configuration.
type HandlerConstructor func(Loader, jsonconfig.Obj) (http.Handler, error)

var mapLock sync.Mutex
var storageConstructors = make(map[string]StorageConstructor)
var handlerConstructors = make(map[string]HandlerConstructor)

func RegisterStorageConstructor(typ string, ctor StorageConstructor) {
	mapLock.Lock()
	defer mapLock.Unlock()
	if _, ok := storageConstructors[typ]; ok {
		panic("blobserver: StorageConstructor already registered for type: " + typ)
	}
	storageConstructors[typ] = ctor
}

func CreateStorage(typ string, loader Loader, config jsonconfig.Obj) (Storage, error) {
	mapLock.Lock()
	ctor, ok := storageConstructors[typ]
	mapLock.Unlock()
	if !ok {
		return nil, fmt.Errorf("Storage type %q not known or loaded", typ)
	}
	return ctor(loader, config)
}

// RegisterHandlerConstructor registers an http Handler constructor function
// for a given handler type.
//
// It is an error to register the same handler type twice.
func RegisterHandlerConstructor(typ string, ctor HandlerConstructor) {
	mapLock.Lock()
	defer mapLock.Unlock()
	if _, ok := handlerConstructors[typ]; ok {
		panic("blobserver: HandlerConstrutor already registered for type: " + typ)
	}
	handlerConstructors[typ] = ctor
}

// CreateHandler instantiates an http Handler of type 'typ' from the
// provided JSON configuration, and finding peer handlers and
// configuration from the environment in 'loader'.
//
// The handler 'typ' must have bee previously registered with
// RegisterHandlerConstructor.
func CreateHandler(typ string, loader Loader, config jsonconfig.Obj) (http.Handler, error) {
	mapLock.Lock()
	ctor, ok := handlerConstructors[typ]
	mapLock.Unlock()
	if !ok {
		return nil, fmt.Errorf("blobserver: Handler type %q not known or loaded", typ)
	}
	return ctor(loader, config)
}
<commit_msg>typo in doc for CreateHandler<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package blobserver

import (
	"errors"
	"fmt"
	"net/http"
	"sync"

	"camlistore.org/pkg/jsonconfig"
)

var ErrHandlerTypeNotFound = errors.New("requested handler type not loaded")

type FindHandlerByTyper interface {
	// FindHandlerByType finds a handler by its handlerType and
	// returns its prefix and handler if it's loaded.  If it's not
	// loaded, the error will be ErrHandlerTypeNotFound.
	//
	// This is used by handlers to find siblings (such as the "ui" type handler)
	// which might have more knowledge about the configuration for discovery, etc.
	//
	// Note that if this is called during handler construction
	// time, only the prefix may be returned with a nil handler
	// and nil err.  Unlike GetHandler and GetStorage, this does
	// not cause the prefix to load immediately. At runtime (after
	// construction of all handlers), then prefix and handler will
	// both be non-nil when err is nil.
	FindHandlerByType(handlerType string) (prefix string, handler interface{}, err error)

	// AllHandlers returns a map from prefix to handler type, and
	// a map from prefix to handler.
	AllHandlers() (map[string]string, map[string]interface{})
}

type Loader interface {
	FindHandlerByTyper

	// MyPrefix returns the prefix of the handler currently being constructed,
	// with both leading and trailing slashes (e.g. "/ui/").
	MyPrefix() string

	// BaseURL returns the server's base URL, without trailing slash, and not including
	// the prefix (as returned by MyPrefix).
	BaseURL() string

	// GetHandlerType returns the handler's configured type, but does
	// not force it to start being loaded yet.
	GetHandlerType(prefix string) string // returns "" if unknown

	// GetHandler returns either a Storage or an http.Handler.
	// It forces the handler to be loaded and returns an error if
	// a cycle is created.
	GetHandler(prefix string) (interface{}, error)

	// GetStorage is like GetHandler but requires that the Handler be
	// a storage Handler.
	GetStorage(prefix string) (Storage, error)
}

// HandlerIniter is an optional interface which can be implemented
// by Storage or http.Handlers (from StorageConstructor or HandlerConstructor)
// to be called once all the handlers have been created.
type HandlerIniter interface {
	InitHandler(FindHandlerByTyper) error
}

// A StorageConstructor returns a Storage implementation from a Loader
// environment and a configuration.
type StorageConstructor func(Loader, jsonconfig.Obj) (Storage, error)

// A HandlerConstructor returns an http.Handler from a Loader
// environment and a configuration.
type HandlerConstructor func(Loader, jsonconfig.Obj) (http.Handler, error)

var mapLock sync.Mutex
var storageConstructors = make(map[string]StorageConstructor)
var handlerConstructors = make(map[string]HandlerConstructor)

func RegisterStorageConstructor(typ string, ctor StorageConstructor) {
	mapLock.Lock()
	defer mapLock.Unlock()
	if _, ok := storageConstructors[typ]; ok {
		panic("blobserver: StorageConstructor already registered for type: " + typ)
	}
	storageConstructors[typ] = ctor
}

func CreateStorage(typ string, loader Loader, config jsonconfig.Obj) (Storage, error) {
	mapLock.Lock()
	ctor, ok := storageConstructors[typ]
	mapLock.Unlock()
	if !ok {
		return nil, fmt.Errorf("Storage type %q not known or loaded", typ)
	}
	return ctor(loader, config)
}

// RegisterHandlerConstructor registers an http Handler constructor function
// for a given handler type.
//
// It is an error to register the same handler type twice.
func RegisterHandlerConstructor(typ string, ctor HandlerConstructor) {
	mapLock.Lock()
	defer mapLock.Unlock()
	if _, ok := handlerConstructors[typ]; ok {
		panic("blobserver: HandlerConstrutor already registered for type: " + typ)
	}
	handlerConstructors[typ] = ctor
}

// CreateHandler instantiates an http Handler of type 'typ' from the
// provided JSON configuration, and finding peer handlers and
// configuration from the environment in 'loader'.
//
// The handler 'typ' must have been previously registered with
// RegisterHandlerConstructor.
func CreateHandler(typ string, loader Loader, config jsonconfig.Obj) (http.Handler, error) {
	mapLock.Lock()
	ctor, ok := handlerConstructors[typ]
	mapLock.Unlock()
	if !ok {
		return nil, fmt.Errorf("blobserver: Handler type %q not known or loaded", typ)
	}
	return ctor(loader, config)
}
<|endoftext|>


<commit_before>package ttime

import (
    "os"
    "testing"
    "time"

    ccar "github.com/jtyr/gbt/pkg/core/car"
)

func TestInit(t *testing.T) {
    fakedDate := time.Date(2018, time.January, 6, 23, 57, 41, 0, time.UTC)
    tnow = func() time.Time {
        return fakedDate
    }

    ccar.Shell = "plain"

    tests := []struct {
        format string
        expectedOutput string
    }{
        {
            format: "{{ DateTime }}",
            expectedOutput: "\x1b[48;5;12m\x1b[38;5;7m\x1b[48;5;12m\x1b[38;5;7m\x1b[48;5;12m\x1b[38;5;7mSat 06 Jan\x1b[48;5;12m\x1b[38;5;7m \x1b[48;5;12m\x1b[38;5;11m23:57:41\x1b[48;5;12m\x1b[38;5;7m\x1b[48;5;12m\x1b[38;5;7m",
        },
        {
            format: "{{ Date }}",
            expectedOutput: "\x1b[48;5;12m\x1b[38;5;7m\x1b[48;5;12m\x1b[38;5;7mSat 06 Jan\x1b[48;5;12m\x1b[38;5;7m",
        },
        {
            format: "{{ Time }}",
            expectedOutput: "\x1b[48;5;12m\x1b[38;5;7m\x1b[48;5;12m\x1b[38;5;11m23:57:41\x1b[48;5;12m\x1b[38;5;7m",
        },
    }

    for i, test := range tests {
        os.Setenv("GBT_CAR_TIME_FORMAT", test.format)

        car := Car{}
        car.Init()
        output := car.Format()

        if output != test.expectedOutput {
            t.Errorf("Test [%d]: Expected '%x', found '%x'.", i, test.expectedOutput, output)
        }
    }
}
<commit_msg>Time car: test the functionality not the formatting<commit_after>package ttime

import (
    "testing"
    "time"

    ccar "github.com/jtyr/gbt/pkg/core/car"
)

func TestInit(t *testing.T) {
    fakedDate := time.Date(2018, time.January, 6, 23, 57, 41, 0, time.UTC)
    tnow = func() time.Time {
        return fakedDate
    }

    ccar.Shell = "plain"

    tests := []struct {
        field string
        expectedOutput string
    }{
        {
            field: "Date",
            expectedOutput: "Sat 06 Jan",
        },
        {
            field: "Time",
            expectedOutput: "23:57:41",
        },
    }

    for i, test := range tests {
        car := Car{}
        car.Init()

        val := car.Model[test.field].Text

        if val != test.expectedOutput {
            t.Errorf("Test [%d]: Expected '%s', found '%s'.", i, test.expectedOutput, val)
        }
    }
}
<|endoftext|>


<commit_before>package sc

import (
	"context"
	"errors"
	"fmt"
	"github.com/apache/servicecomb-service-center/pkg/util"
	"net/http"
	"net/url"

	"github.com/gorilla/websocket"
)

func (c *LBClient) WebsocketDial(ctx context.Context, api string, headers http.Header) (conn *websocket.Conn, err error) {
	dialer := &websocket.Dialer{TLSClientConfig: c.TLS}
	var errs []string
	for i := 0; i < c.Retries; i++ {
		var addr *url.URL
		addr, err = url.Parse(c.Next())
		if err != nil {
			errs = append(errs, fmt.Sprintf("[%s]: %s", addr, err.Error()))
			continue
		}
		if addr.Scheme == "https" {
			addr.Scheme = "wss"
		} else {
			addr.Scheme = "ws"
		}
		conn, _, err = dialer.Dial(addr.String() + api, headers)
		if err == nil {
			break
		}
		errs = append(errs, fmt.Sprintf("[%s]: %s", addr, err.Error()))
	}
	if err != nil {
		err = errors.New(util.StringJoin(errs, ", "))
	}
	return
}
<commit_msg>Update websocket.go<commit_after>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sc

import (
	"context"
	"errors"
	"fmt"
	"github.com/apache/servicecomb-service-center/pkg/util"
	"net/http"
	"net/url"

	"github.com/gorilla/websocket"
)

func (c *LBClient) WebsocketDial(ctx context.Context, api string, headers http.Header) (conn *websocket.Conn, err error) {
	dialer := &websocket.Dialer{TLSClientConfig: c.TLS}
	var errs []string
	for i := 0; i < c.Retries; i++ {
		var addr *url.URL
		addr, err = url.Parse(c.Next())
		if err != nil {
			errs = append(errs, fmt.Sprintf("[%s]: %s", addr, err.Error()))
			continue
		}
		if addr.Scheme == "https" {
			addr.Scheme = "wss"
		} else {
			addr.Scheme = "ws"
		}
		conn, _, err = dialer.Dial(addr.String() + api, headers)
		if err == nil {
			break
		}
		errs = append(errs, fmt.Sprintf("[%s]: %s", addr, err.Error()))
	}
	if err != nil {
		err = errors.New(util.StringJoin(errs, ", "))
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cluster

import (
	"fmt"
	"math"
	"time"

	"github.com/coreos/etcd-operator/pkg/util/etcdutil"
	"github.com/coreos/etcd-operator/pkg/util/k8sutil"
	"github.com/coreos/etcd-operator/pkg/util/retryutil"

	"github.com/pborman/uuid"
)

func (c *Cluster) addOneSelfHostedMember() error {
	c.status.AppendScalingUpCondition(c.members.Size(), c.cluster.Spec.Size)

	newMember := c.newMember(c.memberCounter)
	c.memberCounter++
	peerURL := newMember.PeerURL()
	initialCluster := append(c.members.PeerURLPairs(), newMember.Name+"="+peerURL)

	ns := c.cluster.Metadata.Namespace
	pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, c.members.ClientURLs(), c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner())

	_, err := c.config.KubeCli.CoreV1().Pods(ns).Create(pod)
	if err != nil {
		return err
	}
	// wait for the new pod to start and add itself into the etcd cluster.
	oldN := c.members.Size()
	err = retryutil.Retry(5*time.Second, math.MaxInt64, func() (bool, error) {
		err = c.updateMembers(c.members)
		if err != nil {
			c.logger.Errorf("add self hosted member: fail to update members: %v", err)
			if err == errUnexpectedUnreadyMember {
				return false, nil
			}
			return false, err
		}
		if c.members.Size() > oldN {
			return true, nil
		}
		return false, nil
	})
	if err != nil {
		return err
	}

	c.logger.Infof("added a self-hosted member (%s)", newMember.Name)
	return nil
}

func (c *Cluster) newSelfHostedSeedMember() error {
	newMember := c.newMember(c.memberCounter)
	c.memberCounter++
	initialCluster := []string{newMember.Name + "=" + newMember.PeerURL()}

	pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, nil, c.cluster.Metadata.Name, "new", uuid.New(), c.cluster.Spec, c.cluster.AsOwner())
	_, err := k8sutil.CreateAndWaitPod(c.config.KubeCli, c.cluster.Metadata.Namespace, pod, 30*time.Second)
	if err != nil {
		return err
	}

	c.logger.Infof("self-hosted cluster created with seed member (%s)", newMember.Name)
	return nil
}

func (c *Cluster) migrateBootMember() error {
	endpoint := c.cluster.Spec.SelfHosted.BootMemberClientEndpoint

	c.logger.Infof("migrating boot member (%s)", endpoint)

	resp, err := etcdutil.ListMembers([]string{endpoint}, c.tlsConfig)
	if err != nil {
		return fmt.Errorf("failed to list members from boot member (%v)", err)
	}
	if len(resp.Members) != 1 {
		return fmt.Errorf("boot cluster contains more than one member")
	}
	bootMember := resp.Members[0]

	initialCluster := make([]string, 0)
	for _, purl := range bootMember.PeerURLs {
		initialCluster = append(initialCluster, fmt.Sprintf("%s=%s", bootMember.Name, purl))
	}

	// create the member inside Kubernetes for migration
	newMember := c.newMember(c.memberCounter)
	c.memberCounter++

	peerURL := newMember.PeerURL()
	initialCluster = append(initialCluster, newMember.Name+"="+peerURL)

	pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, []string{endpoint}, c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner())
	ns := c.cluster.Metadata.Namespace
	_, err = k8sutil.CreateAndWaitPod(c.config.KubeCli, ns, pod, 30*time.Second)
	if err != nil {
		return err
	}

	go func() {
		// TODO: a shorter timeout?
		// Waiting here for cluster to get stable:
		// - etcd data are replicated;
		// - cluster TPR state has switched to "Running"
		delay := 60 * time.Second
		c.logger.Infof("wait %v before removing the boot member", delay)
		time.Sleep(delay)

		err = etcdutil.RemoveMember([]string{newMember.ClientAddr()}, c.tlsConfig, bootMember.ID)
		if err != nil {
			c.logger.Errorf("boot member migration: failed to remove the boot member (%v)", err)
		}
	}()

	c.logger.Infof("self-hosted cluster created with boot member (%s)", endpoint)

	return nil
}
<commit_msg>selfHosted: unblock adding member on scheduling issue<commit_after>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cluster

import (
	"fmt"
	"math"
	"time"

	"github.com/coreos/etcd-operator/pkg/util/etcdutil"
	"github.com/coreos/etcd-operator/pkg/util/k8sutil"
	"github.com/coreos/etcd-operator/pkg/util/retryutil"

	"github.com/pborman/uuid"
)

func (c *Cluster) addOneSelfHostedMember() error {
	c.status.AppendScalingUpCondition(c.members.Size(), c.cluster.Spec.Size)

	newMember := c.newMember(c.memberCounter)
	c.memberCounter++
	peerURL := newMember.PeerURL()
	initialCluster := append(c.members.PeerURLPairs(), newMember.Name+"="+peerURL)

	ns := c.cluster.Metadata.Namespace
	pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, c.members.ClientURLs(), c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner())

	_, err := c.config.KubeCli.CoreV1().Pods(ns).Create(pod)
	if err != nil {
		return err
	}
	// wait for the new pod to start and add itself into the etcd cluster.
	oldN := c.members.Size()
	err = retryutil.Retry(10*time.Second, 6, func() (bool, error) {
		err = c.updateMembers(c.members)
		if err != nil {
			c.logger.Warningf("unable to update members: %v", err)
			return false, nil
		}
		if c.members.Size() > oldN {
			return true, nil
		}
		c.logger.Infof("still waiting for the new self hosted member (%s) to start...", newMember.Name)
		return false, nil
	})
	if err != nil {
		c.logger.Warningf("failed to add member (%s) due to scheduling failure. Removing its pod (%s)", newMember.Name, pod.Name)
		// our reconcile loop assumes that no new member is added without control.
		retryutil.Retry(10*time.Second, math.MaxInt64, func() (bool, error) {
			err := c.removePod(newMember.Name)
			if err != nil {
				c.logger.Errorf("failed to delete pod (%s), retry later: %v", newMember.Name, err)
				return false, nil
			}
			return true, nil
		})
		c.logger.Infof("pod (%s) has been removed", pod.Name)
		return nil
	}

	c.logger.Infof("added a self-hosted member (%s)", newMember.Name)
	return nil
}

func (c *Cluster) newSelfHostedSeedMember() error {
	newMember := c.newMember(c.memberCounter)
	c.memberCounter++
	initialCluster := []string{newMember.Name + "=" + newMember.PeerURL()}

	pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, nil, c.cluster.Metadata.Name, "new", uuid.New(), c.cluster.Spec, c.cluster.AsOwner())
	_, err := k8sutil.CreateAndWaitPod(c.config.KubeCli, c.cluster.Metadata.Namespace, pod, 30*time.Second)
	if err != nil {
		return err
	}

	c.logger.Infof("self-hosted cluster created with seed member (%s)", newMember.Name)
	return nil
}

func (c *Cluster) migrateBootMember() error {
	endpoint := c.cluster.Spec.SelfHosted.BootMemberClientEndpoint

	c.logger.Infof("migrating boot member (%s)", endpoint)

	resp, err := etcdutil.ListMembers([]string{endpoint}, c.tlsConfig)
	if err != nil {
		return fmt.Errorf("failed to list members from boot member (%v)", err)
	}
	if len(resp.Members) != 1 {
		return fmt.Errorf("boot cluster contains more than one member")
	}
	bootMember := resp.Members[0]

	initialCluster := make([]string, 0)
	for _, purl := range bootMember.PeerURLs {
		initialCluster = append(initialCluster, fmt.Sprintf("%s=%s", bootMember.Name, purl))
	}

	// create the member inside Kubernetes for migration
	newMember := c.newMember(c.memberCounter)
	c.memberCounter++

	peerURL := newMember.PeerURL()
	initialCluster = append(initialCluster, newMember.Name+"="+peerURL)

	pod := k8sutil.NewSelfHostedEtcdPod(newMember, initialCluster, []string{endpoint}, c.cluster.Metadata.Name, "existing", "", c.cluster.Spec, c.cluster.AsOwner())
	ns := c.cluster.Metadata.Namespace
	_, err = k8sutil.CreateAndWaitPod(c.config.KubeCli, ns, pod, 30*time.Second)
	if err != nil {
		return err
	}

	go func() {
		// TODO: a shorter timeout?
		// Waiting here for cluster to get stable:
		// - etcd data are replicated;
		// - cluster TPR state has switched to "Running"
		delay := 60 * time.Second
		c.logger.Infof("wait %v before removing the boot member", delay)
		time.Sleep(delay)

		err = etcdutil.RemoveMember([]string{newMember.ClientAddr()}, c.tlsConfig, bootMember.ID)
		if err != nil {
			c.logger.Errorf("boot member migration: failed to remove the boot member (%v)", err)
		}
	}()

	c.logger.Infof("self-hosted cluster created with boot member (%s)", endpoint)

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
)

// Build release binaries according to the supplied settings, returning the
// path to a directory containing exactly root-relative file system structure
// we desire.
func buildBinaries(
	version string,
	commit string,
	osys string,
	arch string) (dir string, err error) {
	log.Printf("Building %s at %s for %s (%s).", version, commit, osys, arch)
	// Create a directory to hold our outputs. Kill it if we later return in
	// error.
	dir, err = ioutil.TempDir("", "build_release_binaries")
	if err != nil {
		err = fmt.Errorf("TempDir: %v", err)
		return
	}

	defer func() {
		if err != nil {
			os.RemoveAll(dir)
		}
	}()

	// Create the target structure.
	var binDir string
	switch osys {
	case "darwin":
		binDir = path.Join(dir, "usr/local/bin")

	case "linux":
		binDir = path.Join(dir, "usr/bin")

	default:
		err = fmt.Errorf("Don't know where to put binaries for %s", osys)
		return
	}

	err = os.MkdirAll(binDir, 0700)
	if err != nil {
		err = fmt.Errorf("MkdirAll: %v", err)
		return
	}

	// Create another directory to become GOPATH for our build below.
	gopath, err := ioutil.TempDir("", "build_release_gopath")
	if err != nil {
		err = fmt.Errorf("TempDir: %v", err)
		return
	}

	defer os.RemoveAll(gopath)

	// Create a directory to store the source code.
	gitDir := path.Join(gopath, "src/github.com/googlecloudplatform/gcsfuse")
	err = os.MkdirAll(gitDir, 0700)
	if err != nil {
		err = fmt.Errorf("MkdirAll: %v", err)
		return
	}

	// Clone the source code into that directory.
	log.Printf("Cloning into %s", gitDir)

	cmd := exec.Command(
		"git",
		"clone",
		"https://github.com/GoogleCloudPlatform/gcsfuse.git",
		gitDir)

	output, err := cmd.CombinedOutput()
	if err != nil {
		err = fmt.Errorf("Cloning: %v\nOutput:\n%s", err, output)
		return
	}

	// Check out the appropriate commit.
	cmd = exec.Command("git", "checkout", commit)
	cmd.Dir = gitDir

	output, err = cmd.CombinedOutput()
	if err != nil {
		err = fmt.Errorf("git checkout: %v\nOutput:\n%s", err, output)
		return
	}

	// Overwrite version.go with a file containing the appropriate constant.
	versionContents := fmt.Sprintf(
		"package main\nconst gcsfuseVersion = \"%s (commit %s)\"",
		version,
		commit)

	err = ioutil.WriteFile(
		path.Join(gitDir, "version.go"),
		[]byte(versionContents),
		0400)

	if err != nil {
		err = fmt.Errorf("WriteFile: %v", err)
		return
	}

	// Build the binaries.
	binaries := []string{
		"github.com/googlecloudplatform/gcsfuse",
		"github.com/googlecloudplatform/gcsfuse/tools/mount_gcsfuse",
	}

	for _, bin := range binaries {
		log.Printf("Building %s", bin)

		cmd = exec.Command(
			"go",
			"build",
			"-o",
			path.Join(binDir, path.Base(bin)),
			bin)

		cmd.Env = []string{
			"GO15VENDOREXPERIMENT=1",
			fmt.Sprintf("GOPATH=%s", gopath),
			fmt.Sprintf("GOOS=%s", osys),
			fmt.Sprintf("GOARCH=%s", arch),
		}

		output, err = cmd.CombinedOutput()
		if err != nil {
			err = fmt.Errorf("Building %s: %v\nOutput:\n%s", bin, err, output)
			return
		}
	}

	return
}
<commit_msg>Use better permissions for the file system structure.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
)

// Build release binaries according to the supplied settings, returning the
// path to a directory containing exactly root-relative file system structure
// we desire.
func buildBinaries(
	version string,
	commit string,
	osys string,
	arch string) (dir string, err error) {
	log.Printf("Building %s at %s for %s (%s).", version, commit, osys, arch)
	// Create a directory to hold our outputs. Kill it if we later return in
	// error.
	dir, err = ioutil.TempDir("", "build_release_binaries")
	if err != nil {
		err = fmt.Errorf("TempDir: %v", err)
		return
	}

	defer func() {
		if err != nil {
			os.RemoveAll(dir)
		}
	}()

	// Create the target structure.
	var binDir string
	switch osys {
	case "darwin":
		binDir = path.Join(dir, "usr/local/bin")

	case "linux":
		binDir = path.Join(dir, "usr/bin")

	default:
		err = fmt.Errorf("Don't know where to put binaries for %s", osys)
		return
	}

	err = os.MkdirAll(binDir, 0755)
	if err != nil {
		err = fmt.Errorf("MkdirAll: %v", err)
		return
	}

	// Create another directory to become GOPATH for our build below.
	gopath, err := ioutil.TempDir("", "build_release_gopath")
	if err != nil {
		err = fmt.Errorf("TempDir: %v", err)
		return
	}

	defer os.RemoveAll(gopath)

	// Create a directory to store the source code.
	gitDir := path.Join(gopath, "src/github.com/googlecloudplatform/gcsfuse")
	err = os.MkdirAll(gitDir, 0700)
	if err != nil {
		err = fmt.Errorf("MkdirAll: %v", err)
		return
	}

	// Clone the source code into that directory.
	log.Printf("Cloning into %s", gitDir)

	cmd := exec.Command(
		"git",
		"clone",
		"https://github.com/GoogleCloudPlatform/gcsfuse.git",
		gitDir)

	output, err := cmd.CombinedOutput()
	if err != nil {
		err = fmt.Errorf("Cloning: %v\nOutput:\n%s", err, output)
		return
	}

	// Check out the appropriate commit.
	cmd = exec.Command("git", "checkout", commit)
	cmd.Dir = gitDir

	output, err = cmd.CombinedOutput()
	if err != nil {
		err = fmt.Errorf("git checkout: %v\nOutput:\n%s", err, output)
		return
	}

	// Overwrite version.go with a file containing the appropriate constant.
	versionContents := fmt.Sprintf(
		"package main\nconst gcsfuseVersion = \"%s (commit %s)\"",
		version,
		commit)

	err = ioutil.WriteFile(
		path.Join(gitDir, "version.go"),
		[]byte(versionContents),
		0400)

	if err != nil {
		err = fmt.Errorf("WriteFile: %v", err)
		return
	}

	// Build the binaries.
	binaries := []string{
		"github.com/googlecloudplatform/gcsfuse",
		"github.com/googlecloudplatform/gcsfuse/tools/mount_gcsfuse",
	}

	for _, bin := range binaries {
		log.Printf("Building %s", bin)

		cmd = exec.Command(
			"go",
			"build",
			"-o",
			path.Join(binDir, path.Base(bin)),
			bin)

		cmd.Env = []string{
			"GO15VENDOREXPERIMENT=1",
			fmt.Sprintf("GOPATH=%s", gopath),
			fmt.Sprintf("GOOS=%s", osys),
			fmt.Sprintf("GOARCH=%s", arch),
		}

		output, err = cmd.CombinedOutput()
		if err != nil {
			err = fmt.Errorf("Building %s: %v\nOutput:\n%s", bin, err, output)
			return
		}
	}

	return
}
<|endoftext|>