<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017, 2018 Red Hat, Inc.
 *
 */

package services

import (
	"path/filepath"
	"strconv"
	"strings"

	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/tools/cache"

	"kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/precond"
	"kubevirt.io/kubevirt/pkg/registry-disk"
)

const configMapName = "kube-system/kubevirt-config"
const useEmulationKey = "debug.useEmulation"
const KvmDevice = "devices.kubevirt.io/kvm"
const TunDevice = "devices.kubevirt.io/tun"

type TemplateService interface {
	RenderLaunchManifest(*v1.VirtualMachineInstance) (*k8sv1.Pod, error)
}

type templateService struct {
	launcherImage   string
	virtShareDir    string
	imagePullSecret string
	store           cache.Store
}

func IsEmulationAllowed(store cache.Store) (bool, error) {
	obj, exists, err := store.GetByKey(configMapName)
	if err != nil {
		return false, err
	}
	if !exists {
		return exists, nil
	}
	useEmulation := false
	cm := obj.(*k8sv1.ConfigMap)
	emu, ok := cm.Data[useEmulationKey]
	if ok {
		useEmulation = (strings.ToLower(emu) == "true")
	}
	return useEmulation, nil
}

func (t *templateService) RenderLaunchManifest(vmi *v1.VirtualMachineInstance) (*k8sv1.Pod, error) {
	precond.MustNotBeNil(vmi)
	domain := precond.MustNotBeEmpty(vmi.GetObjectMeta().GetName())
	namespace := precond.MustNotBeEmpty(vmi.GetObjectMeta().GetNamespace())

	initialDelaySeconds := 2
	timeoutSeconds := 5
	periodSeconds := 2
	successThreshold := 1
	failureThreshold := 5

	var volumes []k8sv1.Volume
	var userId int64 = 0
	var privileged bool = false
	var volumesMounts []k8sv1.VolumeMount
	var imagePullSecrets []k8sv1.LocalObjectReference

	gracePeriodSeconds := v1.DefaultGracePeriodSeconds
	if vmi.Spec.TerminationGracePeriodSeconds != nil {
		gracePeriodSeconds = *vmi.Spec.TerminationGracePeriodSeconds
	}

	volumesMounts = append(volumesMounts, k8sv1.VolumeMount{
		Name:      "virt-share-dir",
		MountPath: t.virtShareDir,
	})
	volumesMounts = append(volumesMounts, k8sv1.VolumeMount{
		Name:      "libvirt-runtime",
		MountPath: "/var/run/libvirt",
	})
	for _, volume := range vmi.Spec.Volumes {
		volumeMount := k8sv1.VolumeMount{
			Name:      volume.Name,
			MountPath: filepath.Join("/var/run/kubevirt-private", "vmi-disks", volume.Name),
		}
		if volume.PersistentVolumeClaim != nil {
			volumesMounts = append(volumesMounts, volumeMount)
			volumes = append(volumes, k8sv1.Volume{
				Name: volume.Name,
				VolumeSource: k8sv1.VolumeSource{
					PersistentVolumeClaim: volume.PersistentVolumeClaim,
				},
			})
		}
		if volume.Ephemeral != nil {
			volumesMounts = append(volumesMounts, volumeMount)
			volumes = append(volumes, k8sv1.Volume{
				Name: volume.Name,
				VolumeSource: k8sv1.VolumeSource{
					PersistentVolumeClaim: volume.Ephemeral.PersistentVolumeClaim,
				},
			})
		}
		if volume.RegistryDisk != nil && volume.RegistryDisk.ImagePullSecret != "" {
			imagePullSecrets = appendUniqueImagePullSecret(imagePullSecrets, k8sv1.LocalObjectReference{
				Name: volume.RegistryDisk.ImagePullSecret,
			})
		}
	}

	if t.imagePullSecret != "" {
		imagePullSecrets = appendUniqueImagePullSecret(imagePullSecrets, k8sv1.LocalObjectReference{
			Name: t.imagePullSecret,
		})
	}

	// Pad the virt-launcher grace period.
	// Ideally we want virt-handler to handle tearing down
	// the vmi without virt-launcher's termination forcing
	// the vmi down.
	gracePeriodSeconds = gracePeriodSeconds + int64(15)
	gracePeriodKillAfter := gracePeriodSeconds + int64(15)

	// Get memory overhead
	memoryOverhead := getMemoryOverhead(vmi.Spec.Domain)

	// Consider CPU and memory requests and limits for pod scheduling
	resources := k8sv1.ResourceRequirements{}
	vmiResources := vmi.Spec.Domain.Resources

	resources.Requests = make(k8sv1.ResourceList)

	// Copy vmi resources requests to a container
	for key, value := range vmiResources.Requests {
		resources.Requests[key] = value
	}

	// Copy vmi resources limits to a container
	if vmiResources.Limits != nil {
		resources.Limits = make(k8sv1.ResourceList)
	}

	for key, value := range vmiResources.Limits {
		resources.Limits[key] = value
	}

	// Consider hugepages resource for pod scheduling
	if vmi.Spec.Domain.Memory != nil && vmi.Spec.Domain.Memory.Hugepages != nil {
		if resources.Limits == nil {
			resources.Limits = make(k8sv1.ResourceList)
		}

		hugepageType := k8sv1.ResourceName(k8sv1.ResourceHugePagesPrefix + vmi.Spec.Domain.Memory.Hugepages.PageSize)
		resources.Requests[hugepageType] = resources.Requests[k8sv1.ResourceMemory]
		resources.Limits[hugepageType] = resources.Requests[k8sv1.ResourceMemory]

		// Configure hugepages mount on a pod
		volumesMounts = append(volumesMounts, k8sv1.VolumeMount{
			Name:      "hugepages",
			MountPath: filepath.Join("/dev/hugepages"),
		})
		volumes = append(volumes, k8sv1.Volume{
			Name: "hugepages",
			VolumeSource: k8sv1.VolumeSource{
				EmptyDir: &k8sv1.EmptyDirVolumeSource{
					Medium: k8sv1.StorageMediumHugePages,
				},
			},
		})

		// Set requested memory equals to overhead memory
		resources.Requests[k8sv1.ResourceMemory] = *memoryOverhead
		if _, ok := resources.Limits[k8sv1.ResourceMemory]; ok {
			resources.Limits[k8sv1.ResourceMemory] = *memoryOverhead
		}
	} else {
		// Add overhead memory
		memoryRequest := resources.Requests[k8sv1.ResourceMemory]
		memoryRequest.Add(*memoryOverhead)
		resources.Requests[k8sv1.ResourceMemory] = memoryRequest

		if memoryLimit, ok := resources.Limits[k8sv1.ResourceMemory]; ok {
			memoryLimit.Add(*memoryOverhead)
			resources.Limits[k8sv1.ResourceMemory] = memoryLimit
		}
	}

	command := []string{"/usr/share/kubevirt/virt-launcher/entrypoint.sh",
		"--qemu-timeout", "5m",
		"--name", domain,
		"--namespace", namespace,
		"--kubevirt-share-dir", t.virtShareDir,
		"--readiness-file", "/tmp/healthy",
		"--grace-period-seconds", strconv.Itoa(int(gracePeriodSeconds)),
	}

	useEmulation, err := IsEmulationAllowed(t.store)
	if err != nil {
		return nil, err
	}

	if resources.Limits == nil {
		resources.Limits = make(k8sv1.ResourceList)
	}

	// TODO: This can be hardcoded in the current model, but will need to be revisted
	// once dynamic network device allocation is added
	resources.Limits[TunDevice] = resource.MustParse("1")

	// FIXME: decision point: allow emulation means "it's ok to skip hw acceleration if not present"
	// but if the KVM resource is not requested then it's guaranteed to be not present
	// This code works for now, but the semantics are wrong. revisit this.
	if useEmulation {
		command = append(command, "--use-emulation")
	} else {
		resources.Limits[KvmDevice] = resource.MustParse("1")
	}

	// VirtualMachineInstance target container
	container := k8sv1.Container{
		Name:            "compute",
		Image:           t.launcherImage,
		ImagePullPolicy: k8sv1.PullIfNotPresent,
		SecurityContext: &k8sv1.SecurityContext{
			RunAsUser: &userId,
			// Privileged mode is disabled.
			Privileged: &privileged,
			Capabilities: &k8sv1.Capabilities{
				// NET_ADMIN is needed to set up networking for the VM
				Add: []k8sv1.Capability{"NET_ADMIN"},
			},
		},
		Command:      command,
		VolumeMounts: volumesMounts,
		ReadinessProbe: &k8sv1.Probe{
			Handler: k8sv1.Handler{
				Exec: &k8sv1.ExecAction{
					Command: []string{
						"cat",
						"/tmp/healthy",
					},
				},
			},
			InitialDelaySeconds: int32(initialDelaySeconds),
			PeriodSeconds:       int32(periodSeconds),
			TimeoutSeconds:      int32(timeoutSeconds),
			SuccessThreshold:    int32(successThreshold),
			FailureThreshold:    int32(failureThreshold),
		},
		Resources: resources,
	}

	containers := registrydisk.GenerateContainers(vmi, "libvirt-runtime", "/var/run/libvirt")

	volumes = append(volumes, k8sv1.Volume{
		Name: "virt-share-dir",
		VolumeSource: k8sv1.VolumeSource{
			HostPath: &k8sv1.HostPathVolumeSource{
				Path: t.virtShareDir,
			},
		},
	})
	volumes = append(volumes, k8sv1.Volume{
		Name: "libvirt-runtime",
		VolumeSource: k8sv1.VolumeSource{
			EmptyDir: &k8sv1.EmptyDirVolumeSource{},
		},
	})

	nodeSelector := map[string]string{}
	for k, v := range vmi.Spec.NodeSelector {
		nodeSelector[k] = v

	}
	nodeSelector[v1.NodeSchedulable] = "true"

	podLabels := map[string]string{}

	for k, v := range vmi.Labels {
		podLabels[k] = v
	}
	podLabels[v1.AppLabel] = "virt-launcher"
	podLabels[v1.DomainLabel] = domain

	containers = append(containers, container)

	hostName := vmi.Name
	if vmi.Spec.Hostname != "" {
		hostName = vmi.Spec.Hostname
	}

	// TODO use constants for podLabels
	pod := k8sv1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "virt-launcher-" + domain + "-",
			Labels:       podLabels,
			Annotations: map[string]string{
				v1.CreatedByAnnotation: string(vmi.UID),
				v1.OwnedByAnnotation:   "virt-controller",
			},
		},
		Spec: k8sv1.PodSpec{
			Hostname:  hostName,
			Subdomain: vmi.Spec.Subdomain,
			SecurityContext: &k8sv1.PodSecurityContext{
				RunAsUser: &userId,
			},
			TerminationGracePeriodSeconds: &gracePeriodKillAfter,
			RestartPolicy:                 k8sv1.RestartPolicyNever,
			Containers:                    containers,
			NodeSelector:                  nodeSelector,
			Volumes:                       volumes,
			ImagePullSecrets:              imagePullSecrets,
		},
	}

	if vmi.Spec.Affinity != nil {
		pod.Spec.Affinity = &k8sv1.Affinity{}

		if vmi.Spec.Affinity.NodeAffinity != nil {
			pod.Spec.Affinity.NodeAffinity = vmi.Spec.Affinity.NodeAffinity
		}

		if vmi.Spec.Affinity.PodAffinity != nil {
			pod.Spec.Affinity.PodAffinity = vmi.Spec.Affinity.PodAffinity
		}

		if vmi.Spec.Affinity.PodAntiAffinity != nil {
			pod.Spec.Affinity.PodAntiAffinity = vmi.Spec.Affinity.PodAntiAffinity
		}
	}

	return &pod, nil
}

func appendUniqueImagePullSecret(secrets []k8sv1.LocalObjectReference, newsecret k8sv1.LocalObjectReference) []k8sv1.LocalObjectReference {
	for _, oldsecret := range secrets {
		if oldsecret == newsecret {
			return secrets
		}
	}
	return append(secrets, newsecret)
}

// getMemoryOverhead computes the estimation of total
// memory needed for the domain to operate properly.
// This includes the memory needed for the guest and memory
// for Qemu and OS overhead.
//
// The return value is overhead memory quantity
//
// Note: This is the best estimation we were able to come up with
//       and is still not 100% accurate
func getMemoryOverhead(domain v1.DomainSpec) *resource.Quantity {
	vmiMemoryReq := domain.Resources.Requests.Memory()

	overhead := resource.NewScaledQuantity(0, resource.Kilo)

	// Add the memory needed for pagetables (one bit for every 512b of RAM size)
	pagetableMemory := resource.NewScaledQuantity(vmiMemoryReq.ScaledValue(resource.Kilo), resource.Kilo)
	pagetableMemory.Set(pagetableMemory.Value() / 512)
	overhead.Add(*pagetableMemory)

	// Add fixed overhead for shared libraries and such
	// TODO account for the overhead of kubevirt components running in the pod
	overhead.Add(resource.MustParse("64M"))

	// Add CPU table overhead (8 MiB per vCPU and 8 MiB per IO thread)
	// overhead per vcpu in MiB
	coresMemory := uint32(8)
	if domain.CPU != nil {
		coresMemory *= domain.CPU.Cores
	}
	overhead.Add(resource.MustParse(strconv.Itoa(int(coresMemory)) + "Mi"))

	// static overhead for IOThread
	overhead.Add(resource.MustParse("8Mi"))

	// Add video RAM overhead
	overhead.Add(resource.MustParse("16Mi"))

	return overhead
}

func NewTemplateService(launcherImage string, virtShareDir string, imagePullSecret string, configMapCache cache.Store) TemplateService {
	precond.MustNotBeEmpty(launcherImage)
	svc := templateService{
		launcherImage:   launcherImage,
		virtShareDir:    virtShareDir,
		imagePullSecret: imagePullSecret,
		store:           configMapCache,
	}
	return &svc
}
<commit_msg>Change virt-launcher SELinux type to spc_t<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2017, 2018 Red Hat, Inc.
 *
 */

package services

import (
	"path/filepath"
	"strconv"
	"strings"

	k8sv1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/tools/cache"

	"kubevirt.io/kubevirt/pkg/api/v1"
	"kubevirt.io/kubevirt/pkg/precond"
	"kubevirt.io/kubevirt/pkg/registry-disk"
)

const configMapName = "kube-system/kubevirt-config"
const useEmulationKey = "debug.useEmulation"
const KvmDevice = "devices.kubevirt.io/kvm"
const TunDevice = "devices.kubevirt.io/tun"

type TemplateService interface {
	RenderLaunchManifest(*v1.VirtualMachineInstance) (*k8sv1.Pod, error)
}

type templateService struct {
	launcherImage   string
	virtShareDir    string
	imagePullSecret string
	store           cache.Store
}

func IsEmulationAllowed(store cache.Store) (bool, error) {
	obj, exists, err := store.GetByKey(configMapName)
	if err != nil {
		return false, err
	}
	if !exists {
		return exists, nil
	}
	useEmulation := false
	cm := obj.(*k8sv1.ConfigMap)
	emu, ok := cm.Data[useEmulationKey]
	if ok {
		useEmulation = (strings.ToLower(emu) == "true")
	}
	return useEmulation, nil
}

func (t *templateService) RenderLaunchManifest(vmi *v1.VirtualMachineInstance) (*k8sv1.Pod, error) {
	precond.MustNotBeNil(vmi)
	domain := precond.MustNotBeEmpty(vmi.GetObjectMeta().GetName())
	namespace := precond.MustNotBeEmpty(vmi.GetObjectMeta().GetNamespace())

	initialDelaySeconds := 2
	timeoutSeconds := 5
	periodSeconds := 2
	successThreshold := 1
	failureThreshold := 5

	var volumes []k8sv1.Volume
	var userId int64 = 0
	var privileged bool = false
	var volumesMounts []k8sv1.VolumeMount
	var imagePullSecrets []k8sv1.LocalObjectReference

	gracePeriodSeconds := v1.DefaultGracePeriodSeconds
	if vmi.Spec.TerminationGracePeriodSeconds != nil {
		gracePeriodSeconds = *vmi.Spec.TerminationGracePeriodSeconds
	}

	volumesMounts = append(volumesMounts, k8sv1.VolumeMount{
		Name:      "virt-share-dir",
		MountPath: t.virtShareDir,
	})
	volumesMounts = append(volumesMounts, k8sv1.VolumeMount{
		Name:      "libvirt-runtime",
		MountPath: "/var/run/libvirt",
	})
	for _, volume := range vmi.Spec.Volumes {
		volumeMount := k8sv1.VolumeMount{
			Name:      volume.Name,
			MountPath: filepath.Join("/var/run/kubevirt-private", "vmi-disks", volume.Name),
		}
		if volume.PersistentVolumeClaim != nil {
			volumesMounts = append(volumesMounts, volumeMount)
			volumes = append(volumes, k8sv1.Volume{
				Name: volume.Name,
				VolumeSource: k8sv1.VolumeSource{
					PersistentVolumeClaim: volume.PersistentVolumeClaim,
				},
			})
		}
		if volume.Ephemeral != nil {
			volumesMounts = append(volumesMounts, volumeMount)
			volumes = append(volumes, k8sv1.Volume{
				Name: volume.Name,
				VolumeSource: k8sv1.VolumeSource{
					PersistentVolumeClaim: volume.Ephemeral.PersistentVolumeClaim,
				},
			})
		}
		if volume.RegistryDisk != nil && volume.RegistryDisk.ImagePullSecret != "" {
			imagePullSecrets = appendUniqueImagePullSecret(imagePullSecrets, k8sv1.LocalObjectReference{
				Name: volume.RegistryDisk.ImagePullSecret,
			})
		}
	}

	if t.imagePullSecret != "" {
		imagePullSecrets = appendUniqueImagePullSecret(imagePullSecrets, k8sv1.LocalObjectReference{
			Name: t.imagePullSecret,
		})
	}

	// Pad the virt-launcher grace period.
	// Ideally we want virt-handler to handle tearing down
	// the vmi without virt-launcher's termination forcing
	// the vmi down.
	gracePeriodSeconds = gracePeriodSeconds + int64(15)
	gracePeriodKillAfter := gracePeriodSeconds + int64(15)

	// Get memory overhead
	memoryOverhead := getMemoryOverhead(vmi.Spec.Domain)

	// Consider CPU and memory requests and limits for pod scheduling
	resources := k8sv1.ResourceRequirements{}
	vmiResources := vmi.Spec.Domain.Resources

	resources.Requests = make(k8sv1.ResourceList)

	// Copy vmi resources requests to a container
	for key, value := range vmiResources.Requests {
		resources.Requests[key] = value
	}

	// Copy vmi resources limits to a container
	if vmiResources.Limits != nil {
		resources.Limits = make(k8sv1.ResourceList)
	}

	for key, value := range vmiResources.Limits {
		resources.Limits[key] = value
	}

	// Consider hugepages resource for pod scheduling
	if vmi.Spec.Domain.Memory != nil && vmi.Spec.Domain.Memory.Hugepages != nil {
		if resources.Limits == nil {
			resources.Limits = make(k8sv1.ResourceList)
		}

		hugepageType := k8sv1.ResourceName(k8sv1.ResourceHugePagesPrefix + vmi.Spec.Domain.Memory.Hugepages.PageSize)
		resources.Requests[hugepageType] = resources.Requests[k8sv1.ResourceMemory]
		resources.Limits[hugepageType] = resources.Requests[k8sv1.ResourceMemory]

		// Configure hugepages mount on a pod
		volumesMounts = append(volumesMounts, k8sv1.VolumeMount{
			Name:      "hugepages",
			MountPath: filepath.Join("/dev/hugepages"),
		})
		volumes = append(volumes, k8sv1.Volume{
			Name: "hugepages",
			VolumeSource: k8sv1.VolumeSource{
				EmptyDir: &k8sv1.EmptyDirVolumeSource{
					Medium: k8sv1.StorageMediumHugePages,
				},
			},
		})

		// Set requested memory equals to overhead memory
		resources.Requests[k8sv1.ResourceMemory] = *memoryOverhead
		if _, ok := resources.Limits[k8sv1.ResourceMemory]; ok {
			resources.Limits[k8sv1.ResourceMemory] = *memoryOverhead
		}
	} else {
		// Add overhead memory
		memoryRequest := resources.Requests[k8sv1.ResourceMemory]
		memoryRequest.Add(*memoryOverhead)
		resources.Requests[k8sv1.ResourceMemory] = memoryRequest

		if memoryLimit, ok := resources.Limits[k8sv1.ResourceMemory]; ok {
			memoryLimit.Add(*memoryOverhead)
			resources.Limits[k8sv1.ResourceMemory] = memoryLimit
		}
	}

	command := []string{"/usr/share/kubevirt/virt-launcher/entrypoint.sh",
		"--qemu-timeout", "5m",
		"--name", domain,
		"--namespace", namespace,
		"--kubevirt-share-dir", t.virtShareDir,
		"--readiness-file", "/tmp/healthy",
		"--grace-period-seconds", strconv.Itoa(int(gracePeriodSeconds)),
	}

	useEmulation, err := IsEmulationAllowed(t.store)
	if err != nil {
		return nil, err
	}

	if resources.Limits == nil {
		resources.Limits = make(k8sv1.ResourceList)
	}

	// TODO: This can be hardcoded in the current model, but will need to be revisted
	// once dynamic network device allocation is added
	resources.Limits[TunDevice] = resource.MustParse("1")

	// FIXME: decision point: allow emulation means "it's ok to skip hw acceleration if not present"
	// but if the KVM resource is not requested then it's guaranteed to be not present
	// This code works for now, but the semantics are wrong. revisit this.
	if useEmulation {
		command = append(command, "--use-emulation")
	} else {
		resources.Limits[KvmDevice] = resource.MustParse("1")
	}

	// VirtualMachineInstance target container
	container := k8sv1.Container{
		Name:            "compute",
		Image:           t.launcherImage,
		ImagePullPolicy: k8sv1.PullIfNotPresent,
		SecurityContext: &k8sv1.SecurityContext{
			RunAsUser: &userId,
			// Privileged mode is disabled.
			Privileged: &privileged,
			Capabilities: &k8sv1.Capabilities{
				// NET_ADMIN is needed to set up networking for the VM
				Add: []k8sv1.Capability{"NET_ADMIN"},
			},
		},
		Command:      command,
		VolumeMounts: volumesMounts,
		ReadinessProbe: &k8sv1.Probe{
			Handler: k8sv1.Handler{
				Exec: &k8sv1.ExecAction{
					Command: []string{
						"cat",
						"/tmp/healthy",
					},
				},
			},
			InitialDelaySeconds: int32(initialDelaySeconds),
			PeriodSeconds:       int32(periodSeconds),
			TimeoutSeconds:      int32(timeoutSeconds),
			SuccessThreshold:    int32(successThreshold),
			FailureThreshold:    int32(failureThreshold),
		},
		Resources: resources,
	}

	containers := registrydisk.GenerateContainers(vmi, "libvirt-runtime", "/var/run/libvirt")

	volumes = append(volumes, k8sv1.Volume{
		Name: "virt-share-dir",
		VolumeSource: k8sv1.VolumeSource{
			HostPath: &k8sv1.HostPathVolumeSource{
				Path: t.virtShareDir,
			},
		},
	})
	volumes = append(volumes, k8sv1.Volume{
		Name: "libvirt-runtime",
		VolumeSource: k8sv1.VolumeSource{
			EmptyDir: &k8sv1.EmptyDirVolumeSource{},
		},
	})

	nodeSelector := map[string]string{}
	for k, v := range vmi.Spec.NodeSelector {
		nodeSelector[k] = v

	}
	nodeSelector[v1.NodeSchedulable] = "true"

	podLabels := map[string]string{}

	for k, v := range vmi.Labels {
		podLabels[k] = v
	}
	podLabels[v1.AppLabel] = "virt-launcher"
	podLabels[v1.DomainLabel] = domain

	containers = append(containers, container)

	hostName := vmi.Name
	if vmi.Spec.Hostname != "" {
		hostName = vmi.Spec.Hostname
	}

	// TODO use constants for podLabels
	pod := k8sv1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: "virt-launcher-" + domain + "-",
			Labels:       podLabels,
			Annotations: map[string]string{
				v1.CreatedByAnnotation: string(vmi.UID),
				v1.OwnedByAnnotation:   "virt-controller",
			},
		},
		Spec: k8sv1.PodSpec{
			Hostname:  hostName,
			Subdomain: vmi.Spec.Subdomain,
			SecurityContext: &k8sv1.PodSecurityContext{
				RunAsUser: &userId,
				SELinuxOptions: &k8sv1.SELinuxOptions{
					Type: "spc_t",
				},
			},
			TerminationGracePeriodSeconds: &gracePeriodKillAfter,
			RestartPolicy:                 k8sv1.RestartPolicyNever,
			Containers:                    containers,
			NodeSelector:                  nodeSelector,
			Volumes:                       volumes,
			ImagePullSecrets:              imagePullSecrets,
		},
	}

	if vmi.Spec.Affinity != nil {
		pod.Spec.Affinity = &k8sv1.Affinity{}

		if vmi.Spec.Affinity.NodeAffinity != nil {
			pod.Spec.Affinity.NodeAffinity = vmi.Spec.Affinity.NodeAffinity
		}

		if vmi.Spec.Affinity.PodAffinity != nil {
			pod.Spec.Affinity.PodAffinity = vmi.Spec.Affinity.PodAffinity
		}

		if vmi.Spec.Affinity.PodAntiAffinity != nil {
			pod.Spec.Affinity.PodAntiAffinity = vmi.Spec.Affinity.PodAntiAffinity
		}
	}

	return &pod, nil
}

func appendUniqueImagePullSecret(secrets []k8sv1.LocalObjectReference, newsecret k8sv1.LocalObjectReference) []k8sv1.LocalObjectReference {
	for _, oldsecret := range secrets {
		if oldsecret == newsecret {
			return secrets
		}
	}
	return append(secrets, newsecret)
}

// getMemoryOverhead computes the estimation of total
// memory needed for the domain to operate properly.
// This includes the memory needed for the guest and memory
// for Qemu and OS overhead.
//
// The return value is overhead memory quantity
//
// Note: This is the best estimation we were able to come up with
//       and is still not 100% accurate
func getMemoryOverhead(domain v1.DomainSpec) *resource.Quantity {
	vmiMemoryReq := domain.Resources.Requests.Memory()

	overhead := resource.NewScaledQuantity(0, resource.Kilo)

	// Add the memory needed for pagetables (one bit for every 512b of RAM size)
	pagetableMemory := resource.NewScaledQuantity(vmiMemoryReq.ScaledValue(resource.Kilo), resource.Kilo)
	pagetableMemory.Set(pagetableMemory.Value() / 512)
	overhead.Add(*pagetableMemory)

	// Add fixed overhead for shared libraries and such
	// TODO account for the overhead of kubevirt components running in the pod
	overhead.Add(resource.MustParse("64M"))

	// Add CPU table overhead (8 MiB per vCPU and 8 MiB per IO thread)
	// overhead per vcpu in MiB
	coresMemory := uint32(8)
	if domain.CPU != nil {
		coresMemory *= domain.CPU.Cores
	}
	overhead.Add(resource.MustParse(strconv.Itoa(int(coresMemory)) + "Mi"))

	// static overhead for IOThread
	overhead.Add(resource.MustParse("8Mi"))

	// Add video RAM overhead
	overhead.Add(resource.MustParse("16Mi"))

	return overhead
}

func NewTemplateService(launcherImage string, virtShareDir string, imagePullSecret string, configMapCache cache.Store) TemplateService {
	precond.MustNotBeEmpty(launcherImage)
	svc := templateService{
		launcherImage:   launcherImage,
		virtShareDir:    virtShareDir,
		imagePullSecret: imagePullSecret,
		store:           configMapCache,
	}
	return &svc
}
<|endoftext|>


<commit_before>//go:generate bitfanDoc
// Read events from standard input.
// By default, each event is assumed to be one line. If you want to join lines, you’ll want to use the multiline filter.
package stdin

import (
	"os"
	"time"

	"github.com/vjeantet/bitfan/codecs"
	"github.com/vjeantet/bitfan/processors"
)

func New() processors.Processor {
	return &processor{opt: &options{}}
}

type options struct {
	// Add a field to an event
	Add_field map[string]interface{}

	// Add any number of arbitrary tags to your event.
	// This can help with processing later.
	Tags []string

	// Add a type field to all events handled by this input
	Type string

	// The codec used for input data. Input codecs are a convenient method for decoding
	// your data before it enters the input, without needing a separate filter in your bitfan pipeline
	// @default "line"
	Codec codecs.Codec
}

// Reads events from standard input
type processor struct {
	processors.Base

	opt  *options
	q    chan bool
	host string
}

func (p *processor) Configure(ctx processors.ProcessorContext, conf map[string]interface{}) error {
	defaults := options{
		Codec: codecs.New("line"),
	}
	p.opt = &defaults
	err := p.ConfigureAndValidate(ctx, conf, p.opt)

	if p.host, err = os.Hostname(); err != nil {
		p.Logger.Warnf("can not get hostname : %s", err.Error())
	}

	return err
}
func (p *processor) Start(e processors.IPacket) error {
	p.q = make(chan bool)

	var dec codecs.Decoder
	var err error

	if dec, err = p.opt.Codec.Decoder(os.Stdin); err != nil {
		p.Logger.Errorln("decoder error : ", err.Error())
		return err
	}

	stdinChan := make(chan string)
	go func(p *processor, ch chan string) {
		defer p.Logger.Errorln("XXXXXXXX")
		for {
			if record, err := dec.Decode(); err != nil {
				p.Logger.Errorln("codec error : ", err.Error())
				return
			} else {
				if record == nil {
					p.Logger.Debugln("waiting for more content...")
				} else {
					ch <- record["message"].(string)
				}
			}
		}
	}(p, stdinChan)

	go func(ch chan string) {
		for {
			select {
			case stdin, _ := <-ch:

				ne := p.NewPacket(stdin, map[string]interface{}{
					"host": p.host,
				})

				processors.ProcessCommonFields(ne.Fields(), p.opt.Add_field, p.opt.Tags, p.opt.Type)
				p.Send(ne)

			case <-time.After(5 * time.Second):

			}

			select {
			case <-p.q:
				close(p.q)
				close(ch)
				return
			default:
			}
		}
	}(stdinChan)

	return nil
}

func (p *processor) Stop(e processors.IPacket) error {
	p.q <- true
	<-p.q
	return nil
}
<commit_msg>stdin : fix debug messages<commit_after>//go:generate bitfanDoc
// Read events from standard input.
// By default, each event is assumed to be one line. If you want to join lines, you’ll want to use the multiline filter.
package stdin

import (
	"os"
	"time"

	"github.com/vjeantet/bitfan/codecs"
	"github.com/vjeantet/bitfan/processors"
)

func New() processors.Processor {
	return &processor{opt: &options{}}
}

type options struct {
	// Add a field to an event
	Add_field map[string]interface{}

	// Add any number of arbitrary tags to your event.
	// This can help with processing later.
	Tags []string

	// Add a type field to all events handled by this input
	Type string

	// The codec used for input data. Input codecs are a convenient method for decoding
	// your data before it enters the input, without needing a separate filter in your bitfan pipeline
	// @default "line"
	Codec codecs.Codec
}

// Reads events from standard input
type processor struct {
	processors.Base

	opt  *options
	q    chan bool
	host string
}

func (p *processor) Configure(ctx processors.ProcessorContext, conf map[string]interface{}) error {
	defaults := options{
		Codec: codecs.New("line"),
	}
	p.opt = &defaults
	err := p.ConfigureAndValidate(ctx, conf, p.opt)

	if p.host, err = os.Hostname(); err != nil {
		p.Logger.Warnf("can not get hostname : %s", err.Error())
	}

	return err
}
func (p *processor) Start(e processors.IPacket) error {
	p.q = make(chan bool)

	var dec codecs.Decoder
	var err error

	if dec, err = p.opt.Codec.Decoder(os.Stdin); err != nil {
		p.Logger.Errorln("decoder error : ", err.Error())
		return err
	}

	stdinChan := make(chan string)
	go func(p *processor, ch chan string) {
		for {
			if record, err := dec.Decode(); err != nil {
				p.Logger.Errorln("codec error : ", err.Error())
				return
			} else {
				if record == nil {
					p.Logger.Debugln("waiting for more content...")
				} else {
					ch <- record["message"].(string)
				}
			}
		}
	}(p, stdinChan)

	go func(ch chan string) {
		for {
			select {
			case msg, _ := <-ch:

				ne := p.NewPacket(msg, map[string]interface{}{
					"host": p.host,
				})

				processors.ProcessCommonFields(ne.Fields(), p.opt.Add_field, p.opt.Tags, p.opt.Type)
				p.Send(ne)

			case <-time.After(1 * time.Second):

			case <-p.q:
				close(p.q)
				close(ch)
				return
			}
		}
	}(stdinChan)

	return nil
}

func (p *processor) Stop(e processors.IPacket) error {
	p.q <- true
	<-p.q
	return nil
}
<|endoftext|>


<commit_before>package ngserver

import (
    "encoding/json"
    "fmt"
    //"github.com/ga0/ng/ngnet"
    "golang.org/x/net/websocket"
    "net/http"
)

func init() {
}

type NGClient struct {
    eventChan chan interface{}
    server    *NGServer
    ws        *websocket.Conn
}

func (c *NGClient) RecvAndProcessCommand() {
    for {
        var msg string
        err := websocket.Message.Receive(c.ws, &msg)
        if err != nil {
            return
        }
        if len(msg) > 0 {
            fmt.Println("Cmd: " + msg)
            if msg == "sync" {
                c.server.Sync(c)
            }
        } else {
            panic("empty command")
        }
    }
}

func (c *NGClient) TransmitEvents() {
    defer fmt.Println("Transmit exit")
    for ev := range c.eventChan {
        json, err := json.Marshal(ev)
        if err == nil {
            websocket.Message.Send(c.ws, string(json))
        }
    }
}

func (c *NGClient) Close() {
    close(c.eventChan)
}

func NewNGClient(ws *websocket.Conn, server *NGServer) *NGClient {
    c := new(NGClient)
    c.server = server
    c.ws = ws
    c.eventChan = make(chan interface{}, 16)
    return c
}

type NGServer struct {
    eventChan       chan interface{}
    addr            string
    staticFileDir   string
    connectedClient map[*websocket.Conn]*NGClient
    eventBuffer     []interface{}
}

func (s *NGServer) webHandler(ws *websocket.Conn) {
    c := NewNGClient(ws, s)
    s.connectedClient[ws] = c
    go c.TransmitEvents()
    c.RecvAndProcessCommand()
    c.Close()
    delete(s.connectedClient, ws)
}

func (s *NGServer) DispatchEvent() {
    for ev := range s.eventChan {
        s.eventBuffer = append(s.eventBuffer, ev)
        for _, c := range s.connectedClient {
            c.eventChan <- ev
        }
    }
    fmt.Println("Done")
}

func (s *NGServer) Sync(c *NGClient) {
    fmt.Println("Sync", c.ws.RemoteAddr(), "Event count: ", len(s.eventBuffer))
    for _, ev := range s.eventBuffer {
        c.eventChan <- ev
    }
}

func (s *NGServer) Serve() {
    go s.DispatchEvent()
    http.Handle("/data", websocket.Handler(s.webHandler))
    fs := http.FileServer(http.Dir(s.staticFileDir))
    http.Handle("/", fs)
    fmt.Println("Server runs")
    http.ListenAndServe(s.addr, nil)
}

func NewNGServer(addr string, staticFileDir string, eventChan chan interface{}) *NGServer {
    s := new(NGServer)
    s.eventChan = eventChan
    s.addr = addr
    s.staticFileDir = staticFileDir
    s.connectedClient = make(map[*websocket.Conn]*NGClient)
    return s
}
<commit_msg>make save event configurable<commit_after>package ngserver

import (
    "encoding/json"
    "flag"
    "fmt"
    "golang.org/x/net/websocket"
    "net/http"
)

var saveEvent = flag.Bool("s", true, "save network event locally")

func init() {
}

type NGClient struct {
    eventChan chan interface{}
    server    *NGServer
    ws        *websocket.Conn
}

func (c *NGClient) RecvAndProcessCommand() {
    for {
        var msg string
        err := websocket.Message.Receive(c.ws, &msg)
        if err != nil {
            return
        }
        if len(msg) > 0 {
            fmt.Println("Cmd: " + msg)
            if msg == "sync" {
                c.server.Sync(c)
            }
        } else {
            panic("empty command")
        }
    }
}

func (c *NGClient) TransmitEvents() {
    defer fmt.Println("Transmit exit")
    for ev := range c.eventChan {
        json, err := json.Marshal(ev)
        if err == nil {
            websocket.Message.Send(c.ws, string(json))
        }
    }
}

func (c *NGClient) Close() {
    close(c.eventChan)
}

func NewNGClient(ws *websocket.Conn, server *NGServer) *NGClient {
    c := new(NGClient)
    c.server = server
    c.ws = ws
    c.eventChan = make(chan interface{}, 16)
    return c
}

type NGServer struct {
    eventChan       chan interface{}
    addr            string
    staticFileDir   string
    connectedClient map[*websocket.Conn]*NGClient
    eventBuffer     []interface{}
}

func (s *NGServer) webHandler(ws *websocket.Conn) {
    c := NewNGClient(ws, s)
    s.connectedClient[ws] = c
    go c.TransmitEvents()
    c.RecvAndProcessCommand()
    c.Close()
    delete(s.connectedClient, ws)
}

func (s *NGServer) DispatchEvent() {
    for ev := range s.eventChan {
        if *saveEvent {
            s.eventBuffer = append(s.eventBuffer, ev)
        }
        for _, c := range s.connectedClient {
            c.eventChan <- ev
        }
    }
    fmt.Println("Done")
}

func (s *NGServer) Sync(c *NGClient) {
    fmt.Println("Sync", c.ws.RemoteAddr(), "Event count: ", len(s.eventBuffer))
    for _, ev := range s.eventBuffer {
        c.eventChan <- ev
    }
}

func (s *NGServer) Serve() {
    go s.DispatchEvent()
    http.Handle("/data", websocket.Handler(s.webHandler))
    fs := http.FileServer(http.Dir(s.staticFileDir))
    http.Handle("/", fs)
    fmt.Println("Server runs")
    http.ListenAndServe(s.addr, nil)
}

func NewNGServer(addr string, staticFileDir string, eventChan chan interface{}) *NGServer {
    s := new(NGServer)
    s.eventChan = eventChan
    s.addr = addr
    s.staticFileDir = staticFileDir
    s.connectedClient = make(map[*websocket.Conn]*NGClient)
    return s
}
<|endoftext|>


<commit_before>package proctl

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"strconv"
	"strings"
	"syscall"
	"unsafe"

	"github.com/derekparker/delve/dwarf/op"
	"github.com/derekparker/delve/vendor/dwarf"
)

type Variable struct {
	Name  string
	Value string
	Type  string
}

func (dbp *DebuggedProcess) PrintGoroutinesInfo() error {
	data, err := dbp.Executable.DWARF()
	if err != nil {
		return err
	}

	allglen, err := allglenval(dbp, data)
	if err != nil {
		return err
	}
	goidoffset, err := parsegoidoffset(dbp, data)
	if err != nil {
		return err
	}
	schedoffset, err := parseschedoffset(dbp, data)
	if err != nil {
		return err
	}
	allgentryaddr, err := allgentryptr(dbp, data)
	if err != nil {
		return err
	}
	fmt.Printf("[%d goroutines]\n", allglen)
	faddr, err := dbp.CurrentThread.readMemory(uintptr(allgentryaddr), 8)
	allg := binary.LittleEndian.Uint64(faddr)
	fmt.Println("sched", schedoffset)

	for i := uint64(0); i < allglen; i++ {
		err = printGoroutineInfo(dbp, allg+(i*8), goidoffset, schedoffset)
		if err != nil {
			return err
		}
	}

	return nil
}

func printGoroutineInfo(dbp *DebuggedProcess, addr uint64, goidoffset, schedoffset uint64) error {
	gaddrbytes, err := dbp.CurrentThread.readMemory(uintptr(addr), 8)
	if err != nil {
		return fmt.Errorf("error derefing *G %s", err)
	}
	gaddr := binary.LittleEndian.Uint64(gaddrbytes)

	goidbytes, err := dbp.CurrentThread.readMemory(uintptr(gaddr+goidoffset), 8)
	if err != nil {
		return fmt.Errorf("error reading goid %s", err)
	}
	schedbytes, err := dbp.CurrentThread.readMemory(uintptr(gaddr+schedoffset+8), 8)
	if err != nil {
		return fmt.Errorf("error reading goid %s", err)
	}
	gopc := binary.LittleEndian.Uint64(schedbytes)
	f, l, _ := dbp.GoSymTable.PCToLine(gopc)
	fmt.Printf("Goroutine %d - %s:%d\n", binary.LittleEndian.Uint64(goidbytes), f, l)
	return nil
}

func allglenval(dbp *DebuggedProcess, data *dwarf.Data) (uint64, error) {
	entry, err := findDwarfEntry("runtime.allglen", data)
	if err != nil {
		return 0, err
	}

	instructions, ok := entry.Val(dwarf.AttrLocation).([]byte)
	if !ok {
		return 0, fmt.Errorf("type assertion failed")
	}
	addr, err := op.ExecuteStackProgram(0, instructions)
	if err != nil {
		return 0, err
	}
	val, err := dbp.CurrentThread.readMemory(uintptr(addr), 8)
	if err != nil {
		return 0, err
	}
	return binary.LittleEndian.Uint64(val), nil
}

func allgentryptr(dbp *DebuggedProcess, data *dwarf.Data) (uint64, error) {
	entry, err := findDwarfEntry("runtime.allg", data)
	if err != nil {
		return 0, err
	}

	instructions, ok := entry.Val(dwarf.AttrLocation).([]byte)
	if !ok {
		return 0, fmt.Errorf("type assertion failed")
	}
	addr, err := op.ExecuteStackProgram(0, instructions)
	if err != nil {
		return 0, err
	}

	return uint64(addr), nil
}

func parsegoidoffset(dbp *DebuggedProcess, data *dwarf.Data) (uint64, error) {
	entry, err := findDwarfEntry("goid", data)
	if err != nil {
		return 0, err
	}
	instructions, ok := entry.Val(dwarf.AttrDataMemberLoc).([]byte)
	if !ok {
		return 0, fmt.Errorf("type assertion failed")
	}
	offset, err := op.ExecuteStackProgram(0, instructions)
	if err != nil {
		return 0, err
	}

	return uint64(offset), nil
}

func parseschedoffset(dbp *DebuggedProcess, data *dwarf.Data) (uint64, error) {
	entry, err := findDwarfEntry("sched", data)
	if err != nil {
		return 0, err
	}
	instructions, ok := entry.Val(dwarf.AttrDataMemberLoc).([]byte)
	if !ok {
		return 0, fmt.Errorf("type assertion failed")
	}
	offset, err := op.ExecuteStackProgram(0, instructions)
	if err != nil {
		return 0, err
	}

	return uint64(offset), nil
}

// Returns the value of the named symbol.
func (thread *ThreadContext) EvalSymbol(name string) (*Variable, error) {
	data, err := thread.Process.Executable.DWARF()
	if err != nil {
		return nil, err
	}

	entry, err := findDwarfEntry(name, data)
	if err != nil {
		return nil, err
	}

	offset, ok := entry.Val(dwarf.AttrType).(dwarf.Offset)
	if !ok {
		return nil, fmt.Errorf("type assertion failed")
	}

	t, err := data.Type(offset)
	if err != nil {
		return nil, err
	}

	instructions, ok := entry.Val(dwarf.AttrLocation).([]byte)
	if !ok {
		return nil, fmt.Errorf("type assertion failed")
	}

	val, err := thread.extractValue(instructions, 0, t)
	if err != nil {
		return nil, err
	}

	return &Variable{Name: name, Type: t.String(), Value: val}, nil
}

func findDwarfEntry(name string, data *dwarf.Data) (*dwarf.Entry, error) {
	reader := data.Reader()

	for entry, err := reader.Next(); entry != nil; entry, err = reader.Next() {
		if err != nil {
			return nil, err
		}

		if entry.Tag != dwarf.TagVariable && entry.Tag != dwarf.TagFormalParameter && entry.Tag != dwarf.TagMember {
			continue
		}

		n, ok := entry.Val(dwarf.AttrName).(string)
		if !ok || n != name {
			continue
		}
		return entry, nil
	}
	return nil, fmt.Errorf("could not find symbol value for %s", name)
}

// Extracts the value from the instructions given in the DW_AT_location entry.
// We execute the stack program described in the DW_OP_* instruction stream, and
// then grab the value from the other processes memory.
func (thread *ThreadContext) extractValue(instructions []byte, off int64, typ interface{}) (string, error) {
	regs, err := thread.Registers()
	if err != nil {
		return "", err
	}

	fde, err := thread.Process.FrameEntries.FDEForPC(regs.PC())
	if err != nil {
		return "", err
	}

	fctx := fde.EstablishFrame(regs.PC())
	cfaOffset := fctx.CFAOffset()

	offset := off
	if off == 0 {
		offset, err = op.ExecuteStackProgram(cfaOffset, instructions)
		if err != nil {
			return "", err
		}
		offset = int64(regs.Rsp) + offset
	}

	// If we have a user defined type, find the
	// underlying concrete type and use that.
	if tt, ok := typ.(*dwarf.TypedefType); ok {
		typ = tt.Type
	}

	offaddr := uintptr(offset)
	switch t := typ.(type) {
	case *dwarf.PtrType:
		addr, err := thread.readMemory(offaddr, 8)
		if err != nil {
			return "", err
		}
		adr := binary.LittleEndian.Uint64(addr)
		val, err := thread.extractValue(nil, int64(adr), t.Type)
		if err != nil {
			return "", err
		}

		retstr := fmt.Sprintf("*%s", val)
		return retstr, nil
	case *dwarf.StructType:
		switch t.StructName {
		case "string":
			return thread.readString(offaddr)
		case "[]int":
			return thread.readIntSlice(offaddr)
		default:
			// Recursively call extractValue to grab
			// the value of all the members of the struct.
			fields := make([]string, 0, len(t.Field))
			for _, field := range t.Field {
				val, err := thread.extractValue(nil, field.ByteOffset+offset, field.Type)
				if err != nil {
					return "", err
				}

				fields = append(fields, fmt.Sprintf("%s: %s", field.Name, val))
			}
			retstr := fmt.Sprintf("%s {%s}", t.StructName, strings.Join(fields, ", "))
			return retstr, nil
		}
	case *dwarf.ArrayType:
		return thread.readIntArray(offaddr, t)
	case *dwarf.IntType:
		return thread.readInt(offaddr, t.ByteSize)
	case *dwarf.FloatType:
		return thread.readFloat(offaddr, t.ByteSize)
	}

	return "", fmt.Errorf("could not find value for type %s", typ)
}

func (thread *ThreadContext) readString(addr uintptr) (string, error) {
	val, err := thread.readMemory(addr, 8)
	if err != nil {
		return "", err
	}

	// deref the pointer to the string
	addr = uintptr(binary.LittleEndian.Uint64(val))
	val, err = thread.readMemory(addr, 16)
	if err != nil {
		return "", err
	}

	i := bytes.IndexByte(val, 0x0)
	val = val[:i]
	return *(*string)(unsafe.Pointer(&val)), nil
}

func (thread *ThreadContext) readIntSlice(addr uintptr) (string, error) {
	var number uint64

	val, err := thread.readMemory(addr, uintptr(24))
	if err != nil {
		return "", err
	}

	a := binary.LittleEndian.Uint64(val[:8])
	l := binary.LittleEndian.Uint64(val[8:16])
	c := binary.LittleEndian.Uint64(val[16:24])

	val, err = thread.readMemory(uintptr(a), uintptr(8*l))
	if err != nil {
		return "", err
	}

	members := make([]uint64, 0, l)
	buf := bytes.NewBuffer(val)
	for {
		err := binary.Read(buf, binary.LittleEndian, &number)
		if err != nil {
			break
		}

		members = append(members, number)
	}

	return fmt.Sprintf("len: %d cap: %d %d", l, c, members), nil
}

func (thread *ThreadContext) readIntArray(addr uintptr, t *dwarf.ArrayType) (string, error) {
	var (
		number  uint64
		members = make([]uint64, 0, t.ByteSize)
	)

	val, err := thread.readMemory(addr, uintptr(t.ByteSize))
	if err != nil {
		return "", err
	}

	buf := bytes.NewBuffer(val)
	for {
		err := binary.Read(buf, binary.LittleEndian, &number)
		if err != nil {
			break
		}

		members = append(members, number)
	}

	return fmt.Sprintf("[%d]int %d", t.ByteSize/8, members), nil
}

func (thread *ThreadContext) readInt(addr uintptr, size int64) (string, error) {
	var n int

	val, err := thread.readMemory(addr, uintptr(size))
	if err != nil {
		return "", err
	}

	switch size {
	case 1:
		n = int(val[0])
	case 2:
		n = int(binary.LittleEndian.Uint16(val))
	case 4:
		n = int(binary.LittleEndian.Uint32(val))
	case 8:
		n = int(binary.LittleEndian.Uint64(val))
	}

	return strconv.Itoa(n), nil
}

func (thread *ThreadContext) readFloat(addr uintptr, size int64) (string, error) {
	val, err := thread.readMemory(addr, uintptr(size))
	if err != nil {
		return "", err
	}
	buf := bytes.NewBuffer(val)

	switch size {
	case 4:
		n := float32(0)
		binary.Read(buf, binary.LittleEndian, &n)
		return strconv.FormatFloat(float64(n), 'f', -1, int(size)*8), nil
	case 8:
		n := float64(0)
		binary.Read(buf, binary.LittleEndian, &n)
		return strconv.FormatFloat(n, 'f', -1, int(size)*8), nil
	}

	return "", fmt.Errorf("could not read float")
}

func (thread *ThreadContext) readMemory(addr uintptr, size uintptr) ([]byte, error) {
	buf := make([]byte, size)

	_, err := syscall.PtracePeekData(thread.Id, addr, buf)
	if err != nil {
		return nil, err
	}

	return buf, nil
}
<commit_msg>cleanup<commit_after>package proctl

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"strconv"
	"strings"
	"syscall"
	"unsafe"

	"github.com/derekparker/delve/dwarf/op"
	"github.com/derekparker/delve/vendor/dwarf"
)

type Variable struct {
	Name  string
	Value string
	Type  string
}

func (dbp *DebuggedProcess) PrintGoroutinesInfo() error {
	data, err := dbp.Executable.DWARF()
	if err != nil {
		return err
	}

	allglen, err := allglenval(dbp, data)
	if err != nil {
		return err
	}
	goidoffset, err := parsegoidoffset(dbp, data)
	if err != nil {
		return err
	}
	schedoffset, err := parseschedoffset(dbp, data)
	if err != nil {
		return err
	}
	allgentryaddr, err := allgentryptr(dbp, data)
	if err != nil {
		return err
	}
	fmt.Printf("[%d goroutines]\n", allglen)
	faddr, err := dbp.CurrentThread.readMemory(uintptr(allgentryaddr), 8)
	allg := binary.LittleEndian.Uint64(faddr)

	for i := uint64(0); i < allglen; i++ {
		err = printGoroutineInfo(dbp, allg+(i*8), goidoffset, schedoffset)
		if err != nil {
			return err
		}
	}

	return nil
}

func printGoroutineInfo(dbp *DebuggedProcess, addr uint64, goidoffset, schedoffset uint64) error {
	gaddrbytes, err := dbp.CurrentThread.readMemory(uintptr(addr), 8)
	if err != nil {
		return fmt.Errorf("error derefing *G %s", err)
	}
	gaddr := binary.LittleEndian.Uint64(gaddrbytes)

	goidbytes, err := dbp.CurrentThread.readMemory(uintptr(gaddr+goidoffset), 8)
	if err != nil {
		return fmt.Errorf("error reading goid %s", err)
	}
	schedbytes, err := dbp.CurrentThread.readMemory(uintptr(gaddr+schedoffset+8), 8)
	if err != nil {
		return fmt.Errorf("error reading goid %s", err)
	}
	gopc := binary.LittleEndian.Uint64(schedbytes)
	f, l, _ := dbp.GoSymTable.PCToLine(gopc)
	fmt.Printf("Goroutine %d - %s:%d\n", binary.LittleEndian.Uint64(goidbytes), f, l)
	return nil
}

func allglenval(dbp *DebuggedProcess, data *dwarf.Data) (uint64, error) {
	entry, err := findDwarfEntry("runtime.allglen", data)
	if err != nil {
		return 0, err
	}

	instructions, ok := entry.Val(dwarf.AttrLocation).([]byte)
	if !ok {
		return 0, fmt.Errorf("type assertion failed")
	}
	addr, err := op.ExecuteStackProgram(0, instructions)
	if err != nil {
		return 0, err
	}
	val, err := dbp.CurrentThread.readMemory(uintptr(addr), 8)
	if err != nil {
		return 0, err
	}
	return binary.LittleEndian.Uint64(val), nil
}

func allgentryptr(dbp *DebuggedProcess, data *dwarf.Data) (uint64, error) {
	entry, err := findDwarfEntry("runtime.allg", data)
	if err != nil {
		return 0, err
	}

	instructions, ok := entry.Val(dwarf.AttrLocation).([]byte)
	if !ok {
		return 0, fmt.Errorf("type assertion failed")
	}
	addr, err := op.ExecuteStackProgram(0, instructions)
	if err != nil {
		return 0, err
	}

	return uint64(addr), nil
}

func parsegoidoffset(dbp *DebuggedProcess, data *dwarf.Data) (uint64, error) {
	entry, err := findDwarfEntry("goid", data)
	if err != nil {
		return 0, err
	}
	instructions, ok := entry.Val(dwarf.AttrDataMemberLoc).([]byte)
	if !ok {
		return 0, fmt.Errorf("type assertion failed")
	}
	offset, err := op.ExecuteStackProgram(0, instructions)
	if err != nil {
		return 0, err
	}

	return uint64(offset), nil
}

func parseschedoffset(dbp *DebuggedProcess, data *dwarf.Data) (uint64, error) {
	entry, err := findDwarfEntry("sched", data)
	if err != nil {
		return 0, err
	}
	instructions, ok := entry.Val(dwarf.AttrDataMemberLoc).([]byte)
	if !ok {
		return 0, fmt.Errorf("type assertion failed")
	}
	offset, err := op.ExecuteStackProgram(0, instructions)
	if err != nil {
		return 0, err
	}

	return uint64(offset), nil
}

// Returns the value of the named symbol.
func (thread *ThreadContext) EvalSymbol(name string) (*Variable, error) {
	data, err := thread.Process.Executable.DWARF()
	if err != nil {
		return nil, err
	}

	entry, err := findDwarfEntry(name, data)
	if err != nil {
		return nil, err
	}

	offset, ok := entry.Val(dwarf.AttrType).(dwarf.Offset)
	if !ok {
		return nil, fmt.Errorf("type assertion failed")
	}

	t, err := data.Type(offset)
	if err != nil {
		return nil, err
	}

	instructions, ok := entry.Val(dwarf.AttrLocation).([]byte)
	if !ok {
		return nil, fmt.Errorf("type assertion failed")
	}

	val, err := thread.extractValue(instructions, 0, t)
	if err != nil {
		return nil, err
	}

	return &Variable{Name: name, Type: t.String(), Value: val}, nil
}

func findDwarfEntry(name string, data *dwarf.Data) (*dwarf.Entry, error) {
	reader := data.Reader()

	for entry, err := reader.Next(); entry != nil; entry, err = reader.Next() {
		if err != nil {
			return nil, err
		}

		if entry.Tag != dwarf.TagVariable && entry.Tag != dwarf.TagFormalParameter && entry.Tag != dwarf.TagMember {
			continue
		}

		n, ok := entry.Val(dwarf.AttrName).(string)
		if !ok || n != name {
			continue
		}
		return entry, nil
	}
	return nil, fmt.Errorf("could not find symbol value for %s", name)
}

// Extracts the value from the instructions given in the DW_AT_location entry.
// We execute the stack program described in the DW_OP_* instruction stream, and
// then grab the value from the other processes memory.
func (thread *ThreadContext) extractValue(instructions []byte, off int64, typ interface{}) (string, error) {
	regs, err := thread.Registers()
	if err != nil {
		return "", err
	}

	fde, err := thread.Process.FrameEntries.FDEForPC(regs.PC())
	if err != nil {
		return "", err
	}

	fctx := fde.EstablishFrame(regs.PC())
	cfaOffset := fctx.CFAOffset()

	offset := off
	if off == 0 {
		offset, err = op.ExecuteStackProgram(cfaOffset, instructions)
		if err != nil {
			return "", err
		}
		offset = int64(regs.Rsp) + offset
	}

	// If we have a user defined type, find the
	// underlying concrete type and use that.
	if tt, ok := typ.(*dwarf.TypedefType); ok {
		typ = tt.Type
	}

	offaddr := uintptr(offset)
	switch t := typ.(type) {
	case *dwarf.PtrType:
		addr, err := thread.readMemory(offaddr, 8)
		if err != nil {
			return "", err
		}
		adr := binary.LittleEndian.Uint64(addr)
		val, err := thread.extractValue(nil, int64(adr), t.Type)
		if err != nil {
			return "", err
		}

		retstr := fmt.Sprintf("*%s", val)
		return retstr, nil
	case *dwarf.StructType:
		switch t.StructName {
		case "string":
			return thread.readString(offaddr)
		case "[]int":
			return thread.readIntSlice(offaddr)
		default:
			// Recursively call extractValue to grab
			// the value of all the members of the struct.
			fields := make([]string, 0, len(t.Field))
			for _, field := range t.Field {
				val, err := thread.extractValue(nil, field.ByteOffset+offset, field.Type)
				if err != nil {
					return "", err
				}

				fields = append(fields, fmt.Sprintf("%s: %s", field.Name, val))
			}
			retstr := fmt.Sprintf("%s {%s}", t.StructName, strings.Join(fields, ", "))
			return retstr, nil
		}
	case *dwarf.ArrayType:
		return thread.readIntArray(offaddr, t)
	case *dwarf.IntType:
		return thread.readInt(offaddr, t.ByteSize)
	case *dwarf.FloatType:
		return thread.readFloat(offaddr, t.ByteSize)
	}

	return "", fmt.Errorf("could not find value for type %s", typ)
}

func (thread *ThreadContext) readString(addr uintptr) (string, error) {
	val, err := thread.readMemory(addr, 8)
	if err != nil {
		return "", err
	}

	// deref the pointer to the string
	addr = uintptr(binary.LittleEndian.Uint64(val))
	val, err = thread.readMemory(addr, 16)
	if err != nil {
		return "", err
	}

	i := bytes.IndexByte(val, 0x0)
	val = val[:i]
	return *(*string)(unsafe.Pointer(&val)), nil
}

func (thread *ThreadContext) readIntSlice(addr uintptr) (string, error) {
	var number uint64

	val, err := thread.readMemory(addr, uintptr(24))
	if err != nil {
		return "", err
	}

	a := binary.LittleEndian.Uint64(val[:8])
	l := binary.LittleEndian.Uint64(val[8:16])
	c := binary.LittleEndian.Uint64(val[16:24])

	val, err = thread.readMemory(uintptr(a), uintptr(8*l))
	if err != nil {
		return "", err
	}

	members := make([]uint64, 0, l)
	buf := bytes.NewBuffer(val)
	for {
		err := binary.Read(buf, binary.LittleEndian, &number)
		if err != nil {
			break
		}

		members = append(members, number)
	}

	return fmt.Sprintf("len: %d cap: %d %d", l, c, members), nil
}

func (thread *ThreadContext) readIntArray(addr uintptr, t *dwarf.ArrayType) (string, error) {
	var (
		number  uint64
		members = make([]uint64, 0, t.ByteSize)
	)

	val, err := thread.readMemory(addr, uintptr(t.ByteSize))
	if err != nil {
		return "", err
	}

	buf := bytes.NewBuffer(val)
	for {
		err := binary.Read(buf, binary.LittleEndian, &number)
		if err != nil {
			break
		}

		members = append(members, number)
	}

	return fmt.Sprintf("[%d]int %d", t.ByteSize/8, members), nil
}

func (thread *ThreadContext) readInt(addr uintptr, size int64) (string, error) {
	var n int

	val, err := thread.readMemory(addr, uintptr(size))
	if err != nil {
		return "", err
	}

	switch size {
	case 1:
		n = int(val[0])
	case 2:
		n = int(binary.LittleEndian.Uint16(val))
	case 4:
		n = int(binary.LittleEndian.Uint32(val))
	case 8:
		n = int(binary.LittleEndian.Uint64(val))
	}

	return strconv.Itoa(n), nil
}

func (thread *ThreadContext) readFloat(addr uintptr, size int64) (string, error) {
	val, err := thread.readMemory(addr, uintptr(size))
	if err != nil {
		return "", err
	}
	buf := bytes.NewBuffer(val)

	switch size {
	case 4:
		n := float32(0)
		binary.Read(buf, binary.LittleEndian, &n)
		return strconv.FormatFloat(float64(n), 'f', -1, int(size)*8), nil
	case 8:
		n := float64(0)
		binary.Read(buf, binary.LittleEndian, &n)
		return strconv.FormatFloat(n, 'f', -1, int(size)*8), nil
	}

	return "", fmt.Errorf("could not read float")
}

func (thread *ThreadContext) readMemory(addr uintptr, size uintptr) ([]byte, error) {
	buf := make([]byte, size)

	_, err := syscall.PtracePeekData(thread.Id, addr, buf)
	if err != nil {
		return nil, err
	}

	return buf, nil
}
<|endoftext|>


<commit_before>package git

import (
	"fmt"
	"sync"
	"unsafe"
)

type HandleList struct {
	sync.RWMutex
	// stores the Go pointers
	handles []interface{}
	// indicates which indices are in use
	set map[uintptr]bool
}

func NewHandleList() *HandleList {
	return &HandleList{
		handles: make([]interface{}, 5),
		set:     make(map[uintptr]bool),
	}
}

// findUnusedSlot finds the smallest-index empty space in our
// list. You must only run this function while holding a write lock.
func (v *HandleList) findUnusedSlot() uintptr {
	for i := 1; i < len(v.handles); i++ {
		isUsed := v.set[uintptr(i)]
		if !isUsed {
			return uintptr(i)
		}
	}

	// reaching here means we've run out of entries so append and
	// return the new index, which is equal to the old length.
	slot := len(v.handles)
	v.handles = append(v.handles, nil)

	return uintptr(slot)
}

// Track adds the given pointer to the list of pointers to track and
// returns a pointer value which can be passed to C as an opaque
// pointer.
func (v *HandleList) Track(pointer interface{}) unsafe.Pointer {
	v.Lock()

	slot := v.findUnusedSlot()
	v.handles[slot] = pointer
	v.set[slot] = true

	v.Unlock()

	return unsafe.Pointer(slot)
}

// Untrack stops tracking the pointer given by the handle
func (v *HandleList) Untrack(handle unsafe.Pointer) {
	slot := uintptr(handle)

	v.Lock()

	v.handles[slot] = nil
	delete(v.set, slot)

	v.Unlock()
}

// Get retrieves the pointer from the given handle
func (v *HandleList) Get(handle unsafe.Pointer) interface{} {
	slot := uintptr(handle)

	v.RLock()

	if _, ok := v.set[slot]; !ok {
		panic(fmt.Sprintf("invalid pointer handle: %p", handle))
	}

	ptr := v.handles[slot]

	v.RUnlock()

	return ptr
}
<commit_msg>handles: do not store handles by uintptr<commit_after>package git

import (
	"fmt"
	"sync"
	"unsafe"
)

type HandleList struct {
	sync.RWMutex
	// stores the Go pointers
	handles []interface{}
	// indicates which indices are in use
	set map[int]bool
}

func NewHandleList() *HandleList {
	return &HandleList{
		handles: make([]interface{}, 5),
		set:     make(map[int]bool),
	}
}

// findUnusedSlot finds the smallest-index empty space in our
// list. You must only run this function while holding a write lock.
func (v *HandleList) findUnusedSlot() int {
	for i := 1; i < len(v.handles); i++ {
		isUsed := v.set[i]
		if !isUsed {
			return i
		}
	}

	// reaching here means we've run out of entries so append and
	// return the new index, which is equal to the old length.
	slot := len(v.handles)
	v.handles = append(v.handles, nil)

	return slot
}

// Track adds the given pointer to the list of pointers to track and
// returns a pointer value which can be passed to C as an opaque
// pointer.
func (v *HandleList) Track(pointer interface{}) unsafe.Pointer {
	v.Lock()

	slot := v.findUnusedSlot()
	v.handles[slot] = pointer
	v.set[slot] = true

	v.Unlock()

	return unsafe.Pointer(&slot)
}

// Untrack stops tracking the pointer given by the handle
func (v *HandleList) Untrack(handle unsafe.Pointer) {
	slot := *(*int)(handle)

	v.Lock()

	v.handles[slot] = nil
	delete(v.set, slot)

	v.Unlock()
}

// Get retrieves the pointer from the given handle
func (v *HandleList) Get(handle unsafe.Pointer) interface{} {
	slot := *(*int)(handle)

	v.RLock()

	if _, ok := v.set[slot]; !ok {
		panic(fmt.Sprintf("invalid pointer handle: %p", handle))
	}

	ptr := v.handles[slot]

	v.RUnlock()

	return ptr
}
<|endoftext|>


<commit_before>package acceptance

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/http/httputil"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("upload-stemcell command", func() {
	var (
		stemcellName string
		content      *os.File
		server       *httptest.Server
	)

	BeforeEach(func() {
		var err error
		content, err = ioutil.TempFile("", "cool_name.com")
		Expect(err).NotTo(HaveOccurred())

		_, err = content.WriteString("content so validation does not fail")
		Expect(err).NotTo(HaveOccurred())

		server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			var responseString string
			w.Header().Set("Content-Type", "application/json")

			switch req.URL.Path {
			case "/uaa/oauth/token":
				responseString = `{
				"access_token": "some-opsman-token",
				"token_type": "bearer",
				"expires_in": 3600
			}`
			case "/api/v0/stemcells":
				auth := req.Header.Get("Authorization")
				if auth != "Bearer some-opsman-token" {
					w.WriteHeader(http.StatusUnauthorized)
					return
				}

				err := req.ParseMultipartForm(100)
				if err != nil {
					panic(err)
				}

				stemcellName = req.MultipartForm.File["stemcell[file]"][0].Filename
				responseString = "{}"
			default:
				out, err := httputil.DumpRequest(req, true)
				Expect(err).NotTo(HaveOccurred())
				Fail(fmt.Sprintf("unexpected request: %s", out))
			}

			w.Write([]byte(responseString))
		}))
	})

	AfterEach(func() {
		err := os.Remove(content.Name())
		Expect(err).NotTo(HaveOccurred())
	})

	It("successfully sends the stemcell to ops-manager", func() {
		command := exec.Command(pathToMain,
			"--target", server.URL,
			"--username", "some-username",
			"--password", "some-password",
			"--skip-ssl-validation",
			"upload-stemcell",
			"--stemcell", content.Name(),
		)

		session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
		Expect(err).NotTo(HaveOccurred())

		Eventually(session).Should(gexec.Exit(0))
		Eventually(session.Out).Should(gbytes.Say("processing stemcell"))
		Eventually(session.Out).Should(gbytes.Say("beginning stemcell upload to Ops Manager"))
		Eventually(session.Out).Should(gbytes.Say("finished upload"))

		Expect(stemcellName).To(Equal(filepath.Base(content.Name())))
	})

	Context("when an error occurs", func() {
		Context("when the content to upload is empty", func() {
			var emptyContent *os.File

			BeforeEach(func() {
				var err error
				emptyContent, err = ioutil.TempFile("", "")
				Expect(err).NotTo(HaveOccurred())
			})

			AfterEach(func() {
				err := os.Remove(emptyContent.Name())
				Expect(err).NotTo(HaveOccurred())
			})

			It("returns an error", func() {
				command := exec.Command(pathToMain,
					"--target", server.URL,
					"--username", "some-username",
					"--password", "some-password",
					"--skip-ssl-validation",
					"upload-stemcell",
					"--stemcell", emptyContent.Name(),
				)

				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(session).Should(gexec.Exit(1))
				Eventually(session.Out).Should(gbytes.Say("failed to load stemcell: file provided has no content"))
			})
		})

		Context("when the content cannot be read", func() {
			BeforeEach(func() {
				err := content.Chmod(000)
				Expect(err).NotTo(HaveOccurred())
			})

			It("returns an error", func() {
				command := exec.Command(pathToMain,
					"--target", server.URL,
					"--username", "some-username",
					"--password", "some-password",
					"--skip-ssl-validation",
					"upload-stemcell",
					"--stemcell", content.Name(),
				)

				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(session).Should(gexec.Exit(1))
				Eventually(session.Out).Should(gbytes.Say(`permission denied`))
			})
		})
	})
})
<commit_msg>switch to non-permission based acceptance test<commit_after>package acceptance

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/http/httputil"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("upload-stemcell command", func() {
	var (
		stemcellName string
		content      *os.File
		server       *httptest.Server
	)

	BeforeEach(func() {
		var err error
		content, err = ioutil.TempFile("", "cool_name.com")
		Expect(err).NotTo(HaveOccurred())

		_, err = content.WriteString("content so validation does not fail")
		Expect(err).NotTo(HaveOccurred())

		server = httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			var responseString string
			w.Header().Set("Content-Type", "application/json")

			switch req.URL.Path {
			case "/uaa/oauth/token":
				responseString = `{
				"access_token": "some-opsman-token",
				"token_type": "bearer",
				"expires_in": 3600
			}`
			case "/api/v0/stemcells":
				auth := req.Header.Get("Authorization")
				if auth != "Bearer some-opsman-token" {
					w.WriteHeader(http.StatusUnauthorized)
					return
				}

				err := req.ParseMultipartForm(100)
				if err != nil {
					panic(err)
				}

				stemcellName = req.MultipartForm.File["stemcell[file]"][0].Filename
				responseString = "{}"
			default:
				out, err := httputil.DumpRequest(req, true)
				Expect(err).NotTo(HaveOccurred())
				Fail(fmt.Sprintf("unexpected request: %s", out))
			}

			w.Write([]byte(responseString))
		}))
	})

	AfterEach(func() {
		os.Remove(content.Name())
	})

	It("successfully sends the stemcell to ops-manager", func() {
		command := exec.Command(pathToMain,
			"--target", server.URL,
			"--username", "some-username",
			"--password", "some-password",
			"--skip-ssl-validation",
			"upload-stemcell",
			"--stemcell", content.Name(),
		)

		session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
		Expect(err).NotTo(HaveOccurred())

		Eventually(session).Should(gexec.Exit(0))
		Eventually(session.Out).Should(gbytes.Say("processing stemcell"))
		Eventually(session.Out).Should(gbytes.Say("beginning stemcell upload to Ops Manager"))
		Eventually(session.Out).Should(gbytes.Say("finished upload"))

		Expect(stemcellName).To(Equal(filepath.Base(content.Name())))
	})

	Context("when an error occurs", func() {
		Context("when the content to upload is empty", func() {
			var emptyContent *os.File

			BeforeEach(func() {
				var err error
				emptyContent, err = ioutil.TempFile("", "")
				Expect(err).NotTo(HaveOccurred())
			})

			AfterEach(func() {
				err := os.Remove(emptyContent.Name())
				Expect(err).NotTo(HaveOccurred())
			})

			It("returns an error", func() {
				command := exec.Command(pathToMain,
					"--target", server.URL,
					"--username", "some-username",
					"--password", "some-password",
					"--skip-ssl-validation",
					"upload-stemcell",
					"--stemcell", emptyContent.Name(),
				)

				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(session).Should(gexec.Exit(1))
				Eventually(session.Out).Should(gbytes.Say("failed to load stemcell: file provided has no content"))
			})
		})

		Context("when the content cannot be read", func() {
			BeforeEach(func() {
				err := os.Remove(content.Name())
				Expect(err).NotTo(HaveOccurred())
			})

			It("returns an error", func() {
				command := exec.Command(pathToMain,
					"--target", server.URL,
					"--username", "some-username",
					"--password", "some-password",
					"--skip-ssl-validation",
					"upload-stemcell",
					"--stemcell", content.Name(),
				)

				session, err := gexec.Start(command, GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(session).Should(gexec.Exit(1))
				Eventually(session.Out).Should(gbytes.Say(`no such file or directory`))
			})
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2013 Google Inc.  All rights reserved.
// Copyright 2016 the gousb Authors.  All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package usb

// #include <libusb.h>
import "C"

import (
	"fmt"
	"reflect"
	"sync"
	"time"
	"unsafe"
)

var DefaultReadTimeout = 1 * time.Second
var DefaultWriteTimeout = 1 * time.Second
var DefaultControlTimeout = 250 * time.Millisecond //5 * time.Second

type Device struct {
	handle *C.libusb_device_handle

	// Embed the device information for easy access
	*Descriptor

	// Timeouts
	ReadTimeout    time.Duration
	WriteTimeout   time.Duration
	ControlTimeout time.Duration

	// Claimed interfaces
	lock    *sync.Mutex
	claimed map[uint8]int
}

func newDevice(handle *C.libusb_device_handle, desc *Descriptor) *Device {
	ifaces := 0
	d := &Device{
		handle:         handle,
		Descriptor:     desc,
		ReadTimeout:    DefaultReadTimeout,
		WriteTimeout:   DefaultWriteTimeout,
		ControlTimeout: DefaultControlTimeout,
		lock:           new(sync.Mutex),
		claimed:        make(map[uint8]int, ifaces),
	}

	return d
}

func (d *Device) Reset() error {
	if errno := C.libusb_reset_device(d.handle); errno != 0 {
		return usbError(errno)
	}
	return nil
}

func (d *Device) Control(rType, request uint8, val, idx uint16, data []byte) (int, error) {
	//log.Printf("control xfer: %d:%d/%d:%d %x", idx, rType, request, val, string(data))
	dataSlice := (*reflect.SliceHeader)(unsafe.Pointer(&data))
	n := C.libusb_control_transfer(
		d.handle,
		C.uint8_t(rType),
		C.uint8_t(request),
		C.uint16_t(val),
		C.uint16_t(idx),
		(*C.uchar)(unsafe.Pointer(dataSlice.Data)),
		C.uint16_t(len(data)),
		C.uint(d.ControlTimeout/time.Millisecond))
	if n < 0 {
		return int(n), usbError(n)
	}
	return int(n), nil
}

// ActiveConfig returns the config id (not the index) of the active configuration.
// This corresponds to the ConfigInfo.Config field.
func (d *Device) ActiveConfig() (uint8, error) {
	var cfg C.int
	if errno := C.libusb_get_configuration(d.handle, &cfg); errno < 0 {
		return 0, usbError(errno)
	}
	return uint8(cfg), nil
}

// SetConfig attempts to change the active configuration.
// The cfg provided is the config id (not the index) of the configuration to set,
// which corresponds to the ConfigInfo.Config field.
func (d *Device) SetConfig(cfg uint8) error {
	if errno := C.libusb_set_configuration(d.handle, C.int(cfg)); errno < 0 {
		return usbError(errno)
	}
	return nil
}

// Close the device.
func (d *Device) Close() error {
	if d.handle == nil {
		return fmt.Errorf("usb: double close on device")
	}
	d.lock.Lock()
	defer d.lock.Unlock()
	for iface := range d.claimed {
		C.libusb_release_interface(d.handle, C.int(iface))
	}
	C.libusb_close(d.handle)
	d.handle = nil
	return nil
}

func (d *Device) OpenEndpoint(conf, iface, setup, epoint uint8) (Endpoint, error) {
	end := &endpoint{
		Device: d,
	}

	var setAlternate bool
	for _, c := range d.Configs {
		if c.Config != conf {
			continue
		}
		debug.Printf("found conf: %#v\n", c)
		for _, i := range c.Interfaces {
			if i.Number != iface {
				continue
			}
			debug.Printf("found iface: %#v\n", i)
			for i, s := range i.Setups {
				if s.Alternate != setup {
					continue
				}
				setAlternate = i != 0

				debug.Printf("found setup: %#v [default: %v]\n", s, !setAlternate)
				for _, e := range s.Endpoints {
					debug.Printf("ep %02x search: %#v\n", epoint, s)
					if e.Address != epoint {
						continue
					}
					end.InterfaceSetup = s
					end.EndpointInfo = e
					goto found
				}
				return nil, fmt.Errorf("usb: unknown endpoint %02x", epoint)
			}
			return nil, fmt.Errorf("usb: unknown setup %02x", setup)
		}
		return nil, fmt.Errorf("usb: unknown interface %02x", iface)
	}
	return nil, fmt.Errorf("usb: unknown configuration %02x", conf)

found:

	// Set the configuration
	var activeConf C.int
	if errno := C.libusb_get_configuration(d.handle, &activeConf); errno < 0 {
		return nil, fmt.Errorf("usb: getcfg: %s", usbError(errno))
	}
	if int(activeConf) != int(conf) {
		if errno := C.libusb_set_configuration(d.handle, C.int(conf)); errno < 0 {
			return nil, fmt.Errorf("usb: setcfg: %s", usbError(errno))
		}
	}

	// Claim the interface
	if errno := C.libusb_claim_interface(d.handle, C.int(iface)); errno < 0 {
		return nil, fmt.Errorf("usb: claim: %s", usbError(errno))
	}

	// Increment the claim count
	d.lock.Lock()
	d.claimed[iface]++
	d.lock.Unlock() // unlock immediately because the next calls may block

	// Choose the alternate
	if setAlternate {
		if errno := C.libusb_set_interface_alt_setting(d.handle, C.int(iface), C.int(setup)); errno < 0 {
			debug.Printf("altsetting error: %s", usbError(errno))
			return nil, fmt.Errorf("usb: setalt: %s", usbError(errno))
		}
	}

	return end, nil
}

func (d *Device) GetStringDescriptor(desc_index int) (string, error) {

	// allocate 200-byte array limited the length of string descriptor
	goBuffer := make([]byte, 200)

	// get string descriptor from libusb. if errno < 0 then there are any errors.
	// if errno >= 0; it is a length of result string descriptor
	errno := C.libusb_get_string_descriptor_ascii(
		d.handle,
		C.uint8_t(desc_index),
		(*C.uchar)(unsafe.Pointer(&goBuffer[0])),
		200)

	// if any errors occur
	if errno < 0 {
		return "", fmt.Errorf("usb: getstr: %s", usbError(errno))
	}
	// convert slice of byte to string with limited length from errno
	stringDescriptor := string(goBuffer[:errno])

	return stringDescriptor, nil
}

// SetAutoDetach Enables/disables libusb's automatic kernel driver detachment.
// When autodetach is enabled libusb will automatically detach the kernel driver
// on the interface and reattach it when releasing the interface.
// Automatic kernel driver detachment is disabled on newly opened device handles by default.
func (d *Device) SetAutoDetach(autodetach bool) error {
	autodetachInt := 0
	if autodetach {
		autodetachInt = 1
	}

	err := C.libusb_set_auto_detach_kernel_driver(
		d.handle,
		C.int(autodetachInt),
	)

	// TODO LIBUSB_ERROR_NOT_SUPPORTED (-12) handling
	// if any errors occur
	if err != C.int(SUCCESS) {
		return fmt.Errorf("usb: setautodetach: %s", usbError(err))
	}
	return nil
}
<commit_msg>comment nitpicking :)<commit_after>// Copyright 2013 Google Inc.  All rights reserved.
// Copyright 2016 the gousb Authors.  All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package usb

// #include <libusb.h>
import "C"

import (
	"fmt"
	"reflect"
	"sync"
	"time"
	"unsafe"
)

var DefaultReadTimeout = 1 * time.Second
var DefaultWriteTimeout = 1 * time.Second
var DefaultControlTimeout = 250 * time.Millisecond //5 * time.Second

type Device struct {
	handle *C.libusb_device_handle

	// Embed the device information for easy access
	*Descriptor

	// Timeouts
	ReadTimeout    time.Duration
	WriteTimeout   time.Duration
	ControlTimeout time.Duration

	// Claimed interfaces
	lock    *sync.Mutex
	claimed map[uint8]int
}

func newDevice(handle *C.libusb_device_handle, desc *Descriptor) *Device {
	ifaces := 0
	d := &Device{
		handle:         handle,
		Descriptor:     desc,
		ReadTimeout:    DefaultReadTimeout,
		WriteTimeout:   DefaultWriteTimeout,
		ControlTimeout: DefaultControlTimeout,
		lock:           new(sync.Mutex),
		claimed:        make(map[uint8]int, ifaces),
	}

	return d
}

func (d *Device) Reset() error {
	if errno := C.libusb_reset_device(d.handle); errno != 0 {
		return usbError(errno)
	}
	return nil
}

func (d *Device) Control(rType, request uint8, val, idx uint16, data []byte) (int, error) {
	//log.Printf("control xfer: %d:%d/%d:%d %x", idx, rType, request, val, string(data))
	dataSlice := (*reflect.SliceHeader)(unsafe.Pointer(&data))
	n := C.libusb_control_transfer(
		d.handle,
		C.uint8_t(rType),
		C.uint8_t(request),
		C.uint16_t(val),
		C.uint16_t(idx),
		(*C.uchar)(unsafe.Pointer(dataSlice.Data)),
		C.uint16_t(len(data)),
		C.uint(d.ControlTimeout/time.Millisecond))
	if n < 0 {
		return int(n), usbError(n)
	}
	return int(n), nil
}

// ActiveConfig returns the config id (not the index) of the active configuration.
// This corresponds to the ConfigInfo.Config field.
func (d *Device) ActiveConfig() (uint8, error) {
	var cfg C.int
	if errno := C.libusb_get_configuration(d.handle, &cfg); errno < 0 {
		return 0, usbError(errno)
	}
	return uint8(cfg), nil
}

// SetConfig attempts to change the active configuration.
// The cfg provided is the config id (not the index) of the configuration to set,
// which corresponds to the ConfigInfo.Config field.
func (d *Device) SetConfig(cfg uint8) error {
	if errno := C.libusb_set_configuration(d.handle, C.int(cfg)); errno < 0 {
		return usbError(errno)
	}
	return nil
}

// Close the device.
func (d *Device) Close() error {
	if d.handle == nil {
		return fmt.Errorf("usb: double close on device")
	}
	d.lock.Lock()
	defer d.lock.Unlock()
	for iface := range d.claimed {
		C.libusb_release_interface(d.handle, C.int(iface))
	}
	C.libusb_close(d.handle)
	d.handle = nil
	return nil
}

func (d *Device) OpenEndpoint(conf, iface, setup, epoint uint8) (Endpoint, error) {
	end := &endpoint{
		Device: d,
	}

	var setAlternate bool
	for _, c := range d.Configs {
		if c.Config != conf {
			continue
		}
		debug.Printf("found conf: %#v\n", c)
		for _, i := range c.Interfaces {
			if i.Number != iface {
				continue
			}
			debug.Printf("found iface: %#v\n", i)
			for i, s := range i.Setups {
				if s.Alternate != setup {
					continue
				}
				setAlternate = i != 0

				debug.Printf("found setup: %#v [default: %v]\n", s, !setAlternate)
				for _, e := range s.Endpoints {
					debug.Printf("ep %02x search: %#v\n", epoint, s)
					if e.Address != epoint {
						continue
					}
					end.InterfaceSetup = s
					end.EndpointInfo = e
					goto found
				}
				return nil, fmt.Errorf("usb: unknown endpoint %02x", epoint)
			}
			return nil, fmt.Errorf("usb: unknown setup %02x", setup)
		}
		return nil, fmt.Errorf("usb: unknown interface %02x", iface)
	}
	return nil, fmt.Errorf("usb: unknown configuration %02x", conf)

found:

	// Set the configuration
	var activeConf C.int
	if errno := C.libusb_get_configuration(d.handle, &activeConf); errno < 0 {
		return nil, fmt.Errorf("usb: getcfg: %s", usbError(errno))
	}
	if int(activeConf) != int(conf) {
		if errno := C.libusb_set_configuration(d.handle, C.int(conf)); errno < 0 {
			return nil, fmt.Errorf("usb: setcfg: %s", usbError(errno))
		}
	}

	// Claim the interface
	if errno := C.libusb_claim_interface(d.handle, C.int(iface)); errno < 0 {
		return nil, fmt.Errorf("usb: claim: %s", usbError(errno))
	}

	// Increment the claim count
	d.lock.Lock()
	d.claimed[iface]++
	d.lock.Unlock() // unlock immediately because the next calls may block

	// Choose the alternate
	if setAlternate {
		if errno := C.libusb_set_interface_alt_setting(d.handle, C.int(iface), C.int(setup)); errno < 0 {
			debug.Printf("altsetting error: %s", usbError(errno))
			return nil, fmt.Errorf("usb: setalt: %s", usbError(errno))
		}
	}

	return end, nil
}

func (d *Device) GetStringDescriptor(desc_index int) (string, error) {

	// allocate 200-byte array limited the length of string descriptor
	goBuffer := make([]byte, 200)

	// get string descriptor from libusb. if errno < 0 then there are any errors.
	// if errno >= 0; it is a length of result string descriptor
	errno := C.libusb_get_string_descriptor_ascii(
		d.handle,
		C.uint8_t(desc_index),
		(*C.uchar)(unsafe.Pointer(&goBuffer[0])),
		200)

	// if any errors occur
	if errno < 0 {
		return "", fmt.Errorf("usb: getstr: %s", usbError(errno))
	}
	// convert slice of byte to string with limited length from errno
	stringDescriptor := string(goBuffer[:errno])

	return stringDescriptor, nil
}

// SetAutoDetach enables/disables libusb's automatic kernel driver detachment.
// When autodetach is enabled libusb will automatically detach the kernel driver
// on the interface and reattach it when releasing the interface.
// Automatic kernel driver detachment is disabled on newly opened device handles by default.
func (d *Device) SetAutoDetach(autodetach bool) error {
	autodetachInt := 0
	if autodetach {
		autodetachInt = 1
	}

	err := C.libusb_set_auto_detach_kernel_driver(
		d.handle,
		C.int(autodetachInt),
	)

	// TODO LIBUSB_ERROR_NOT_SUPPORTED (-12) handling
	// if any errors occur
	if err != C.int(SUCCESS) {
		return fmt.Errorf("usb: setautodetach: %s", usbError(err))
	}
	return nil
}
<|endoftext|>


<commit_before><commit_msg>fix remote file read<commit_after><|endoftext|>


<commit_before>package collectors

import (
	"errors"
	"fmt"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"golang.org/x/net/context"
	"google.golang.org/api/monitoring/v3"

	"github.com/frodenas/stackdriver_exporter/utils"
)

type MonitoringCollector struct {
	projectID                       string
	metricsTypePrefixes             []string
	metricsInterval                 time.Duration
	monitoringService               *monitoring.Service
	apiCallsTotalMetric             prometheus.Counter
	scrapesTotalMetric              prometheus.Counter
	scrapeErrorsTotalMetric         prometheus.Counter
	lastScrapeErrorMetric           prometheus.Gauge
	lastScrapeTimestampMetric       prometheus.Gauge
	lastScrapeDurationSecondsMetric prometheus.Gauge
}

func NewMonitoringCollector(projectID string, metricsTypePrefixes []string, metricsInterval time.Duration, monitoringService *monitoring.Service) (*MonitoringCollector, error) {
	apiCallsTotalMetric := prometheus.NewCounter(
		prometheus.CounterOpts{
			Namespace:   "stackdriver",
			Subsystem:   "monitoring",
			Name:        "api_calls_total",
			Help:        "Total number of Google Stackdriver Monitoring API calls made.",
			ConstLabels: prometheus.Labels{"project_id": projectID},
		},
	)

	scrapesTotalMetric := prometheus.NewCounter(
		prometheus.CounterOpts{
			Namespace:   "stackdriver",
			Subsystem:   "monitoring",
			Name:        "scrapes_total",
			Help:        "Total number of Google Stackdriver Monitoring metrics scrapes.",
			ConstLabels: prometheus.Labels{"project_id": projectID},
		},
	)

	scrapeErrorsTotalMetric := prometheus.NewCounter(
		prometheus.CounterOpts{
			Namespace:   "stackdriver",
			Subsystem:   "monitoring",
			Name:        "scrape_errors_total",
			Help:        "Total number of Google Stackdriver Monitoring metrics scrape errors.",
			ConstLabels: prometheus.Labels{"project_id": projectID},
		},
	)

	lastScrapeErrorMetric := prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace:   "stackdriver",
			Subsystem:   "monitoring",
			Name:        "last_scrape_error",
			Help:        "Whether the last metrics scrape from Google Stackdriver Monitoring resulted in an error (1 for error, 0 for success).",
			ConstLabels: prometheus.Labels{"project_id": projectID},
		},
	)

	lastScrapeTimestampMetric := prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace:   "stackdriver",
			Subsystem:   "monitoring",
			Name:        "last_scrape_timestamp",
			Help:        "Number of seconds since 1970 since last metrics scrape from Google Stackdriver Monitoring.",
			ConstLabels: prometheus.Labels{"project_id": projectID},
		},
	)

	lastScrapeDurationSecondsMetric := prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace:   "stackdriver",
			Subsystem:   "monitoring",
			Name:        "last_scrape_duration_seconds",
			Help:        "Duration of the last metrics scrape from Google Stackdriver Monitoring.",
			ConstLabels: prometheus.Labels{"project_id": projectID},
		},
	)

	monitoringCollector := &MonitoringCollector{
		projectID:                       projectID,
		metricsTypePrefixes:             metricsTypePrefixes,
		metricsInterval:                 metricsInterval,
		monitoringService:               monitoringService,
		apiCallsTotalMetric:             apiCallsTotalMetric,
		scrapesTotalMetric:              scrapesTotalMetric,
		scrapeErrorsTotalMetric:         scrapeErrorsTotalMetric,
		lastScrapeErrorMetric:           lastScrapeErrorMetric,
		lastScrapeTimestampMetric:       lastScrapeTimestampMetric,
		lastScrapeDurationSecondsMetric: lastScrapeDurationSecondsMetric,
	}

	return monitoringCollector, nil
}

func (c *MonitoringCollector) Describe(ch chan<- *prometheus.Desc) {
	c.apiCallsTotalMetric.Describe(ch)
	c.scrapesTotalMetric.Describe(ch)
	c.scrapeErrorsTotalMetric.Describe(ch)
	c.lastScrapeErrorMetric.Describe(ch)
	c.lastScrapeTimestampMetric.Describe(ch)
	c.lastScrapeDurationSecondsMetric.Describe(ch)
}

func (c *MonitoringCollector) Collect(ch chan<- prometheus.Metric) {
	var begun = time.Now()

	errorMetric := float64(0)
	if err := c.reportMonitoringMetrics(ch); err != nil {
		errorMetric = float64(1)
		c.scrapeErrorsTotalMetric.Inc()
		log.Errorf("Error while getting Google Stackdriver Monitoring metrics: %s", err)
	}
	c.scrapeErrorsTotalMetric.Collect(ch)

	c.apiCallsTotalMetric.Collect(ch)

	c.scrapesTotalMetric.Inc()
	c.scrapesTotalMetric.Collect(ch)

	c.lastScrapeErrorMetric.Set(errorMetric)
	c.lastScrapeErrorMetric.Collect(ch)

	c.lastScrapeTimestampMetric.Set(float64(time.Now().Unix()))
	c.lastScrapeTimestampMetric.Collect(ch)

	c.lastScrapeDurationSecondsMetric.Set(time.Since(begun).Seconds())
	c.lastScrapeDurationSecondsMetric.Collect(ch)
}

func (c *MonitoringCollector) reportMonitoringMetrics(ch chan<- prometheus.Metric) error {
	metricDescriptorsFunction := func(page *monitoring.ListMetricDescriptorsResponse) error {
		var wg = &sync.WaitGroup{}

		c.apiCallsTotalMetric.Inc()

		doneChannel := make(chan bool, 1)
		errChannel := make(chan error, 1)

		startTime := time.Now().UTC().Add(c.metricsInterval * -1)
		endTime := time.Now().UTC()

		for _, metricDescriptor := range page.MetricDescriptors {
			wg.Add(1)
			go func(metricDescriptor *monitoring.MetricDescriptor, ch chan<- prometheus.Metric) {
				defer wg.Done()
				log.Debugf("Retrieving Google Stackdriver Monitoring metrics for descriptor `%s`...", metricDescriptor.Type)
				timeSeriesListCall := c.monitoringService.Projects.TimeSeries.List(utils.ProjectResource(c.projectID)).
					Filter(fmt.Sprintf("metric.type=\"%s\"", metricDescriptor.Type)).
					IntervalStartTime(startTime.Format(time.RFC3339Nano)).
					IntervalEndTime(endTime.Format(time.RFC3339Nano))

				for {
					c.apiCallsTotalMetric.Inc()
					page, err := timeSeriesListCall.Do()
					if err != nil {
						errChannel <- err
						break
					}
					if err := c.reportTimeSeriesMetrics(page, metricDescriptor, ch); err != nil {
						errChannel <- err
						break
					}
					if page.NextPageToken == "" {
						break
					}
					timeSeriesListCall.PageToken(page.NextPageToken)
				}
			}(metricDescriptor, ch)
		}

		go func() {
			wg.Wait()
			close(doneChannel)
		}()

		select {
		case <-doneChannel:
		case err := <-errChannel:
			return err
		}

		return nil
	}

	var wg = &sync.WaitGroup{}

	doneChannel := make(chan bool, 1)
	errChannel := make(chan error, 1)

	for _, metricsTypePrefix := range c.metricsTypePrefixes {
		wg.Add(1)
		go func(metricsTypePrefix string) {
			defer wg.Done()
			log.Debugf("Listing Google Stackdriver Monitoring metric descriptors starting with `%s`...", metricsTypePrefix)
			ctx := context.Background()
			if err := c.monitoringService.Projects.MetricDescriptors.List(utils.ProjectResource(c.projectID)).
				Filter(fmt.Sprintf("metric.type = starts_with(\"%s\")", metricsTypePrefix)).
				Pages(ctx, metricDescriptorsFunction); err != nil {
				errChannel <- err
			}
		}(metricsTypePrefix)
	}

	go func() {
		wg.Wait()
		close(doneChannel)
	}()

	select {
	case <-doneChannel:
	case err := <-errChannel:
		return err
	}

	return nil
}

func (c *MonitoringCollector) reportTimeSeriesMetrics(page *monitoring.ListTimeSeriesResponse, metricDescriptor *monitoring.MetricDescriptor, ch chan<- prometheus.Metric) error {
	var metricValue float64
	var metricValueType prometheus.ValueType
	var newestTSPoint *monitoring.Point

	for _, timeSeries := range page.TimeSeries {
		newestEndTime := time.Unix(0, 0)
		for _, point := range timeSeries.Points {
			endTime, err := time.Parse(time.RFC3339Nano, point.Interval.EndTime)
			if err != nil {
				return errors.New(fmt.Sprintf("Error parsing TimeSeries Point interval end time `%s`: %s", point.Interval.EndTime, err))
			}
			if endTime.After(newestEndTime) {
				newestEndTime = endTime
				newestTSPoint = point
			}
		}

		switch timeSeries.MetricKind {
		case "GAUGE":
			metricValueType = prometheus.GaugeValue
		case "DELTA":
			metricValueType = prometheus.CounterValue
		case "CUMULATIVE":
			metricValueType = prometheus.CounterValue
		default:
			continue
		}

		switch timeSeries.ValueType {
		case "BOOL":
			metricValue = 0
			if *newestTSPoint.Value.BoolValue {
				metricValue = 1
			}
		case "INT64":
			metricValue = float64(*newestTSPoint.Value.Int64Value)
		case "DOUBLE":
			metricValue = *newestTSPoint.Value.DoubleValue
		default:
			log.Debugf("Discarding `%s` metric: %+v", timeSeries.ValueType, timeSeries)
			continue
		}

		labelKeys := []string{"unit", "resource_type"}
		labelValues := []string{metricDescriptor.Unit, timeSeries.Resource.Type}
		for key, value := range timeSeries.Metric.Labels {
			labelKeys = append(labelKeys, key)
			labelValues = append(labelValues, value)
		}
		for key, value := range timeSeries.Resource.Labels {
			labelKeys = append(labelKeys, key)
			labelValues = append(labelValues, value)
		}

		ch <- prometheus.MustNewConstMetric(
			prometheus.NewDesc(
				prometheus.BuildFQName("stackdriver", "monitoring", utils.NormalizeMetricName(timeSeries.Metric.Type)),
				metricDescriptor.Description,
				labelKeys,
				prometheus.Labels{},
			),
			metricValueType,
			metricValue,
			labelValues...,
		)
	}

	return nil
}
<commit_msg>Exit loop if page is nil<commit_after>package collectors

import (
	"errors"
	"fmt"
	"sync"
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/log"
	"golang.org/x/net/context"
	"google.golang.org/api/monitoring/v3"

	"github.com/frodenas/stackdriver_exporter/utils"
)

type MonitoringCollector struct {
	projectID                       string
	metricsTypePrefixes             []string
	metricsInterval                 time.Duration
	monitoringService               *monitoring.Service
	apiCallsTotalMetric             prometheus.Counter
	scrapesTotalMetric              prometheus.Counter
	scrapeErrorsTotalMetric         prometheus.Counter
	lastScrapeErrorMetric           prometheus.Gauge
	lastScrapeTimestampMetric       prometheus.Gauge
	lastScrapeDurationSecondsMetric prometheus.Gauge
}

func NewMonitoringCollector(projectID string, metricsTypePrefixes []string, metricsInterval time.Duration, monitoringService *monitoring.Service) (*MonitoringCollector, error) {
	apiCallsTotalMetric := prometheus.NewCounter(
		prometheus.CounterOpts{
			Namespace:   "stackdriver",
			Subsystem:   "monitoring",
			Name:        "api_calls_total",
			Help:        "Total number of Google Stackdriver Monitoring API calls made.",
			ConstLabels: prometheus.Labels{"project_id": projectID},
		},
	)

	scrapesTotalMetric := prometheus.NewCounter(
		prometheus.CounterOpts{
			Namespace:   "stackdriver",
			Subsystem:   "monitoring",
			Name:        "scrapes_total",
			Help:        "Total number of Google Stackdriver Monitoring metrics scrapes.",
			ConstLabels: prometheus.Labels{"project_id": projectID},
		},
	)

	scrapeErrorsTotalMetric := prometheus.NewCounter(
		prometheus.CounterOpts{
			Namespace:   "stackdriver",
			Subsystem:   "monitoring",
			Name:        "scrape_errors_total",
			Help:        "Total number of Google Stackdriver Monitoring metrics scrape errors.",
			ConstLabels: prometheus.Labels{"project_id": projectID},
		},
	)

	lastScrapeErrorMetric := prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace:   "stackdriver",
			Subsystem:   "monitoring",
			Name:        "last_scrape_error",
			Help:        "Whether the last metrics scrape from Google Stackdriver Monitoring resulted in an error (1 for error, 0 for success).",
			ConstLabels: prometheus.Labels{"project_id": projectID},
		},
	)

	lastScrapeTimestampMetric := prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace:   "stackdriver",
			Subsystem:   "monitoring",
			Name:        "last_scrape_timestamp",
			Help:        "Number of seconds since 1970 since last metrics scrape from Google Stackdriver Monitoring.",
			ConstLabels: prometheus.Labels{"project_id": projectID},
		},
	)

	lastScrapeDurationSecondsMetric := prometheus.NewGauge(
		prometheus.GaugeOpts{
			Namespace:   "stackdriver",
			Subsystem:   "monitoring",
			Name:        "last_scrape_duration_seconds",
			Help:        "Duration of the last metrics scrape from Google Stackdriver Monitoring.",
			ConstLabels: prometheus.Labels{"project_id": projectID},
		},
	)

	monitoringCollector := &MonitoringCollector{
		projectID:                       projectID,
		metricsTypePrefixes:             metricsTypePrefixes,
		metricsInterval:                 metricsInterval,
		monitoringService:               monitoringService,
		apiCallsTotalMetric:             apiCallsTotalMetric,
		scrapesTotalMetric:              scrapesTotalMetric,
		scrapeErrorsTotalMetric:         scrapeErrorsTotalMetric,
		lastScrapeErrorMetric:           lastScrapeErrorMetric,
		lastScrapeTimestampMetric:       lastScrapeTimestampMetric,
		lastScrapeDurationSecondsMetric: lastScrapeDurationSecondsMetric,
	}

	return monitoringCollector, nil
}

func (c *MonitoringCollector) Describe(ch chan<- *prometheus.Desc) {
	c.apiCallsTotalMetric.Describe(ch)
	c.scrapesTotalMetric.Describe(ch)
	c.scrapeErrorsTotalMetric.Describe(ch)
	c.lastScrapeErrorMetric.Describe(ch)
	c.lastScrapeTimestampMetric.Describe(ch)
	c.lastScrapeDurationSecondsMetric.Describe(ch)
}

func (c *MonitoringCollector) Collect(ch chan<- prometheus.Metric) {
	var begun = time.Now()

	errorMetric := float64(0)
	if err := c.reportMonitoringMetrics(ch); err != nil {
		errorMetric = float64(1)
		c.scrapeErrorsTotalMetric.Inc()
		log.Errorf("Error while getting Google Stackdriver Monitoring metrics: %s", err)
	}
	c.scrapeErrorsTotalMetric.Collect(ch)

	c.apiCallsTotalMetric.Collect(ch)

	c.scrapesTotalMetric.Inc()
	c.scrapesTotalMetric.Collect(ch)

	c.lastScrapeErrorMetric.Set(errorMetric)
	c.lastScrapeErrorMetric.Collect(ch)

	c.lastScrapeTimestampMetric.Set(float64(time.Now().Unix()))
	c.lastScrapeTimestampMetric.Collect(ch)

	c.lastScrapeDurationSecondsMetric.Set(time.Since(begun).Seconds())
	c.lastScrapeDurationSecondsMetric.Collect(ch)
}

func (c *MonitoringCollector) reportMonitoringMetrics(ch chan<- prometheus.Metric) error {
	metricDescriptorsFunction := func(page *monitoring.ListMetricDescriptorsResponse) error {
		var wg = &sync.WaitGroup{}

		c.apiCallsTotalMetric.Inc()

		doneChannel := make(chan bool, 1)
		errChannel := make(chan error, 1)

		startTime := time.Now().UTC().Add(c.metricsInterval * -1)
		endTime := time.Now().UTC()

		for _, metricDescriptor := range page.MetricDescriptors {
			wg.Add(1)
			go func(metricDescriptor *monitoring.MetricDescriptor, ch chan<- prometheus.Metric) {
				defer wg.Done()
				log.Debugf("Retrieving Google Stackdriver Monitoring metrics for descriptor `%s`...", metricDescriptor.Type)
				timeSeriesListCall := c.monitoringService.Projects.TimeSeries.List(utils.ProjectResource(c.projectID)).
					Filter(fmt.Sprintf("metric.type=\"%s\"", metricDescriptor.Type)).
					IntervalStartTime(startTime.Format(time.RFC3339Nano)).
					IntervalEndTime(endTime.Format(time.RFC3339Nano))

				for {
					c.apiCallsTotalMetric.Inc()
					page, err := timeSeriesListCall.Do()
					if err != nil {
						errChannel <- err
						break
					}
					if page == nil {
						break
					}
					if err := c.reportTimeSeriesMetrics(page, metricDescriptor, ch); err != nil {
						errChannel <- err
						break
					}
					if page.NextPageToken == "" {
						break
					}
					timeSeriesListCall.PageToken(page.NextPageToken)
				}
			}(metricDescriptor, ch)
		}

		go func() {
			wg.Wait()
			close(doneChannel)
		}()

		select {
		case <-doneChannel:
		case err := <-errChannel:
			return err
		}

		return nil
	}

	var wg = &sync.WaitGroup{}

	doneChannel := make(chan bool, 1)
	errChannel := make(chan error, 1)

	for _, metricsTypePrefix := range c.metricsTypePrefixes {
		wg.Add(1)
		go func(metricsTypePrefix string) {
			defer wg.Done()
			log.Debugf("Listing Google Stackdriver Monitoring metric descriptors starting with `%s`...", metricsTypePrefix)
			ctx := context.Background()
			if err := c.monitoringService.Projects.MetricDescriptors.List(utils.ProjectResource(c.projectID)).
				Filter(fmt.Sprintf("metric.type = starts_with(\"%s\")", metricsTypePrefix)).
				Pages(ctx, metricDescriptorsFunction); err != nil {
				errChannel <- err
			}
		}(metricsTypePrefix)
	}

	go func() {
		wg.Wait()
		close(doneChannel)
	}()

	select {
	case <-doneChannel:
	case err := <-errChannel:
		return err
	}

	return nil
}

func (c *MonitoringCollector) reportTimeSeriesMetrics(page *monitoring.ListTimeSeriesResponse, metricDescriptor *monitoring.MetricDescriptor, ch chan<- prometheus.Metric) error {
	var metricValue float64
	var metricValueType prometheus.ValueType
	var newestTSPoint *monitoring.Point

	for _, timeSeries := range page.TimeSeries {
		newestEndTime := time.Unix(0, 0)
		for _, point := range timeSeries.Points {
			endTime, err := time.Parse(time.RFC3339Nano, point.Interval.EndTime)
			if err != nil {
				return errors.New(fmt.Sprintf("Error parsing TimeSeries Point interval end time `%s`: %s", point.Interval.EndTime, err))
			}
			if endTime.After(newestEndTime) {
				newestEndTime = endTime
				newestTSPoint = point
			}
		}

		switch timeSeries.MetricKind {
		case "GAUGE":
			metricValueType = prometheus.GaugeValue
		case "DELTA":
			metricValueType = prometheus.CounterValue
		case "CUMULATIVE":
			metricValueType = prometheus.CounterValue
		default:
			continue
		}

		switch timeSeries.ValueType {
		case "BOOL":
			metricValue = 0
			if *newestTSPoint.Value.BoolValue {
				metricValue = 1
			}
		case "INT64":
			metricValue = float64(*newestTSPoint.Value.Int64Value)
		case "DOUBLE":
			metricValue = *newestTSPoint.Value.DoubleValue
		default:
			log.Debugf("Discarding `%s` metric: %+v", timeSeries.ValueType, timeSeries)
			continue
		}

		labelKeys := []string{"unit", "resource_type"}
		labelValues := []string{metricDescriptor.Unit, timeSeries.Resource.Type}
		for key, value := range timeSeries.Metric.Labels {
			labelKeys = append(labelKeys, key)
			labelValues = append(labelValues, value)
		}
		for key, value := range timeSeries.Resource.Labels {
			labelKeys = append(labelKeys, key)
			labelValues = append(labelValues, value)
		}

		ch <- prometheus.MustNewConstMetric(
			prometheus.NewDesc(
				prometheus.BuildFQName("stackdriver", "monitoring", utils.NormalizeMetricName(timeSeries.Metric.Type)),
				metricDescriptor.Description,
				labelKeys,
				prometheus.Labels{},
			),
			metricValueType,
			metricValue,
			labelValues...,
		)
	}

	return nil
}
<|endoftext|>