<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package etcd

import (
	"fmt"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/klog/v2"
	utilsnet "k8s.io/utils/net"

	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	"k8s.io/kubernetes/cmd/kubeadm/app/images"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
	staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
)

const (
	etcdVolumeName           = "etcd-data"
	certsVolumeName          = "etcd-certs"
	etcdHealthyCheckInterval = 5 * time.Second
	etcdHealthyCheckRetries  = 8
)

// CreateLocalEtcdStaticPodManifestFile will write local etcd static pod manifest file.
// This function is used by init - when the etcd cluster is empty - or by kubeadm
// upgrade - when the etcd cluster is already up and running (and the --initial-cluster flag have no impact)
func CreateLocalEtcdStaticPodManifestFile(manifestDir, patchesDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error {
	if cfg.Etcd.External != nil {
		return errors.New("etcd static pod manifest cannot be generated for cluster using external etcd")
	}
	// gets etcd StaticPodSpec
	spec := GetEtcdPodSpec(cfg, endpoint, nodeName, []etcdutil.Member{})

	// if patchesDir is defined, patch the static Pod manifest
	if patchesDir != "" {
		patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout)
		if err != nil {
			return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", kubeadmconstants.Etcd)
		}
		spec = *patchedSpec
	}

	// writes etcd StaticPod to disk
	if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
		return err
	}

	klog.V(1).Infof("[etcd] wrote Static Pod manifest for a local etcd member to %q\n", kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.Etcd, manifestDir))
	return nil
}

// CheckLocalEtcdClusterStatus verifies health state of local/stacked etcd cluster before installing a new etcd member
func CheckLocalEtcdClusterStatus(client clientset.Interface, cfg *kubeadmapi.ClusterConfiguration) error {
	klog.V(1).Info("[etcd] Checking etcd cluster health")

	// creates an etcd client that connects to all the local/stacked etcd members
	klog.V(1).Info("creating etcd client that connects to etcd pods")
	etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir)
	if err != nil {
		return err
	}

	// Checking health state
	err = etcdClient.CheckClusterHealth()
	if err != nil {
		return errors.Wrap(err, "etcd cluster is not healthy")
	}

	return nil
}

// RemoveStackedEtcdMemberFromCluster will remove a local etcd member from etcd cluster,
// when reset the control plane node.
func RemoveStackedEtcdMemberFromCluster(client clientset.Interface, cfg *kubeadmapi.InitConfiguration) error {
	// creates an etcd client that connects to all the local/stacked etcd members
	klog.V(1).Info("[etcd] creating etcd client that connects to etcd pods")
	etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir)
	if err != nil {
		return err
	}

	members, err := etcdClient.ListMembers()
	if err != nil {
		return err
	}
	// If this is the only remaining stacked etcd member in the cluster, calling RemoveMember()
	// is not needed.
	if len(members) == 1 {
		etcdClientAddress := etcdutil.GetClientURL(&cfg.LocalAPIEndpoint)
		for _, endpoint := range etcdClient.Endpoints {
			if endpoint == etcdClientAddress {
				klog.V(1).Info("[etcd] This is the only remaining etcd member in the etcd cluster, skip removing it")
				return nil
			}
		}
	}

	// notifies the other members of the etcd cluster about the removing member
	etcdPeerAddress := etcdutil.GetPeerURL(&cfg.LocalAPIEndpoint)

	klog.V(2).Infof("[etcd] get the member id from peer: %s", etcdPeerAddress)
	id, err := etcdClient.GetMemberID(etcdPeerAddress)
	if err != nil {
		return err
	}

	klog.V(1).Infof("[etcd] removing etcd member: %s, id: %d", etcdPeerAddress, id)
	members, err = etcdClient.RemoveMember(id)
	if err != nil {
		return err
	}
	klog.V(1).Infof("[etcd] Updated etcd member list: %v", members)

	return nil
}

// CreateStackedEtcdStaticPodManifestFile will write local etcd static pod manifest file
// for an additional etcd member that is joining an existing local/stacked etcd cluster.
// Other members of the etcd cluster will be notified of the joining node in beforehand as well.
func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir, patchesDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error {
	// creates an etcd client that connects to all the local/stacked etcd members
	klog.V(1).Info("creating etcd client that connects to etcd pods")
	etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir)
	if err != nil {
		return err
	}

	etcdPeerAddress := etcdutil.GetPeerURL(endpoint)

	klog.V(1).Infoln("[etcd] Getting the list of existing members")
	initialCluster, err := etcdClient.ListMembers()
	if err != nil {
		return err
	}

	// only add the new member if it doesn't already exists
	var exists bool
	klog.V(1).Infof("[etcd] Checking if the etcd member already exists: %s", etcdPeerAddress)
	for i := range initialCluster {
		if initialCluster[i].PeerURL == etcdPeerAddress {
			exists = true
			if len(initialCluster[i].Name) == 0 {
				klog.V(1).Infof("[etcd] etcd member name is empty. Setting it to the node name: %s", nodeName)
				initialCluster[i].Name = nodeName
			}
			break
		}
	}

	if exists {
		klog.V(1).Infof("[etcd] Etcd member already exists: %s", endpoint)
	} else {
		klog.V(1).Infof("[etcd] Adding etcd member: %s", etcdPeerAddress)
		initialCluster, err = etcdClient.AddMember(nodeName, etcdPeerAddress)
		if err != nil {
			return err
		}

		fmt.Println("[etcd] Announced new etcd member joining to the existing etcd cluster")
		klog.V(1).Infof("Updated etcd member list: %v", initialCluster)
	}

	fmt.Printf("[etcd] Creating static Pod manifest for %q\n", kubeadmconstants.Etcd)

	// gets etcd StaticPodSpec, actualized for the current InitConfiguration and the new list of etcd members
	spec := GetEtcdPodSpec(cfg, endpoint, nodeName, initialCluster)

	// if patchesDir is defined, patch the static Pod manifest
	if patchesDir != "" {
		patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout)
		if err != nil {
			return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", kubeadmconstants.Etcd)
		}
		spec = *patchedSpec
	}

	// writes etcd StaticPod to disk
	if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
		return err
	}

	fmt.Printf("[etcd] Waiting for the new etcd member to join the cluster. This can take up to %v\n", etcdHealthyCheckInterval*etcdHealthyCheckRetries)
	if _, err := etcdClient.WaitForClusterAvailable(etcdHealthyCheckRetries, etcdHealthyCheckInterval); err != nil {
		return err
	}

	return nil
}

// GetEtcdPodSpec returns the etcd static Pod actualized to the context of the current configuration
// NB. GetEtcdPodSpec methods holds the information about how kubeadm creates etcd static pod manifests.
func GetEtcdPodSpec(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member) v1.Pod {
	pathType := v1.HostPathDirectoryOrCreate
	etcdMounts := map[string]v1.Volume{
		etcdVolumeName:  staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.Local.DataDir, &pathType),
		certsVolumeName: staticpodutil.NewVolume(certsVolumeName, cfg.CertificatesDir+"/etcd", &pathType),
	}
	// probeHostname returns the correct localhost IP address family based on the endpoint AdvertiseAddress
	probeHostname, probePort, probeScheme := staticpodutil.GetEtcdProbeEndpoint(&cfg.Etcd, utilsnet.IsIPv6String(endpoint.AdvertiseAddress))
	return staticpodutil.ComponentPod(
		v1.Container{
			Name:            kubeadmconstants.Etcd,
			Command:         getEtcdCommand(cfg, endpoint, nodeName, initialCluster),
			Image:           images.GetEtcdImage(cfg),
			ImagePullPolicy: v1.PullIfNotPresent,
			// Mount the etcd datadir path read-write so etcd can store data in a more persistent manner
			VolumeMounts: []v1.VolumeMount{
				staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.Local.DataDir, false),
				staticpodutil.NewVolumeMount(certsVolumeName, cfg.CertificatesDir+"/etcd", false),
			},
			Resources: v1.ResourceRequirements{
				Requests: v1.ResourceList{
					v1.ResourceCPU:              resource.MustParse("100m"),
					v1.ResourceMemory:           resource.MustParse("100Mi"),
					v1.ResourceEphemeralStorage: resource.MustParse("100Mi"),
				},
			},
			LivenessProbe: staticpodutil.LivenessProbe(probeHostname, "/health", probePort, probeScheme),
			StartupProbe:  staticpodutil.StartupProbe(probeHostname, "/health", probePort, probeScheme, cfg.APIServer.TimeoutForControlPlane),
		},
		etcdMounts,
		// etcd will listen on the advertise address of the API server, in a different port (2379)
		map[string]string{kubeadmconstants.EtcdAdvertiseClientUrlsAnnotationKey: etcdutil.GetClientURL(endpoint)},
	)
}

// getEtcdCommand builds the right etcd command from the given config object
func getEtcdCommand(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member) []string {
	// localhost IP family should be the same that the AdvertiseAddress
	etcdLocalhostAddress := "127.0.0.1"
	if utilsnet.IsIPv6String(endpoint.AdvertiseAddress) {
		etcdLocalhostAddress = "::1"
	}
	defaultArguments := map[string]string{
		"name":                        nodeName,
		"listen-client-urls":          fmt.Sprintf("%s,%s", etcdutil.GetClientURLByIP(etcdLocalhostAddress), etcdutil.GetClientURL(endpoint)),
		"advertise-client-urls":       etcdutil.GetClientURL(endpoint),
		"listen-peer-urls":            etcdutil.GetPeerURL(endpoint),
		"initial-advertise-peer-urls": etcdutil.GetPeerURL(endpoint),
		"data-dir":                    cfg.Etcd.Local.DataDir,
		"cert-file":                   filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerCertName),
		"key-file":                    filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName),
		"trusted-ca-file":             filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName),
		"client-cert-auth":            "true",
		"peer-cert-file":              filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerCertName),
		"peer-key-file":               filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName),
		"peer-trusted-ca-file":        filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName),
		"peer-client-cert-auth":       "true",
		"snapshot-count":              "10000",
		"listen-metrics-urls":         fmt.Sprintf("http://%s", net.JoinHostPort(etcdLocalhostAddress, strconv.Itoa(kubeadmconstants.EtcdMetricsPort))),
	}

	if len(initialCluster) == 0 {
		defaultArguments["initial-cluster"] = fmt.Sprintf("%s=%s", nodeName, etcdutil.GetPeerURL(endpoint))
	} else {
		// NB. the joining etcd member should be part of the initialCluster list
		endpoints := []string{}
		for _, member := range initialCluster {
			endpoints = append(endpoints, fmt.Sprintf("%s=%s", member.Name, member.PeerURL))
		}

		defaultArguments["initial-cluster"] = strings.Join(endpoints, ",")
		defaultArguments["initial-cluster-state"] = "existing"
	}

	command := []string{"etcd"}
	command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.Etcd.Local.ExtraArgs)...)
	return command
}
<commit_msg>feat: remove ephemeral-storage etcd requirement<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package etcd

import (
	"fmt"
	"net"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	clientset "k8s.io/client-go/kubernetes"
	"k8s.io/klog/v2"
	utilsnet "k8s.io/utils/net"

	kubeadmapi "k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
	kubeadmconstants "k8s.io/kubernetes/cmd/kubeadm/app/constants"
	"k8s.io/kubernetes/cmd/kubeadm/app/images"
	kubeadmutil "k8s.io/kubernetes/cmd/kubeadm/app/util"
	etcdutil "k8s.io/kubernetes/cmd/kubeadm/app/util/etcd"
	staticpodutil "k8s.io/kubernetes/cmd/kubeadm/app/util/staticpod"
)

const (
	etcdVolumeName           = "etcd-data"
	certsVolumeName          = "etcd-certs"
	etcdHealthyCheckInterval = 5 * time.Second
	etcdHealthyCheckRetries  = 8
)

// CreateLocalEtcdStaticPodManifestFile will write local etcd static pod manifest file.
// This function is used by init - when the etcd cluster is empty - or by kubeadm
// upgrade - when the etcd cluster is already up and running (and the --initial-cluster flag have no impact)
func CreateLocalEtcdStaticPodManifestFile(manifestDir, patchesDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error {
	if cfg.Etcd.External != nil {
		return errors.New("etcd static pod manifest cannot be generated for cluster using external etcd")
	}
	// gets etcd StaticPodSpec
	spec := GetEtcdPodSpec(cfg, endpoint, nodeName, []etcdutil.Member{})

	// if patchesDir is defined, patch the static Pod manifest
	if patchesDir != "" {
		patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout)
		if err != nil {
			return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", kubeadmconstants.Etcd)
		}
		spec = *patchedSpec
	}

	// writes etcd StaticPod to disk
	if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
		return err
	}

	klog.V(1).Infof("[etcd] wrote Static Pod manifest for a local etcd member to %q\n", kubeadmconstants.GetStaticPodFilepath(kubeadmconstants.Etcd, manifestDir))
	return nil
}

// CheckLocalEtcdClusterStatus verifies health state of local/stacked etcd cluster before installing a new etcd member
func CheckLocalEtcdClusterStatus(client clientset.Interface, cfg *kubeadmapi.ClusterConfiguration) error {
	klog.V(1).Info("[etcd] Checking etcd cluster health")

	// creates an etcd client that connects to all the local/stacked etcd members
	klog.V(1).Info("creating etcd client that connects to etcd pods")
	etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir)
	if err != nil {
		return err
	}

	// Checking health state
	err = etcdClient.CheckClusterHealth()
	if err != nil {
		return errors.Wrap(err, "etcd cluster is not healthy")
	}

	return nil
}

// RemoveStackedEtcdMemberFromCluster will remove a local etcd member from etcd cluster,
// when reset the control plane node.
func RemoveStackedEtcdMemberFromCluster(client clientset.Interface, cfg *kubeadmapi.InitConfiguration) error {
	// creates an etcd client that connects to all the local/stacked etcd members
	klog.V(1).Info("[etcd] creating etcd client that connects to etcd pods")
	etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir)
	if err != nil {
		return err
	}

	members, err := etcdClient.ListMembers()
	if err != nil {
		return err
	}
	// If this is the only remaining stacked etcd member in the cluster, calling RemoveMember()
	// is not needed.
	if len(members) == 1 {
		etcdClientAddress := etcdutil.GetClientURL(&cfg.LocalAPIEndpoint)
		for _, endpoint := range etcdClient.Endpoints {
			if endpoint == etcdClientAddress {
				klog.V(1).Info("[etcd] This is the only remaining etcd member in the etcd cluster, skip removing it")
				return nil
			}
		}
	}

	// notifies the other members of the etcd cluster about the removing member
	etcdPeerAddress := etcdutil.GetPeerURL(&cfg.LocalAPIEndpoint)

	klog.V(2).Infof("[etcd] get the member id from peer: %s", etcdPeerAddress)
	id, err := etcdClient.GetMemberID(etcdPeerAddress)
	if err != nil {
		return err
	}

	klog.V(1).Infof("[etcd] removing etcd member: %s, id: %d", etcdPeerAddress, id)
	members, err = etcdClient.RemoveMember(id)
	if err != nil {
		return err
	}
	klog.V(1).Infof("[etcd] Updated etcd member list: %v", members)

	return nil
}

// CreateStackedEtcdStaticPodManifestFile will write local etcd static pod manifest file
// for an additional etcd member that is joining an existing local/stacked etcd cluster.
// Other members of the etcd cluster will be notified of the joining node in beforehand as well.
func CreateStackedEtcdStaticPodManifestFile(client clientset.Interface, manifestDir, patchesDir string, nodeName string, cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint) error {
	// creates an etcd client that connects to all the local/stacked etcd members
	klog.V(1).Info("creating etcd client that connects to etcd pods")
	etcdClient, err := etcdutil.NewFromCluster(client, cfg.CertificatesDir)
	if err != nil {
		return err
	}

	etcdPeerAddress := etcdutil.GetPeerURL(endpoint)

	klog.V(1).Infoln("[etcd] Getting the list of existing members")
	initialCluster, err := etcdClient.ListMembers()
	if err != nil {
		return err
	}

	// only add the new member if it doesn't already exists
	var exists bool
	klog.V(1).Infof("[etcd] Checking if the etcd member already exists: %s", etcdPeerAddress)
	for i := range initialCluster {
		if initialCluster[i].PeerURL == etcdPeerAddress {
			exists = true
			if len(initialCluster[i].Name) == 0 {
				klog.V(1).Infof("[etcd] etcd member name is empty. Setting it to the node name: %s", nodeName)
				initialCluster[i].Name = nodeName
			}
			break
		}
	}

	if exists {
		klog.V(1).Infof("[etcd] Etcd member already exists: %s", endpoint)
	} else {
		klog.V(1).Infof("[etcd] Adding etcd member: %s", etcdPeerAddress)
		initialCluster, err = etcdClient.AddMember(nodeName, etcdPeerAddress)
		if err != nil {
			return err
		}

		fmt.Println("[etcd] Announced new etcd member joining to the existing etcd cluster")
		klog.V(1).Infof("Updated etcd member list: %v", initialCluster)
	}

	fmt.Printf("[etcd] Creating static Pod manifest for %q\n", kubeadmconstants.Etcd)

	// gets etcd StaticPodSpec, actualized for the current InitConfiguration and the new list of etcd members
	spec := GetEtcdPodSpec(cfg, endpoint, nodeName, initialCluster)

	// if patchesDir is defined, patch the static Pod manifest
	if patchesDir != "" {
		patchedSpec, err := staticpodutil.PatchStaticPod(&spec, patchesDir, os.Stdout)
		if err != nil {
			return errors.Wrapf(err, "failed to patch static Pod manifest file for %q", kubeadmconstants.Etcd)
		}
		spec = *patchedSpec
	}

	// writes etcd StaticPod to disk
	if err := staticpodutil.WriteStaticPodToDisk(kubeadmconstants.Etcd, manifestDir, spec); err != nil {
		return err
	}

	fmt.Printf("[etcd] Waiting for the new etcd member to join the cluster. This can take up to %v\n", etcdHealthyCheckInterval*etcdHealthyCheckRetries)
	if _, err := etcdClient.WaitForClusterAvailable(etcdHealthyCheckRetries, etcdHealthyCheckInterval); err != nil {
		return err
	}

	return nil
}

// GetEtcdPodSpec returns the etcd static Pod actualized to the context of the current configuration
// NB. GetEtcdPodSpec methods holds the information about how kubeadm creates etcd static pod manifests.
func GetEtcdPodSpec(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member) v1.Pod {
	pathType := v1.HostPathDirectoryOrCreate
	etcdMounts := map[string]v1.Volume{
		etcdVolumeName:  staticpodutil.NewVolume(etcdVolumeName, cfg.Etcd.Local.DataDir, &pathType),
		certsVolumeName: staticpodutil.NewVolume(certsVolumeName, cfg.CertificatesDir+"/etcd", &pathType),
	}
	// probeHostname returns the correct localhost IP address family based on the endpoint AdvertiseAddress
	probeHostname, probePort, probeScheme := staticpodutil.GetEtcdProbeEndpoint(&cfg.Etcd, utilsnet.IsIPv6String(endpoint.AdvertiseAddress))
	return staticpodutil.ComponentPod(
		v1.Container{
			Name:            kubeadmconstants.Etcd,
			Command:         getEtcdCommand(cfg, endpoint, nodeName, initialCluster),
			Image:           images.GetEtcdImage(cfg),
			ImagePullPolicy: v1.PullIfNotPresent,
			// Mount the etcd datadir path read-write so etcd can store data in a more persistent manner
			VolumeMounts: []v1.VolumeMount{
				staticpodutil.NewVolumeMount(etcdVolumeName, cfg.Etcd.Local.DataDir, false),
				staticpodutil.NewVolumeMount(certsVolumeName, cfg.CertificatesDir+"/etcd", false),
			},
			Resources: v1.ResourceRequirements{
				Requests: v1.ResourceList{
					v1.ResourceCPU:    resource.MustParse("100m"),
					v1.ResourceMemory: resource.MustParse("100Mi"),
				},
			},
			LivenessProbe: staticpodutil.LivenessProbe(probeHostname, "/health", probePort, probeScheme),
			StartupProbe:  staticpodutil.StartupProbe(probeHostname, "/health", probePort, probeScheme, cfg.APIServer.TimeoutForControlPlane),
		},
		etcdMounts,
		// etcd will listen on the advertise address of the API server, in a different port (2379)
		map[string]string{kubeadmconstants.EtcdAdvertiseClientUrlsAnnotationKey: etcdutil.GetClientURL(endpoint)},
	)
}

// getEtcdCommand builds the right etcd command from the given config object
func getEtcdCommand(cfg *kubeadmapi.ClusterConfiguration, endpoint *kubeadmapi.APIEndpoint, nodeName string, initialCluster []etcdutil.Member) []string {
	// localhost IP family should be the same that the AdvertiseAddress
	etcdLocalhostAddress := "127.0.0.1"
	if utilsnet.IsIPv6String(endpoint.AdvertiseAddress) {
		etcdLocalhostAddress = "::1"
	}
	defaultArguments := map[string]string{
		"name":                        nodeName,
		"listen-client-urls":          fmt.Sprintf("%s,%s", etcdutil.GetClientURLByIP(etcdLocalhostAddress), etcdutil.GetClientURL(endpoint)),
		"advertise-client-urls":       etcdutil.GetClientURL(endpoint),
		"listen-peer-urls":            etcdutil.GetPeerURL(endpoint),
		"initial-advertise-peer-urls": etcdutil.GetPeerURL(endpoint),
		"data-dir":                    cfg.Etcd.Local.DataDir,
		"cert-file":                   filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerCertName),
		"key-file":                    filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdServerKeyName),
		"trusted-ca-file":             filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName),
		"client-cert-auth":            "true",
		"peer-cert-file":              filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerCertName),
		"peer-key-file":               filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdPeerKeyName),
		"peer-trusted-ca-file":        filepath.Join(cfg.CertificatesDir, kubeadmconstants.EtcdCACertName),
		"peer-client-cert-auth":       "true",
		"snapshot-count":              "10000",
		"listen-metrics-urls":         fmt.Sprintf("http://%s", net.JoinHostPort(etcdLocalhostAddress, strconv.Itoa(kubeadmconstants.EtcdMetricsPort))),
	}

	if len(initialCluster) == 0 {
		defaultArguments["initial-cluster"] = fmt.Sprintf("%s=%s", nodeName, etcdutil.GetPeerURL(endpoint))
	} else {
		// NB. the joining etcd member should be part of the initialCluster list
		endpoints := []string{}
		for _, member := range initialCluster {
			endpoints = append(endpoints, fmt.Sprintf("%s=%s", member.Name, member.PeerURL))
		}

		defaultArguments["initial-cluster"] = strings.Join(endpoints, ",")
		defaultArguments["initial-cluster-state"] = "existing"
	}

	command := []string{"etcd"}
	command = append(command, kubeadmutil.BuildArgumentListFromMap(defaultArguments, cfg.Etcd.Local.ExtraArgs)...)
	return command
}
<|endoftext|>


<commit_before>// Copyright 2017 Mathew Robinson <mrobinson@praelatus.io>. All rights reserved.
// Use of this source code is governed by the AGPLv3 license that can be found in
// the LICENSE file.

package middleware

import (
	"errors"
	"fmt"
	"net/http"
	"time"

	jwt "github.com/dgrijalva/jwt-go"
	"github.com/praelatus/praelatus/models"
)

// TODO: create a way to invalidate a session
// TODO: prevent session hijacking
// TODO: Make this actually secure
var signingKey = []byte("CHANGE ME")

func makeClaims(user models.User) jwt.MapClaims {
	now := time.Now()
	return jwt.MapClaims{
		"username": user.Username,
		"email":    user.Email,
		"is_admin": user.IsAdmin,
		"iat":      now,
		"exp":      now.Add(time.Hour * 24),
	}
}

func userFromClaims(claims jwt.MapClaims) models.User {
	maybeUsername, ok := claims["username"]
	if !ok {
		return models.User{}
	}

	maybeEmail, ok := claims["email"]
	if !ok {
		return models.User{}
	}

	maybeAdmin, ok := claims["is_admin"]
	if !ok {
		return models.User{}
	}

	username, ok := maybeUsername.(string)
	if !ok {
		return models.User{}
	}

	email, ok := maybeEmail.(string)
	if !ok {
		return models.User{}
	}

	isAdmin, ok := maybeAdmin.(bool)
	if !ok {
		return models.User{}
	}

	return models.User{
		Username: username,
		Email:    email,
		IsAdmin:  isAdmin,
	}
}

func getToken(r *http.Request) *jwt.Token {
	auth := r.Header.Get("Authorization")
	fmt.Println(r.Header)
	if len(auth) == 0 {
		return nil
	}

	tokenString := auth[len("Bearer "):]
	if tokenString == "" {
		return nil
	}

	token, err := jwt.Parse(tokenString,
		func(t *jwt.Token) (interface{}, error) {
			return signingKey, nil
		})
	if err != nil {
		fmt.Println("ERROR: [TOKEN_VALIDATION]", err.Error())
		return nil
	}

	return token
}

func getClaims(token *jwt.Token) jwt.MapClaims {
	claims, ok := token.Claims.(jwt.MapClaims)
	if !ok {
		fmt.Println("ERROR: [TOKEN_VALIDATION] Invalid Claims", claims)
		return nil
	}

	return claims
}

// GetUserSession will check the given http.Request for a session token and if
// found it will return the corresponding user.
func GetUserSession(r *http.Request) *models.User {
	token := getToken(r)
	if token == nil {
		return nil
	}

	claims := getClaims(token)
	if claims == nil {
		return nil
	}

	user := userFromClaims(claims)
	return &user
}

// SetUserSession will generate a secure cookie for user u, will set the cookie
// on the response w and will add the user session to the session store
func SetUserSession(u models.User, w http.ResponseWriter) error {
	token := jwt.NewWithClaims(jwt.SigningMethodHS256, makeClaims(u))
	signed, err := token.SignedString(signingKey)
	if err != nil {
		return err
	}

	// TODO: Store the session with the client specific ID for session hijacking
	w.Header().Set("X-Praelatus-Token", signed)
	return nil
}

// RefreshSession will reset the expiry on the session for the given request
func RefreshSession(r *http.Request) error {
	token := getToken(r)
	if token == nil {
		return errors.New("no session on this request")
	}

	claims := getClaims(token)
	if claims == nil {
		return errors.New("no claims on token")
	}

	claims["exp"] = time.Now().Add(time.Hour * 24)
	return nil
}
<commit_msg>remove debugging<commit_after>// Copyright 2017 Mathew Robinson <mrobinson@praelatus.io>. All rights reserved.
// Use of this source code is governed by the AGPLv3 license that can be found in
// the LICENSE file.

package middleware

import (
	"errors"
	"fmt"
	"net/http"
	"time"

	jwt "github.com/dgrijalva/jwt-go"
	"github.com/praelatus/praelatus/models"
)

// TODO: create a way to invalidate a session
// TODO: prevent session hijacking
// TODO: Make this actually secure
var signingKey = []byte("CHANGE ME")

func makeClaims(user models.User) jwt.MapClaims {
	now := time.Now()
	return jwt.MapClaims{
		"username": user.Username,
		"email":    user.Email,
		"is_admin": user.IsAdmin,
		"iat":      now,
		"exp":      now.Add(time.Hour * 24),
	}
}

func userFromClaims(claims jwt.MapClaims) models.User {
	maybeUsername, ok := claims["username"]
	if !ok {
		return models.User{}
	}

	maybeEmail, ok := claims["email"]
	if !ok {
		return models.User{}
	}

	maybeAdmin, ok := claims["is_admin"]
	if !ok {
		return models.User{}
	}

	username, ok := maybeUsername.(string)
	if !ok {
		return models.User{}
	}

	email, ok := maybeEmail.(string)
	if !ok {
		return models.User{}
	}

	isAdmin, ok := maybeAdmin.(bool)
	if !ok {
		return models.User{}
	}

	return models.User{
		Username: username,
		Email:    email,
		IsAdmin:  isAdmin,
	}
}

func getToken(r *http.Request) *jwt.Token {
	auth := r.Header.Get("Authorization")
	if len(auth) == 0 {
		return nil
	}

	tokenString := auth[len("Bearer "):]
	if tokenString == "" {
		return nil
	}

	token, err := jwt.Parse(tokenString,
		func(t *jwt.Token) (interface{}, error) {
			return signingKey, nil
		})
	if err != nil {
		fmt.Println("ERROR: [TOKEN_VALIDATION]", err.Error())
		return nil
	}

	return token
}

func getClaims(token *jwt.Token) jwt.MapClaims {
	claims, ok := token.Claims.(jwt.MapClaims)
	if !ok {
		fmt.Println("ERROR: [TOKEN_VALIDATION] Invalid Claims", claims)
		return nil
	}

	return claims
}

// GetUserSession will check the given http.Request for a session token and if
// found it will return the corresponding user.
func GetUserSession(r *http.Request) *models.User {
	token := getToken(r)
	if token == nil {
		return nil
	}

	claims := getClaims(token)
	if claims == nil {
		return nil
	}

	user := userFromClaims(claims)
	return &user
}

// SetUserSession will generate a secure cookie for user u, will set the cookie
// on the response w and will add the user session to the session store
func SetUserSession(u models.User, w http.ResponseWriter) error {
	token := jwt.NewWithClaims(jwt.SigningMethodHS256, makeClaims(u))
	signed, err := token.SignedString(signingKey)
	if err != nil {
		return err
	}

	// TODO: Store the session with the client specific ID for session hijacking
	w.Header().Set("X-Praelatus-Token", signed)
	return nil
}

// RefreshSession will reset the expiry on the session for the given request
func RefreshSession(r *http.Request) error {
	token := getToken(r)
	if token == nil {
		return errors.New("no session on this request")
	}

	claims := getClaims(token)
	if claims == nil {
		return errors.New("no claims on token")
	}

	claims["exp"] = time.Now().Add(time.Hour * 24)
	return nil
}
<|endoftext|>


<commit_before>package auth

import (
	"fmt"
	"net/http"
)

type anonymousHandler struct {
	groups   []string
	username string
}

// NewAnonymousHandler creates a new anonymous authenticator using
// the provided username and groups
func NewAnonymousHandler(username string, groups []string) Authenticator {

	return &anonymousHandler{
		username: username,
		groups:   groups,
	}
}

// Name returns the name of this authenticator
func (a *anonymousHandler) Name() string {
	return a.username
}

// Description returns the user-friendly description of this authenticator
func (a *anonymousHandler) Description() string {
	return "Anonymous user"
}

// Type returns the type of this authenticator
func (a *anonymousHandler) Type() string {
	return "anonymous"
}

// LoginURL returns the initial login URL for this handler
func (a *anonymousHandler) LoginURL() string {
	_type := a.Type()
	name := a.Name()
	return fmt.Sprintf("/auth/%s/%s", _type, name)
	// return path.Join("/", "auth", _type, name)
}

// PostWithCredentials returns true if this authenticator expects username/password credentials be POST'd
func (a *anonymousHandler) PostWithCredentials() bool {
	return false
}

// IconURL returns an icon URL to signify this login method; empty string implies a default can be used
func (a *anonymousHandler) IconURL() string {
	return ""
}

func (a *anonymousHandler) Authenticate(w http.ResponseWriter, r *http.Request) (*SessionToken, error) {
	return NewSessionToken(a.username, a.groups, nil), nil
}
<commit_msg>change description to 'Anonymous access'<commit_after>package auth

import (
	"fmt"
	"net/http"
)

type anonymousHandler struct {
	groups   []string
	username string
}

// NewAnonymousHandler creates a new anonymous authenticator using
// the provided username and groups
func NewAnonymousHandler(username string, groups []string) Authenticator {

	return &anonymousHandler{
		username: username,
		groups:   groups,
	}
}

// Name returns the name of this authenticator
func (a *anonymousHandler) Name() string {
	return a.username
}

// Description returns the user-friendly description of this authenticator
func (a *anonymousHandler) Description() string {
	return "Anonymous access"
}

// Type returns the type of this authenticator
func (a *anonymousHandler) Type() string {
	return "anonymous"
}

// LoginURL returns the initial login URL for this handler
func (a *anonymousHandler) LoginURL() string {
	_type := a.Type()
	name := a.Name()
	return fmt.Sprintf("/auth/%s/%s", _type, name)
	// return path.Join("/", "auth", _type, name)
}

// PostWithCredentials returns true if this authenticator expects username/password credentials be POST'd
func (a *anonymousHandler) PostWithCredentials() bool {
	return false
}

// IconURL returns an icon URL to signify this login method; empty string implies a default can be used
func (a *anonymousHandler) IconURL() string {
	return ""
}

func (a *anonymousHandler) Authenticate(w http.ResponseWriter, r *http.Request) (*SessionToken, error) {
	return NewSessionToken(a.username, a.groups, nil), nil
}
<|endoftext|>


<commit_before>package collectors

import (
	"strings"
	"time"

	"github.com/StackExchange/scollector/opentsdb"
)

func init() {
	collectors = append(collectors, &IntervalCollector{F: c_omreport_chassis, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_system, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_storage_enclosure, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_storage_vdisk, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_storage_controller, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_storage_battery, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_ps, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_ps_amps, Interval: time.Minute * 5})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_ps_volts, Interval: time.Minute * 5})
}

func c_omreport_chassis() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) != 2 || fields[0] == "SEVERITY" {
			return
		}
		sev := 0
		if fields[0] != "Ok" && fields[0] != "Non-Critical" {
			sev = 1
		}
		component := strings.Replace(fields[1], " ", "_", -1)
		Add(&md, "hw.chassis", sev, opentsdb.TagSet{"component": component})
	}, "omreport", "chassis", "-fmt", "ssv")
	return md
}

func c_omreport_system() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) != 2 || fields[0] == "SEVERITY" {
			return
		}
		sev := 0
		if fields[0] != "Ok" && fields[0] != "Non-Critical" {
			sev = 1
		}
		component := strings.Replace(fields[1], " ", "_", -1)
		Add(&md, "hw.system", sev, opentsdb.TagSet{"component": component})
	}, "omreport", "system", "-fmt", "ssv")
	return md
}

func c_omreport_storage_enclosure() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.storage.enclosure", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "enclosure", "-fmt", "ssv")
	return md
}

func c_omreport_storage_vdisk() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.storage.vdisk", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "vdisk", "-fmt", "ssv")
	return md
}

func c_omreport_ps() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "Index" {
			return
		}
		sev := 0
		if fields[1] != "Ok" {
			sev = 1
		}
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.ps", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "chassis", "pwrsupplies", "-fmt", "ssv")
	return md
}

func c_omreport_ps_amps() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) != 2 || !strings.Contains(fields[0], "Current") {
			return
		}
		i_fields := strings.Split(fields[0], "Current")
		v_fields := strings.Fields(fields[1])
		if len(i_fields) < 2 && len(v_fields) < 2 {
			return
		}
		id := strings.Replace(i_fields[0], " ", "", -1)
		Add(&md, "hw.ps.current", v_fields[0], opentsdb.TagSet{"id": id})
	}, "omreport", "chassis", "pwrmonitoring", "-fmt", "ssv")
	return md
}

func c_omreport_ps_volts() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) != 8 || !strings.Contains(fields[2], "Voltage") || fields[3] == "[N/A]" {
			return
		}
		i_fields := strings.Split(fields[2], "Voltage")
		v_fields := strings.Fields(fields[3])
		if len(i_fields) < 2 && len(v_fields) < 2 {
			return
		}
		id := strings.Replace(i_fields[0], " ", "", -1)
		Add(&md, "hw.ps.volts", v_fields[0], opentsdb.TagSet{"id": id})
	}, "omreport", "chassis", "volts", "-fmt", "ssv")
	return md
}

func c_omreport_storage_battery() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.storage.battery", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "battery", "-fmt", "ssv")
	return md
}

func c_omreport_storage_controller() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		c_omreport_storage_pdisk(fields[0], &md)
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.storage.controller", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "controller", "-fmt", "ssv")
	return md
}

// c_omreport_storage_pdisk is called from the controller func, since it needs the encapsulating id.
func c_omreport_storage_pdisk(id string, md *opentsdb.MultiDataPoint) {
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		//Need to find out what the various ID formats might be
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(md, "hw.storage.pdisk", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "pdisk", "controller="+id, "-fmt", "ssv")
}
<commit_msg>cmd/scollector: Use a common interval<commit_after>package collectors

import (
	"strings"
	"time"

	"github.com/StackExchange/scollector/opentsdb"
)

func init() {
	const interval = time.Minute * 5
	collectors = append(collectors, &IntervalCollector{F: c_omreport_chassis, Interval: interval})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_system, Interval: interval})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_storage_enclosure, Interval: interval})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_storage_vdisk, Interval: interval})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_storage_controller, Interval: interval})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_storage_battery, Interval: interval})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_ps, Interval: interval})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_ps_amps, Interval: interval})
	collectors = append(collectors, &IntervalCollector{F: c_omreport_ps_volts, Interval: interval})
}

func c_omreport_chassis() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) != 2 || fields[0] == "SEVERITY" {
			return
		}
		sev := 0
		if fields[0] != "Ok" && fields[0] != "Non-Critical" {
			sev = 1
		}
		component := strings.Replace(fields[1], " ", "_", -1)
		Add(&md, "hw.chassis", sev, opentsdb.TagSet{"component": component})
	}, "omreport", "chassis", "-fmt", "ssv")
	return md
}

func c_omreport_system() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) != 2 || fields[0] == "SEVERITY" {
			return
		}
		sev := 0
		if fields[0] != "Ok" && fields[0] != "Non-Critical" {
			sev = 1
		}
		component := strings.Replace(fields[1], " ", "_", -1)
		Add(&md, "hw.system", sev, opentsdb.TagSet{"component": component})
	}, "omreport", "system", "-fmt", "ssv")
	return md
}

func c_omreport_storage_enclosure() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.storage.enclosure", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "enclosure", "-fmt", "ssv")
	return md
}

func c_omreport_storage_vdisk() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.storage.vdisk", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "vdisk", "-fmt", "ssv")
	return md
}

func c_omreport_ps() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "Index" {
			return
		}
		sev := 0
		if fields[1] != "Ok" {
			sev = 1
		}
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.ps", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "chassis", "pwrsupplies", "-fmt", "ssv")
	return md
}

func c_omreport_ps_amps() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) != 2 || !strings.Contains(fields[0], "Current") {
			return
		}
		i_fields := strings.Split(fields[0], "Current")
		v_fields := strings.Fields(fields[1])
		if len(i_fields) < 2 && len(v_fields) < 2 {
			return
		}
		id := strings.Replace(i_fields[0], " ", "", -1)
		Add(&md, "hw.ps.current", v_fields[0], opentsdb.TagSet{"id": id})
	}, "omreport", "chassis", "pwrmonitoring", "-fmt", "ssv")
	return md
}

func c_omreport_ps_volts() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) != 8 || !strings.Contains(fields[2], "Voltage") || fields[3] == "[N/A]" {
			return
		}
		i_fields := strings.Split(fields[2], "Voltage")
		v_fields := strings.Fields(fields[3])
		if len(i_fields) < 2 && len(v_fields) < 2 {
			return
		}
		id := strings.Replace(i_fields[0], " ", "", -1)
		Add(&md, "hw.ps.volts", v_fields[0], opentsdb.TagSet{"id": id})
	}, "omreport", "chassis", "volts", "-fmt", "ssv")
	return md
}

func c_omreport_storage_battery() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.storage.battery", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "battery", "-fmt", "ssv")
	return md
}

func c_omreport_storage_controller() opentsdb.MultiDataPoint {
	var md opentsdb.MultiDataPoint
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		c_omreport_storage_pdisk(fields[0], &md)
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(&md, "hw.storage.controller", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "controller", "-fmt", "ssv")
	return md
}

// c_omreport_storage_pdisk is called from the controller func, since it needs the encapsulating id.
func c_omreport_storage_pdisk(id string, md *opentsdb.MultiDataPoint) {
	readCommand(func(line string) {
		fields := strings.Split(line, ";")
		if len(fields) < 3 || fields[0] == "ID" {
			return
		}
		sev := 0
		if fields[1] != "Ok" && fields[1] != "Non-Critical" {
			sev = 1
		}
		//Need to find out what the various ID formats might be
		id := strings.Replace(fields[0], ":", "_", -1)
		Add(md, "hw.storage.pdisk", sev, opentsdb.TagSet{"id": id})
	}, "omreport", "storage", "pdisk", "controller="+id, "-fmt", "ssv")
}
<|endoftext|>


<commit_before>// offers api
// https://github.com/topfreegames/offers
//
// Licensed under the MIT license:
// http://www.opensource.org/licenses/mit-license
// Copyright © 2017 Top Free Games <backend@tfgco.com>

package api

import (
	"context"
	"fmt"
	"net/http"

	newrelic "github.com/newrelic/go-agent"
)

//NewRelicMiddleware handles logging
type NewRelicMiddleware struct {
	App  *App
	Next http.Handler
}

const newRelicTransactionKey = contextKey("newRelicTransaction")

func newContextWithNewRelicTransaction(ctx context.Context, txn newrelic.Transaction, r *http.Request) context.Context {
	c := context.WithValue(ctx, newRelicTransactionKey, txn)
	return c
}

func (m *NewRelicMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	if m.App.NewRelic != nil {
		txn := m.App.NewRelic.StartTransaction(fmt.Sprintf("%s %s", r.Method, r.URL.Path), w, r)
		defer txn.End()
		ctx = newContextWithNewRelicTransaction(r.Context(), txn, r)

		mr := metricsReporterFromCtx(ctx)
		if mr != nil {
			mr.AddReporter(&NewRelicMetricsReporter{
				App:         m.App,
				Transaction: txn,
			})
		}
	}

	// Call the next middleware/handler in chain
	m.Next.ServeHTTP(w, r.WithContext(ctx))
}

//SetNext middleware
func (m *NewRelicMiddleware) SetNext(next http.Handler) {
	m.Next = next
}
<commit_msg>Fix new relic transaction names.<commit_after>// offers api
// https://github.com/topfreegames/offers
//
// Licensed under the MIT license:
// http://www.opensource.org/licenses/mit-license
// Copyright © 2017 Top Free Games <backend@tfgco.com>

package api

import (
	"context"
	"fmt"
	"net/http"

	"github.com/gorilla/mux"
	newrelic "github.com/newrelic/go-agent"
)

//NewRelicMiddleware handles logging
type NewRelicMiddleware struct {
	App  *App
	Next http.Handler
}

const newRelicTransactionKey = contextKey("newRelicTransaction")

func newContextWithNewRelicTransaction(ctx context.Context, txn newrelic.Transaction, r *http.Request) context.Context {
	c := context.WithValue(ctx, newRelicTransactionKey, txn)
	return c
}

func (m *NewRelicMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	if m.App.NewRelic != nil {
		route, _ := mux.CurrentRoute(r).GetPathTemplate()
		txn := m.App.NewRelic.StartTransaction(fmt.Sprintf("%s %s", r.Method, route), w, r)
		defer txn.End()
		ctx = newContextWithNewRelicTransaction(r.Context(), txn, r)

		mr := metricsReporterFromCtx(ctx)
		if mr != nil {
			mr.AddReporter(&NewRelicMetricsReporter{
				App:         m.App,
				Transaction: txn,
			})
		}
	}

	// Call the next middleware/handler in chain
	m.Next.ServeHTTP(w, r.WithContext(ctx))
}

//SetNext middleware
func (m *NewRelicMiddleware) SetNext(next http.Handler) {
	m.Next = next
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package arcanist

// This is far from ideal.
//
// Phabricator does not currently provide any sort of API for quering the code review comments.
// To work around this, we directly query the underlying database tables.
//
// There are three tables from which we need to read, all under the "phabricator_differential" schema:
//  differential_transaction stores the top level code review actions, like commenting.
//  differential_transaction_comment stores the actual contents of comments.
//  differential_changeset stores the diffs against which a comment was made.

import (
	"bytes"
	"fmt"
	"log"
	"os/exec"
	"source.developers.google.com/id/AOYtBqJZlBK.git/mirror/review/comment"
	"strconv"
	"strings"
	"time"
)

const (
	// SQL query for differential "transactions". These are atomic operations on a review.
	selectTransactionsQueryTemplate = `
select id, phid, authorPHID, dateCreated, transactionType, newValue, commentPHID
  from phabricator_differential.differential_transaction
	where objectPHID="%s"
		and viewPolicy="public"
		and (transactionType = "differential:action" or
     transactionType = "differential:inline" or
     transactionType = "core:comment")
  order by id;`
	// SQL query for differential "transaction comments". These are always tied
	// to a differential "transaction" and include the body of a review comment.
	selectTransactionCommentsQueryTemplate = `
select phid, changesetID, lineNumber, replyToCommentPHID
	from phabricator_differential.differential_transaction_comment
	where viewPolicy = "public" and transactionPHID = "%s";`
	// SQL query for the contents of a differential "transaction comment". This
	// is separated from the query for the other fields so that we don't have to
	// worry about contents that include tabs, which mysql uses as the separator
	// between multiple column values.
	selectCommentContentsQueryTemplate = `
select content from phabricator_differential.differential_transaction_comment
	where phid = "%s";`
	// SQL query to read the filename for a diff.
	selectChangesetFilenameTemplate = `
select filename from phabricator_differential.differential_changeset
	where id = "%d";`
	// SQL query to read the ID for a diff. We need this in order to be able
	// to read the commit hash for a diff (which we do using the Differential API).
	selectChangesetDiffTemplate = `
select diffID from phabricator_differential.differential_changeset
	where id = "%d";`

	// Timeout used for all SQL queries
	sqlQueryTimeout = 1 * time.Minute
)

// runRawSqlCommandOrDie runs the given SQL command with no additional formatting
// included in the output.
//
// Any errors that could occur here would be a sign of something being seriously
// wrong, so they are treated as fatal. This makes it more evident that something
// has gone wrong when the command is manually run by a user, and gives further
// operations a clean-slate when this is run by supervisord with automatic restarts.
func runRawSqlCommandOrDie(command string) string {
	cmd := exec.Command("mysql", "-Ns", "-r", "-e", command)
	var stdout bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Start()
	go func() {
		time.Sleep(sqlQueryTimeout)
		cmd.Process.Kill()
	}()
	if err := cmd.Wait(); err != nil {
		log.Println("Ran SQL command: ", command)
		log.Fatal(err)
	}
	result := strings.TrimSuffix(stdout.String(), "\n")
	return result
}

// runRawSqlCommandOrDie runs the given SQL command.
//
// Any errors that could occur here would be a sign of something being seriously
// wrong, so they are treated as fatal. This makes it more evident that something
// has gone wrong when the command is manually run by a user, and gives further
// operations a clean-slate when this is run by supervisord with automatic restarts.
func runSqlCommandOrDie(command string) string {
	cmd := exec.Command("mysql", "-Ns", "-e", command)
	var stdout bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Start()
	go func() {
		time.Sleep(sqlQueryTimeout)
		cmd.Process.Kill()
	}()
	if err := cmd.Wait(); err != nil {
		log.Println("Ran SQL command: ", command)
		log.Fatal(err)
	}
	result := strings.Trim(stdout.String(), "\n")
	return result
}

// differentialDatabaseTransaction represents a user action on a code review.
//
// This includes things like approving or rejecting the change and commenting.
// However, when a transaction represents a comment, it does not contain the actual
// contents of the comment; those are stored in a diffferentialDatabaseTransactionComment.
type differentialDatabaseTransaction struct {
	PHID        string
	AuthorPHID  string
	DateCreated uint32
	Type        string
	NewValue    *string
	CommentPHID *string
}

type ReadTransactions func(reviewID string) ([]differentialDatabaseTransaction, error)

func readDatabaseTransactions(reviewID string) ([]differentialDatabaseTransaction, error) {
	var transactions []differentialDatabaseTransaction
	result := runSqlCommandOrDie(fmt.Sprintf(selectTransactionsQueryTemplate, reviewID))
	if strings.Trim(result, " ") == "" {
		// There were no matching transactions
		return nil, nil
	}
	// result will be a line-separated list of query results, each of which has 7 columns.
	for _, line := range strings.Split(result, "\n") {
		lineParts := strings.Split(line, "\t")
		if len(lineParts) != 7 {
			return nil, fmt.Errorf("Unexpected number of transaction parts: %v", lineParts)
		}
		var transaction differentialDatabaseTransaction
		transaction.PHID = lineParts[1]
		transaction.AuthorPHID = lineParts[2]
		timestamp, err := strconv.ParseUint(lineParts[3], 10, 32)
		if err != nil {
			return nil, err
		}
		transaction.DateCreated = uint32(timestamp)
		transaction.Type = lineParts[4]
		if lineParts[5] != "NULL" {
			transaction.NewValue = &lineParts[5]
		}
		if lineParts[6] != "NULL" {
			transaction.CommentPHID = &lineParts[6]
		}
		transactions = append(transactions, transaction)
	}
	return transactions, nil
}

// differentialDatabaseTransactionComment stores the actual contents of a code review comment.
type differentialDatabaseTransactionComment struct {
	PHID               string
	Commit             string
	FileName           string
	LineNumber         uint32
	ReplyToCommentPHID *string
	Content            string
}

type ReadTransactionComment func(transactionID string) (*differentialDatabaseTransactionComment, error)

func readDatabaseTransactionComment(transactionID string) (*differentialDatabaseTransactionComment, error) {
	result := runSqlCommandOrDie(fmt.Sprintf(selectTransactionCommentsQueryTemplate, transactionID))
	// result will be a line separated list of query results, each of which includes 4 columns.
	lines := strings.Split(result, "\n")
	if len(lines) != 1 {
		return nil, fmt.Errorf("Unexpected number of query results: %v", lines)
	}
	lineParts := strings.Split(lines[0], "\t")
	if len(lineParts) != 4 {
		return nil, fmt.Errorf("Unexpected size of query results: %v", lineParts)
	}
	var comment differentialDatabaseTransactionComment
	comment.PHID = lineParts[0]
	if lineParts[1] != "NULL" {
		changesetID, err := strconv.ParseUint(lineParts[1], 10, 32)
		if err != nil {
			return nil, err
		}
		changesetResult := runSqlCommandOrDie(fmt.Sprintf(selectChangesetFilenameTemplate, changesetID))
		// changesetResult should have a single query result, which only includes the filename.
		comment.FileName = changesetResult
		diffIDResult := runSqlCommandOrDie(fmt.Sprintf(selectChangesetDiffTemplate, changesetID))
		diffID, err := strconv.Atoi(diffIDResult)
		if err != nil {
			log.Println(diffIDResult)
			log.Fatal(err)
		}
		diff, err := readDiff(diffID)
		if err != nil {
			log.Fatal(err)
		}
		comment.Commit = diff.findLastCommit()
	}
	lineNumber, err := strconv.ParseUint(lineParts[2], 10, 32)
	if err != nil {
		return nil, err
	}
	comment.LineNumber = uint32(lineNumber)
	if lineParts[3] != "NULL" {
		comment.ReplyToCommentPHID = &lineParts[3]
	}
	// The next SQL command is structured to return a single result with a single column, so we
	// don't need to parse it in any way.
	comment.Content = runRawSqlCommandOrDie(fmt.Sprintf(selectCommentContentsQueryTemplate, comment.PHID))
	return &comment, nil
}

// LoadComments takes in a differentialReview and returns the associated comments.
func (review differentialReview) LoadComments() []comment.Comment {
	return LoadComments(review, readDatabaseTransactions, readDatabaseTransactionComment, lookupUser)
}

func LoadComments(review differentialReview, readTransactions ReadTransactions, readTransactionComment ReadTransactionComment, lookupUser UserLookup) []comment.Comment {

	allTransactions, err := readTransactions(review.PHID)
	if err != nil {
		log.Fatal(err)
	}
	var comments []comment.Comment
	commentsByPHID := make(map[string]comment.Comment)
	rejectionCommentsByUser := make(map[string][]string)

	for _, transaction := range allTransactions {
		author, err := lookupUser(transaction.AuthorPHID)
		if err != nil {
			log.Fatal(err)
		}
		c := comment.Comment{
			Author:    author.Email,
			Timestamp: fmt.Sprintf("%d", transaction.DateCreated),
		}
		if author.Email != "" {
			c.Author = author.Email
		} else {
			c.Author = author.UserName
		}

		if transaction.CommentPHID != nil {
			transactionComment, err := readTransactionComment(transaction.PHID)
			if err != nil {
				log.Fatal(err)
			}
			if transactionComment.FileName != "" {
				c.Location = &comment.CommentLocation{
					Commit: transactionComment.Commit,
					Path:   transactionComment.FileName,
				}
				if transactionComment.LineNumber != 0 {
					c.Location.Range = &comment.CommentRange{
						StartLine: transactionComment.LineNumber,
					}
				}
			}
			c.Description = transactionComment.Content
			if transactionComment.ReplyToCommentPHID != nil {
				// We assume that the parent has to have been processed before the child,
				// and enforce that by ordering the transactions in our queries.
				if replyTo, ok := commentsByPHID[*transactionComment.ReplyToCommentPHID]; ok {
					parentHash, err := replyTo.Hash()
					if err != nil {
						log.Fatal(err)
					}
					c.Parent = parentHash
				}
			}
		}

		// Set the resolved bit based on whether the change was approved or not.
		if transaction.Type == "differential:action" && transaction.NewValue != nil {
			action := *transaction.NewValue
			var resolved bool
			if action == "\"accept\"" {
				resolved = true
				c.Resolved = &resolved

				// Add child comments to all previous rejects by this user and make them accepts
				for _, rejectionCommentHash := range rejectionCommentsByUser[author.UserName] {
					approveComment := comment.Comment{
						Author:    c.Author,
						Timestamp: c.Timestamp,
						Resolved:  &resolved,
						Parent:    rejectionCommentHash,
					}
					comments = append(comments, approveComment)
				}
			} else if action == "\"reject\"" {
				resolved = false
				c.Resolved = &resolved
			}

		}

		// Phabricator only publishes inline comments when you publish a top-level comment.
		// This results in a lot of empty top-level comments, which we do not want to mirror.
		// To work around this, we only return comments that are non-empty.
		if c.Parent != "" || c.Location != nil || c.Description != "" || c.Resolved != nil {
			comments = append(comments, c)
			commentsByPHID[transaction.PHID] = c

			//If this was a rejection comment, add it to ordered comment hash
			if *c.Resolved == false {
				commentHash, err := c.Hash()
				if err != nil {
					log.Fatal(err)
				}
				rejectionCommentsByUser[author.UserName] = append(rejectionCommentsByUser[author.UserName], commentHash)
			}

		}
	}
	return comments
}
<commit_msg>Fixing a null dereference on this line.<commit_after>/*
Copyright 2015 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package arcanist

// This is far from ideal.
//
// Phabricator does not currently provide any sort of API for quering the code review comments.
// To work around this, we directly query the underlying database tables.
//
// There are three tables from which we need to read, all under the "phabricator_differential" schema:
//  differential_transaction stores the top level code review actions, like commenting.
//  differential_transaction_comment stores the actual contents of comments.
//  differential_changeset stores the diffs against which a comment was made.

import (
	"bytes"
	"fmt"
	"log"
	"os/exec"
	"source.developers.google.com/id/AOYtBqJZlBK.git/mirror/review/comment"
	"strconv"
	"strings"
	"time"
)

const (
	// SQL query for differential "transactions". These are atomic operations on a review.
	selectTransactionsQueryTemplate = `
select id, phid, authorPHID, dateCreated, transactionType, newValue, commentPHID
  from phabricator_differential.differential_transaction
	where objectPHID="%s"
		and viewPolicy="public"
		and (transactionType = "differential:action" or
     transactionType = "differential:inline" or
     transactionType = "core:comment")
  order by id;`
	// SQL query for differential "transaction comments". These are always tied
	// to a differential "transaction" and include the body of a review comment.
	selectTransactionCommentsQueryTemplate = `
select phid, changesetID, lineNumber, replyToCommentPHID
	from phabricator_differential.differential_transaction_comment
	where viewPolicy = "public" and transactionPHID = "%s";`
	// SQL query for the contents of a differential "transaction comment". This
	// is separated from the query for the other fields so that we don't have to
	// worry about contents that include tabs, which mysql uses as the separator
	// between multiple column values.
	selectCommentContentsQueryTemplate = `
select content from phabricator_differential.differential_transaction_comment
	where phid = "%s";`
	// SQL query to read the filename for a diff.
	selectChangesetFilenameTemplate = `
select filename from phabricator_differential.differential_changeset
	where id = "%d";`
	// SQL query to read the ID for a diff. We need this in order to be able
	// to read the commit hash for a diff (which we do using the Differential API).
	selectChangesetDiffTemplate = `
select diffID from phabricator_differential.differential_changeset
	where id = "%d";`

	// Timeout used for all SQL queries
	sqlQueryTimeout = 1 * time.Minute
)

// runRawSqlCommandOrDie runs the given SQL command with no additional formatting
// included in the output.
//
// Any errors that could occur here would be a sign of something being seriously
// wrong, so they are treated as fatal. This makes it more evident that something
// has gone wrong when the command is manually run by a user, and gives further
// operations a clean-slate when this is run by supervisord with automatic restarts.
func runRawSqlCommandOrDie(command string) string {
	cmd := exec.Command("mysql", "-Ns", "-r", "-e", command)
	var stdout bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Start()
	go func() {
		time.Sleep(sqlQueryTimeout)
		cmd.Process.Kill()
	}()
	if err := cmd.Wait(); err != nil {
		log.Println("Ran SQL command: ", command)
		log.Fatal(err)
	}
	result := strings.TrimSuffix(stdout.String(), "\n")
	return result
}

// runRawSqlCommandOrDie runs the given SQL command.
//
// Any errors that could occur here would be a sign of something being seriously
// wrong, so they are treated as fatal. This makes it more evident that something
// has gone wrong when the command is manually run by a user, and gives further
// operations a clean-slate when this is run by supervisord with automatic restarts.
func runSqlCommandOrDie(command string) string {
	cmd := exec.Command("mysql", "-Ns", "-e", command)
	var stdout bytes.Buffer
	cmd.Stdout = &stdout
	cmd.Start()
	go func() {
		time.Sleep(sqlQueryTimeout)
		cmd.Process.Kill()
	}()
	if err := cmd.Wait(); err != nil {
		log.Println("Ran SQL command: ", command)
		log.Fatal(err)
	}
	result := strings.Trim(stdout.String(), "\n")
	return result
}

// differentialDatabaseTransaction represents a user action on a code review.
//
// This includes things like approving or rejecting the change and commenting.
// However, when a transaction represents a comment, it does not contain the actual
// contents of the comment; those are stored in a diffferentialDatabaseTransactionComment.
type differentialDatabaseTransaction struct {
	PHID        string
	AuthorPHID  string
	DateCreated uint32
	Type        string
	NewValue    *string
	CommentPHID *string
}

type ReadTransactions func(reviewID string) ([]differentialDatabaseTransaction, error)

func readDatabaseTransactions(reviewID string) ([]differentialDatabaseTransaction, error) {
	var transactions []differentialDatabaseTransaction
	result := runSqlCommandOrDie(fmt.Sprintf(selectTransactionsQueryTemplate, reviewID))
	if strings.Trim(result, " ") == "" {
		// There were no matching transactions
		return nil, nil
	}
	// result will be a line-separated list of query results, each of which has 7 columns.
	for _, line := range strings.Split(result, "\n") {
		lineParts := strings.Split(line, "\t")
		if len(lineParts) != 7 {
			return nil, fmt.Errorf("Unexpected number of transaction parts: %v", lineParts)
		}
		var transaction differentialDatabaseTransaction
		transaction.PHID = lineParts[1]
		transaction.AuthorPHID = lineParts[2]
		timestamp, err := strconv.ParseUint(lineParts[3], 10, 32)
		if err != nil {
			return nil, err
		}
		transaction.DateCreated = uint32(timestamp)
		transaction.Type = lineParts[4]
		if lineParts[5] != "NULL" {
			transaction.NewValue = &lineParts[5]
		}
		if lineParts[6] != "NULL" {
			transaction.CommentPHID = &lineParts[6]
		}
		transactions = append(transactions, transaction)
	}
	return transactions, nil
}

// differentialDatabaseTransactionComment stores the actual contents of a code review comment.
type differentialDatabaseTransactionComment struct {
	PHID               string
	Commit             string
	FileName           string
	LineNumber         uint32
	ReplyToCommentPHID *string
	Content            string
}

type ReadTransactionComment func(transactionID string) (*differentialDatabaseTransactionComment, error)

func readDatabaseTransactionComment(transactionID string) (*differentialDatabaseTransactionComment, error) {
	result := runSqlCommandOrDie(fmt.Sprintf(selectTransactionCommentsQueryTemplate, transactionID))
	// result will be a line separated list of query results, each of which includes 4 columns.
	lines := strings.Split(result, "\n")
	if len(lines) != 1 {
		return nil, fmt.Errorf("Unexpected number of query results: %v", lines)
	}
	lineParts := strings.Split(lines[0], "\t")
	if len(lineParts) != 4 {
		return nil, fmt.Errorf("Unexpected size of query results: %v", lineParts)
	}
	var comment differentialDatabaseTransactionComment
	comment.PHID = lineParts[0]
	if lineParts[1] != "NULL" {
		changesetID, err := strconv.ParseUint(lineParts[1], 10, 32)
		if err != nil {
			return nil, err
		}
		changesetResult := runSqlCommandOrDie(fmt.Sprintf(selectChangesetFilenameTemplate, changesetID))
		// changesetResult should have a single query result, which only includes the filename.
		comment.FileName = changesetResult
		diffIDResult := runSqlCommandOrDie(fmt.Sprintf(selectChangesetDiffTemplate, changesetID))
		diffID, err := strconv.Atoi(diffIDResult)
		if err != nil {
			log.Println(diffIDResult)
			log.Fatal(err)
		}
		diff, err := readDiff(diffID)
		if err != nil {
			log.Fatal(err)
		}
		comment.Commit = diff.findLastCommit()
	}
	lineNumber, err := strconv.ParseUint(lineParts[2], 10, 32)
	if err != nil {
		return nil, err
	}
	comment.LineNumber = uint32(lineNumber)
	if lineParts[3] != "NULL" {
		comment.ReplyToCommentPHID = &lineParts[3]
	}
	// The next SQL command is structured to return a single result with a single column, so we
	// don't need to parse it in any way.
	comment.Content = runRawSqlCommandOrDie(fmt.Sprintf(selectCommentContentsQueryTemplate, comment.PHID))
	return &comment, nil
}

// LoadComments takes in a differentialReview and returns the associated comments.
func (review differentialReview) LoadComments() []comment.Comment {
	return LoadComments(review, readDatabaseTransactions, readDatabaseTransactionComment, lookupUser)
}

func LoadComments(review differentialReview, readTransactions ReadTransactions, readTransactionComment ReadTransactionComment, lookupUser UserLookup) []comment.Comment {

	allTransactions, err := readTransactions(review.PHID)
	if err != nil {
		log.Fatal(err)
	}
	var comments []comment.Comment
	commentsByPHID := make(map[string]comment.Comment)
	rejectionCommentsByUser := make(map[string][]string)

	for _, transaction := range allTransactions {
		author, err := lookupUser(transaction.AuthorPHID)
		if err != nil {
			log.Fatal(err)
		}
		c := comment.Comment{
			Author:    author.Email,
			Timestamp: fmt.Sprintf("%d", transaction.DateCreated),
		}
		if author.Email != "" {
			c.Author = author.Email
		} else {
			c.Author = author.UserName
		}

		if transaction.CommentPHID != nil {
			transactionComment, err := readTransactionComment(transaction.PHID)
			if err != nil {
				log.Fatal(err)
			}
			if transactionComment.FileName != "" {
				c.Location = &comment.CommentLocation{
					Commit: transactionComment.Commit,
					Path:   transactionComment.FileName,
				}
				if transactionComment.LineNumber != 0 {
					c.Location.Range = &comment.CommentRange{
						StartLine: transactionComment.LineNumber,
					}
				}
			}
			c.Description = transactionComment.Content
			if transactionComment.ReplyToCommentPHID != nil {
				// We assume that the parent has to have been processed before the child,
				// and enforce that by ordering the transactions in our queries.
				if replyTo, ok := commentsByPHID[*transactionComment.ReplyToCommentPHID]; ok {
					parentHash, err := replyTo.Hash()
					if err != nil {
						log.Fatal(err)
					}
					c.Parent = parentHash
				}
			}
		}

		// Set the resolved bit based on whether the change was approved or not.
		if transaction.Type == "differential:action" && transaction.NewValue != nil {
			action := *transaction.NewValue
			var resolved bool
			if action == "\"accept\"" {
				resolved = true
				c.Resolved = &resolved

				// Add child comments to all previous rejects by this user and make them accepts
				for _, rejectionCommentHash := range rejectionCommentsByUser[author.UserName] {
					approveComment := comment.Comment{
						Author:    c.Author,
						Timestamp: c.Timestamp,
						Resolved:  &resolved,
						Parent:    rejectionCommentHash,
					}
					comments = append(comments, approveComment)
				}
			} else if action == "\"reject\"" {
				resolved = false
				c.Resolved = &resolved
			}

		}

		// Phabricator only publishes inline comments when you publish a top-level comment.
		// This results in a lot of empty top-level comments, which we do not want to mirror.
		// To work around this, we only return comments that are non-empty.
		if c.Parent != "" || c.Location != nil || c.Description != "" || c.Resolved != nil {
			comments = append(comments, c)
			commentsByPHID[transaction.PHID] = c

			//If this was a rejection comment, add it to ordered comment hash
			if c.Resolved != nil && *c.Resolved == false {
				commentHash, err := c.Hash()
				if err != nil {
					log.Fatal(err)
				}
				rejectionCommentsByUser[author.UserName] = append(rejectionCommentsByUser[author.UserName], commentHash)
			}

		}
	}
	return comments
}
<|endoftext|>


<commit_before>package controller

import (
	"bytes"
	"errors"
	"fmt"
	"time"

	"github.com/appscode/log"
	"github.com/ghodss/yaml"
	"github.com/graymeta/stow"
	_ "github.com/graymeta/stow/google"
	_ "github.com/graymeta/stow/s3"
	tapi "github.com/k8sdb/apimachinery/api"
	kapi "k8s.io/kubernetes/pkg/api"
	k8serr "k8s.io/kubernetes/pkg/api/errors"
	kapps "k8s.io/kubernetes/pkg/apis/apps"
	"k8s.io/kubernetes/pkg/labels"
)

func (c *Controller) ValidateStorageSpec(spec *tapi.StorageSpec) (*tapi.StorageSpec, error) {
	if spec == nil {
		return nil, nil
	}

	if spec.Class == "" {
		return nil, fmt.Errorf(`Object 'Class' is missing in '%v'`, *spec)
	}

	if _, err := c.Client.Storage().StorageClasses().Get(spec.Class); err != nil {
		if k8serr.IsNotFound(err) {
			return nil, fmt.Errorf(`Spec.Storage.Class "%v" not found`, spec.Class)
		}
		return nil, err
	}

	if len(spec.AccessModes) == 0 {
		spec.AccessModes = []kapi.PersistentVolumeAccessMode{
			kapi.ReadWriteOnce,
		}
		log.Infof(`Using "%v" as AccessModes in "%v"`, kapi.ReadWriteOnce, *spec)
	}

	if val, found := spec.Resources.Requests[kapi.ResourceStorage]; found {
		if val.Value() <= 0 {
			return nil, errors.New("Invalid ResourceStorage request")
		}
	} else {
		return nil, errors.New("Missing ResourceStorage request")
	}

	return spec, nil
}

func (c *Controller) ValidateBackupSchedule(spec *tapi.BackupScheduleSpec) error {
	if spec == nil {
		return nil
	}
	// CronExpression can't be empty
	if spec.CronExpression == "" {
		return errors.New("Invalid cron expression")
	}

	return c.ValidateSnapshotSpec(spec.SnapshotSpec)
}

func (c *Controller) ValidateSnapshotSpec(spec tapi.SnapshotSpec) error {
	// BucketName can't be empty
	bucketName := spec.BucketName
	if bucketName == "" {
		return fmt.Errorf(`Object 'BucketName' is missing in '%v'`, spec)
	}

	// Need to provide Storage credential secret
	storageSecret := spec.StorageSecret
	if storageSecret == nil {
		return fmt.Errorf(`Object 'StorageSecret' is missing in '%v'`, spec)
	}

	// Credential SecretName  can't be empty
	storageSecretName := storageSecret.SecretName
	if storageSecretName == "" {
		return fmt.Errorf(`Object 'SecretName' is missing in '%v'`, *spec.StorageSecret)
	}
	return nil
}

const (
	keyProvider = "provider"
	keyConfig   = "config"
)

func (c *Controller) CheckBucketAccess(dbSnapshot *tapi.DatabaseSnapshot) error {
	secret, err := c.Client.Core().Secrets(dbSnapshot.Namespace).Get(dbSnapshot.Spec.StorageSecret.SecretName)
	if err != nil {
		return err
	}

	provider := secret.Data[keyProvider]
	if provider == nil {
		return errors.New("Missing provider key")
	}
	configData := secret.Data[keyConfig]
	if configData == nil {
		return errors.New("Missing config key")
	}

	var config stow.ConfigMap
	if err := yaml.Unmarshal(configData, &config); err != nil {
		return err
	}

	loc, err := stow.Dial(string(provider), config)
	if err != nil {
		return err
	}

	container, err := loc.Container(dbSnapshot.Spec.BucketName)
	if err != nil {
		return err
	}

	r := bytes.NewReader([]byte("CheckBucketAccess"))
	item, err := container.Put(".k8sdb", r, r.Size(), nil)
	if err != nil {
		return err
	}

	if err := container.RemoveItem(item.ID()); err != nil {
		return err
	}
	return nil
}

func (c *Controller) CreateGoverningServiceAccount(name, namespace string) error {
	var err error
	if _, err = c.Client.Core().ServiceAccounts(namespace).Get(name); err == nil {
		return nil
	}
	if !k8serr.IsNotFound(err) {
		return err
	}

	serviceAccount := &kapi.ServiceAccount{
		ObjectMeta: kapi.ObjectMeta{
			Name: name,
		},
	}
	_, err = c.Client.Core().ServiceAccounts(namespace).Create(serviceAccount)
	return err
}

func (c *Controller) CheckStatefulSetPodStatus(statefulSet *kapps.StatefulSet, checkDuration time.Duration) error {
	podName := fmt.Sprintf("%v-%v", statefulSet.Name, 0)

	podReady := false
	then := time.Now()
	now := time.Now()
	for now.Sub(then) < checkDuration {
		pod, err := c.Client.Core().Pods(statefulSet.Namespace).Get(podName)
		if err != nil {
			if k8serr.IsNotFound(err) {
				_, err := c.Client.Apps().StatefulSets(statefulSet.Namespace).Get(statefulSet.Name)
				if k8serr.IsNotFound(err) {
					break
				}

				time.Sleep(time.Second * 10)
				now = time.Now()
				continue
			} else {
				return err
			}
		}
		log.Debugf("Pod Phase: %v", pod.Status.Phase)

		// If job is success
		if pod.Status.Phase == kapi.PodRunning {
			podReady = true
			break
		}

		time.Sleep(time.Minute)
		now = time.Now()
	}
	if !podReady {
		return errors.New("Database fails to be Ready")
	}
	return nil
}

func (c *Controller) DeletePersistentVolumeClaims(namespace string, selector labels.Selector) error {
	pvcList, err := c.Client.Core().PersistentVolumeClaims(namespace).List(
		kapi.ListOptions{
			LabelSelector: selector,
		},
	)
	if err != nil {
		return err
	}

	for _, pvc := range pvcList.Items {
		if err := c.Client.Core().PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, nil); err != nil {
			return err
		}
	}
	return nil
}

func (c *Controller) DeleteSnapshotData(dbSnapshot *tapi.DatabaseSnapshot) error {
	secret, err := c.Client.Core().Secrets(dbSnapshot.Namespace).Get(dbSnapshot.Spec.StorageSecret.SecretName)
	if err != nil {
		return err
	}

	provider := secret.Data[keyProvider]
	if provider == nil {
		return errors.New("Missing provider key")
	}
	configData := secret.Data[keyConfig]
	if configData == nil {
		return errors.New("Missing config key")
	}

	var config stow.ConfigMap
	if err := yaml.Unmarshal(configData, &config); err != nil {
		return err
	}

	loc, err := stow.Dial(string(provider), config)
	if err != nil {
		return err
	}

	container, err := loc.Container(dbSnapshot.Spec.BucketName)
	if err != nil {
		return err
	}

	folderName := dbSnapshot.Labels[LabelDatabaseType] + "-" + dbSnapshot.Spec.DatabaseName
	prefix := fmt.Sprintf("%v/%v", folderName, dbSnapshot.Name)
	cursor := stow.CursorStart
	for {
		items, next, err := container.Items(prefix, cursor, 50)
		if err != nil {
			return err
		}
		for _, item := range items {
			if err := container.RemoveItem(item.ID()); err != nil {
				return err
			}
		}
		cursor = next
		if stow.IsCursorEnd(cursor) {
			break
		}
	}

	return nil
}

func (c *Controller) DeleteDatabaseSnapshots(namespace string, selector labels.Selector) error {
	dbSnapshotList, err := c.ExtClient.DatabaseSnapshots(namespace).List(
		kapi.ListOptions{
			LabelSelector: selector,
		},
	)
	if err != nil {
		return err
	}

	for _, dbsnapshot := range dbSnapshotList.Items {
		if err := c.ExtClient.DatabaseSnapshots(dbsnapshot.Namespace).Delete(dbsnapshot.Name); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Modify CheckBucketAccess() parameters<commit_after>package controller

import (
	"bytes"
	"errors"
	"fmt"
	"time"

	"github.com/appscode/log"
	"github.com/ghodss/yaml"
	"github.com/graymeta/stow"
	_ "github.com/graymeta/stow/google"
	_ "github.com/graymeta/stow/s3"
	tapi "github.com/k8sdb/apimachinery/api"
	kapi "k8s.io/kubernetes/pkg/api"
	k8serr "k8s.io/kubernetes/pkg/api/errors"
	kapps "k8s.io/kubernetes/pkg/apis/apps"
	"k8s.io/kubernetes/pkg/labels"
)

func (c *Controller) ValidateStorageSpec(spec *tapi.StorageSpec) (*tapi.StorageSpec, error) {
	if spec == nil {
		return nil, nil
	}

	if spec.Class == "" {
		return nil, fmt.Errorf(`Object 'Class' is missing in '%v'`, *spec)
	}

	if _, err := c.Client.Storage().StorageClasses().Get(spec.Class); err != nil {
		if k8serr.IsNotFound(err) {
			return nil, fmt.Errorf(`Spec.Storage.Class "%v" not found`, spec.Class)
		}
		return nil, err
	}

	if len(spec.AccessModes) == 0 {
		spec.AccessModes = []kapi.PersistentVolumeAccessMode{
			kapi.ReadWriteOnce,
		}
		log.Infof(`Using "%v" as AccessModes in "%v"`, kapi.ReadWriteOnce, *spec)
	}

	if val, found := spec.Resources.Requests[kapi.ResourceStorage]; found {
		if val.Value() <= 0 {
			return nil, errors.New("Invalid ResourceStorage request")
		}
	} else {
		return nil, errors.New("Missing ResourceStorage request")
	}

	return spec, nil
}

func (c *Controller) ValidateBackupSchedule(spec *tapi.BackupScheduleSpec) error {
	if spec == nil {
		return nil
	}
	// CronExpression can't be empty
	if spec.CronExpression == "" {
		return errors.New("Invalid cron expression")
	}

	return c.ValidateSnapshotSpec(spec.SnapshotSpec)
}

func (c *Controller) ValidateSnapshotSpec(spec tapi.SnapshotSpec) error {
	// BucketName can't be empty
	bucketName := spec.BucketName
	if bucketName == "" {
		return fmt.Errorf(`Object 'BucketName' is missing in '%v'`, spec)
	}

	// Need to provide Storage credential secret
	storageSecret := spec.StorageSecret
	if storageSecret == nil {
		return fmt.Errorf(`Object 'StorageSecret' is missing in '%v'`, spec)
	}

	// Credential SecretName  can't be empty
	storageSecretName := storageSecret.SecretName
	if storageSecretName == "" {
		return fmt.Errorf(`Object 'SecretName' is missing in '%v'`, *spec.StorageSecret)
	}
	return nil
}

const (
	keyProvider = "provider"
	keyConfig   = "config"
)

func (c *Controller) CheckBucketAccess(snapshotSpec *tapi.SnapshotSpec, namespace string) error {
	secret, err := c.Client.Core().Secrets(namespace).Get(snapshotSpec.StorageSecret.SecretName)
	if err != nil {
		return err
	}

	provider := secret.Data[keyProvider]
	if provider == nil {
		return errors.New("Missing provider key")
	}
	configData := secret.Data[keyConfig]
	if configData == nil {
		return errors.New("Missing config key")
	}

	var config stow.ConfigMap
	if err := yaml.Unmarshal(configData, &config); err != nil {
		return err
	}

	loc, err := stow.Dial(string(provider), config)
	if err != nil {
		return err
	}

	container, err := loc.Container(snapshotSpec.BucketName)
	if err != nil {
		return err
	}

	r := bytes.NewReader([]byte("CheckBucketAccess"))
	item, err := container.Put(".k8sdb", r, r.Size(), nil)
	if err != nil {
		return err
	}

	if err := container.RemoveItem(item.ID()); err != nil {
		return err
	}
	return nil
}

func (c *Controller) CreateGoverningServiceAccount(name, namespace string) error {
	var err error
	if _, err = c.Client.Core().ServiceAccounts(namespace).Get(name); err == nil {
		return nil
	}
	if !k8serr.IsNotFound(err) {
		return err
	}

	serviceAccount := &kapi.ServiceAccount{
		ObjectMeta: kapi.ObjectMeta{
			Name: name,
		},
	}
	_, err = c.Client.Core().ServiceAccounts(namespace).Create(serviceAccount)
	return err
}

func (c *Controller) CheckStatefulSetPodStatus(statefulSet *kapps.StatefulSet, checkDuration time.Duration) error {
	podName := fmt.Sprintf("%v-%v", statefulSet.Name, 0)

	podReady := false
	then := time.Now()
	now := time.Now()
	for now.Sub(then) < checkDuration {
		pod, err := c.Client.Core().Pods(statefulSet.Namespace).Get(podName)
		if err != nil {
			if k8serr.IsNotFound(err) {
				_, err := c.Client.Apps().StatefulSets(statefulSet.Namespace).Get(statefulSet.Name)
				if k8serr.IsNotFound(err) {
					break
				}

				time.Sleep(time.Second * 10)
				now = time.Now()
				continue
			} else {
				return err
			}
		}
		log.Debugf("Pod Phase: %v", pod.Status.Phase)

		// If job is success
		if pod.Status.Phase == kapi.PodRunning {
			podReady = true
			break
		}

		time.Sleep(time.Minute)
		now = time.Now()
	}
	if !podReady {
		return errors.New("Database fails to be Ready")
	}
	return nil
}

func (c *Controller) DeletePersistentVolumeClaims(namespace string, selector labels.Selector) error {
	pvcList, err := c.Client.Core().PersistentVolumeClaims(namespace).List(
		kapi.ListOptions{
			LabelSelector: selector,
		},
	)
	if err != nil {
		return err
	}

	for _, pvc := range pvcList.Items {
		if err := c.Client.Core().PersistentVolumeClaims(pvc.Namespace).Delete(pvc.Name, nil); err != nil {
			return err
		}
	}
	return nil
}

func (c *Controller) DeleteSnapshotData(dbSnapshot *tapi.DatabaseSnapshot) error {
	secret, err := c.Client.Core().Secrets(dbSnapshot.Namespace).Get(dbSnapshot.Spec.StorageSecret.SecretName)
	if err != nil {
		return err
	}

	provider := secret.Data[keyProvider]
	if provider == nil {
		return errors.New("Missing provider key")
	}
	configData := secret.Data[keyConfig]
	if configData == nil {
		return errors.New("Missing config key")
	}

	var config stow.ConfigMap
	if err := yaml.Unmarshal(configData, &config); err != nil {
		return err
	}

	loc, err := stow.Dial(string(provider), config)
	if err != nil {
		return err
	}

	container, err := loc.Container(dbSnapshot.Spec.BucketName)
	if err != nil {
		return err
	}

	folderName := dbSnapshot.Labels[LabelDatabaseType] + "-" + dbSnapshot.Spec.DatabaseName
	prefix := fmt.Sprintf("%v/%v", folderName, dbSnapshot.Name)
	cursor := stow.CursorStart
	for {
		items, next, err := container.Items(prefix, cursor, 50)
		if err != nil {
			return err
		}
		for _, item := range items {
			if err := container.RemoveItem(item.ID()); err != nil {
				return err
			}
		}
		cursor = next
		if stow.IsCursorEnd(cursor) {
			break
		}
	}

	return nil
}

func (c *Controller) DeleteDatabaseSnapshots(namespace string, selector labels.Selector) error {
	dbSnapshotList, err := c.ExtClient.DatabaseSnapshots(namespace).List(
		kapi.ListOptions{
			LabelSelector: selector,
		},
	)
	if err != nil {
		return err
	}

	for _, dbsnapshot := range dbSnapshotList.Items {
		if err := c.ExtClient.DatabaseSnapshots(dbsnapshot.Namespace).Delete(dbsnapshot.Name); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package collectors

import (
	"bufio"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/StackExchange/slog"
	"github.com/bosun-monitor/scollector/opentsdb"
	"github.com/bosun-monitor/scollector/util"
)

type ProgramCollector struct {
	Path     string
	Interval time.Duration
}

func InitPrograms(cpath string) {
	cdir, err := os.Open(cpath)
	if err != nil {
		slog.Infoln(err)
		return
	}
	idirs, err := cdir.Readdir(0)
	if err != nil {
		slog.Infoln(err)
		return
	}
	for _, idir := range idirs {
		i, err := strconv.Atoi(idir.Name())
		if err != nil || i < 0 {
			slog.Infoln("invalid collector folder name:", idir.Name())
			continue
		}
		interval := time.Second * time.Duration(i)
		dir, err := os.Open(filepath.Join(cdir.Name(), idir.Name()))
		if err != nil {
			slog.Infoln(err)
			continue
		}
		files, err := dir.Readdir(0)
		if err != nil {
			slog.Infoln(err)
			continue
		}
		for _, file := range files {
			if !isExecutable(file) {
				continue
			}
			collectors = append(collectors, &ProgramCollector{
				Path:     filepath.Join(dir.Name(), file.Name()),
				Interval: interval,
			})
		}
	}
}

func isExecutable(f os.FileInfo) bool {
	switch runtime.GOOS {
	case "windows":
		exts := strings.Split(os.Getenv("PATHEXT"), ";")
		fileExt := filepath.Ext(strings.ToUpper(f.Name()))
		for _, ext := range exts {
			if filepath.Ext(strings.ToUpper(ext)) == fileExt {
				return true
			}
		}
		return false
	default:
		return f.Mode()&0111 != 0
	}
}

func (c *ProgramCollector) Run(dpchan chan<- *opentsdb.DataPoint) {
	if c.Interval == 0 {
		for {
			next := time.After(DefaultFreq)
			if err := c.runProgram(dpchan); err != nil {
				slog.Infoln(err)
			}
			<-next
			slog.Infoln("restarting", c.Path)
		}
	} else {
		for {
			next := time.After(c.Interval)
			c.runProgram(dpchan)
			<-next
		}
	}
}

func (c *ProgramCollector) Init() {
}

func (c *ProgramCollector) runProgram(dpchan chan<- *opentsdb.DataPoint) (progError error) {
	cmd := exec.Command(c.Path)
	pr, pw := io.Pipe()
	s := bufio.NewScanner(pr)
	cmd.Stdout = pw
	er, ew := io.Pipe()
	cmd.Stderr = ew
	if err := cmd.Start(); err != nil {
		return err
	}
	go func() {
		progError = cmd.Wait()
		pw.Close()
		ew.Close()
	}()
	go func() {
		es := bufio.NewScanner(er)
		for es.Scan() {
			line := strings.TrimSpace(es.Text())
			slog.Error(line)
		}
	}()
Loop:
	for s.Scan() {
		line := strings.TrimSpace(s.Text())
		sp := strings.Fields(line)
		if len(sp) < 3 {
			slog.Errorf("bad line in program %s: %s", c.Path, line)
			continue
		}
		ts, err := strconv.ParseInt(sp[1], 10, 64)
		if err != nil {
			slog.Errorf("bad timestamp in program %s: %s", c.Path, sp[1])
			continue
		}
		val, err := strconv.ParseInt(sp[2], 10, 64)
		if err != nil {
			slog.Errorf("bad value in program %s: %s", c.Path, sp[2])
			continue
		}
		dp := opentsdb.DataPoint{
			Metric:    sp[0],
			Timestamp: ts,
			Value:     val,
			Tags:      opentsdb.TagSet{"host": util.Hostname},
		}
		for _, tag := range sp[3:] {
			tags, err := opentsdb.ParseTags(tag)
			if v, ok := tags["host"]; ok && v == "" {
				delete(dp.Tags, "host")
			} else if err != nil {
				slog.Errorf("bad tag in program %s, metric %s: %v", c.Path, sp[0], tag)
				continue Loop
			} else {
				dp.Tags.Merge(tags)
			}
		}
		dpchan <- &dp
	}
	if err := s.Err(); err != nil {
		return err
	}
	return
}

func (c *ProgramCollector) Name() string {
	return c.Path
}
<commit_msg>cmd/scollector: Values are floats, not inst<commit_after>package collectors

import (
	"bufio"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/StackExchange/slog"
	"github.com/bosun-monitor/scollector/opentsdb"
	"github.com/bosun-monitor/scollector/util"
)

type ProgramCollector struct {
	Path     string
	Interval time.Duration
}

func InitPrograms(cpath string) {
	cdir, err := os.Open(cpath)
	if err != nil {
		slog.Infoln(err)
		return
	}
	idirs, err := cdir.Readdir(0)
	if err != nil {
		slog.Infoln(err)
		return
	}
	for _, idir := range idirs {
		i, err := strconv.Atoi(idir.Name())
		if err != nil || i < 0 {
			slog.Infoln("invalid collector folder name:", idir.Name())
			continue
		}
		interval := time.Second * time.Duration(i)
		dir, err := os.Open(filepath.Join(cdir.Name(), idir.Name()))
		if err != nil {
			slog.Infoln(err)
			continue
		}
		files, err := dir.Readdir(0)
		if err != nil {
			slog.Infoln(err)
			continue
		}
		for _, file := range files {
			if !isExecutable(file) {
				continue
			}
			collectors = append(collectors, &ProgramCollector{
				Path:     filepath.Join(dir.Name(), file.Name()),
				Interval: interval,
			})
		}
	}
}

func isExecutable(f os.FileInfo) bool {
	switch runtime.GOOS {
	case "windows":
		exts := strings.Split(os.Getenv("PATHEXT"), ";")
		fileExt := filepath.Ext(strings.ToUpper(f.Name()))
		for _, ext := range exts {
			if filepath.Ext(strings.ToUpper(ext)) == fileExt {
				return true
			}
		}
		return false
	default:
		return f.Mode()&0111 != 0
	}
}

func (c *ProgramCollector) Run(dpchan chan<- *opentsdb.DataPoint) {
	if c.Interval == 0 {
		for {
			next := time.After(DefaultFreq)
			if err := c.runProgram(dpchan); err != nil {
				slog.Infoln(err)
			}
			<-next
			slog.Infoln("restarting", c.Path)
		}
	} else {
		for {
			next := time.After(c.Interval)
			c.runProgram(dpchan)
			<-next
		}
	}
}

func (c *ProgramCollector) Init() {
}

func (c *ProgramCollector) runProgram(dpchan chan<- *opentsdb.DataPoint) (progError error) {
	cmd := exec.Command(c.Path)
	pr, pw := io.Pipe()
	s := bufio.NewScanner(pr)
	cmd.Stdout = pw
	er, ew := io.Pipe()
	cmd.Stderr = ew
	if err := cmd.Start(); err != nil {
		return err
	}
	go func() {
		progError = cmd.Wait()
		pw.Close()
		ew.Close()
	}()
	go func() {
		es := bufio.NewScanner(er)
		for es.Scan() {
			line := strings.TrimSpace(es.Text())
			slog.Error(line)
		}
	}()
Loop:
	for s.Scan() {
		line := strings.TrimSpace(s.Text())
		sp := strings.Fields(line)
		if len(sp) < 3 {
			slog.Errorf("bad line in program %s: %s", c.Path, line)
			continue
		}
		ts, err := strconv.ParseInt(sp[1], 10, 64)
		if err != nil {
			slog.Errorf("bad timestamp in program %s: %s", c.Path, sp[1])
			continue
		}
		val, err := strconv.ParseFloat(sp[2], 64)
		if err != nil {
			slog.Errorf("bad value in program %s: %s", c.Path, sp[2])
			continue
		}
		dp := opentsdb.DataPoint{
			Metric:    sp[0],
			Timestamp: ts,
			Value:     val,
			Tags:      opentsdb.TagSet{"host": util.Hostname},
		}
		for _, tag := range sp[3:] {
			tags, err := opentsdb.ParseTags(tag)
			if v, ok := tags["host"]; ok && v == "" {
				delete(dp.Tags, "host")
			} else if err != nil {
				slog.Errorf("bad tag in program %s, metric %s: %v", c.Path, sp[0], tag)
				continue Loop
			} else {
				dp.Tags.Merge(tags)
			}
		}
		dpchan <- &dp
	}
	if err := s.Err(); err != nil {
		return err
	}
	return
}

func (c *ProgramCollector) Name() string {
	return c.Path
}
<|endoftext|>


<commit_before>// BaruwaAPI Golang bindings for Baruwa REST API
// Copyright (C) 2019 Andrew Colin Kissa <andrew@topdog.za.net>

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

// Package api Golang bindings for Baruwa REST API
package api
<commit_msg>FET: Organization relay skeleton<commit_after>// BaruwaAPI Golang bindings for Baruwa REST API
// Copyright (C) 2019 Andrew Colin Kissa <andrew@topdog.za.net>

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

// Package api Golang bindings for Baruwa REST API
package api

// RelaySetting holds relay settings
type RelaySetting struct {
	ID              int     `json:"id,omitempty"`
	Address         string  `json:"address"`
	Username        string  `json:"username"`
	Enabled         bool    `json:"enabled"`
	RequireTLS      bool    `json:"require_tls"`
	Password1       string  `json:"password1,omitempty"`
	Password2       string  `json:"password2,omitempty"`
	Description     string  `json:"description"`
	LowScore        float64 `json:"low_score"`
	HighScore       float64 `json:"high_score"`
	SpamActions     int     `json:"spam_actions"`
	HighSpamActions int     `json:"highspam_actions"`
	BlockMacros     bool    `json:"block_macros"`
	RateLimit       int     `json:"ratelimit"`
}

// GetRelaySetting returns radius settings
// https://www.baruwa.com/docs/api/#retrieve-relay-settings
func (c *Client) GetRelaySetting(id int) (server *RelaySetting, err error) {
	return
}

// CreateRelaySetting creates radius settings
// https://www.baruwa.com/docs/api/#create-relay-settings
func (c *Client) CreateRelaySetting(server *RelaySetting) (err error) {
	return
}

// UpdateRelaySetting updates radius settings
// https://www.baruwa.com/docs/api/#update-relay-settings
func (c *Client) UpdateRelaySetting(server *RelaySetting) (err error) {
	return
}

// DeleteRelaySetting deletes radius settings
// https://www.baruwa.com/docs/api/#delete-relay-settings
func (c *Client) DeleteRelaySetting(id int) (err error) {
	return
}
<|endoftext|>