<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"

	"github.com/jessevdk/go-flags"
	"github.com/mackerelio/checkers"
)

type clusterInfoOpts struct {
	orchestratorOpts
	URI string `short:"U" long:"uri" default:"api/clusters-info" description:"URI"`
}

type ClusterInfoResponse struct {
	ClusterName                            string
	ClusterAlias                           string
	ClusterDomain                          string
	CountInstances                         int
	HeuristicLag                           int
	HasAutomatedMasterRecovery             bool
	HasAutomatedIntermediateMasterRecovery bool
}

func checkClusterInfo(args []string) *checkers.Checker {

	opts := clusterInfoOpts{}
	psr := flags.NewParser(&opts, flags.Default)
	psr.Usage = "clusterinfo [OPTIONS]"
	_, err := psr.ParseArgs(args)
	if err != nil {
		os.Exit(1)
	}

	uri := fmt.Sprintf("%s://%s:%s/%s", sslPrefix(opts.SSL), opts.Host, opts.Port, opts.URI)
	client := &http.Client{Transport: getHttpTransport(opts.NoCert)}
	resp, err := client.Get(uri)
	if err != nil {
		return checkers.NewChecker(checkers.UNKNOWN, fmt.Sprintf("Could not connect to Orchestrator API on %s", uri))
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	var r []ClusterInfoResponse
	err = json.Unmarshal(body, &r)

	if err != nil {
		return checkers.NewChecker(checkers.UNKNOWN, fmt.Sprintf("Could read content for the Orchestrator API on %s\n%s", uri, err))
	}

	var aliases []string
	for _, s := range r {
		alias := fmt.Sprintf("%s (HasAutomatedMasterRecovery = %t) (HasAutomtedIntermediateMasterRecovery = %t)",
			s.ClusterAlias, s.HasAutomatedMasterRecovery, s.HasAutomatedIntermediateMasterRecovery)
		aliases = append(aliases, alias)
	}

	if len(aliases) > 0 {
		return checkers.NewChecker(checkers.OK, fmt.Sprintf("This instance manages following clusters:\n%s", strings.Join(aliases, "\n")))
	}

	return checkers.NewChecker(checkers.WARNING, "This Orchestrator is responding correctly but is not managing any clusters.")
}
<commit_msg>show clusteraliases in summary<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"

	"github.com/jessevdk/go-flags"
	"github.com/mackerelio/checkers"
)

type clusterInfoOpts struct {
	orchestratorOpts
	URI string `short:"U" long:"uri" default:"api/clusters-info" description:"URI"`
}

type ClusterInfoResponse struct {
	ClusterName                            string
	ClusterAlias                           string
	ClusterDomain                          string
	CountInstances                         int
	HeuristicLag                           int
	HasAutomatedMasterRecovery             bool
	HasAutomatedIntermediateMasterRecovery bool
}

func checkClusterInfo(args []string) *checkers.Checker {

	opts := clusterInfoOpts{}
	psr := flags.NewParser(&opts, flags.Default)
	psr.Usage = "clusterinfo [OPTIONS]"
	_, err := psr.ParseArgs(args)
	if err != nil {
		os.Exit(1)
	}

	uri := fmt.Sprintf("%s://%s:%s/%s", sslPrefix(opts.SSL), opts.Host, opts.Port, opts.URI)
	client := &http.Client{Transport: getHttpTransport(opts.NoCert)}
	resp, err := client.Get(uri)
	if err != nil {
		return checkers.NewChecker(checkers.UNKNOWN, fmt.Sprintf("Could not connect to Orchestrator API on %s", uri))
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)

	var r []ClusterInfoResponse
	err = json.Unmarshal(body, &r)

	if err != nil {
		return checkers.NewChecker(checkers.UNKNOWN, fmt.Sprintf("Could read content for the Orchestrator API on %s\n%s", uri, err))
	}

	var aliases []string
	var aliasdetails []string
	for _, s := range r {
		alias := fmt.Sprintf("%s (HasAutomatedMasterRecovery = %t) (HasAutomtedIntermediateMasterRecovery = %t)",
			s.ClusterAlias, s.HasAutomatedMasterRecovery, s.HasAutomatedIntermediateMasterRecovery)

		aliases = append(aliases, s.ClusterAlias)
		aliasdetails = append(aliasdetails, alias)
	}

	if len(aliases) > 0 {
		return checkers.NewChecker(checkers.OK, fmt.Sprintf("This instance manages following clusters: %s\n%s", strings.Join(aliases, ", "), strings.Join(aliasdetails, "\n")))
	}

	return checkers.NewChecker(checkers.WARNING, "This Orchestrator is responding correctly but is not managing any clusters.")
}
<|endoftext|>


<commit_before>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2017 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package cmd

import (
	"os"
	"path/filepath"

	"github.com/loadimpact/k6/converter/har"
	"github.com/spf13/cobra"
)

var output = "har-script.js"

var (
	enableChecks bool
	threshold    uint
	only         []string
	skip         []string
)

var convertCmd = &cobra.Command{
	Use:   "convert",
	Short: "Convert a HAR file to a k6 script",
	Long:  "Convert a HAR (HTTP Archive) file to a k6 script",
	Example: `
  # Convert a HAR file to a k6 script.
  k6 convert -O har-session.js session.har

  # Convert a HAR file to a k6 script creating requests only for the given domain/s.
  k6 convert -O har-session.js --only yourdomain.com,additionaldomain.com session.har

  # Run the k6 script.
  k6 run har-session.js`[1:],
	Args: cobra.ExactArgs(1),
	RunE: func(cmd *cobra.Command, args []string) error {
		// Parse the HAR file
		filePath, err := filepath.Abs(args[0])
		if err != nil {
			return err
		}
		r, err := os.Open(filePath)
		if err != nil {
			return err
		}
		h, err := har.Decode(r)
		if err != nil {
			return err
		}
		if err := r.Close(); err != nil {
			return err
		}

		script, err := har.Convert(h, enableChecks, threshold, only, skip)
		if err != nil {
			return err
		}

		// Write script content to output
		f, err := os.Create(output)
		if err != nil {
			return err
		}
		if _, err := f.WriteString(script); err != nil {
			return err
		}
		if err := f.Sync(); err != nil {
			return err
		}
		if err := f.Close(); err != nil {
			return err
		}
		return nil
	},
}

func init() {
	RootCmd.AddCommand(convertCmd)
	convertCmd.Flags().SortFlags = false
	convertCmd.Flags().StringVarP(&output, "output", "O", output, "k6 script output filename")
	convertCmd.Flags().StringSliceVarP(&only, "only", "", []string{}, "include only requests from the given domains")
	convertCmd.Flags().StringSliceVarP(&skip, "skip", "", []string{}, "skip requests from the given domains")
	convertCmd.Flags().UintVarP(&threshold, "batch-threshold", "", 500, "split requests in different batch statements when the start time difference between subsequent requests is smaller than the given value in ms. A sleep will be added between the batch statements.")
	convertCmd.Flags().BoolVarP(&enableChecks, "enable-status-code-checks", "", false, "add a check for each http status response")
}
<commit_msg>Fix convert help text, closes #473<commit_after>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2017 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package cmd

import (
	"os"
	"path/filepath"

	"github.com/loadimpact/k6/converter/har"
	"github.com/spf13/cobra"
)

var output = "har-script.js"

var (
	enableChecks bool
	threshold    uint
	only         []string
	skip         []string
)

var convertCmd = &cobra.Command{
	Use:   "convert",
	Short: "Convert a HAR file to a k6 script",
	Long:  "Convert a HAR (HTTP Archive) file to a k6 script",
	Example: `
  # Convert a HAR file to a k6 script.
  k6 convert -O har-session.js session.har

  # Convert a HAR file to a k6 script creating requests only for the given domain/s.
  k6 convert -O har-session.js --only yourdomain.com,additionaldomain.com session.har

  # Convert a HAR file. Batching requests together as long as idle time between requests <800ms
  k6 convert --batch-threshold 800 session.har

  # Run the k6 script.
  k6 run har-session.js`[1:],
	Args: cobra.ExactArgs(1),
	RunE: func(cmd *cobra.Command, args []string) error {
		// Parse the HAR file
		filePath, err := filepath.Abs(args[0])
		if err != nil {
			return err
		}
		r, err := os.Open(filePath)
		if err != nil {
			return err
		}
		h, err := har.Decode(r)
		if err != nil {
			return err
		}
		if err := r.Close(); err != nil {
			return err
		}

		script, err := har.Convert(h, enableChecks, threshold, only, skip)
		if err != nil {
			return err
		}

		// Write script content to output
		f, err := os.Create(output)
		if err != nil {
			return err
		}
		if _, err := f.WriteString(script); err != nil {
			return err
		}
		if err := f.Sync(); err != nil {
			return err
		}
		if err := f.Close(); err != nil {
			return err
		}
		return nil
	},
}

func init() {
	RootCmd.AddCommand(convertCmd)
	convertCmd.Flags().SortFlags = false
	convertCmd.Flags().StringVarP(&output, "output", "O", output, "k6 script output filename")
	convertCmd.Flags().StringSliceVarP(&only, "only", "", []string{}, "include only requests from the given domains")
	convertCmd.Flags().StringSliceVarP(&skip, "skip", "", []string{}, "skip requests from the given domains")
	convertCmd.Flags().UintVarP(&threshold, "batch-threshold", "", 500, "batch request idle time threshold (see example)")
	convertCmd.Flags().BoolVarP(&enableChecks, "enable-status-code-checks", "", false, "add a check for each http status response")
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"io/ioutil"
	"os"
	"runtime"
	"strconv"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/clientauth"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/golang/glog"
)

var (
	authConfig = flag.String("auth_config", os.Getenv("HOME")+"/.kubernetes_auth", "Path to the auth info file.")
	host       = flag.String("host", "", "The host to connect to")
)

func waitForPodRunning(c *client.Client, id string) {
	for {
		time.Sleep(5 * time.Second)
		pod, err := c.Pods(api.NamespaceDefault).Get(id)
		if err != nil {
			glog.Warningf("Get pod failed: %v", err)
			continue
		}
		if pod.Status.Phase == api.PodRunning {
			break
		}
		glog.Infof("Waiting for pod status to be running (%s)", pod.Status.Phase)
	}
}

func loadObjectOrDie(filePath string) interface{} {
	data, err := ioutil.ReadFile(filePath)
	if err != nil {
		glog.Fatalf("Failed to read pod: %v", err)
	}
	obj, err := latest.Codec.Decode(data)
	if err != nil {
		glog.Fatalf("Failed to decode pod: %v", err)
	}
	return obj
}

func loadPodOrDie(filePath string) *api.Pod {
	obj := loadObjectOrDie(filePath)
	pod, ok := obj.(*api.Pod)
	if !ok {
		glog.Fatalf("Failed to load pod: %v", obj)
	}
	return pod
}

func loadClientOrDie() *client.Client {
	config := client.Config{
		Host: *host,
	}
	auth, err := clientauth.LoadFromFile(*authConfig)
	if err != nil {
		glog.Fatalf("Error loading auth: %v", err)
	}
	config, err = auth.MergeWithConfig(config)
	if err != nil {
		glog.Fatalf("Error creating client")
	}
	c, err := client.New(&config)
	if err != nil {
		glog.Fatalf("Error creating client")
	}
	return c
}

func TestKubernetesROService(c *client.Client) bool {
	svc := api.ServiceList{}
	err := c.Get().
		Namespace("default").
		AbsPath("/api/v1beta1/proxy/services/kubernetes-ro/api/v1beta1/services").
		Do().
		Into(&svc)
	if err != nil {
		glog.Errorf("unexpected error listing services using ro service: %v", err)
		return false
	}
	var foundRW, foundRO bool
	for i := range svc.Items {
		if svc.Items[i].Name == "kubernetes" {
			foundRW = true
		}
		if svc.Items[i].Name == "kubernetes-ro" {
			foundRO = true
		}
	}
	if !foundRW {
		glog.Error("no RW service found")
	}
	if !foundRO {
		glog.Error("no RO service found")
	}
	if !foundRW || !foundRO {
		return false
	}
	return true
}

func TestPodUpdate(c *client.Client) bool {
	podClient := c.Pods(api.NamespaceDefault)

	pod := loadPodOrDie("./api/examples/pod.json")
	value := strconv.Itoa(time.Now().Nanosecond())
	pod.Labels["time"] = value

	_, err := podClient.Create(pod)
	if err != nil {
		glog.Errorf("Failed to create pod: %v", err)
		return false
	}
	defer podClient.Delete(pod.Name)
	waitForPodRunning(c, pod.Name)
	pods, err := podClient.List(labels.SelectorFromSet(labels.Set(map[string]string{"time": value})))
	if len(pods.Items) != 1 {
		glog.Errorf("Failed to find the correct pod")
		return false
	}

	podOut, err := podClient.Get(pod.Name)
	if err != nil {
		glog.Errorf("Failed to get pod: %v", err)
		return false
	}
	value = "time" + value
	pod.Labels["time"] = value
	pod.ResourceVersion = podOut.ResourceVersion
	pod.UID = podOut.UID
	pod, err = podClient.Update(pod)
	if err != nil {
		glog.Errorf("Failed to update pod: %v", err)
		return false
	}
	waitForPodRunning(c, pod.Name)
	pods, err = podClient.List(labels.SelectorFromSet(labels.Set(map[string]string{"time": value})))
	if len(pods.Items) != 1 {
		glog.Errorf("Failed to find the correct pod after update.")
		return false
	}
	glog.Infof("pod update OK")
	return true
}

// TestKubeletSendsEvent checks that kubelets and scheduler send events about pods scheduling and running.
func TestKubeletSendsEvent(c *client.Client) bool {
	provider := os.Getenv("KUBERNETES_PROVIDER")
	if provider != "gce" {
		glog.Infof("skipping TestKubeletSendsEvent on cloud provider %s", provider)
		return true
	}
	if provider == "" {
		glog.Info("KUBERNETES_PROVIDER is unset assuming \"gce\"")
	}

	podClient := c.Pods(api.NamespaceDefault)

	pod := loadPodOrDie("./cmd/e2e/pod.json")
	value := strconv.Itoa(time.Now().Nanosecond())
	pod.Labels["time"] = value

	_, err := podClient.Create(pod)
	if err != nil {
		glog.Errorf("Failed to create pod: %v", err)
		return false
	}
	defer podClient.Delete(pod.Name)
	waitForPodRunning(c, pod.Name)
	pods, err := podClient.List(labels.SelectorFromSet(labels.Set(map[string]string{"time": value})))
	if len(pods.Items) != 1 {
		glog.Errorf("Failed to find the correct pod")
		return false
	}

	_, err = podClient.Get(pod.Name)
	if err != nil {
		glog.Errorf("Failed to get pod: %v", err)
		return false
	}

	// Check for scheduler event about the pod.
	events, err := c.Events(api.NamespaceDefault).List(
		labels.Everything(),
		labels.Set{
			"involvedObject.name":      pod.Name,
			"involvedObject.kind":      "Pod",
			"involvedObject.namespace": api.NamespaceDefault,
			"source":                   "scheduler",
			"time":                     value,
		}.AsSelector(),
	)
	if err != nil {
		glog.Error("Error while listing events:", err)
		return false
	}
	if len(events.Items) == 0 {
		glog.Error("Didn't see any scheduler events even though pod was running.")
		return false
	}
	glog.Info("Saw scheduler event for our pod.")

	// Check for kubelet event about the pod.
	events, err = c.Events(api.NamespaceDefault).List(
		labels.Everything(),
		labels.Set{
			"involvedObject.name":      pod.Name,
			"involvedObject.kind":      "BoundPod",
			"involvedObject.namespace": api.NamespaceDefault,
			"source":                   "kubelet",
		}.AsSelector(),
	)
	if err != nil {
		glog.Error("Error while listing events:", err)
		return false
	}
	if len(events.Items) == 0 {
		glog.Error("Didn't see any kubelet events even though pod was running.")
		return false
	}
	glog.Info("Saw kubelet event for our pod.")
	return true
}

func main() {
	flag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU())
	util.ReallyCrash = true
	util.InitLogs()
	defer util.FlushLogs()

	go func() {
		defer util.FlushLogs()
		time.Sleep(3 * time.Minute)
		glog.Fatalf("This test has timed out.")
	}()

	c := loadClientOrDie()

	tests := []func(c *client.Client) bool{
		TestKubernetesROService,
		TestKubeletSendsEvent,
		// TODO(brendandburns): fix this test and re-add it: TestPodUpdate,
	}

	passed := true
	for _, test := range tests {
		testPassed := test(c)
		if !testPassed {
			passed = false
		}
		// TODO: clean up objects created during a test after the test, so cases
		// are independent.
	}
	if !passed {
		glog.Fatalf("Tests failed")
	}
}
<commit_msg>Add a test for important URLs.<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"io/ioutil"
	"os"
	"runtime"
	"strconv"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/api/latest"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/clientauth"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/golang/glog"
)

var (
	authConfig = flag.String("auth_config", os.Getenv("HOME")+"/.kubernetes_auth", "Path to the auth info file.")
	host       = flag.String("host", "", "The host to connect to")
)

func waitForPodRunning(c *client.Client, id string) {
	for {
		time.Sleep(5 * time.Second)
		pod, err := c.Pods(api.NamespaceDefault).Get(id)
		if err != nil {
			glog.Warningf("Get pod failed: %v", err)
			continue
		}
		if pod.Status.Phase == api.PodRunning {
			break
		}
		glog.Infof("Waiting for pod status to be running (%s)", pod.Status.Phase)
	}
}

func loadObjectOrDie(filePath string) interface{} {
	data, err := ioutil.ReadFile(filePath)
	if err != nil {
		glog.Fatalf("Failed to read pod: %v", err)
	}
	obj, err := latest.Codec.Decode(data)
	if err != nil {
		glog.Fatalf("Failed to decode pod: %v", err)
	}
	return obj
}

func loadPodOrDie(filePath string) *api.Pod {
	obj := loadObjectOrDie(filePath)
	pod, ok := obj.(*api.Pod)
	if !ok {
		glog.Fatalf("Failed to load pod: %v", obj)
	}
	return pod
}

func loadClientOrDie() *client.Client {
	config := client.Config{
		Host: *host,
	}
	auth, err := clientauth.LoadFromFile(*authConfig)
	if err != nil {
		glog.Fatalf("Error loading auth: %v", err)
	}
	config, err = auth.MergeWithConfig(config)
	if err != nil {
		glog.Fatalf("Error creating client")
	}
	c, err := client.New(&config)
	if err != nil {
		glog.Fatalf("Error creating client")
	}
	return c
}

func TestKubernetesROService(c *client.Client) bool {
	svc := api.ServiceList{}
	err := c.Get().
		Namespace("default").
		AbsPath("/api/v1beta1/proxy/services/kubernetes-ro/api/v1beta1/services").
		Do().
		Into(&svc)
	if err != nil {
		glog.Errorf("unexpected error listing services using ro service: %v", err)
		return false
	}
	var foundRW, foundRO bool
	for i := range svc.Items {
		if svc.Items[i].Name == "kubernetes" {
			foundRW = true
		}
		if svc.Items[i].Name == "kubernetes-ro" {
			foundRO = true
		}
	}
	if !foundRW {
		glog.Error("no RW service found")
	}
	if !foundRO {
		glog.Error("no RO service found")
	}
	if !foundRW || !foundRO {
		return false
	}
	return true
}

func TestPodUpdate(c *client.Client) bool {
	podClient := c.Pods(api.NamespaceDefault)

	pod := loadPodOrDie("./api/examples/pod.json")
	value := strconv.Itoa(time.Now().Nanosecond())
	pod.Labels["time"] = value

	_, err := podClient.Create(pod)
	if err != nil {
		glog.Errorf("Failed to create pod: %v", err)
		return false
	}
	defer podClient.Delete(pod.Name)
	waitForPodRunning(c, pod.Name)
	pods, err := podClient.List(labels.SelectorFromSet(labels.Set(map[string]string{"time": value})))
	if len(pods.Items) != 1 {
		glog.Errorf("Failed to find the correct pod")
		return false
	}

	podOut, err := podClient.Get(pod.Name)
	if err != nil {
		glog.Errorf("Failed to get pod: %v", err)
		return false
	}
	value = "time" + value
	pod.Labels["time"] = value
	pod.ResourceVersion = podOut.ResourceVersion
	pod.UID = podOut.UID
	pod, err = podClient.Update(pod)
	if err != nil {
		glog.Errorf("Failed to update pod: %v", err)
		return false
	}
	waitForPodRunning(c, pod.Name)
	pods, err = podClient.List(labels.SelectorFromSet(labels.Set(map[string]string{"time": value})))
	if len(pods.Items) != 1 {
		glog.Errorf("Failed to find the correct pod after update.")
		return false
	}
	glog.Infof("pod update OK")
	return true
}

// TestImportantURLs validates that URLs that people depend on haven't moved.
// ***IMPORTANT*** Do *not* fix this test just by changing the path.  If you moved a URL
// you can break upstream dependencies.
func TestImportantURLs(c *client.Client) bool {
	tests := []struct {
		path string
	}{}
	ok := true
	for _, test := range tests {
		glog.Infof("testing: %s", test.path)
		data, err := c.RESTClient.Get().
			AbsPath(test.path).
			Do().
			Raw()
	}
	if err != nil {
		glog.Errorf("Failed: %v\nBody: %s", err, string(data))
		ok := false
	}
	return ok
}

// TestKubeletSendsEvent checks that kubelets and scheduler send events about pods scheduling and running.
func TestKubeletSendsEvent(c *client.Client) bool {
	provider := os.Getenv("KUBERNETES_PROVIDER")
	if provider != "gce" {
		glog.Infof("skipping TestKubeletSendsEvent on cloud provider %s", provider)
		return true
	}
	if provider == "" {
		glog.Info("KUBERNETES_PROVIDER is unset assuming \"gce\"")
	}

	podClient := c.Pods(api.NamespaceDefault)

	pod := loadPodOrDie("./cmd/e2e/pod.json")
	value := strconv.Itoa(time.Now().Nanosecond())
	pod.Labels["time"] = value

	_, err := podClient.Create(pod)
	if err != nil {
		glog.Errorf("Failed to create pod: %v", err)
		return false
	}
	defer podClient.Delete(pod.Name)
	waitForPodRunning(c, pod.Name)
	pods, err := podClient.List(labels.SelectorFromSet(labels.Set(map[string]string{"time": value})))
	if len(pods.Items) != 1 {
		glog.Errorf("Failed to find the correct pod")
		return false
	}

	_, err = podClient.Get(pod.Name)
	if err != nil {
		glog.Errorf("Failed to get pod: %v", err)
		return false
	}

	// Check for scheduler event about the pod.
	events, err := c.Events(api.NamespaceDefault).List(
		labels.Everything(),
		labels.Set{
			"involvedObject.name":      pod.Name,
			"involvedObject.kind":      "Pod",
			"involvedObject.namespace": api.NamespaceDefault,
			"source":                   "scheduler",
			"time":                     value,
		}.AsSelector(),
	)
	if err != nil {
		glog.Error("Error while listing events:", err)
		return false
	}
	if len(events.Items) == 0 {
		glog.Error("Didn't see any scheduler events even though pod was running.")
		return false
	}
	glog.Info("Saw scheduler event for our pod.")

	// Check for kubelet event about the pod.
	events, err = c.Events(api.NamespaceDefault).List(
		labels.Everything(),
		labels.Set{
			"involvedObject.name":      pod.Name,
			"involvedObject.kind":      "BoundPod",
			"involvedObject.namespace": api.NamespaceDefault,
			"source":                   "kubelet",
		}.AsSelector(),
	)
	if err != nil {
		glog.Error("Error while listing events:", err)
		return false
	}
	if len(events.Items) == 0 {
		glog.Error("Didn't see any kubelet events even though pod was running.")
		return false
	}
	glog.Info("Saw kubelet event for our pod.")
	return true
}

func main() {
	flag.Parse()
	runtime.GOMAXPROCS(runtime.NumCPU())
	util.ReallyCrash = true
	util.InitLogs()
	defer util.FlushLogs()

	go func() {
		defer util.FlushLogs()
		time.Sleep(3 * time.Minute)
		glog.Fatalf("This test has timed out.")
	}()

	c := loadClientOrDie()

	tests := []func(c *client.Client) bool{
		TestKubernetesROService,
		TestKubeletSendsEvent,
		// TODO(brendandburns): fix this test and re-add it: TestPodUpdate,
	}

	passed := true
	for _, test := range tests {
		testPassed := test(c)
		if !testPassed {
			passed = false
		}
		// TODO: clean up objects created during a test after the test, so cases
		// are independent.
	}
	if !passed {
		glog.Fatalf("Tests failed")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"

	"github.com/constabulary/gb"
	"github.com/constabulary/gb/cmd"
	"github.com/constabulary/gb/test"
)

func init() {
	registerCommand(TestCmd)
}

var (
	tfs           []string // Arguments passed to the test binary
	testProfile   bool
	testCover     bool
	testCoverMode string
	testCoverPkg  string
)

func addTestFlags(fs *flag.FlagSet) {
	addBuildFlags(fs)
	fs.BoolVar(&testCover, "cover", false, "enable coverage analysis")
	fs.StringVar(&testCoverMode, "covermode", "set", "Set covermode: set (default), count, atomic")
	fs.StringVar(&testCoverPkg, "coverpkg", "", "enable coverage analysis")
}

var TestCmd = &cmd.Command{
	Name:      "test",
	UsageLine: "test [build flags] [packages] [flags for test binary]",
	Short:     "test packages",
	Long: `
Test automates testing the packages named by the import paths.

'gb test' recompiles each package along with any files with names matching
the file pattern "*_test.go".

See 'go help test'.
`,
	Run: func(ctx *gb.Context, args []string) error {
		ctx.Force = F
		ctx.SkipInstall = FF
		r := test.TestResolver(ctx)
		pkgs, err := gb.ResolvePackages(r, args...)
		if err != nil {
			return err
		}

		test, err := test.TestPackages(TestFlags(tfs), pkgs...)
		if err != nil {
			return err
		}

		if dotfile != "" {
			f, err := os.Create(dotfile)
			if err != nil {
				return err
			}
			defer f.Close()
			printActions(f, test)
		}

		return gb.ExecuteConcurrent(test, P)
	},
	AddFlags: addTestFlags,
	FlagParse: func(flags *flag.FlagSet, args []string) error {
		var err error
		args, tfs, err = TestFlagsExtraParse(args[2:])
		if err != nil {
			fmt.Fprintf(os.Stderr, "gb test: %s\n", err)
			fmt.Fprintf(os.Stderr, `run "go help test" or "go help testflag" for more information`+"\n")
			os.Exit(2)
		}
		return flags.Parse(args)
	},
}
<commit_msg>Execute tests in a stable order.<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"sort"

	"github.com/constabulary/gb"
	"github.com/constabulary/gb/cmd"
	"github.com/constabulary/gb/test"
)

func init() {
	registerCommand(TestCmd)
}

var (
	tfs           []string // Arguments passed to the test binary
	testProfile   bool
	testCover     bool
	testCoverMode string
	testCoverPkg  string
)

func addTestFlags(fs *flag.FlagSet) {
	addBuildFlags(fs)
	fs.BoolVar(&testCover, "cover", false, "enable coverage analysis")
	fs.StringVar(&testCoverMode, "covermode", "set", "Set covermode: set (default), count, atomic")
	fs.StringVar(&testCoverPkg, "coverpkg", "", "enable coverage analysis")
}

var TestCmd = &cmd.Command{
	Name:      "test",
	UsageLine: "test [build flags] [packages] [flags for test binary]",
	Short:     "test packages",
	Long: `
Test automates testing the packages named by the import paths.

'gb test' recompiles each package along with any files with names matching
the file pattern "*_test.go".

See 'go help test'.
`,
	Run: func(ctx *gb.Context, args []string) error {
		ctx.Force = F
		ctx.SkipInstall = FF
		r := test.TestResolver(ctx)

		// gb build builds packages in dependency order, however
		// gb test tests packages in alpha order. This matches the
		// expected behaviour from go test; tests are executed in
		// stable order.
		sort.Strings(args)

		pkgs, err := gb.ResolvePackages(r, args...)
		if err != nil {
			return err
		}

		test, err := test.TestPackages(TestFlags(tfs), pkgs...)
		if err != nil {
			return err
		}

		if dotfile != "" {
			f, err := os.Create(dotfile)
			if err != nil {
				return err
			}
			defer f.Close()
			printActions(f, test)
		}

		return gb.ExecuteConcurrent(test, P)
	},
	AddFlags: addTestFlags,
	FlagParse: func(flags *flag.FlagSet, args []string) error {
		var err error
		args, tfs, err = TestFlagsExtraParse(args[2:])
		if err != nil {
			fmt.Fprintf(os.Stderr, "gb test: %s\n", err)
			fmt.Fprintf(os.Stderr, `run "go help test" or "go help testflag" for more information`+"\n")
			os.Exit(2)
		}
		return flags.Parse(args)
	},
}
<|endoftext|>


<commit_before>package cmd

import (
	"context"
	"fmt"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/jessevdk/go-flags"
	"github.com/mitchellh/cli"
	"github.com/mitchellh/go-homedir"
	"github.com/pkg/errors"

	"github.com/nerdalize/nerd/pkg/transfer"
	"github.com/nerdalize/nerd/svc"
)

//JobRun command
type JobRun struct {
	KubeOpts
	TransferOpts
	Name    string   `long:"name" short:"n" description:"assign a name to the job"`
	Env     []string `long:"env" short:"e" description:"environment variables to use"`
	Memory  string   `long:"memory" short:"m" description:"memory to use for this job, expressed in gigabytes" default:"3"`
	VCPU    string   `long:"vcpu" description:"number of vcpus to use for this job" default:"2"`
	Inputs  []string `long:"input" description:"specify one or more inputs that will be used for the job using the following format: <DIR|DATASET_NAME>:<JOB_DIR>"`
	Outputs []string `long:"output" description:"specify one or more output folders that will be stored as datasets after the job is finished using the following format: <DIR>:<JOB_DIR>"`

	*command
}

//JobRunFactory creates the command
func JobRunFactory(ui cli.Ui) cli.CommandFactory {
	cmd := &JobRun{}
	cmd.command = createCommand(ui, cmd.Execute, cmd.Description, cmd.Usage, cmd, flags.PassAfterNonOption, "nerd job run")
	return func() (cli.Command, error) {
		return cmd, nil
	}
}

//ParseInputSpecification will look at an input string and return its parts if valid
func ParseInputSpecification(input string) (parts []string, err error) {
	parts = strings.Split(input, ":")

	//Two accepted cases:
	//- Two unix paths with a colon separating them, e.g. ~/data:/input
	//- Windows path with a disk specification, e.g. C:/data:/input
	if len(parts) != 2 && len(parts) != 3 {
		return nil, fmt.Errorf("invalid input specified, expected '<DIR|DATASET_ID>:<JOB_DIR>' format, got: %s", input)
	}

	//Handle Windows paths where DIR may contain colons
	//e.g. C:/foo/bar:/input will be parsed into []string{"C", "/foo/bar", "/input"}
	//and should be turned into []string{"C:/foo/bar", "/input"}
	//We assume that POSIX paths will never have colons
	parts = []string{strings.Join(parts[:len(parts)-1], ":"), parts[len(parts)-1]}

	//Expand tilde for homedir
	parts[0], err = homedir.Expand(parts[0])
	if err != nil {
		return nil, errors.Wrap(err, "failed to expand home directory in dataset local path")
	}

	//Normalize all slashes to native platform slashes (e.g. / to \ on Windows)
	parts[0] = filepath.FromSlash(parts[0])

	// Ensure that all parts are non-empty
	if len(strings.TrimSpace(parts[0])) == 0 {
		return nil, errors.New("input source is empty")
	} else if len(strings.TrimSpace(parts[1])) == 0 {
		return nil, errors.New("input mount path is empty")
	}

	return parts, nil
}

// dsHandle keeps handles to update the job froms and to.
// newDs helps us to keep track if the dataset used is an ad-hoc dataset or a dataset that was previously submitted,
// so we know which ones we should delete in case of problems with `nerd job run`.
type dsHandle struct {
	handle transfer.Handle
	newDs  bool
}

//Execute runs the command
func (cmd *JobRun) Execute(args []string) (err error) {
	if len(args) < 1 {
		return errShowUsage(fmt.Sprintf(MessageNotEnoughArguments, 1, ""))
	}

	kopts := cmd.KubeOpts
	deps, err := NewDeps(cmd.Logger(), kopts)
	if err != nil {
		return renderConfigError(err, "failed to configure")
	}

	//setup a context with a timeout
	ctx := context.TODO()

	err = checkResources(cmd.Memory, cmd.VCPU)
	if err != nil {
		return err
	}

	//setup job arguments
	jargs := []string{}
	if len(args) > 1 {
		jargs = args[1:]
	}

	jenv := map[string]string{}
	for _, l := range cmd.Env {
		split := strings.SplitN(l, "=", 2)
		if len(split) < 2 {
			return fmt.Errorf("invalid environment variable format, expected 'FOO=bar' format, got: %v", l)
		}
		jenv[split[0]] = split[1]
	}

	//setup the transfer manager
	kube := svc.NewKube(deps)
	mgr, sto, sta, err := cmd.TransferOpts.TransferManager(kube)
	if err != nil {
		return errors.Wrap(err, "failed to setup transfer manager")
	}

	//keep handles to update the job froms and to
	inputs := []dsHandle{}
	outputs := []dsHandle{}

	//start with input volumes
	vols := map[string]*svc.JobVolume{}
	for _, input := range cmd.Inputs {
		var parts []string
		parts, err = ParseInputSpecification(input)
		if err != nil {
			return cmd.rollbackDatasets(ctx, mgr, inputs, outputs, errors.Wrap(err, "failed to parse input specification"))
		}

		//if the input spec has a path-like string, try to upload it for the user
		var h dsHandle
		if strings.Contains(parts[0], string(filepath.Separator)) {
			//the user has provided a path as its input, clean it and make it absolute
			parts[0], err = filepath.Abs(parts[0])
			if err != nil {
				return cmd.rollbackDatasets(ctx, mgr, inputs, outputs, errors.Wrap(err, "failed to turn local dataset path into absolute path"))
			}

			h.handle, err = mgr.Create(ctx, "", *sto, *sta)
			if err != nil {
				return renderServiceError(
					cmd.rollbackDatasets(ctx, mgr, inputs, outputs, err),
					"failed to create dataset",
				)
			}

			h.newDs = true
			err = h.handle.Push(ctx, parts[0], &progressBarReporter{})
			if err != nil {
				return renderServiceError(
					cmd.rollbackDatasets(ctx, mgr, append(inputs, h), outputs, err),
					"failed to upload dataset",
				)
			}

			cmd.out.Infof("Uploaded input dataset: '%s'", h.handle.Name())
		} else { //open an existing dataset
			h.handle, err = mgr.Open(ctx, parts[0])
			if err != nil {
				return renderServiceError(
					cmd.rollbackDatasets(ctx, mgr, inputs, outputs, err),
					"failed to open dataset '%s'", parts[0],
				)
			}
			h.newDs = false
		}

		//add handler for job mapping
		inputs = append(inputs, h)
		defer h.handle.Close()

		vols[parts[1]] = &svc.JobVolume{
			MountPath:    parts[1],
			InputDataset: h.handle.Name(),
		}

		err = deps.val.Struct(vols[parts[1]])
		if err != nil {
			return cmd.rollbackDatasets(ctx, mgr, inputs, outputs, errors.Wrap(err, "incorrect input"))
		}
	}

	for _, output := range cmd.Outputs {
		parts := strings.Split(output, ":")
		if len(parts) < 1 || len(parts) > 2 {
			return cmd.rollbackDatasets(ctx, mgr, inputs, outputs, fmt.Errorf("invalid output specified, expected '<JOB_DIR>:[DATASET_NAME]' format, got: %s", output))
		}

		vol, ok := vols[parts[0]]
		if !ok {
			vol = &svc.JobVolume{MountPath: parts[0]}
			vols[parts[0]] = vol
		}

		err = deps.val.Struct(vol)
		if err != nil {
			return cmd.rollbackDatasets(ctx, mgr, inputs, outputs, errors.Wrap(err, "incorrect output"))
		}

		//if the second part is provided we want to upload the output to a specific  dataset
		var h dsHandle
		if len(parts) == 2 { //open an existing dataset
			h.handle, err = mgr.Open(ctx, parts[1])
			if err != nil {
				return renderServiceError(
					cmd.rollbackDatasets(ctx, mgr, inputs, outputs, err),
					"failed to open dataset '%s'", parts[1],
				)
			}

			h.newDs = false

		} else { //create an empty dataset for the output
			h.newDs = true
			h.handle, err = mgr.Create(ctx, "", *sto, *sta)
			if err != nil {
				return renderServiceError(
					cmd.rollbackDatasets(ctx, mgr, inputs, append(outputs, h), err),
					"failed to create dataset",
				)
			}

			cmd.out.Infof("Setup empty output dataset: '%s'", h.handle.Name())
		}

		//register for job mapping and cleanup
		outputs = append(outputs, h)
		defer h.handle.Close()

		vol.OutputDataset = h.handle.Name()
	}

	//continue with actuall creating the job
	in := &svc.RunJobInput{
		Image:  args[0],
		Name:   cmd.Name,
		Env:    jenv,
		Args:   jargs,
		Memory: fmt.Sprintf("%sGi", cmd.Memory),
		VCPU:   cmd.VCPU,
	}

	for _, vol := range vols {
		in.Volumes = append(in.Volumes, *vol)
	}

	out, err := kube.RunJob(ctx, in)
	if err != nil {
		cmd.rollbackDatasets(ctx, mgr, inputs, outputs, nil)
		return renderServiceError(err, "failed to run job")
	}

	err = updateDatasets(ctx, kube, inputs, outputs, out.Name)
	if err != nil {
		return err
	}

	cmd.out.Infof("Submitted job: '%s'", out.Name)
	cmd.out.Infof("To see whats happening, use: 'nerd job list'")
	return nil
}

func checkResources(memory, vcpu string) error {
	if memory != "" {
		m, err := strconv.ParseFloat(memory, 64)
		if err != nil {
			return fmt.Errorf("invalid memory option format, %v", err)
		}
		if m > 60 || m <= 0 {
			return fmt.Errorf("invalid value for memory parameter. Memory request must be greater than 0 and lower than 60Gbs")
		}
	}
	if vcpu != "" {
		v, err := strconv.ParseFloat(vcpu, 64)
		if err != nil {
			return fmt.Errorf("invalid vcpu option format, %v", err)
		}
		if v > 40 || v <= 0 {
			return fmt.Errorf("invalid value for vcpu parameter. VCPU request must be greater than 0 and lower than 40")
		}
	}
	return nil
}

func (cmd *JobRun) rollbackDatasets(ctx context.Context, mgr transfer.Manager, inputs, outputs []dsHandle, err error) error {
	for _, input := range inputs {
		if input.newDs {
			mgr.Remove(ctx, input.handle.Name())
		}
	}
	for _, output := range outputs {
		if output.newDs {
			mgr.Remove(ctx, output.handle.Name())
		}
	}

	return err
}

func updateDatasets(ctx context.Context, kube *svc.Kube, inputs, outputs []dsHandle, name string) error {
	//add job to each dataset's InputFor
	for _, input := range inputs {
		_, err := kube.UpdateDataset(ctx, &svc.UpdateDatasetInput{Name: input.handle.Name(), InputFor: name})
		if err != nil {
			return err
		}
	}
	//add job to each dataset's OutputOf
	for _, output := range outputs {
		_, err := kube.UpdateDataset(ctx, &svc.UpdateDatasetInput{Name: output.handle.Name(), OutputFrom: name})
		if err != nil {
			return err
		}
	}
	return nil
}

// Description returns long-form help text
func (cmd *JobRun) Description() string { return cmd.Synopsis() }

// Synopsis returns a one-line
func (cmd *JobRun) Synopsis() string { return "Runs a job on your compute cluster" }

// Usage shows usage
func (cmd *JobRun) Usage() string { return "nerd job run [OPTIONS] IMAGE [ARG...]" }
<commit_msg>Create dataset if output ds doesn't exist<commit_after>package cmd

import (
	"context"
	"fmt"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/jessevdk/go-flags"
	"github.com/mitchellh/cli"
	"github.com/mitchellh/go-homedir"
	"github.com/pkg/errors"

	"github.com/nerdalize/nerd/pkg/transfer"
	"github.com/nerdalize/nerd/svc"
)

//JobRun command
type JobRun struct {
	KubeOpts
	TransferOpts
	Name    string   `long:"name" short:"n" description:"assign a name to the job"`
	Env     []string `long:"env" short:"e" description:"environment variables to use"`
	Memory  string   `long:"memory" short:"m" description:"memory to use for this job, expressed in gigabytes" default:"3"`
	VCPU    string   `long:"vcpu" description:"number of vcpus to use for this job" default:"2"`
	Inputs  []string `long:"input" description:"specify one or more inputs that will be used for the job using the following format: <DIR|DATASET_NAME>:<JOB_DIR>"`
	Outputs []string `long:"output" description:"specify one or more output folders that will be stored as datasets after the job is finished using the following format: <DIR>:<JOB_DIR>"`

	*command
}

//JobRunFactory creates the command
func JobRunFactory(ui cli.Ui) cli.CommandFactory {
	cmd := &JobRun{}
	cmd.command = createCommand(ui, cmd.Execute, cmd.Description, cmd.Usage, cmd, flags.PassAfterNonOption, "nerd job run")
	return func() (cli.Command, error) {
		return cmd, nil
	}
}

//ParseInputSpecification will look at an input string and return its parts if valid
func ParseInputSpecification(input string) (parts []string, err error) {
	parts = strings.Split(input, ":")

	//Two accepted cases:
	//- Two unix paths with a colon separating them, e.g. ~/data:/input
	//- Windows path with a disk specification, e.g. C:/data:/input
	if len(parts) != 2 && len(parts) != 3 {
		return nil, fmt.Errorf("invalid input specified, expected '<DIR|DATASET_ID>:<JOB_DIR>' format, got: %s", input)
	}

	//Handle Windows paths where DIR may contain colons
	//e.g. C:/foo/bar:/input will be parsed into []string{"C", "/foo/bar", "/input"}
	//and should be turned into []string{"C:/foo/bar", "/input"}
	//We assume that POSIX paths will never have colons
	parts = []string{strings.Join(parts[:len(parts)-1], ":"), parts[len(parts)-1]}

	//Expand tilde for homedir
	parts[0], err = homedir.Expand(parts[0])
	if err != nil {
		return nil, errors.Wrap(err, "failed to expand home directory in dataset local path")
	}

	//Normalize all slashes to native platform slashes (e.g. / to \ on Windows)
	parts[0] = filepath.FromSlash(parts[0])

	// Ensure that all parts are non-empty
	if len(strings.TrimSpace(parts[0])) == 0 {
		return nil, errors.New("input source is empty")
	} else if len(strings.TrimSpace(parts[1])) == 0 {
		return nil, errors.New("input mount path is empty")
	}

	return parts, nil
}

// dsHandle keeps handles to update the job froms and to.
// newDs helps us to keep track if the dataset used is an ad-hoc dataset or a dataset that was previously submitted,
// so we know which ones we should delete in case of problems with `nerd job run`.
type dsHandle struct {
	handle transfer.Handle
	newDs  bool
}

//Execute runs the command
func (cmd *JobRun) Execute(args []string) (err error) {
	if len(args) < 1 {
		return errShowUsage(fmt.Sprintf(MessageNotEnoughArguments, 1, ""))
	}

	kopts := cmd.KubeOpts
	deps, err := NewDeps(cmd.Logger(), kopts)
	if err != nil {
		return renderConfigError(err, "failed to configure")
	}

	//setup a context with a timeout
	ctx := context.TODO()

	err = checkResources(cmd.Memory, cmd.VCPU)
	if err != nil {
		return err
	}

	//setup job arguments
	jargs := []string{}
	if len(args) > 1 {
		jargs = args[1:]
	}

	jenv := map[string]string{}
	for _, l := range cmd.Env {
		split := strings.SplitN(l, "=", 2)
		if len(split) < 2 {
			return fmt.Errorf("invalid environment variable format, expected 'FOO=bar' format, got: %v", l)
		}
		jenv[split[0]] = split[1]
	}

	//setup the transfer manager
	kube := svc.NewKube(deps)
	mgr, sto, sta, err := cmd.TransferOpts.TransferManager(kube)
	if err != nil {
		return errors.Wrap(err, "failed to setup transfer manager")
	}

	//keep handles to update the job froms and to
	inputs := []dsHandle{}
	outputs := []dsHandle{}

	//start with input volumes
	vols := map[string]*svc.JobVolume{}
	for _, input := range cmd.Inputs {
		var parts []string
		parts, err = ParseInputSpecification(input)
		if err != nil {
			return cmd.rollbackDatasets(ctx, mgr, inputs, outputs, errors.Wrap(err, "failed to parse input specification"))
		}

		//if the input spec has a path-like string, try to upload it for the user
		var h dsHandle
		if strings.Contains(parts[0], string(filepath.Separator)) {
			//the user has provided a path as its input, clean it and make it absolute
			parts[0], err = filepath.Abs(parts[0])
			if err != nil {
				return cmd.rollbackDatasets(ctx, mgr, inputs, outputs, errors.Wrap(err, "failed to turn local dataset path into absolute path"))
			}

			h.handle, err = mgr.Create(ctx, "", *sto, *sta)
			if err != nil {
				return renderServiceError(
					cmd.rollbackDatasets(ctx, mgr, inputs, outputs, err),
					"failed to create dataset",
				)
			}

			h.newDs = true
			err = h.handle.Push(ctx, parts[0], &progressBarReporter{})
			if err != nil {
				return renderServiceError(
					cmd.rollbackDatasets(ctx, mgr, append(inputs, h), outputs, err),
					"failed to upload dataset",
				)
			}

			cmd.out.Infof("Uploaded input dataset: '%s'", h.handle.Name())
		} else { //open an existing dataset
			h.handle, err = mgr.Open(ctx, parts[0])
			if err != nil {
				return renderServiceError(
					cmd.rollbackDatasets(ctx, mgr, inputs, outputs, err),
					"failed to open dataset '%s'", parts[0],
				)
			}
			h.newDs = false
		}

		//add handler for job mapping
		inputs = append(inputs, h)
		defer h.handle.Close()

		vols[parts[1]] = &svc.JobVolume{
			MountPath:    parts[1],
			InputDataset: h.handle.Name(),
		}

		err = deps.val.Struct(vols[parts[1]])
		if err != nil {
			return cmd.rollbackDatasets(ctx, mgr, inputs, outputs, errors.Wrap(err, "incorrect input"))
		}
	}

	for _, output := range cmd.Outputs {
		parts := strings.Split(output, ":")
		if len(parts) < 1 || len(parts) > 2 {
			return cmd.rollbackDatasets(ctx, mgr, inputs, outputs, fmt.Errorf("invalid output specified, expected '<JOB_DIR>:[DATASET_NAME]' format, got: %s", output))
		}

		vol, ok := vols[parts[len(parts)-1]]
		if !ok {
			vol = &svc.JobVolume{MountPath: parts[len(parts)-1]}
			vols[parts[len(parts)-1]] = vol
		}

		err = deps.val.Struct(vol)
		if err != nil {
			return cmd.rollbackDatasets(ctx, mgr, inputs, outputs, errors.Wrap(err, "incorrect output"))
		}

		//if the second part is provided we want to upload the output to a specific  dataset
		var h dsHandle
		if len(parts) == 2 { //open an existing dataset
			h.handle, err = mgr.Open(ctx, parts[0])
			h.newDs = false
			// @TODO check if the error is "dataset doesn't exist", and only in this case we should create a new one
			if err != nil {
				h.handle, err = mgr.Create(ctx, parts[0], *sto, *sta)
				if err != nil {
					return renderServiceError(
						cmd.rollbackDatasets(ctx, mgr, inputs, outputs, err),
						"failed to open/create dataset '%s'", parts[0],
					)
				}
				cmd.out.Infof("Setup empty output dataset: '%s'", h.handle.Name())
			}

		} else { //create an empty dataset for the output
			h.newDs = true
			h.handle, err = mgr.Create(ctx, "", *sto, *sta)
			if err != nil {
				return renderServiceError(
					cmd.rollbackDatasets(ctx, mgr, inputs, append(outputs, h), err),
					"failed to create dataset",
				)
			}

			cmd.out.Infof("Setup empty output dataset: '%s'", h.handle.Name())
		}

		//register for job mapping and cleanup
		outputs = append(outputs, h)
		defer h.handle.Close()

		vol.OutputDataset = h.handle.Name()
	}

	//continue with actuall creating the job
	in := &svc.RunJobInput{
		Image:  args[0],
		Name:   cmd.Name,
		Env:    jenv,
		Args:   jargs,
		Memory: fmt.Sprintf("%sGi", cmd.Memory),
		VCPU:   cmd.VCPU,
	}

	for _, vol := range vols {
		in.Volumes = append(in.Volumes, *vol)
	}

	out, err := kube.RunJob(ctx, in)
	if err != nil {
		cmd.rollbackDatasets(ctx, mgr, inputs, outputs, nil)
		return renderServiceError(err, "failed to run job")
	}

	err = updateDatasets(ctx, kube, inputs, outputs, out.Name)
	if err != nil {
		return err
	}

	cmd.out.Infof("Submitted job: '%s'", out.Name)
	cmd.out.Infof("To see whats happening, use: 'nerd job list'")
	return nil
}

func checkResources(memory, vcpu string) error {
	if memory != "" {
		m, err := strconv.ParseFloat(memory, 64)
		if err != nil {
			return fmt.Errorf("invalid memory option format, %v", err)
		}
		if m > 60 || m <= 0 {
			return fmt.Errorf("invalid value for memory parameter. Memory request must be greater than 0 and lower than 60Gbs")
		}
	}
	if vcpu != "" {
		v, err := strconv.ParseFloat(vcpu, 64)
		if err != nil {
			return fmt.Errorf("invalid vcpu option format, %v", err)
		}
		if v > 40 || v <= 0 {
			return fmt.Errorf("invalid value for vcpu parameter. VCPU request must be greater than 0 and lower than 40")
		}
	}
	return nil
}

func (cmd *JobRun) rollbackDatasets(ctx context.Context, mgr transfer.Manager, inputs, outputs []dsHandle, err error) error {
	for _, input := range inputs {
		if input.newDs {
			mgr.Remove(ctx, input.handle.Name())
		}
	}
	for _, output := range outputs {
		if output.newDs {
			mgr.Remove(ctx, output.handle.Name())
		}
	}

	return err
}

func updateDatasets(ctx context.Context, kube *svc.Kube, inputs, outputs []dsHandle, name string) error {
	//add job to each dataset's InputFor
	for _, input := range inputs {
		_, err := kube.UpdateDataset(ctx, &svc.UpdateDatasetInput{Name: input.handle.Name(), InputFor: name})
		if err != nil {
			return err
		}
	}
	//add job to each dataset's OutputOf
	for _, output := range outputs {
		_, err := kube.UpdateDataset(ctx, &svc.UpdateDatasetInput{Name: output.handle.Name(), OutputFrom: name})
		if err != nil {
			return err
		}
	}
	return nil
}

// Description returns long-form help text
func (cmd *JobRun) Description() string { return cmd.Synopsis() }

// Synopsis returns a one-line
func (cmd *JobRun) Synopsis() string { return "Runs a job on your compute cluster" }

// Usage shows usage
func (cmd *JobRun) Usage() string { return "nerd job run [OPTIONS] IMAGE [ARG...]" }
<|endoftext|>


<commit_before>package cmd

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"runtime/pprof"
	"strings"
	"syscall"
	"time"

	homedir "github.com/mitchellh/go-homedir"
	"github.com/pkg/errors"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"github.com/vektra/mockery/v2/pkg"
	"github.com/vektra/mockery/v2/pkg/config"
	"github.com/vektra/mockery/v2/pkg/logging"
	"golang.org/x/crypto/ssh/terminal"
	"golang.org/x/tools/go/packages"
)

var (
	cfgFile = ""
	rootCmd = &cobra.Command{
		Use:   "mockery",
		Short: "Generate mock objects for your Golang interfaces",
		RunE: func(cmd *cobra.Command, args []string) error {
			r, err := GetRootAppFromViper(viper.GetViper())
			if err != nil {
				printStackTrace(err)
				return nil
			}
			return r.Run()
		},
	}
)

type stackTracer interface {
	StackTrace() errors.StackTrace
}

func printStackTrace(e error) {
	fmt.Printf("%v\n", e)
	if err, ok := e.(stackTracer); ok {
		for _, f := range err.StackTrace() {
			fmt.Printf("%+s:%d\n", f, f)
		}
	}

}

// Execute executes the cobra CLI workflow
func Execute() {
	if err := rootCmd.Execute(); err != nil {
		//printStackTrace(err)
		os.Exit(1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)

	pFlags := rootCmd.PersistentFlags()
	pFlags.StringVar(&cfgFile, "config", "", "config file to use")
	pFlags.String("name", "", "name or matching regular expression of interface to generate mock for")
	pFlags.Bool("print", false, "print the generated mock to stdout")
	pFlags.String("output", "./mocks", "directory to write mocks to")
	pFlags.String("outpkg", "mocks", "name of generated package")
	pFlags.String("dir", ".", "directory to search for interfaces")
	pFlags.BoolP("recursive", "r", false, "recurse search into sub-directories")
	pFlags.Bool("all", false, "generates mocks for all found interfaces in all sub-directories")
	pFlags.Bool("inpackage", false, "generate a mock that goes inside the original package")
	pFlags.Bool("testonly", false, "generate a mock in a _test.go file")
	pFlags.String("case", "camel", "name the mocked file using casing convention [camel, snake, underscore]")
	pFlags.String("note", "", "comment to insert into prologue of each generated file")
	pFlags.String("cpuprofile", "", "write cpu profile to file")
	pFlags.Bool("version", false, "prints the installed version of mockery")
	pFlags.Bool("quiet", false, "suppress output to stdout")
	pFlags.Bool("keeptree", false, "keep the tree structure of the original interface files into a different repository. Must be used with XX")
	pFlags.String("tags", "", "space-separated list of additional build tags to use")
	pFlags.String("filename", "", "name of generated file (only works with -name and no regex)")
	pFlags.String("structname", "", "name of generated struct (only works with -name and no regex)")
	pFlags.String("log-level", "info", "Level of logging")
	pFlags.String("srcpkg", "", "source pkg to search for interfaces")
	pFlags.BoolP("dry-run", "d", false, "Do a dry run, don't modify any files")

	viper.BindPFlags(pFlags)
}

func initConfig() {
	if cfgFile != "" {
		// Use config file from the flag.
		viper.SetConfigFile(cfgFile)
	} else {
		// Find home directory.
		home, err := homedir.Dir()
		if err != nil {
			log.Fatal().Err(err).Msgf("Failed to find homedir")
		}

		// Search config in home directory with name ".cobra" (without extension).
		viper.AddConfigPath(".")
		viper.AddConfigPath(home)
		viper.SetConfigName(".mockery")
	}

	viper.SetEnvPrefix("mockery")
	viper.AutomaticEnv()

	// Note we purposely ignore the error. Don't care if we can't find a config file.
	if err := viper.ReadInConfig(); err == nil {
		fmt.Fprintf(os.Stderr, "Using config file: %s\n", viper.ConfigFileUsed())
	}
}

const regexMetadataChars = "\\.+*?()|[]{}^$"

type RootApp struct {
	config.Config
}

func GetRootAppFromViper(v *viper.Viper) (*RootApp, error) {
	r := &RootApp{}
	if err := v.UnmarshalExact(&r.Config); err != nil {
		return nil, errors.Wrapf(err, "failed to get config")
	}
	return r, nil
}

func (r *RootApp) Run() error {
	var recursive bool
	var filter *regexp.Regexp
	var err error
	var limitOne bool

	if r.Quiet {
		// if "quiet" flag is set, set os.Stdout to /dev/null to suppress all output to Stdout
		os.Stdout = os.NewFile(uintptr(syscall.Stdout), os.DevNull)
		r.Config.LogLevel = ""
	}

	log, err := getLogger(r.Config.LogLevel)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to initialize logger: %v\n", err)
		return err
	}
	log = log.With().Bool(logging.LogKeyDryRun, r.Config.DryRun).Logger()
	log.Info().Msgf("Starting mockery")
	ctx := log.WithContext(context.Background())

	if r.Config.Version {
		fmt.Println(config.SemVer)
		return nil
	} else if r.Config.Name != "" && r.Config.All {
		log.Fatal().Msgf("Specify --name or --all, but not both")
	} else if (r.Config.FileName != "" || r.Config.StructName != "") && r.Config.All {
		log.Fatal().Msgf("Cannot specify --filename or --structname with --all")
	} else if r.Config.Dir != "" && r.Config.Dir != "." && r.Config.SrcPkg != "" {
		log.Fatal().Msgf("Specify -dir or -srcpkg, but not both")
	} else if r.Config.Name != "" {
		recursive = r.Config.Recursive
		if strings.ContainsAny(r.Config.Name, regexMetadataChars) {
			if filter, err = regexp.Compile(r.Config.Name); err != nil {
				log.Fatal().Err(err).Msgf("Invalid regular expression provided to -name")
			} else if r.Config.FileName != "" || r.Config.StructName != "" {
				log.Fatal().Msgf("Cannot specify --filename or --structname with regex in --name")
			}
		} else {
			filter = regexp.MustCompile(fmt.Sprintf("^%s$", r.Config.Name))
			limitOne = true
		}
	} else if r.Config.All {
		recursive = true
		filter = regexp.MustCompile(".*")
	} else {
		log.Fatal().Msgf("Use --name to specify the name of the interface or --all for all interfaces found")
	}
	if r.Config.KeepTree && r.Config.InPackage {
		log.Fatal().Msgf("--keeptree and --inpackage are mutually exclusive")
	}

	if r.Config.Profile != "" {
		f, err := os.Create(r.Config.Profile)
		if err != nil {
			return errors.Wrapf(err, "Failed to create profile file")
		}

		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	var osp pkg.OutputStreamProvider
	if r.Config.Print {
		osp = &pkg.StdoutStreamProvider{}
	} else {
		osp = &pkg.FileOutputStreamProvider{
			Config:                    r.Config,
			BaseDir:                   r.Config.Output,
			InPackage:                 r.Config.InPackage,
			TestOnly:                  r.Config.TestOnly,
			Case:                      r.Config.Case,
			KeepTree:                  r.Config.KeepTree,
			KeepTreeOriginalDirectory: r.Config.Dir,
			FileName:                  r.Config.FileName,
		}
	}

	baseDir := r.Config.Dir

	if r.Config.SrcPkg != "" {
		pkgs, err := packages.Load(&packages.Config{
			Mode: packages.NeedFiles,
		}, r.Config.SrcPkg)
		if err != nil || len(pkgs) == 0 {
			log.Fatal().Err(err).Msgf("Failed to load package %s", r.Config.SrcPkg)
		}

		// NOTE: we only pass one package name (config.SrcPkg) to packages.Load
		// it should return one package at most
		pkg := pkgs[0]

		if pkg.Errors != nil {
			log.Fatal().Err(pkg.Errors[0]).Msgf("Failed to load package %s", r.Config.SrcPkg)
		}

		if len(pkg.GoFiles) == 0 {
			log.Fatal().Msgf("No go files in package %s", r.Config.SrcPkg)
		}
		baseDir = filepath.Dir(pkg.GoFiles[0])
	}

	visitor := &pkg.GeneratorVisitor{
		Config:      r.Config,
		InPackage:   r.Config.InPackage,
		Note:        r.Config.Note,
		Osp:         osp,
		PackageName: r.Config.Outpkg,
		StructName:  r.Config.StructName,
	}

	walker := pkg.Walker{
		Config:    r.Config,
		BaseDir:   baseDir,
		Recursive: recursive,
		Filter:    filter,
		LimitOne:  limitOne,
		BuildTags: strings.Split(r.Config.BuildTags, " "),
	}

	generated := walker.Walk(ctx, visitor)

	if r.Config.Name != "" && !generated {
		log.Fatal().Msgf("Unable to find '%s' in any go files under this path", r.Config.Name)
	}
	return nil
}

type timeHook struct{}

func (t timeHook) Run(e *zerolog.Event, level zerolog.Level, msg string) {
	e.Time("time", time.Now())
}

func getLogger(levelStr string) (zerolog.Logger, error) {
	level, err := zerolog.ParseLevel(levelStr)
	if err != nil {
		return zerolog.Logger{}, errors.Wrapf(err, "Couldn't parse log level")
	}
	out := os.Stderr
	writer := zerolog.ConsoleWriter{
		Out:        out,
		TimeFormat: time.RFC822,
	}
	if !terminal.IsTerminal(int(out.Fd())) {
		writer.NoColor = true
	}
	log := zerolog.New(writer).
		Hook(timeHook{}).
		Level(level).
		With().
		Str("version", config.SemVer).
		Logger()

	return log, nil
}
<commit_msg>Return error if configuration parsing fails<commit_after>package cmd

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"runtime/pprof"
	"strings"
	"syscall"
	"time"

	homedir "github.com/mitchellh/go-homedir"
	"github.com/pkg/errors"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"github.com/vektra/mockery/v2/pkg"
	"github.com/vektra/mockery/v2/pkg/config"
	"github.com/vektra/mockery/v2/pkg/logging"
	"golang.org/x/crypto/ssh/terminal"
	"golang.org/x/tools/go/packages"
)

var (
	cfgFile = ""
	rootCmd = &cobra.Command{
		Use:   "mockery",
		Short: "Generate mock objects for your Golang interfaces",
		RunE: func(cmd *cobra.Command, args []string) error {
			r, err := GetRootAppFromViper(viper.GetViper())
			if err != nil {
				printStackTrace(err)
				return err
			}
			return r.Run()
		},
	}
)

type stackTracer interface {
	StackTrace() errors.StackTrace
}

func printStackTrace(e error) {
	fmt.Printf("%v\n", e)
	if err, ok := e.(stackTracer); ok {
		for _, f := range err.StackTrace() {
			fmt.Printf("%+s:%d\n", f, f)
		}
	}

}

// Execute executes the cobra CLI workflow
func Execute() {
	if err := rootCmd.Execute(); err != nil {
		//printStackTrace(err)
		os.Exit(1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)

	pFlags := rootCmd.PersistentFlags()
	pFlags.StringVar(&cfgFile, "config", "", "config file to use")
	pFlags.String("name", "", "name or matching regular expression of interface to generate mock for")
	pFlags.Bool("print", false, "print the generated mock to stdout")
	pFlags.String("output", "./mocks", "directory to write mocks to")
	pFlags.String("outpkg", "mocks", "name of generated package")
	pFlags.String("dir", ".", "directory to search for interfaces")
	pFlags.BoolP("recursive", "r", false, "recurse search into sub-directories")
	pFlags.Bool("all", false, "generates mocks for all found interfaces in all sub-directories")
	pFlags.Bool("inpackage", false, "generate a mock that goes inside the original package")
	pFlags.Bool("testonly", false, "generate a mock in a _test.go file")
	pFlags.String("case", "camel", "name the mocked file using casing convention [camel, snake, underscore]")
	pFlags.String("note", "", "comment to insert into prologue of each generated file")
	pFlags.String("cpuprofile", "", "write cpu profile to file")
	pFlags.Bool("version", false, "prints the installed version of mockery")
	pFlags.Bool("quiet", false, "suppress output to stdout")
	pFlags.Bool("keeptree", false, "keep the tree structure of the original interface files into a different repository. Must be used with XX")
	pFlags.String("tags", "", "space-separated list of additional build tags to use")
	pFlags.String("filename", "", "name of generated file (only works with -name and no regex)")
	pFlags.String("structname", "", "name of generated struct (only works with -name and no regex)")
	pFlags.String("log-level", "info", "Level of logging")
	pFlags.String("srcpkg", "", "source pkg to search for interfaces")
	pFlags.BoolP("dry-run", "d", false, "Do a dry run, don't modify any files")

	viper.BindPFlags(pFlags)
}

func initConfig() {
	if cfgFile != "" {
		// Use config file from the flag.
		viper.SetConfigFile(cfgFile)
	} else {
		// Find home directory.
		home, err := homedir.Dir()
		if err != nil {
			log.Fatal().Err(err).Msgf("Failed to find homedir")
		}

		// Search config in home directory with name ".cobra" (without extension).
		viper.AddConfigPath(".")
		viper.AddConfigPath(home)
		viper.SetConfigName(".mockery")
	}

	viper.SetEnvPrefix("mockery")
	viper.AutomaticEnv()

	// Note we purposely ignore the error. Don't care if we can't find a config file.
	if err := viper.ReadInConfig(); err == nil {
		fmt.Fprintf(os.Stderr, "Using config file: %s\n", viper.ConfigFileUsed())
	}
}

const regexMetadataChars = "\\.+*?()|[]{}^$"

type RootApp struct {
	config.Config
}

func GetRootAppFromViper(v *viper.Viper) (*RootApp, error) {
	r := &RootApp{}
	if err := v.UnmarshalExact(&r.Config); err != nil {
		return nil, errors.Wrapf(err, "failed to get config")
	}
	return r, nil
}

func (r *RootApp) Run() error {
	var recursive bool
	var filter *regexp.Regexp
	var err error
	var limitOne bool

	if r.Quiet {
		// if "quiet" flag is set, set os.Stdout to /dev/null to suppress all output to Stdout
		os.Stdout = os.NewFile(uintptr(syscall.Stdout), os.DevNull)
		r.Config.LogLevel = ""
	}

	log, err := getLogger(r.Config.LogLevel)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to initialize logger: %v\n", err)
		return err
	}
	log = log.With().Bool(logging.LogKeyDryRun, r.Config.DryRun).Logger()
	log.Info().Msgf("Starting mockery")
	ctx := log.WithContext(context.Background())

	if r.Config.Version {
		fmt.Println(config.SemVer)
		return nil
	} else if r.Config.Name != "" && r.Config.All {
		log.Fatal().Msgf("Specify --name or --all, but not both")
	} else if (r.Config.FileName != "" || r.Config.StructName != "") && r.Config.All {
		log.Fatal().Msgf("Cannot specify --filename or --structname with --all")
	} else if r.Config.Dir != "" && r.Config.Dir != "." && r.Config.SrcPkg != "" {
		log.Fatal().Msgf("Specify -dir or -srcpkg, but not both")
	} else if r.Config.Name != "" {
		recursive = r.Config.Recursive
		if strings.ContainsAny(r.Config.Name, regexMetadataChars) {
			if filter, err = regexp.Compile(r.Config.Name); err != nil {
				log.Fatal().Err(err).Msgf("Invalid regular expression provided to -name")
			} else if r.Config.FileName != "" || r.Config.StructName != "" {
				log.Fatal().Msgf("Cannot specify --filename or --structname with regex in --name")
			}
		} else {
			filter = regexp.MustCompile(fmt.Sprintf("^%s$", r.Config.Name))
			limitOne = true
		}
	} else if r.Config.All {
		recursive = true
		filter = regexp.MustCompile(".*")
	} else {
		log.Fatal().Msgf("Use --name to specify the name of the interface or --all for all interfaces found")
	}
	if r.Config.KeepTree && r.Config.InPackage {
		log.Fatal().Msgf("--keeptree and --inpackage are mutually exclusive")
	}

	if r.Config.Profile != "" {
		f, err := os.Create(r.Config.Profile)
		if err != nil {
			return errors.Wrapf(err, "Failed to create profile file")
		}

		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	var osp pkg.OutputStreamProvider
	if r.Config.Print {
		osp = &pkg.StdoutStreamProvider{}
	} else {
		osp = &pkg.FileOutputStreamProvider{
			Config:                    r.Config,
			BaseDir:                   r.Config.Output,
			InPackage:                 r.Config.InPackage,
			TestOnly:                  r.Config.TestOnly,
			Case:                      r.Config.Case,
			KeepTree:                  r.Config.KeepTree,
			KeepTreeOriginalDirectory: r.Config.Dir,
			FileName:                  r.Config.FileName,
		}
	}

	baseDir := r.Config.Dir

	if r.Config.SrcPkg != "" {
		pkgs, err := packages.Load(&packages.Config{
			Mode: packages.NeedFiles,
		}, r.Config.SrcPkg)
		if err != nil || len(pkgs) == 0 {
			log.Fatal().Err(err).Msgf("Failed to load package %s", r.Config.SrcPkg)
		}

		// NOTE: we only pass one package name (config.SrcPkg) to packages.Load
		// it should return one package at most
		pkg := pkgs[0]

		if pkg.Errors != nil {
			log.Fatal().Err(pkg.Errors[0]).Msgf("Failed to load package %s", r.Config.SrcPkg)
		}

		if len(pkg.GoFiles) == 0 {
			log.Fatal().Msgf("No go files in package %s", r.Config.SrcPkg)
		}
		baseDir = filepath.Dir(pkg.GoFiles[0])
	}

	visitor := &pkg.GeneratorVisitor{
		Config:      r.Config,
		InPackage:   r.Config.InPackage,
		Note:        r.Config.Note,
		Osp:         osp,
		PackageName: r.Config.Outpkg,
		StructName:  r.Config.StructName,
	}

	walker := pkg.Walker{
		Config:    r.Config,
		BaseDir:   baseDir,
		Recursive: recursive,
		Filter:    filter,
		LimitOne:  limitOne,
		BuildTags: strings.Split(r.Config.BuildTags, " "),
	}

	generated := walker.Walk(ctx, visitor)

	if r.Config.Name != "" && !generated {
		log.Fatal().Msgf("Unable to find '%s' in any go files under this path", r.Config.Name)
	}
	return nil
}

type timeHook struct{}

func (t timeHook) Run(e *zerolog.Event, level zerolog.Level, msg string) {
	e.Time("time", time.Now())
}

func getLogger(levelStr string) (zerolog.Logger, error) {
	level, err := zerolog.ParseLevel(levelStr)
	if err != nil {
		return zerolog.Logger{}, errors.Wrapf(err, "Couldn't parse log level")
	}
	out := os.Stderr
	writer := zerolog.ConsoleWriter{
		Out:        out,
		TimeFormat: time.RFC822,
	}
	if !terminal.IsTerminal(int(out.Fd())) {
		writer.NoColor = true
	}
	log := zerolog.New(writer).
		Hook(timeHook{}).
		Level(level).
		With().
		Str("version", config.SemVer).
		Logger()

	return log, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/nightlyone/lockfile"
	"io"
	"log"
	"log/syslog"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
	"time"
)

// Avoid thundering herd problem on remote services used by this command.
// Interval will be 0, if this is not an issue.
func SpreadWait(interval time.Duration) {
	if interval > 0 {
		// Seed random generator with current process ID
		rand.Seed(int64(os.Getpid()))
		// Sleep for random amount of time within interval
		time.Sleep(time.Duration(rand.Int63n(int64(interval))))
	}
}

// Ok states that execution went well. Logs debug output and reports ok to
// monitoring.
func Ok() {
	log.Println("Ok")
	monitor(monitorOk, "")
}

// NotAvailable states that the command could not be started successfully. It
// might not be installed or has other problems.
func NotAvailable(err error) {
	s := fmt.Sprintln("Cannot start command: ", err)
	log.Println("FATAL:", s)
	monitor(monitorUnknown, s)
}

// TimedOut states that the command took too long and reports failure to the
// monitoring.
func TimedOut() {
	s := "execution took too long"
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

// Busy states that the command hangs and reports failure to the monitoring.
// Those tasks should be automatically killed, if it happens often.
func Busy() {
	s := "previous invocation of command still running"
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

// Failed states that the command didn't execute successfully and reports
// failure to the monitoring. Also Logs error output.
func Failed(err error) {
	s := fmt.Sprintln("Failed to execute: ", err)
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

var useSyslog bool

// derive logger
func getLogger() (logger io.Writer, err error) {
	if useSyslog {
		logger, err = syslog.New(syslog.LOG_NOTICE, monitoringEvent)
	} else {
		logger = os.Stderr
	}
	if err != nil {
		log.SetOutput(logger)
	}
	return logger, err
}

func main() {
	var cmd *exec.Cmd
	var interval, timeout time.Duration

	// FIXME(mlafeldt) add command-line options for kill or wait on busy
	// state
	log.SetFlags(0)

	flag.DurationVar(&interval, "i", -1,
		"set execution interval for command, e.g. 45s, 2m, 1h30m, default: 1/10 of timeout")
	flag.DurationVar(&timeout, "t", 1*time.Minute,
		"set execution timeout for command, e.g. 45s, 2m, 1h30m, default: 1m")
	flag.BoolVar(&useSyslog, "l", false, "log via syslog")
	flag.Parse()

	if flag.NArg() < 1 {
		log.Fatal("FATAL: no command to execute")
		return
	}

	command := flag.Arg(0)
	monitoringEvent = filepath.Base(command)
	_, err := getLogger()

	if interval >= timeout {
		log.Fatal("FATAL: interval >= timeout, no time left for actual command execution")
		return
	}

	if interval == -1 {
		interval = timeout / 10
	}

	loadMonitoringCommands()

	// FIXME(nightlyone) try two intervals instead of one?
	timer := time.AfterFunc(timeout, func() {
		TimedOut()
		if cmd != nil && cmd.Process != nil {
			cmd.Process.Kill()
			// FIXME(nightlyone) log the kill
		}
		os.Exit(0)
	})

	SpreadWait(interval)

	// Ensures that only one of these command runs concurrently on this
	// machine.  Also cleans up stale locks of dead instances.
	lock_dir := os.TempDir()
	os.Mkdir(filepath.Join(lock_dir, monitoringEvent), 0700)
	lock, _ := lockfile.New(filepath.Join(lock_dir, monitoringEvent, monitoringEvent+".lock"))
	if err := lock.TryLock(); err != nil {
		if err != lockfile.ErrBusy {
			log.Printf("ERROR: locking %s: reason: %v\n", lock, err)
		}
		timer.Stop()
		Busy()
		return
	}
	defer lock.Unlock()

	cmd = exec.Command(command, flag.Args()[1:]...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}

	wg := sync.WaitGroup{}
	wg.Add(1)

	go func() {
		for {
			if _, err := io.Copy(os.Stdout, stdout); err != nil {
				break
			}
		}
		wg.Done()
	}()

	if err := cmd.Start(); err != nil {
		timer.Stop()
		NotAvailable(err)
		return
	}

	if err := cmd.Wait(); err != nil {
		timer.Stop()
		Failed(err)
	} else {
		timer.Stop()
		Ok()
	}

	wg.Wait()
}
<commit_msg>log stderr and stdout<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/nightlyone/lockfile"
	"io"
	"log"
	"log/syslog"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"sync"
	"time"
)

// Avoid thundering herd problem on remote services used by this command.
// Interval will be 0, if this is not an issue.
func SpreadWait(interval time.Duration) {
	if interval > 0 {
		// Seed random generator with current process ID
		rand.Seed(int64(os.Getpid()))
		// Sleep for random amount of time within interval
		time.Sleep(time.Duration(rand.Int63n(int64(interval))))
	}
}

// Ok states that execution went well. Logs debug output and reports ok to
// monitoring.
func Ok() {
	log.Println("Ok")
	monitor(monitorOk, "")
}

// NotAvailable states that the command could not be started successfully. It
// might not be installed or has other problems.
func NotAvailable(err error) {
	s := fmt.Sprintln("Cannot start command: ", err)
	log.Println("FATAL:", s)
	monitor(monitorUnknown, s)
}

// TimedOut states that the command took too long and reports failure to the
// monitoring.
func TimedOut() {
	s := "execution took too long"
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

// Busy states that the command hangs and reports failure to the monitoring.
// Those tasks should be automatically killed, if it happens often.
func Busy() {
	s := "previous invocation of command still running"
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

// Failed states that the command didn't execute successfully and reports
// failure to the monitoring. Also Logs error output.
func Failed(err error) {
	s := fmt.Sprintln("Failed to execute: ", err)
	log.Println("FATAL:", s)
	monitor(monitorCritical, s)
}

var useSyslog bool

// derive logger
func getLogger() (logger io.Writer, err error) {
	if useSyslog {
		logger, err = syslog.New(syslog.LOG_NOTICE, monitoringEvent)
	} else {
		logger = os.Stderr
	}
	if err != nil {
		log.SetOutput(logger)
	}
	return logger, err
}

// pipe r to logger in the background
func logStream(r io.Reader, logger io.Writer, wg *sync.WaitGroup) {
	wg.Add(1)

	go func() {
		for {
			if _, err := io.Copy(logger, r); err != nil {
				break
			}
		}
		wg.Done()
	}()
}

func main() {
	var cmd *exec.Cmd
	var interval, timeout time.Duration
	var wg sync.WaitGroup

	// FIXME(mlafeldt) add command-line options for kill or wait on busy
	// state
	log.SetFlags(0)

	flag.DurationVar(&interval, "i", -1,
		"set execution interval for command, e.g. 45s, 2m, 1h30m, default: 1/10 of timeout")
	flag.DurationVar(&timeout, "t", 1*time.Minute,
		"set execution timeout for command, e.g. 45s, 2m, 1h30m, default: 1m")
	flag.BoolVar(&useSyslog, "l", false, "log via syslog")
	flag.Parse()

	if flag.NArg() < 1 {
		log.Fatal("FATAL: no command to execute")
		return
	}

	command := flag.Arg(0)
	monitoringEvent = filepath.Base(command)
	logger, err := getLogger()

	if interval >= timeout {
		log.Fatal("FATAL: interval >= timeout, no time left for actual command execution")
		return
	}

	if interval == -1 {
		interval = timeout / 10
	}

	loadMonitoringCommands()

	// FIXME(nightlyone) try two intervals instead of one?
	timer := time.AfterFunc(timeout, func() {
		TimedOut()
		if cmd != nil && cmd.Process != nil {
			cmd.Process.Kill()
			// FIXME(nightlyone) log the kill
		}
		os.Exit(0)
	})

	SpreadWait(interval)

	// Ensures that only one of these command runs concurrently on this
	// machine.  Also cleans up stale locks of dead instances.
	lock_dir := os.TempDir()
	os.Mkdir(filepath.Join(lock_dir, monitoringEvent), 0700)
	lock, _ := lockfile.New(filepath.Join(lock_dir, monitoringEvent, monitoringEvent+".lock"))
	if err := lock.TryLock(); err != nil {
		if err != lockfile.ErrBusy {
			log.Printf("ERROR: locking %s: reason: %v\n", lock, err)
		}
		timer.Stop()
		Busy()
		return
	}
	defer lock.Unlock()

	cmd = exec.Command(command, flag.Args()[1:]...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	logStream(stdout, logger, &wg)

	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}
	logStream(stderr, logger, &wg)

	if err := cmd.Start(); err != nil {
		timer.Stop()
		NotAvailable(err)
		return
	}

	if err := cmd.Wait(); err != nil {
		timer.Stop()
		Failed(err)
	} else {
		timer.Stop()
		Ok()
	}

	wg.Wait()
}
<|endoftext|>


<commit_before>package cmd

import (
	"strconv"
	"strings"
	"time"

	"github.com/mobingi/mobingi-cli/client/timeout"
	"github.com/mobingi/mobingi-cli/pkg/cli"
	"github.com/mobingi/mobingi-cli/pkg/cli/confmap"
	"github.com/mobingilabs/mobingi-sdk-go/client"
	"github.com/mobingilabs/mobingi-sdk-go/mobingi/session"
	d "github.com/mobingilabs/mobingi-sdk-go/pkg/debug"
	"github.com/mobingilabs/mobingi-sdk-go/pkg/nativestore"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
)

func getApiVersionInt() int {
	v := 3
	vparam := viper.GetString(confmap.ConfigKey("apiver"))
	in, err := strconv.Atoi(strings.TrimLeft(vparam, "v"))
	if err != nil {
		return -1
	}

	v = in
	return v
}

func clisession() (*session.Session, error) {
	v := getApiVersionInt()
	if v < 0 {
		return nil, errors.New("cannot get api version")
	}

	// check if we have credentials in nativestore
	user, secret, err := nativestore.Get(cli.CliUrl)
	if err == nil {
		if user != "" && secret != "" {
			if cli.Verbose {
				d.Info("use credentials from native store")
			}

			return session.New(&session.Config{
				ClientId:        user,
				ClientSecret:    secret,
				ApiVersion:      v,
				BaseApiUrl:      viper.GetString(confmap.ConfigKey("url")),
				BaseRegistryUrl: viper.GetString(confmap.ConfigKey("rurl")),
				HttpClientConfig: &client.Config{
					Timeout: time.Second * time.Duration(timeout.Timeout),
					Verbose: cli.Verbose,
				},
			})
		}
	}

	if cli.Verbose {
		d.Info("cannot access native store, use config file token")
	}

	return session.New(&session.Config{
		ApiVersion:      v,
		AccessToken:     viper.GetString(confmap.ConfigKey("token")),
		BaseApiUrl:      viper.GetString(confmap.ConfigKey("url")),
		BaseRegistryUrl: viper.GetString(confmap.ConfigKey("rurl")),
		HttpClientConfig: &client.Config{
			Timeout: time.Second * time.Duration(timeout.Timeout),
			Verbose: cli.Verbose,
		},
	})
}
<commit_msg>Add verbose in session.<commit_after>package cmd

import (
	"strconv"
	"strings"
	"time"

	"github.com/mobingi/mobingi-cli/client/timeout"
	"github.com/mobingi/mobingi-cli/pkg/cli"
	"github.com/mobingi/mobingi-cli/pkg/cli/confmap"
	"github.com/mobingilabs/mobingi-sdk-go/client"
	"github.com/mobingilabs/mobingi-sdk-go/mobingi/session"
	d "github.com/mobingilabs/mobingi-sdk-go/pkg/debug"
	"github.com/mobingilabs/mobingi-sdk-go/pkg/nativestore"
	"github.com/pkg/errors"
	"github.com/spf13/viper"
)

func getApiVersionInt() int {
	v := 3
	vparam := viper.GetString(confmap.ConfigKey("apiver"))
	in, err := strconv.Atoi(strings.TrimLeft(vparam, "v"))
	if err != nil {
		return -1
	}

	v = in
	return v
}

func clisession() (*session.Session, error) {
	v := getApiVersionInt()
	if v < 0 {
		return nil, errors.New("cannot get api version")
	}

	// check if we have credentials in nativestore
	user, secret, err := nativestore.Get(cli.CliUrl)
	if err == nil {
		if user != "" && secret != "" {
			if cli.Verbose {
				d.Info("use credentials from native store")
			}

			return session.New(&session.Config{
				ClientId:        user,
				ClientSecret:    secret,
				ApiVersion:      v,
				BaseApiUrl:      viper.GetString(confmap.ConfigKey("url")),
				BaseRegistryUrl: viper.GetString(confmap.ConfigKey("rurl")),
				HttpClientConfig: &client.Config{
					Timeout: time.Second * time.Duration(timeout.Timeout),
					Verbose: cli.Verbose,
				},
			})
		}
	}

	if cli.Verbose {
		if cli.Debug {
			d.Error(err)
		}

		d.Info("cannot access native store, use config file token")
	}

	return session.New(&session.Config{
		ApiVersion:      v,
		AccessToken:     viper.GetString(confmap.ConfigKey("token")),
		BaseApiUrl:      viper.GetString(confmap.ConfigKey("url")),
		BaseRegistryUrl: viper.GetString(confmap.ConfigKey("rurl")),
		HttpClientConfig: &client.Config{
			Timeout: time.Second * time.Duration(timeout.Timeout),
			Verbose: cli.Verbose,
		},
	})
}
<|endoftext|>


<commit_before>/*
 * Copyright 2013 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package proto

type Message struct {
	Id            string            `json:"id,omitempty"`
	Sender        string            `json:"sender,omitempty"`
	SenderService string            `json:"service,omitempty"`
	Header        map[string]string `json:"header,omitempty"`
	Body          []byte            `json:"body,omitempty"`
}

func (self *Message) IsEmpty() bool {
	return len(self.Header) == 0 && len(self.Body) == 0
}

func (self *Message) Size() int {
	ret := len(self.Body)
	for k, v := range self.Header {
		ret += len(k) + 1
		ret += len(v) + 1
	}
	ret += 8
	return ret
}

func (a *Message) EqContent(b *Message) bool {
	if len(a.Header) != len(b.Header) {
		return false
	}
	for k, v := range a.Header {
		if bv, ok := b.Header[k]; ok {
			if bv != v {
				return false
			}
		} else {
			return false
		}
	}
	return bytesEq(a.Body, b.Body)
}

func (a *Message) Eq(b *Message) bool {
	if !a.EqContent(b) {
		return false
	}
	if a.Id != b.Id {
		return false
	}
	if a.Sender != b.Sender {
		return false
	}
	if a.SenderService != b.SenderService {
		return false
	}
	return true
}
<commit_msg>meta-data should be in MessageContainer. fixing #5<commit_after>/*
 * Copyright 2013 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package proto

// MessageContainer is used to represent a message inside
// the program. It has meta-data about a message like:
// the message id, the sender and the service of the sender.
type MessageContainer struct {
	Message       *Message `json:"msg"`
	Id            string   `json:"id,omitempty"`
	Sender        string   `json:"sender,omitempty"`
	SenderService string   `json:"service,omitempty"`
}

func (self *MessageContainer) FromServer() bool {
	return len(self.Sender) == 0
}

func (self *MessageContainer) FromUser() bool {
	return !self.FromServer()
}

type Message struct {
	Id            string            `json:"id,omitempty"`
	Sender        string            `json:"sender,omitempty"`
	SenderService string            `json:"service,omitempty"`
	Header        map[string]string `json:"header,omitempty"`
	Body          []byte            `json:"body,omitempty"`
}

func (self *Message) IsEmpty() bool {
	return len(self.Header) == 0 && len(self.Body) == 0
}

func (self *Message) Size() int {
	ret := len(self.Body)
	for k, v := range self.Header {
		ret += len(k) + 1
		ret += len(v) + 1
	}
	ret += 8
	return ret
}

func (a *Message) EqContent(b *Message) bool {
	if len(a.Header) != len(b.Header) {
		return false
	}
	for k, v := range a.Header {
		if bv, ok := b.Header[k]; ok {
			if bv != v {
				return false
			}
		} else {
			return false
		}
	}
	return bytesEq(a.Body, b.Body)
}

func (a *Message) Eq(b *Message) bool {
	if !a.EqContent(b) {
		return false
	}
	if a.Id != b.Id {
		return false
	}
	if a.Sender != b.Sender {
		return false
	}
	if a.SenderService != b.SenderService {
		return false
	}
	return true
}
<|endoftext|>