<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

package cmd

import (
	"os"
	"path/filepath"
	"strings"

	"arete/pkg/utils"

	"github.com/manifoldco/promptui"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"gopkg.in/yaml.v3"
)

var region, project, billing string
var gcloud string = "gcloud"
var currentRegions = []string{"us-central1", "us-east1", "northamerica-northeast1", "europe-north1", "australia-southeast1", "asia-northeast1"}

type createSteps struct {
	Steps []step `yaml:"steps"`
}

type step struct {
	Step string `yaml:"step"`
}

// Does the step exist in the .create yaml file
func (cs *createSteps) stepExists(step string) bool {
	for _, stp := range cs.Steps {
		if stp.Step == step {
			return true
		}
	}

	return false
}

// createCmd represents the create command
var createCmd = &cobra.Command{
	Use:   "create <instance-name>",
	Short: "Create a new Config Controller instance",
	Long: `
 The arete create command will create a new config controller instance for you, it can either use an existing project or create
 a new one for you. Arete will check to make sure that the project exists, if it does not exist then arete will attempt to
 create project and either prompt for a billing account or if the --billing flag is present it will use that billing ID`,
	Example: ` arete create my-awesome-kcc --region=us-central1 # This will create a new project and prompt for billing ID
 arete create my-awesome-kcc --region=us-central1 --project=my-awesome-project # This will utilize an existing project`,
	Args: cobra.MinimumNArgs(1),
	Run: func(cmd *cobra.Command, args []string) {
		log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})

		regionFound := false
		regionMsg := "Unsupported region. Please choose one of: \n"

		for i := 0; i < len(currentRegions); i++ {
			regionMsg += " - " + currentRegions[i] + "\n"
			if region == currentRegions[i] {
				regionFound = true
			}
		}

		if !regionFound {
			log.Fatal().Msg(regionMsg)
		}

		// check to see if the .create yaml file exists, if not create it
		// The .create file keeps track of steps that have already been completed for repeated calls to the create function.
		createStepsFile := filepath.Join(viper.GetString("cache"), ".create")
		createSteps := createSteps{}

		if _, err := os.Stat(createStepsFile); err != nil {
			createYaml := "steps:\n"

			utils.WriteToCache(&createYaml, ".create", false)
		} else {
			stepsFileContents, err := os.ReadFile(createStepsFile)

			if err == nil {
				yaml.Unmarshal(stepsFileContents, &createSteps)
			}
		}

		// project flag not provided to create a random project name based on the cluster name and random string
		if project == "" {
			project = args[0] + "-" + utils.RandomString(5)
			log.Info().Msgf("Project name will be set to: %s", project)
		}

		// Test if project exists
		rtr, err := utils.CallCommand(gcloud, []string{"projects", "list", "--filter=projectId:" + project , "--format=\"yaml\""}, false)

		if err != nil {
			log.Fatal().Err(err).Msg("")
		}

		// If no project exists (or the user doesn't have access to it) let's create one
		if len(string(rtr)) == 0 {
			// billing flag not used, pull billing accounts and prompt the user to select one
			if billing == "" {
				billing, err = selectGcloudPrompt([]string{"alpha", "billing", "accounts", "list", "--format=value[separator=' - '](NAME, ACCOUNT_ID)"}, "billing account")

				if err != nil {
					log.Fatal().Err(err).Msg("Unable to generate billing prompt")
				}
			}

			err := createProject(project, billing)

			if err != nil {
				log.Fatal().Err(err).Msg("")
			}

			cmdArgs := []string{"beta", "billing", "projects", "link", project, "--billing-account", billing}

			billRes, err := utils.CallCommand(gcloud, cmdArgs, false)

			if err != nil {
				log.Fatal().Err(err).Msg("Unable to assign billing account to project: " + string(billRes))
			}
		}

		// If the GCP services have not already been enabled by the cli then let's do it.
		if !createSteps.stepExists("services") {
			log.Info().Msg("Enabling required services...")

			cmdArgs := []string{"services", "enable", "krmapihosting.googleapis.com", "container.googleapis.com", "cloudresourcemanager.googleapis.com", "cloudbilling.googleapis.com", "--project=" + project}

			_, err := utils.CallCommand(gcloud, cmdArgs, true)

			if err != nil {
				log.Fatal().Err(err).Msg("")
			}

			saveStep("services")
		}

		networkConfig := map[string]interface{}{"name": "kcc-controller", "subnet-name": "kcc-regional-subnet", "cidr": "192.168.0.0/16"}

		if viper.InConfig("network") {
			networkConfig = viper.GetStringMap("network")
		}

		// If the VPC has not already been created by the cli then let's do it
		if !createSteps.stepExists("network") {
			// BUT first let's check to make sure that the network with the same name doesn't already exist
			cmdArgs := []string{"compute", "networks", "list", "--project=" + project, "--filter=name:" + networkConfig["name"].(string), "--format=\"yaml\""}

			ret, err := utils.CallCommand(gcloud, cmdArgs, false)

			if err != nil {
				log.Fatal().Err(err).Msg(string(ret))
			}

			if len(string(ret)) == 0 {
				cmdArgs = []string{"compute", "networks", "create", networkConfig["name"].(string), "--project=" + project, "--subnet-mode=custom"}

				log.Info().Msg("Creating Network...")

				ret, err = utils.CallCommand(gcloud, cmdArgs, false)

				if err != nil {
					log.Fatal().Err(err).Msg(string(ret))
				}

				saveStep("network")
			}
		}

		// If the subnet on the VPC has not already been created by the cli then let's do it
		if !createSteps.stepExists("subnet") {
			// BUT first let's check to make sure that a subnet with the same name doesn't already exist
			cmdArgs := []string{"compute", "networks", "subnets", "list", "--project=" + project, "--filter=name:" + networkConfig["subnet-name"].(string), "--format=\"yaml\""}

			ret, err := utils.CallCommand(gcloud, cmdArgs, false)

			if err != nil {
				log.Fatal().Err(err).Msg(string(ret))
			}

			if len(string(ret)) == 0 || strings.Contains(strings.ToLower(string(ret)), "warning") {
				cmdArgs = []string{"compute", "networks", "subnets", "create", networkConfig["subnet-name"].(string), "--network=" + networkConfig["name"].(string), "--range=" + networkConfig["cidr"].(string), "--enable-private-ip-google-access", "--region=" + region, "--project=" + project}

				log.Info().Msg("Creating subnet....")

				ret, err = utils.CallCommand(gcloud, cmdArgs, false)

				if err != nil {
					log.Fatal().Err(err).Msg(string(ret))
				}

				saveStep("subnet")
			}
		}

		// If config controller hasn't already been setup by the cli then let's do it
		if !createSteps.stepExists("config-controller") {
			cmdArgs := []string{"anthos", "config", "controller", "create", args[0], "--location=" + region, "--network=" + networkConfig["name"].(string), "--subnet=" + networkConfig["subnet-name"].(string), "--project=" + project}

			log.Info().Msg("Creating Config Controller Cluster....")

			_, err := utils.CallCommand(gcloud, cmdArgs, true)

			if err != nil {
				log.Fatal().Err(err).Msg("")
			}

			saveStep("config-controller")
		}

		// Adding service account to the owners role
		if !createSteps.stepExists("add-policy") {
			cmdArgs := []string{"get", "ConfigConnectorContext", "-n", "config-control", "-o", "jsonpath='{.items[0].spec.googleServiceAccount}'"}

			ret, err := utils.CallCommand("kubectl", cmdArgs, false)

			if err != nil {
				log.Fatal().Err(err).Msg("Unable to get configconnectorcontext")
			}

			sa := strings.Replace(string(ret), "'", "", 2)

			cmdArgs = []string{"projects", "add-iam-policy-binding", project, `--member=serviceAccount:` + sa, `--role=roles/owner`, `--condition=None`}

			log.Info().Msg("Add SA to roles/owner role...")
			ret, err = utils.CallCommand(gcloud, cmdArgs, false)

			if err != nil {
				log.Fatal().Err(err).Msg("Unable to add roles/owner to " + sa + string(ret))
			}

			saveStep("add-policy")
		}

		log.Info().Msg("Config Controller setup complete")
	},
}

// init the command and add flags
func init() {
	rootCmd.AddCommand(createCmd)

	createCmd.Flags().StringVarP(&project, "project", "p", "", "The new project ID to create for the Config Controller cluster. Default: <instance-name>")

	createCmd.Flags().StringVarP(&region, "region", "r", "", "REQUIRED: The GCP region to use when creating the instance")
	createCmd.MarkFlagRequired("region")

	createCmd.Flags().StringVarP(&billing, "billing", "b", "", "Billing account to be used for project and resources")
}

// Create a project at either the org or at a folder level.
func createProject(project string, billing string) error {
	var folderId string

	orgId, err := selectGcloudPrompt([]string{"organizations", "list", "--format=value[separator=' - '](DISPLAY_NAME, ID)"}, "organization ID")

	if err != nil {
		return err
	}

	prompt := promptui.Select{
		Label: "Would you like the project to be created at the Org level or in a folder",
		Items: []string{"Organization Level", "Folder Level"},
	}

	_, result, err := prompt.Run()

	if err != nil {
		return err
	}

	cmdArgs := []string{"projects", "create", project, "--set-as-default", "--labels=created-with-arete=true"}

	if result == "Folder Level" {
		folderId, err = selectGcloudPrompt([]string{"resource-manager", "folders", "list", "--organization="+orgId,  "--format=value[separator=' - '](DISPLAY_NAME, ID)"}, "folder")

		if err != nil {
			return err
		}

		cmdArgs = append(cmdArgs, "--folder="+folderId)
	} else {
		cmdArgs = append(cmdArgs, "--organization="+orgId)
	}

	_, err = utils.CallCommand(gcloud, cmdArgs, true)

	if err != nil {
		return err
	}

	return nil
}

// Generic select prompt from a gcloud list command
func selectGcloudPrompt(args []string, label string) (string, error) {
	var opts []string

	if verbose {
		log.Debug().Msg("Calling gcloud "+label+" list")
	}

	out, err := utils.CallCommand("gcloud", args, false)

	if err != nil {
		return "", err
	}

	sp := strings.Split(string(out), "\n")
	sp = sp[:len(sp)-1] // pop off the last element as it's a blank newline element

	for i := 0; i < len(sp); i++ {
		opts = append(opts, sp[i])
	}

	prompt := promptui.Select{
		Label: "Choose a " + label,
		Items: opts,
	}

	_, result, err := prompt.Run()

	if err != nil {
		return "", err
	}

	return result[strings.Index(result, " - ")+3:], nil
}

// Save a step to the cache .create file for future tracking
func saveStep(step string) error {
	step = "- step: " + step + "\n"

	if err := utils.WriteToCache(&step, ".create", true); err != nil {
		return err
	}

	return nil
}
<commit_msg>Fixed auth warning on kubectl blocking call<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

package cmd

import (
	"os"
	"path/filepath"
	"strings"

	"arete/pkg/utils"

	"github.com/manifoldco/promptui"
	"github.com/rs/zerolog"
	"github.com/rs/zerolog/log"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
	"gopkg.in/yaml.v3"
)

var region, project, billing string
var gcloud string = "gcloud"
var currentRegions = []string{"us-central1", "us-east1", "northamerica-northeast1", "europe-north1", "australia-southeast1", "asia-northeast1"}

type createSteps struct {
	Steps []step `yaml:"steps"`
}

type step struct {
	Step string `yaml:"step"`
}

// Does the step exist in the .create yaml file
func (cs *createSteps) stepExists(step string) bool {
	for _, stp := range cs.Steps {
		if stp.Step == step {
			return true
		}
	}

	return false
}

// createCmd represents the create command
var createCmd = &cobra.Command{
	Use:   "create <instance-name>",
	Short: "Create a new Config Controller instance",
	Long: `
 The arete create command will create a new config controller instance for you, it can either use an existing project or create
 a new one for you. Arete will check to make sure that the project exists, if it does not exist then arete will attempt to
 create project and either prompt for a billing account or if the --billing flag is present it will use that billing ID`,
	Example: ` arete create my-awesome-kcc --region=us-central1 # This will create a new project and prompt for billing ID
 arete create my-awesome-kcc --region=us-central1 --project=my-awesome-project # This will utilize an existing project`,
	Args: cobra.MinimumNArgs(1),
	Run: func(cmd *cobra.Command, args []string) {
		log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})

		regionFound := false
		regionMsg := "Unsupported region. Please choose one of: \n"

		for i := 0; i < len(currentRegions); i++ {
			regionMsg += " - " + currentRegions[i] + "\n"
			if region == currentRegions[i] {
				regionFound = true
			}
		}

		if !regionFound {
			log.Fatal().Msg(regionMsg)
		}

		// check to see if the .create yaml file exists, if not create it
		// The .create file keeps track of steps that have already been completed for repeated calls to the create function.
		createStepsFile := filepath.Join(viper.GetString("cache"), ".create")
		createSteps := createSteps{}

		if _, err := os.Stat(createStepsFile); err != nil {
			createYaml := "steps:\n"

			utils.WriteToCache(&createYaml, ".create", false)
		} else {
			stepsFileContents, err := os.ReadFile(createStepsFile)

			if err == nil {
				yaml.Unmarshal(stepsFileContents, &createSteps)
			}
		}

		// project flag not provided to create a random project name based on the cluster name and random string
		if project == "" {
			project = args[0] + "-" + utils.RandomString(5)
			log.Info().Msgf("Project name will be set to: %s", project)
		}

		// Test if project exists
		rtr, err := utils.CallCommand(gcloud, []string{"projects", "list", "--filter=projectId:" + project , "--format=\"yaml\""}, false)

		if err != nil {
			log.Fatal().Err(err).Msg("")
		}

		// If no project exists (or the user doesn't have access to it) let's create one
		if len(string(rtr)) == 0 {
			// billing flag not used, pull billing accounts and prompt the user to select one
			if billing == "" {
				billing, err = selectGcloudPrompt([]string{"alpha", "billing", "accounts", "list", "--format=value[separator=' - '](NAME, ACCOUNT_ID)"}, "billing account")

				if err != nil {
					log.Fatal().Err(err).Msg("Unable to generate billing prompt")
				}
			}

			err := createProject(project, billing)

			if err != nil {
				log.Fatal().Err(err).Msg("")
			}

			cmdArgs := []string{"beta", "billing", "projects", "link", project, "--billing-account", billing}

			billRes, err := utils.CallCommand(gcloud, cmdArgs, false)

			if err != nil {
				log.Fatal().Err(err).Msg("Unable to assign billing account to project: " + string(billRes))
			}
		}

		// If the GCP services have not already been enabled by the cli then let's do it.
		if !createSteps.stepExists("services") {
			log.Info().Msg("Enabling required services...")

			cmdArgs := []string{"services", "enable", "krmapihosting.googleapis.com", "container.googleapis.com", "cloudresourcemanager.googleapis.com", "cloudbilling.googleapis.com", "--project=" + project}

			_, err := utils.CallCommand(gcloud, cmdArgs, true)

			if err != nil {
				log.Fatal().Err(err).Msg("")
			}

			saveStep("services")
		}

		networkConfig := map[string]interface{}{"name": "kcc-controller", "subnet-name": "kcc-regional-subnet", "cidr": "192.168.0.0/16"}

		if viper.InConfig("network") {
			networkConfig = viper.GetStringMap("network")
		}

		// If the VPC has not already been created by the cli then let's do it
		if !createSteps.stepExists("network") {
			// BUT first let's check to make sure that the network with the same name doesn't already exist
			cmdArgs := []string{"compute", "networks", "list", "--project=" + project, "--filter=name:" + networkConfig["name"].(string), "--format=\"yaml\""}

			ret, err := utils.CallCommand(gcloud, cmdArgs, false)

			if err != nil {
				log.Fatal().Err(err).Msg(string(ret))
			}

			if len(string(ret)) == 0 {
				cmdArgs = []string{"compute", "networks", "create", networkConfig["name"].(string), "--project=" + project, "--subnet-mode=custom"}

				log.Info().Msg("Creating Network...")

				ret, err = utils.CallCommand(gcloud, cmdArgs, false)

				if err != nil {
					log.Fatal().Err(err).Msg(string(ret))
				}

				saveStep("network")
			} else {
				saveStep("network")
			}
		}

		// If the subnet on the VPC has not already been created by the cli then let's do it
		if !createSteps.stepExists("subnet") {
			// BUT first let's check to make sure that a subnet with the same name doesn't already exist
			cmdArgs := []string{"compute", "networks", "subnets", "list", "--project=" + project, "--filter=name:" + networkConfig["subnet-name"].(string), "--format=\"yaml\""}

			ret, err := utils.CallCommand(gcloud, cmdArgs, false)

			if err != nil {
				log.Fatal().Err(err).Msg(string(ret))
			}

			if len(string(ret)) == 0 || strings.Contains(strings.ToLower(string(ret)), "warning") {
				cmdArgs = []string{"compute", "networks", "subnets", "create", networkConfig["subnet-name"].(string), "--network=" + networkConfig["name"].(string), "--range=" + networkConfig["cidr"].(string), "--enable-private-ip-google-access", "--region=" + region, "--project=" + project}

				log.Info().Msg("Creating subnet....")

				ret, err = utils.CallCommand(gcloud, cmdArgs, false)

				if err != nil {
					log.Fatal().Err(err).Msg(string(ret))
				}

				saveStep("subnet")
			} else {
				saveStep("subnet")
			}
		}

		// If config controller hasn't already been setup by the cli then let's do it
		if !createSteps.stepExists("config-controller") {
			cmdArgs := []string{"anthos", "config", "controller", "create", args[0], "--location=" + region, "--network=" + networkConfig["name"].(string), "--subnet=" + networkConfig["subnet-name"].(string), "--project=" + project}

			log.Info().Msg("Creating Config Controller Cluster....")

			_, err := utils.CallCommand(gcloud, cmdArgs, true)

			if err != nil {
				log.Fatal().Err(err).Msg("")
			}

			saveStep("config-controller")
		}

		// Adding service account to the owners role
		if !createSteps.stepExists("add-policy") {
			cmdArgs := []string{"container", "clusters", "get-credentials", "krmapihost-" + args[0], "--region=" + region, "--project=" + project}

			_, err := utils.CallCommand(gcloud, cmdArgs, false)

			if err != nil {
				log.Fatal().Err(err).Msg("Unable to get configconnectorcontext")
			}

			cmdArgs = []string{"get", "ConfigConnectorContext", "-n", "config-control", "-o", "jsonpath='{.items[0].spec.googleServiceAccount}'"}

			ret, err := utils.CallCommand("kubectl", cmdArgs, false)

			if err != nil {
				log.Fatal().Err(err).Msg("Unable to get configconnectorcontext")
			}

			sa := strings.Replace(strings.Split(strings.Split(string(ret), "@")[0], "'")[1] + "@" + strings.Split(string(ret), "@")[1], "'", "",1)

			cmdArgs = []string{"projects", "add-iam-policy-binding", project, `--member=serviceAccount:` + sa, `--role=roles/owner`, `--condition=None`}

			log.Info().Msg("Add SA to roles/owner role...")
			ret, err = utils.CallCommand(gcloud, cmdArgs, false)

			if err != nil {
				log.Fatal().Err(err).Msg("Unable to add roles/owner to " + sa + string(ret))
			}

			saveStep("add-policy")
		}

		log.Info().Msg("Config Controller setup complete")
	},
}

// init the command and add flags
func init() {
	rootCmd.AddCommand(createCmd)

	createCmd.Flags().StringVarP(&project, "project", "p", "", "The new project ID to create for the Config Controller cluster. Default: <instance-name>")

	createCmd.Flags().StringVarP(&region, "region", "r", "", "REQUIRED: The GCP region to use when creating the instance")
	createCmd.MarkFlagRequired("region")

	createCmd.Flags().StringVarP(&billing, "billing", "b", "", "Billing account to be used for project and resources")
}

// Create a project at either the org or at a folder level.
func createProject(project string, billing string) error {
	var folderId string

	orgId, err := selectGcloudPrompt([]string{"organizations", "list", "--format=value[separator=' - '](DISPLAY_NAME, ID)"}, "organization ID")

	if err != nil {
		return err
	}

	prompt := promptui.Select{
		Label: "Would you like the project to be created at the Org level or in a folder",
		Items: []string{"Organization Level", "Folder Level"},
	}

	_, result, err := prompt.Run()

	if err != nil {
		return err
	}

	cmdArgs := []string{"projects", "create", project, "--set-as-default", "--labels=created-with-arete=true"}

	if result == "Folder Level" {
		folderId, err = selectGcloudPrompt([]string{"resource-manager", "folders", "list", "--organization="+orgId,  "--format=value[separator=' - '](DISPLAY_NAME, ID)"}, "folder")

		if err != nil {
			return err
		}

		cmdArgs = append(cmdArgs, "--folder="+folderId)
	} else {
		cmdArgs = append(cmdArgs, "--organization="+orgId)
	}

	_, err = utils.CallCommand(gcloud, cmdArgs, true)

	if err != nil {
		return err
	}

	return nil
}

// Generic select prompt from a gcloud list command
func selectGcloudPrompt(args []string, label string) (string, error) {
	var opts []string

	if verbose {
		log.Debug().Msg("Calling gcloud "+label+" list")
	}

	out, err := utils.CallCommand("gcloud", args, false)

	if err != nil {
		return "", err
	}

	sp := strings.Split(string(out), "\n")
	sp = sp[:len(sp)-1] // pop off the last element as it's a blank newline element

	for i := 0; i < len(sp); i++ {
		opts = append(opts, sp[i])
	}

	prompt := promptui.Select{
		Label: "Choose a " + label,
		Items: opts,
	}

	_, result, err := prompt.Run()

	if err != nil {
		return "", err
	}

	return result[strings.Index(result, " - ")+3:], nil
}

// Save a step to the cache .create file for future tracking
func saveStep(step string) error {
	step = "- step: " + step + "\n"

	if err := utils.WriteToCache(&step, ".create", true); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package client

import (
	"io"
	"os/exec"
)

// Program is an interface to execute external programs.
type Program interface {
	Output() ([]byte, error)
	Input(in io.Reader)
}

// ProgramFunc is a type of function that initializes programs based on arguments.
type ProgramFunc func(args ...string) Program

// NewShellProgramFunc creates programs that are executed in a Shell.
func NewShellProgramFunc(name string) ProgramFunc {
	return func(args ...string) Program {
		return &Shell{cmd: exec.Command(name, args...)}
	}
}

// Shell invokes shell commands to talk with a remote credentials helper.
type Shell struct {
	cmd *exec.Cmd
}

// Output returns responses from the remote credentials helper.
func (s *Shell) Output() ([]byte, error) {
	return s.cmd.Output()
}

// Input sets the input to send to a remote credentials helper.
func (s *Shell) Input(in io.Reader) {
	s.cmd.Stdin = in
}
<commit_msg>Caller of cred helpers would now see the stderr returns from cred helpers<commit_after>package client

import (
	"io"
	"os"
	"os/exec"
)

// Program is an interface to execute external programs.
type Program interface {
	Output() ([]byte, error)
	Input(in io.Reader)
}

// ProgramFunc is a type of function that initializes programs based on arguments.
type ProgramFunc func(args ...string) Program

// NewShellProgramFunc creates programs that are executed in a Shell.
func NewShellProgramFunc(name string) ProgramFunc {
	return func(args ...string) Program {
		return &Shell{cmd: newCmdRedirectErr(name, args)}
	}
}

func newCmdRedirectErr(name string, args []string) *exec.Cmd {
	newCmd := exec.Command(name, args...)
	newCmd.Stderr = os.Stderr
	return newCmd
}

// Shell invokes shell commands to talk with a remote credentials helper.
type Shell struct {
	cmd *exec.Cmd
}

// Output returns responses from the remote credentials helper.
func (s *Shell) Output() ([]byte, error) {
	return s.cmd.Output()
}

// Input sets the input to send to a remote credentials helper.
func (s *Shell) Input(in io.Reader) {
	s.cmd.Stdin = in
}
<|endoftext|>


<commit_before>// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"fmt"
	"mime"
	"net/http"
	"net/http/httputil"
	"os"
	"path/filepath"
	"strings"

	"github.com/go-swagger/go-swagger/client"
	"github.com/go-swagger/go-swagger/httpkit"
	"github.com/go-swagger/go-swagger/spec"
	"github.com/go-swagger/go-swagger/strfmt"
)

// Runtime represents an API client that uses the transport
// to make http requests based on a swagger specification.
type Runtime struct {
	DefaultMediaType      string
	DefaultAuthentication client.AuthInfoWriter
	Consumers             map[string]httpkit.Consumer
	Producers             map[string]httpkit.Producer

	Transport http.RoundTripper
	Spec      *spec.Document
	Host      string
	BasePath  string
	Formats   strfmt.Registry
	Debug     bool

	client          *http.Client
	methodsAndPaths map[string]methodAndPath
}

// New creates a new default runtime for a swagger api client.
func New(swaggerSpec *spec.Document) *Runtime {
	var rt Runtime
	rt.DefaultMediaType = httpkit.JSONMime

	// TODO: actually infer this stuff from the spec
	rt.Consumers = map[string]httpkit.Consumer{
		httpkit.JSONMime: httpkit.JSONConsumer(),
	}
	rt.Producers = map[string]httpkit.Producer{
		httpkit.JSONMime: httpkit.JSONProducer(),
	}
	rt.Spec = swaggerSpec
	rt.Transport = http.DefaultTransport
	rt.client = http.DefaultClient
	rt.client.Transport = rt.Transport
	rt.Host = swaggerSpec.Host()
	rt.BasePath = swaggerSpec.BasePath()
	if !strings.HasPrefix(rt.BasePath, "/") {
		rt.BasePath = "/" + rt.BasePath
	}
	rt.Debug = os.Getenv("DEBUG") == "1"
	schemes := swaggerSpec.Spec().Schemes
	if len(schemes) == 0 {
		schemes = append(schemes, "http")
	}
	rt.methodsAndPaths = make(map[string]methodAndPath)
	for mth, pathItem := range rt.Spec.Operations() {
		for pth, op := range pathItem {
			if len(op.Schemes) > 0 {
				rt.methodsAndPaths[op.ID] = methodAndPath{mth, pth, op.Schemes}
			} else {
				rt.methodsAndPaths[op.ID] = methodAndPath{mth, pth, schemes}
			}
		}
	}
	return &rt
}

// Submit a request and when there is a body on success it will turn that into the result
// all other things are turned into an api error for swagger which retains the status code
func (r *Runtime) Submit(context *client.Operation) (interface{}, error) {
	operationID, params, readResponse, auth := context.ID, context.Params, context.Reader, context.AuthInfo
	mthPth, ok := r.methodsAndPaths[operationID]
	if !ok {
		return nil, fmt.Errorf("unknown operation: %q", operationID)
	}
	request, err := newRequest(mthPth.Method, mthPth.PathPattern, params)
	if err != nil {
		return nil, err
	}

	// TODO: infer most appropriate content type
	request.SetHeaderParam(httpkit.HeaderContentType, r.DefaultMediaType)
	var accept []string
	for k := range r.Consumers {
		accept = append(accept, k)
	}

	request.SetHeaderParam(httpkit.HeaderAccept, accept...)

	if auth == nil && r.DefaultAuthentication != nil {
		auth = r.DefaultAuthentication
	}
	if auth != nil {
		if err := auth.AuthenticateRequest(request, r.Formats); err != nil {
			return nil, err
		}
	}

	req, err := request.BuildHTTP(r.Producers[r.DefaultMediaType], r.Formats)

	// set the scheme
	if req.URL.Scheme == "" {
		req.URL.Scheme = "http"
	}
	schLen := len(mthPth.Schemes)
	if schLen > 0 {
		scheme := mthPth.Schemes[0]
		// prefer https, but skip when not possible
		if scheme != "https" && schLen > 1 {
			for _, sch := range mthPth.Schemes {
				if sch == "https" {
					scheme = sch
					break
				}
			}
		}
		req.URL.Scheme = scheme
	}

	req.URL.Host = r.Host
	req.URL.Path = filepath.Join(r.BasePath, req.URL.Path)
	if err != nil {
		return nil, err
	}

	r.client.Transport = r.Transport
	if r.Debug {
		b, err := httputil.DumpRequestOut(req, true)
		if err != nil {
			return nil, err
		}
		fmt.Println(string(b))
	}
	res, err := r.client.Do(req) // make requests, by default follows 10 redirects before failing
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if r.Debug {
		b, err := httputil.DumpResponse(res, true)
		if err != nil {
			return nil, err
		}
		fmt.Println(string(b))
	}
	ct := res.Header.Get(httpkit.HeaderContentType)
	if ct == "" { // this should really really never occur
		ct = r.DefaultMediaType
	}

	mt, _, err := mime.ParseMediaType(ct)
	if err != nil {
		return nil, fmt.Errorf("parse content type: %s", err)
	}
	cons, ok := r.Consumers[mt]
	if !ok {
		// scream about not knowing what to do
		return nil, fmt.Errorf("no consumer: %q", ct)
	}
	return readResponse.ReadResponse(response{res}, cons)
}
<commit_msg>fix url path generation in windows are using backslash instead of  /<commit_after>// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package client

import (
	"fmt"
	"mime"
	"net/http"
	"net/http/httputil"
	"os"
	"path"
	"strings"

	"github.com/go-swagger/go-swagger/client"
	"github.com/go-swagger/go-swagger/httpkit"
	"github.com/go-swagger/go-swagger/spec"
	"github.com/go-swagger/go-swagger/strfmt"
)

// Runtime represents an API client that uses the transport
// to make http requests based on a swagger specification.
type Runtime struct {
	DefaultMediaType      string
	DefaultAuthentication client.AuthInfoWriter
	Consumers             map[string]httpkit.Consumer
	Producers             map[string]httpkit.Producer

	Transport http.RoundTripper
	Spec      *spec.Document
	Host      string
	BasePath  string
	Formats   strfmt.Registry
	Debug     bool

	client          *http.Client
	methodsAndPaths map[string]methodAndPath
}

// New creates a new default runtime for a swagger api client.
func New(swaggerSpec *spec.Document) *Runtime {
	var rt Runtime
	rt.DefaultMediaType = httpkit.JSONMime

	// TODO: actually infer this stuff from the spec
	rt.Consumers = map[string]httpkit.Consumer{
		httpkit.JSONMime: httpkit.JSONConsumer(),
	}
	rt.Producers = map[string]httpkit.Producer{
		httpkit.JSONMime: httpkit.JSONProducer(),
	}
	rt.Spec = swaggerSpec
	rt.Transport = http.DefaultTransport
	rt.client = http.DefaultClient
	rt.client.Transport = rt.Transport
	rt.Host = swaggerSpec.Host()
	rt.BasePath = swaggerSpec.BasePath()
	if !strings.HasPrefix(rt.BasePath, "/") {
		rt.BasePath = "/" + rt.BasePath
	}
	rt.Debug = os.Getenv("DEBUG") == "1"
	schemes := swaggerSpec.Spec().Schemes
	if len(schemes) == 0 {
		schemes = append(schemes, "http")
	}
	rt.methodsAndPaths = make(map[string]methodAndPath)
	for mth, pathItem := range rt.Spec.Operations() {
		for pth, op := range pathItem {
			if len(op.Schemes) > 0 {
				rt.methodsAndPaths[op.ID] = methodAndPath{mth, pth, op.Schemes}
			} else {
				rt.methodsAndPaths[op.ID] = methodAndPath{mth, pth, schemes}
			}
		}
	}
	return &rt
}

// Submit a request and when there is a body on success it will turn that into the result
// all other things are turned into an api error for swagger which retains the status code
func (r *Runtime) Submit(context *client.Operation) (interface{}, error) {
	operationID, params, readResponse, auth := context.ID, context.Params, context.Reader, context.AuthInfo
	mthPth, ok := r.methodsAndPaths[operationID]
	if !ok {
		return nil, fmt.Errorf("unknown operation: %q", operationID)
	}
	request, err := newRequest(mthPth.Method, mthPth.PathPattern, params)
	if err != nil {
		return nil, err
	}

	// TODO: infer most appropriate content type
	request.SetHeaderParam(httpkit.HeaderContentType, r.DefaultMediaType)
	var accept []string
	for k := range r.Consumers {
		accept = append(accept, k)
	}

	request.SetHeaderParam(httpkit.HeaderAccept, accept...)

	if auth == nil && r.DefaultAuthentication != nil {
		auth = r.DefaultAuthentication
	}
	if auth != nil {
		if err := auth.AuthenticateRequest(request, r.Formats); err != nil {
			return nil, err
		}
	}

	req, err := request.BuildHTTP(r.Producers[r.DefaultMediaType], r.Formats)

	// set the scheme
	if req.URL.Scheme == "" {
		req.URL.Scheme = "http"
	}
	schLen := len(mthPth.Schemes)
	if schLen > 0 {
		scheme := mthPth.Schemes[0]
		// prefer https, but skip when not possible
		if scheme != "https" && schLen > 1 {
			for _, sch := range mthPth.Schemes {
				if sch == "https" {
					scheme = sch
					break
				}
			}
		}
		req.URL.Scheme = scheme
	}

	req.URL.Host = r.Host
	req.URL.Path = path.Join(r.BasePath, req.URL.Path)
	if err != nil {
		return nil, err
	}

	r.client.Transport = r.Transport
	if r.Debug {
		b, err := httputil.DumpRequestOut(req, true)
		if err != nil {
			return nil, err
		}
		fmt.Println(string(b))
	}
	res, err := r.client.Do(req) // make requests, by default follows 10 redirects before failing
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	if r.Debug {
		b, err := httputil.DumpResponse(res, true)
		if err != nil {
			return nil, err
		}
		fmt.Println(string(b))
	}
	ct := res.Header.Get(httpkit.HeaderContentType)
	if ct == "" { // this should really really never occur
		ct = r.DefaultMediaType
	}

	mt, _, err := mime.ParseMediaType(ct)
	if err != nil {
		return nil, fmt.Errorf("parse content type: %s", err)
	}
	cons, ok := r.Consumers[mt]
	if !ok {
		// scream about not knowing what to do
		return nil, fmt.Errorf("no consumer: %q", ct)
	}
	return readResponse.ReadResponse(response{res}, cons)
}
<|endoftext|>


<commit_before>// Package config for config files.
package client

// LibraryVersion specifies the current version of twilio-go.
const LibraryVersion = "0.18.0"
<commit_msg>Release v0.18.1<commit_after>// Package config for config files.
package client

// LibraryVersion specifies the current version of twilio-go.
const LibraryVersion = "0.18.1"
<|endoftext|>


<commit_before>/*
 * MinIO Client (C) 2016, 2017, 2018 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"fmt"
	"net/url"
	"sort"
	"strings"
	"time"

	humanize "github.com/dustin/go-humanize"
	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/mc/pkg/colorjson"
	"github.com/minio/mc/pkg/console"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/minio/pkg/madmin"
)

var (
	adminInfoFlags = []cli.Flag{}
)

var adminInfoCmd = cli.Command{
	Name:   "info",
	Usage:  "display MinIO server information",
	Action: mainAdminInfo,
	Before: setGlobalsFromContext,
	Flags:  append(adminInfoFlags, globalFlags...),
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} TARGET

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Get server information of the 'play' MinIO server.
       $ {{.HelpName}} play/

`,
}

// backendType - indicates the type of backend storage
type backendType string

const (
	fsType      = backendType("FS")
	erasureType = backendType("Erasure")
)

// fsBackend contains specific FS storage information
type fsBackend struct {
	Type backendType `json:"backendType"`
}

// xlBackend contains specific erasure storage information
type xlBackend struct {
	Type         backendType `json:"backendType"`
	OnlineDisks  int         `json:"onlineDisks"`
	OfflineDisks int         `json:"offlineDisks"`
	// Data disks for currently configured Standard storage class.
	StandardSCData int `json:"standardSCData"`
	// Parity disks for currently configured Standard storage class.
	StandardSCParity int `json:"standardSCParity"`
	// Data disks for currently configured Reduced Redundancy storage class.
	RRSCData int `json:"rrSCData"`
	// Parity disks for currently configured Reduced Redundancy storage class.
	RRSCParity int `json:"rrSCParity"`

	// List of all disk status.
	Sets [][]madmin.DriveInfo `json:"sets"`
}

// backendStatus represents the overall information of all backend storage types
type backendStatus struct {
	Used    uint64      `json:"used"`
	Backend interface{} `json:"backend"`
}

// ServerInfo holds the whole server information that will be
// returned by ServerInfo API.
type ServerInfo struct {
	StorageInfo backendStatus           `json:"storage"`
	ConnStats   madmin.ServerConnStats  `json:"network"`
	Properties  madmin.ServerProperties `json:"server"`
}

// infoMessage container to hold service status information.
type infoMessage struct {
	Status  string `json:"status"`
	Service string `json:"service"`
	Addr    string `json:"address"`
	Err     string `json:"error"`
	*ServerInfo
}

// String colorized service status message.
func (u infoMessage) String() (msg string) {
	dot := "●"

	// When service is offline
	if u.Service == "off" {
		msg += fmt.Sprintf("%s  %s\n", dot, console.Colorize("PrintB", u.Addr))
		msg += fmt.Sprintf("  Status : %s", console.Colorize("InfoFail", "offline"))
		return
	}

	// Print error if any and exit
	if u.Err != "" {
		msg += fmt.Sprintf("%s  %s\n", dot, console.Colorize("PrintB", u.Addr))
		msg += fmt.Sprintf("   Status : %s\n", console.Colorize("InfoFail", "offline"))
		e := u.Err
		if strings.Trim(e, " ") == "rpc: retry error" {
			e = "unreachable"
		}
		msg += fmt.Sprintf("    Error : %s", console.Colorize("InfoFail", e))
		return
	}

	// Print server title
	msg += fmt.Sprintf("%s  %s\n", dot, console.Colorize("PrintB", u.Addr))

	// Status
	msg += fmt.Sprintf("   Status : %s\n", console.Colorize("Info", "online"))

	// Uptime
	msg += fmt.Sprintf("   Uptime : %s\n", humanize.RelTime(time.Now(), time.Now().Add(-u.ServerInfo.Properties.Uptime), "", ""))
	// Version
	version := u.ServerInfo.Properties.Version
	if u.ServerInfo.Properties.Version == "DEVELOPMENT.GOGET" {
		version = "<development>"
	}
	msg += fmt.Sprintf("  Version : %s\n", version)
	// Region
	if u.ServerInfo.Properties.Region != "" {
		msg += fmt.Sprintf("   Region : %s\n", u.ServerInfo.Properties.Region)
	}
	// ARNs
	sqsARNs := ""
	for _, v := range u.ServerInfo.Properties.SQSARN {
		sqsARNs += fmt.Sprintf("%s ", v)
	}
	if sqsARNs != "" {
		msg += fmt.Sprintf(" SQS ARNs : %s\n", sqsARNs)
	}

	// Incoming/outgoing
	msg += fmt.Sprintf("  Storage : Used %s", humanize.IBytes(u.StorageInfo.Used))
	if v, ok := u.ServerInfo.StorageInfo.Backend.(xlBackend); ok {
		upBackends := 0
		downBackends := 0
		for _, set := range v.Sets {
			for _, s := range set {
				if strings.Contains(s.Endpoint, u.Addr) || s.Endpoint[0] == '/' {
					if s.State == "ok" {
						upBackends++
					} else {
						downBackends++
					}
				}
			}
		}
		upBackendsString := fmt.Sprintf("%d", upBackends)
		if downBackends != 0 {
			upBackendsString = console.Colorize("InfoFail", fmt.Sprintf("%d", upBackends))
		}
		msg += fmt.Sprintf("\n   Drives : %s/%d OK\n", upBackendsString, upBackends+downBackends)
	}
	return
}

// JSON jsonified service status Message message.
func (u infoMessage) JSON() string {
	u.Status = "success"
	statusJSONBytes, e := json.MarshalIndent(u, "", " ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

	return string(statusJSONBytes)
}

// checkAdminInfoSyntax - validate all the passed arguments
func checkAdminInfoSyntax(ctx *cli.Context) {
	if len(ctx.Args()) == 0 || len(ctx.Args()) > 2 {
		cli.ShowCommandHelpAndExit(ctx, "info", 1) // last argument is exit code
	}
}

func mainAdminInfo(ctx *cli.Context) error {
	// Validate service status syntax.
	checkAdminInfoSyntax(ctx)

	console.SetColor("Info", color.New(color.FgGreen, color.Bold))
	console.SetColor("InfoDegraded", color.New(color.FgYellow, color.Bold))
	console.SetColor("InfoFail", color.New(color.FgRed, color.Bold))

	// Get the alias parameter from cli
	args := ctx.Args()
	aliasedURL := args.Get(0)

	// Create a new MinIO Admin Client
	client, err := newAdminClient(aliasedURL)
	fatalIf(err, "Cannot get a configured admin connection.")

	// Fetch info of all servers (cluster or single server)
	serversInfo, e := client.ServerInfo()

	// Check the availability of the server: online or offline. A server is considered
	// offline if we can't get any response or we get a bad format response
	var serviceOffline bool
	switch e.(type) {
	case *json.SyntaxError:
		serviceOffline = true
	case *url.Error:
		serviceOffline = true
	}

	if serviceOffline {
		printMsg(infoMessage{Addr: aliasedURL, Service: "off"})
		return nil
	}

	// If the error is not nil and not unrecognizable, just print it and exit
	fatalIf(probe.NewError(e), "Cannot get service status.")

	infoMessages := []infoMessage{}
	for _, serverInfo := range serversInfo {
		// Print the error if exists and jump to the next server
		if serverInfo.Error != "" {
			infoMessages = append(infoMessages, infoMessage{
				Service: "on",
				Addr:    serverInfo.Addr,
				Err:     serverInfo.Error,
			})
			continue
		}

		// Construct the backend status
		storageInfo := backendStatus{
			Used: serverInfo.Data.StorageInfo.Used,
		}

		if serverInfo.Data.StorageInfo.Backend.Type == madmin.Erasure {
			storageInfo.Backend = xlBackend{
				Type:             erasureType,
				OnlineDisks:      serverInfo.Data.StorageInfo.Backend.OnlineDisks,
				OfflineDisks:     serverInfo.Data.StorageInfo.Backend.OfflineDisks,
				StandardSCData:   serverInfo.Data.StorageInfo.Backend.StandardSCData,
				StandardSCParity: serverInfo.Data.StorageInfo.Backend.StandardSCParity,
				RRSCData:         serverInfo.Data.StorageInfo.Backend.RRSCData,
				RRSCParity:       serverInfo.Data.StorageInfo.Backend.RRSCParity,
				Sets:             serverInfo.Data.StorageInfo.Backend.Sets,
			}
		} else {
			storageInfo.Backend = fsBackend{
				Type: fsType,
			}
		}

		infoMessages = append(infoMessages, (infoMessage{
			Service: "on",
			Addr:    serverInfo.Addr,
			Err:     serverInfo.Error,
			ServerInfo: &ServerInfo{
				StorageInfo: storageInfo,
				ConnStats:   serverInfo.Data.ConnStats,
				Properties:  serverInfo.Data.Properties,
			},
		}))

	}

	sort.Stable(&sortInfoWrapper{infoMessages})
	for _, s := range infoMessages {
		printMsg(s)
	}

	return nil
}

type sortInfoWrapper struct {
	infos []infoMessage
}

func (s *sortInfoWrapper) Len() int {
	return len(s.infos)
}

func (s *sortInfoWrapper) Swap(i, j int) {
	if s.infos != nil {
		s.infos[i], s.infos[j] = s.infos[j], s.infos[i]
	}
}

func (s *sortInfoWrapper) Less(i, j int) bool {
	if s.infos != nil {
		return s.infos[i].Addr < s.infos[j].Addr
	}
	return false

}
<commit_msg>Fix admin info dot coloring (#2830)<commit_after>/*
 * MinIO Client (C) 2016-2019 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"fmt"
	"net/url"
	"sort"
	"strings"
	"time"

	humanize "github.com/dustin/go-humanize"
	"github.com/fatih/color"
	"github.com/minio/cli"
	json "github.com/minio/mc/pkg/colorjson"
	"github.com/minio/mc/pkg/console"
	"github.com/minio/mc/pkg/probe"
	"github.com/minio/minio/pkg/madmin"
)

var (
	adminInfoFlags = []cli.Flag{}
)

var adminInfoCmd = cli.Command{
	Name:   "info",
	Usage:  "display MinIO server information",
	Action: mainAdminInfo,
	Before: setGlobalsFromContext,
	Flags:  append(adminInfoFlags, globalFlags...),
	CustomHelpTemplate: `NAME:
  {{.HelpName}} - {{.Usage}}

USAGE:
  {{.HelpName}} TARGET

FLAGS:
  {{range .VisibleFlags}}{{.}}
  {{end}}
EXAMPLES:
  1. Get server information of the 'play' MinIO server.
       $ {{.HelpName}} play/

`,
}

// backendType - indicates the type of backend storage
type backendType string

const (
	fsType      = backendType("FS")
	erasureType = backendType("Erasure")
)

// fsBackend contains specific FS storage information
type fsBackend struct {
	Type backendType `json:"backendType"`
}

// xlBackend contains specific erasure storage information
type xlBackend struct {
	Type         backendType `json:"backendType"`
	OnlineDisks  int         `json:"onlineDisks"`
	OfflineDisks int         `json:"offlineDisks"`
	// Data disks for currently configured Standard storage class.
	StandardSCData int `json:"standardSCData"`
	// Parity disks for currently configured Standard storage class.
	StandardSCParity int `json:"standardSCParity"`
	// Data disks for currently configured Reduced Redundancy storage class.
	RRSCData int `json:"rrSCData"`
	// Parity disks for currently configured Reduced Redundancy storage class.
	RRSCParity int `json:"rrSCParity"`

	// List of all disk status.
	Sets [][]madmin.DriveInfo `json:"sets"`
}

// backendStatus represents the overall information of all backend storage types
type backendStatus struct {
	Used    uint64      `json:"used"`
	Backend interface{} `json:"backend"`
}

// ServerInfo holds the whole server information that will be
// returned by ServerInfo API.
type ServerInfo struct {
	StorageInfo backendStatus           `json:"storage"`
	ConnStats   madmin.ServerConnStats  `json:"network"`
	Properties  madmin.ServerProperties `json:"server"`
}

// infoMessage container to hold service status information.
type infoMessage struct {
	Status  string `json:"status"`
	Service string `json:"service"`
	Addr    string `json:"address"`
	Err     string `json:"error"`
	*ServerInfo
}

// String colorized service status message.
func (u infoMessage) String() (msg string) {
	dot := "●"

	// When service is offline
	if u.Service == "off" {
		msg += fmt.Sprintf("%s  %s\n", console.Colorize("InfoFail", dot), console.Colorize("PrintB", u.Addr))
		msg += fmt.Sprintf("   Uptime: %s\n", console.Colorize("InfoFail", "offline"))
		return
	}

	// Print error if any and exit
	if u.Err != "" {
		msg += fmt.Sprintf("%s  %s\n", console.Colorize("InfoFail", dot), console.Colorize("PrintB", u.Addr))
		msg += fmt.Sprintf("   Uptime: %s\n", console.Colorize("InfoFail", "offline"))
		e := u.Err
		if strings.Trim(e, " ") == "rpc: retry error" {
			e = "unreachable"
		}
		msg += fmt.Sprintf("    Error: %s", console.Colorize("InfoFail", e))
		return
	}

	// Print server title
	msg += fmt.Sprintf("%s  %s\n", console.Colorize("Info", dot), console.Colorize("PrintB", u.Addr))

	// Uptime
	msg += fmt.Sprintf("   Uptime: %s\n", console.Colorize("Info",
		humanize.RelTime(time.Now(), time.Now().Add(-u.ServerInfo.Properties.Uptime), "", "")))

	// Version
	version := u.ServerInfo.Properties.Version
	if u.ServerInfo.Properties.Version == "DEVELOPMENT.GOGET" {
		version = "<development>"
	}
	msg += fmt.Sprintf("  Version: %s\n", version)
	// Region
	if u.ServerInfo.Properties.Region != "" {
		msg += fmt.Sprintf("   Region: %s\n", u.ServerInfo.Properties.Region)
	}
	// ARNs
	sqsARNs := ""
	for _, v := range u.ServerInfo.Properties.SQSARN {
		sqsARNs += fmt.Sprintf("%s ", v)
	}
	if sqsARNs != "" {
		msg += fmt.Sprintf(" SQS ARNs: %s\n", sqsARNs)
	}

	// Incoming/outgoing
	msg += fmt.Sprintf("  Storage: Used %s", humanize.IBytes(u.StorageInfo.Used))
	if v, ok := u.ServerInfo.StorageInfo.Backend.(xlBackend); ok {
		upBackends := 0
		downBackends := 0
		for _, set := range v.Sets {
			for _, s := range set {
				if strings.Contains(s.Endpoint, u.Addr) || s.Endpoint[0] == '/' {
					if s.State == "ok" {
						upBackends++
					} else {
						downBackends++
					}
				}
			}
		}
		upBackendsString := fmt.Sprintf("%d", upBackends)
		if downBackends != 0 {
			upBackendsString = console.Colorize("InfoFail", fmt.Sprintf("%d", upBackends))
		}
		msg += fmt.Sprintf("\n   Drives: %s/%d %s\n", upBackendsString,
			upBackends+downBackends, console.Colorize("Info", "OK"))
	}
	return
}

// JSON jsonified service status Message message.
func (u infoMessage) JSON() string {
	u.Status = "success"
	statusJSONBytes, e := json.MarshalIndent(u, "", " ")
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

	return string(statusJSONBytes)
}

// checkAdminInfoSyntax - validate all the passed arguments
func checkAdminInfoSyntax(ctx *cli.Context) {
	if len(ctx.Args()) == 0 || len(ctx.Args()) > 2 {
		cli.ShowCommandHelpAndExit(ctx, "info", 1) // last argument is exit code
	}
}

func mainAdminInfo(ctx *cli.Context) error {
	// Validate service status syntax.
	checkAdminInfoSyntax(ctx)

	console.SetColor("Info", color.New(color.FgGreen, color.Bold))
	console.SetColor("InfoDegraded", color.New(color.FgYellow, color.Bold))
	console.SetColor("InfoFail", color.New(color.FgRed, color.Bold))

	// Get the alias parameter from cli
	args := ctx.Args()
	aliasedURL := args.Get(0)

	// Create a new MinIO Admin Client
	client, err := newAdminClient(aliasedURL)
	fatalIf(err, "Cannot get a configured admin connection.")

	// Fetch info of all servers (cluster or single server)
	serversInfo, e := client.ServerInfo()

	// Check the availability of the server: online or offline. A server is considered
	// offline if we can't get any response or we get a bad format response
	var serviceOffline bool
	switch e.(type) {
	case *json.SyntaxError:
		serviceOffline = true
	case *url.Error:
		serviceOffline = true
	}

	if serviceOffline {
		printMsg(infoMessage{Addr: aliasedURL, Service: "off"})
		return nil
	}

	// If the error is not nil and not unrecognizable, just print it and exit
	fatalIf(probe.NewError(e), "Cannot get service status.")

	infoMessages := []infoMessage{}
	for _, serverInfo := range serversInfo {
		// Print the error if exists and jump to the next server
		if serverInfo.Error != "" {
			infoMessages = append(infoMessages, infoMessage{
				Service: "on",
				Addr:    serverInfo.Addr,
				Err:     serverInfo.Error,
			})
			continue
		}

		// Construct the backend status
		storageInfo := backendStatus{
			Used: serverInfo.Data.StorageInfo.Used,
		}

		if serverInfo.Data.StorageInfo.Backend.Type == madmin.Erasure {
			storageInfo.Backend = xlBackend{
				Type:             erasureType,
				OnlineDisks:      serverInfo.Data.StorageInfo.Backend.OnlineDisks,
				OfflineDisks:     serverInfo.Data.StorageInfo.Backend.OfflineDisks,
				StandardSCData:   serverInfo.Data.StorageInfo.Backend.StandardSCData,
				StandardSCParity: serverInfo.Data.StorageInfo.Backend.StandardSCParity,
				RRSCData:         serverInfo.Data.StorageInfo.Backend.RRSCData,
				RRSCParity:       serverInfo.Data.StorageInfo.Backend.RRSCParity,
				Sets:             serverInfo.Data.StorageInfo.Backend.Sets,
			}
		} else {
			storageInfo.Backend = fsBackend{
				Type: fsType,
			}
		}

		infoMessages = append(infoMessages, (infoMessage{
			Service: "on",
			Addr:    serverInfo.Addr,
			Err:     serverInfo.Error,
			ServerInfo: &ServerInfo{
				StorageInfo: storageInfo,
				ConnStats:   serverInfo.Data.ConnStats,
				Properties:  serverInfo.Data.Properties,
			},
		}))

	}

	sort.Stable(&sortInfoWrapper{infoMessages})
	for _, s := range infoMessages {
		printMsg(s)
	}

	return nil
}

type sortInfoWrapper struct {
	infos []infoMessage
}

func (s *sortInfoWrapper) Len() int {
	return len(s.infos)
}

func (s *sortInfoWrapper) Swap(i, j int) {
	if s.infos != nil {
		s.infos[i], s.infos[j] = s.infos[j], s.infos[i]
	}
}

func (s *sortInfoWrapper) Less(i, j int) bool {
	if s.infos != nil {
		return s.infos[i].Addr < s.infos[j].Addr
	}
	return false

}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"sort"
	"text/tabwriter"

	"github.com/golang/dep"
	"github.com/golang/dep/internal/gps"
	"github.com/golang/dep/internal/gps/paths"
	"github.com/golang/dep/internal/gps/pkgtree"
	"github.com/pkg/errors"
)

const statusShortHelp = `Report the status of the project's dependencies`
const statusLongHelp = `
With no arguments, print the status of each dependency of the project.

  PROJECT     Import path
  CONSTRAINT  Version constraint, from the manifest
  VERSION     Version chosen, from the lock
  REVISION    VCS revision of the chosen version
  LATEST      Latest VCS revision available
  PKGS USED   Number of packages from this project that are actually used

With one or more explicitly specified packages, or with the -detailed flag,
print an extended status output for each dependency of the project.

  TODO    Another column description
  FOOBAR  Another column description

Status returns exit code zero if all dependencies are in a "good state".
`

func (cmd *statusCommand) Name() string      { return "status" }
func (cmd *statusCommand) Args() string      { return "[package...]" }
func (cmd *statusCommand) ShortHelp() string { return statusShortHelp }
func (cmd *statusCommand) LongHelp() string  { return statusLongHelp }
func (cmd *statusCommand) Hidden() bool      { return false }

func (cmd *statusCommand) Register(fs *flag.FlagSet) {
	fs.BoolVar(&cmd.detailed, "detailed", false, "report more detailed status")
	fs.BoolVar(&cmd.json, "json", false, "output in JSON format")
	fs.StringVar(&cmd.template, "f", "", "output in text/template format")
	fs.BoolVar(&cmd.dot, "dot", false, "output the dependency graph in GraphViz format")
	fs.BoolVar(&cmd.old, "old", false, "only show out-of-date dependencies")
	fs.BoolVar(&cmd.missing, "missing", false, "only show missing dependencies")
	fs.BoolVar(&cmd.unused, "unused", false, "only show unused dependencies")
	fs.BoolVar(&cmd.modified, "modified", false, "only show modified dependencies")
}

type statusCommand struct {
	detailed bool
	json     bool
	template string
	output   string
	dot      bool
	old      bool
	missing  bool
	unused   bool
	modified bool
}

type outputter interface {
	BasicHeader()
	BasicLine(*BasicStatus)
	BasicFooter()
	MissingHeader()
	MissingLine(*MissingStatus)
	MissingFooter()
}

type tableOutput struct{ w *tabwriter.Writer }

func (out *tableOutput) BasicHeader() {
	fmt.Fprintf(out.w, "PROJECT\tCONSTRAINT\tVERSION\tREVISION\tLATEST\tPKGS USED\n")
}

func (out *tableOutput) BasicFooter() {
	out.w.Flush()
}

func (out *tableOutput) BasicLine(bs *BasicStatus) {
	var constraint string
	if v, ok := bs.Constraint.(gps.Version); ok {
		constraint = formatVersion(v)
	} else {
		constraint = bs.Constraint.String()
	}
	fmt.Fprintf(out.w,
		"%s\t%s\t%s\t%s\t%s\t%d\t\n",
		bs.ProjectRoot,
		constraint,
		formatVersion(bs.Version),
		formatVersion(bs.Revision),
		formatVersion(bs.Latest),
		bs.PackageCount,
	)
}

func (out *tableOutput) MissingHeader() {
	fmt.Fprintln(out.w, "PROJECT\tMISSING PACKAGES")
}

func (out *tableOutput) MissingLine(ms *MissingStatus) {
	fmt.Fprintf(out.w,
		"%s\t%s\t\n",
		ms.ProjectRoot,
		ms.MissingPackages,
	)
}

func (out *tableOutput) MissingFooter() {
	out.w.Flush()
}

type jsonOutput struct {
	w       io.Writer
	basic   []*BasicStatus
	missing []*MissingStatus
}

func (out *jsonOutput) BasicHeader() {
	out.basic = []*BasicStatus{}
}

func (out *jsonOutput) BasicFooter() {
	json.NewEncoder(out.w).Encode(out.basic)
}

func (out *jsonOutput) BasicLine(bs *BasicStatus) {
	out.basic = append(out.basic, bs)
}

func (out *jsonOutput) MissingHeader() {
	out.missing = []*MissingStatus{}
}

func (out *jsonOutput) MissingLine(ms *MissingStatus) {
	out.missing = append(out.missing, ms)
}

func (out *jsonOutput) MissingFooter() {
	json.NewEncoder(out.w).Encode(out.missing)
}

type dotOutput struct {
	w io.Writer
	o string
	g *graphviz
	p *dep.Project
}

func (out *dotOutput) BasicHeader() {
	out.g = new(graphviz).New()

	ptree, _ := pkgtree.ListPackages(out.p.AbsRoot, string(out.p.ImportRoot))
	prm, _ := ptree.ToReachMap(true, false, false, nil)

	out.g.createNode(string(out.p.ImportRoot), "", prm.FlattenFn(paths.IsStandardImportPath))
}

func (out *dotOutput) BasicFooter() {
	gvo := out.g.output()
	fmt.Fprintf(out.w, gvo.String())
}

func (out *dotOutput) BasicLine(bs *BasicStatus) {
	out.g.createNode(bs.ProjectRoot, bs.Version.String(), bs.Children)
}

func (out *dotOutput) MissingHeader()                {}
func (out *dotOutput) MissingLine(ms *MissingStatus) {}
func (out *dotOutput) MissingFooter()                {}

func (cmd *statusCommand) Run(ctx *dep.Ctx, args []string) error {
	p, err := ctx.LoadProject("")
	if err != nil {
		return err
	}

	sm, err := ctx.SourceManager()
	if err != nil {
		return err
	}
	sm.UseDefaultSignalHandling()
	defer sm.Release()

	var buf bytes.Buffer
	var out outputter
	switch {
	case cmd.detailed:
		return errors.Errorf("not implemented")
	case cmd.json:
		out = &jsonOutput{
			w: &buf,
		}
	case cmd.dot:
		out = &dotOutput{
			p: p,
			o: cmd.output,
			w: &buf,
		}
	default:
		out = &tableOutput{
			w: tabwriter.NewWriter(&buf, 0, 4, 2, ' ', 0),
		}
	}

	if err := runStatusAll(ctx.Loggers, out, p, sm); err != nil {
		return err
	}

	ctx.Loggers.Out.Print(buf.String())
	return nil
}

// BasicStatus contains all the information reported about a single dependency
// in the summary/list status output mode.
type BasicStatus struct {
	ProjectRoot  string
	Children     []string
	Constraint   gps.Constraint
	Version      gps.UnpairedVersion
	Revision     gps.Revision
	Latest       gps.Version
	PackageCount int
}

type MissingStatus struct {
	ProjectRoot     string
	MissingPackages []string
}

func runStatusAll(loggers *dep.Loggers, out outputter, p *dep.Project, sm gps.SourceManager) error {
	if p.Lock == nil {
		// TODO if we have no lock file, do...other stuff
		return nil
	}

	// While the network churns on ListVersions() requests, statically analyze
	// code from the current project.
	ptree, err := pkgtree.ListPackages(p.AbsRoot, string(p.ImportRoot))
	if err != nil {
		return errors.Errorf("analysis of local packages failed: %v", err)
	}

	// Set up a solver in order to check the InputHash.
	params := gps.SolveParameters{
		ProjectAnalyzer: dep.Analyzer{},
		RootDir:         p.AbsRoot,
		RootPackageTree: ptree,
		Manifest:        p.Manifest,
		// Locks aren't a part of the input hash check, so we can omit it.
	}
	if loggers.Verbose {
		params.TraceLogger = loggers.Err
	}

	s, err := gps.Prepare(params, sm)
	if err != nil {
		return errors.Errorf("could not set up solver for input hashing: %s", err)
	}

	cm := collectConstraints(ptree, p, sm)

	// Get the project list and sort it so that the printed output users see is
	// deterministically ordered. (This may be superfluous if the lock is always
	// written in alpha order, but it doesn't hurt to double down.)
	slp := p.Lock.Projects()
	sort.Sort(dep.SortedLockedProjects(slp))

	if bytes.Equal(s.HashInputs(), p.Lock.SolveMeta.InputsDigest) {
		// If these are equal, we're guaranteed that the lock is a transitively
		// complete picture of all deps. That eliminates the need for at least
		// some checks.

		out.BasicHeader()

		for _, proj := range slp {
			bs := BasicStatus{
				ProjectRoot:  string(proj.Ident().ProjectRoot),
				PackageCount: len(proj.Packages()),
			}

			// Get children only for specific outputers
			// in order to avoid slower status process
			switch out.(type) {
			case *dotOutput:
				ptr, err := sm.ListPackages(proj.Ident(), proj.Version())

				if err != nil {
					return fmt.Errorf("analysis of %s package failed: %v", proj.Ident().ProjectRoot, err)
				}

				prm, _ := ptr.ToReachMap(true, false, false, nil)
				bs.Children = prm.FlattenFn(paths.IsStandardImportPath)
			}

			// Split apart the version from the lock into its constituent parts
			switch tv := proj.Version().(type) {
			case gps.UnpairedVersion:
				bs.Version = tv
			case gps.Revision:
				bs.Revision = tv
			case gps.PairedVersion:
				bs.Version = tv.Unpair()
				bs.Revision = tv.Underlying()
			}

			// Check if the manifest has an override for this project. If so,
			// set that as the constraint.
			if pp, has := p.Manifest.Ovr[proj.Ident().ProjectRoot]; has && pp.Constraint != nil {
				// TODO note somehow that it's overridden
				bs.Constraint = pp.Constraint
			} else {
				bs.Constraint = gps.Any()
				for _, c := range cm[bs.ProjectRoot] {
					bs.Constraint = c.Intersect(bs.Constraint)
				}
			}

			// Only if we have a non-rev and non-plain version do/can we display
			// anything wrt the version's updateability.
			if bs.Version != nil && bs.Version.Type() != gps.IsVersion {
				c, has := p.Manifest.Constraints[proj.Ident().ProjectRoot]
				if !has {
					c.Constraint = gps.Any()
				}
				// TODO: This constraint is only the constraint imposed by the
				// current project, not by any transitive deps. As a result,
				// transitive project deps will always show "any" here.
				bs.Constraint = c.Constraint

				vl, err := sm.ListVersions(proj.Ident())
				if err == nil {
					gps.SortPairedForUpgrade(vl)

					for _, v := range vl {
						// Because we've sorted the version list for
						// upgrade, the first version we encounter that
						// matches our constraint will be what we want.
						if c.Constraint.Matches(v) {
							bs.Latest = v.Underlying()
							break
						}
					}
				}
			}

			out.BasicLine(&bs)
		}
		out.BasicFooter()

		return nil
	}

	// Hash digest mismatch may indicate that some deps are no longer
	// needed, some are missing, or that some constraints or source
	// locations have changed.
	//
	// It's possible for digests to not match, but still have a correct
	// lock.
	rm, _ := ptree.ToReachMap(true, true, false, nil)

	external := rm.FlattenFn(paths.IsStandardImportPath)
	roots := make(map[gps.ProjectRoot][]string, len(external))

	type fail struct {
		ex  string
		err error
	}
	var errs []fail
	for _, e := range external {
		root, err := sm.DeduceProjectRoot(e)
		if err != nil {
			errs = append(errs, fail{
				ex:  e,
				err: err,
			})
			continue
		}

		roots[root] = append(roots[root], e)
	}

	if len(errs) != 0 {
		// TODO this is just a fix quick so staticcheck doesn't complain.
		// Visually reconciling failure to deduce project roots with the rest of
		// the mismatch output is a larger problem.
		loggers.Err.Printf("Failed to deduce project roots for import paths:\n")
		for _, fail := range errs {
			loggers.Err.Printf("\t%s: %s\n", fail.ex, fail.err.Error())
		}

		return errors.New("address issues with undeducable import paths to get more status information.")
	}

	out.MissingHeader()

outer:
	for root, pkgs := range roots {
		// TODO also handle the case where the project is present, but there
		// are items missing from just the package list
		for _, lp := range slp {
			if lp.Ident().ProjectRoot == root {
				continue outer
			}
		}

		out.MissingLine(&MissingStatus{ProjectRoot: string(root), MissingPackages: pkgs})
	}
	out.MissingFooter()

	return nil
}

func formatVersion(v gps.Version) string {
	if v == nil {
		return ""
	}
	switch v.Type() {
	case gps.IsBranch:
		return "branch " + v.String()
	case gps.IsRevision:
		r := v.String()
		if len(r) > 7 {
			r = r[:7]
		}
		return r
	}
	return v.String()
}

func collectConstraints(ptree pkgtree.PackageTree, p *dep.Project, sm gps.SourceManager) map[string][]gps.Constraint {
	// TODO
	return map[string][]gps.Constraint{}
}
<commit_msg>Add status feedback on memo mismatch<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"sort"
	"text/tabwriter"

	"github.com/golang/dep"
	"github.com/golang/dep/internal/gps"
	"github.com/golang/dep/internal/gps/paths"
	"github.com/golang/dep/internal/gps/pkgtree"
	"github.com/pkg/errors"
)

const statusShortHelp = `Report the status of the project's dependencies`
const statusLongHelp = `
With no arguments, print the status of each dependency of the project.

  PROJECT     Import path
  CONSTRAINT  Version constraint, from the manifest
  VERSION     Version chosen, from the lock
  REVISION    VCS revision of the chosen version
  LATEST      Latest VCS revision available
  PKGS USED   Number of packages from this project that are actually used

With one or more explicitly specified packages, or with the -detailed flag,
print an extended status output for each dependency of the project.

  TODO    Another column description
  FOOBAR  Another column description

Status returns exit code zero if all dependencies are in a "good state".
`

func (cmd *statusCommand) Name() string      { return "status" }
func (cmd *statusCommand) Args() string      { return "[package...]" }
func (cmd *statusCommand) ShortHelp() string { return statusShortHelp }
func (cmd *statusCommand) LongHelp() string  { return statusLongHelp }
func (cmd *statusCommand) Hidden() bool      { return false }

func (cmd *statusCommand) Register(fs *flag.FlagSet) {
	fs.BoolVar(&cmd.detailed, "detailed", false, "report more detailed status")
	fs.BoolVar(&cmd.json, "json", false, "output in JSON format")
	fs.StringVar(&cmd.template, "f", "", "output in text/template format")
	fs.BoolVar(&cmd.dot, "dot", false, "output the dependency graph in GraphViz format")
	fs.BoolVar(&cmd.old, "old", false, "only show out-of-date dependencies")
	fs.BoolVar(&cmd.missing, "missing", false, "only show missing dependencies")
	fs.BoolVar(&cmd.unused, "unused", false, "only show unused dependencies")
	fs.BoolVar(&cmd.modified, "modified", false, "only show modified dependencies")
}

type statusCommand struct {
	detailed bool
	json     bool
	template string
	output   string
	dot      bool
	old      bool
	missing  bool
	unused   bool
	modified bool
}

type outputter interface {
	BasicHeader()
	BasicLine(*BasicStatus)
	BasicFooter()
	MissingHeader()
	MissingLine(*MissingStatus)
	MissingFooter()
}

type tableOutput struct{ w *tabwriter.Writer }

func (out *tableOutput) BasicHeader() {
	fmt.Fprintf(out.w, "PROJECT\tCONSTRAINT\tVERSION\tREVISION\tLATEST\tPKGS USED\n")
}

func (out *tableOutput) BasicFooter() {
	out.w.Flush()
}

func (out *tableOutput) BasicLine(bs *BasicStatus) {
	var constraint string
	if v, ok := bs.Constraint.(gps.Version); ok {
		constraint = formatVersion(v)
	} else {
		constraint = bs.Constraint.String()
	}
	fmt.Fprintf(out.w,
		"%s\t%s\t%s\t%s\t%s\t%d\t\n",
		bs.ProjectRoot,
		constraint,
		formatVersion(bs.Version),
		formatVersion(bs.Revision),
		formatVersion(bs.Latest),
		bs.PackageCount,
	)
}

func (out *tableOutput) MissingHeader() {
	fmt.Fprintln(out.w, "PROJECT\tMISSING PACKAGES")
}

func (out *tableOutput) MissingLine(ms *MissingStatus) {
	fmt.Fprintf(out.w,
		"%s\t%s\t\n",
		ms.ProjectRoot,
		ms.MissingPackages,
	)
}

func (out *tableOutput) MissingFooter() {
	out.w.Flush()
}

type jsonOutput struct {
	w       io.Writer
	basic   []*BasicStatus
	missing []*MissingStatus
}

func (out *jsonOutput) BasicHeader() {
	out.basic = []*BasicStatus{}
}

func (out *jsonOutput) BasicFooter() {
	json.NewEncoder(out.w).Encode(out.basic)
}

func (out *jsonOutput) BasicLine(bs *BasicStatus) {
	out.basic = append(out.basic, bs)
}

func (out *jsonOutput) MissingHeader() {
	out.missing = []*MissingStatus{}
}

func (out *jsonOutput) MissingLine(ms *MissingStatus) {
	out.missing = append(out.missing, ms)
}

func (out *jsonOutput) MissingFooter() {
	json.NewEncoder(out.w).Encode(out.missing)
}

type dotOutput struct {
	w io.Writer
	o string
	g *graphviz
	p *dep.Project
}

func (out *dotOutput) BasicHeader() {
	out.g = new(graphviz).New()

	ptree, _ := pkgtree.ListPackages(out.p.AbsRoot, string(out.p.ImportRoot))
	prm, _ := ptree.ToReachMap(true, false, false, nil)

	out.g.createNode(string(out.p.ImportRoot), "", prm.FlattenFn(paths.IsStandardImportPath))
}

func (out *dotOutput) BasicFooter() {
	gvo := out.g.output()
	fmt.Fprintf(out.w, gvo.String())
}

func (out *dotOutput) BasicLine(bs *BasicStatus) {
	out.g.createNode(bs.ProjectRoot, bs.Version.String(), bs.Children)
}

func (out *dotOutput) MissingHeader()                {}
func (out *dotOutput) MissingLine(ms *MissingStatus) {}
func (out *dotOutput) MissingFooter()                {}

func (cmd *statusCommand) Run(ctx *dep.Ctx, args []string) error {
	p, err := ctx.LoadProject("")
	if err != nil {
		return err
	}

	sm, err := ctx.SourceManager()
	if err != nil {
		return err
	}
	sm.UseDefaultSignalHandling()
	defer sm.Release()

	var buf bytes.Buffer
	var out outputter
	switch {
	case cmd.detailed:
		return errors.Errorf("not implemented")
	case cmd.json:
		out = &jsonOutput{
			w: &buf,
		}
	case cmd.dot:
		out = &dotOutput{
			p: p,
			o: cmd.output,
			w: &buf,
		}
	default:
		out = &tableOutput{
			w: tabwriter.NewWriter(&buf, 0, 4, 2, ' ', 0),
		}
	}

	memoMismatch, hasMissingPkgs, err := runStatusAll(ctx.Loggers, out, p, sm)
	if err != nil {
		return err
	}

	if memoMismatch {
		if hasMissingPkgs {
			ctx.Loggers.Err.Println("Lock memo mismatch due to the following packages missing from the lock:\n")
			ctx.Loggers.Out.Print(buf.String())
			ctx.Loggers.Err.Println("\nThis happens when a new import is added. Run `dep ensure` to install the missing packages.")
		} else {
			ctx.Loggers.Err.Printf("Lock memo mismatch. This happens when Gopkg.toml is modified.\n" +
				"Run `dep ensure` to regenerate the memo.")
		}
	} else {
		ctx.Loggers.Out.Print(buf.String())
	}

	return nil
}

// BasicStatus contains all the information reported about a single dependency
// in the summary/list status output mode.
type BasicStatus struct {
	ProjectRoot  string
	Children     []string
	Constraint   gps.Constraint
	Version      gps.UnpairedVersion
	Revision     gps.Revision
	Latest       gps.Version
	PackageCount int
}

type MissingStatus struct {
	ProjectRoot     string
	MissingPackages []string
}

func runStatusAll(loggers *dep.Loggers, out outputter, p *dep.Project, sm gps.SourceManager) (bool, bool, error) {
	var memoMismatch, hasMissingPkgs bool

	if p.Lock == nil {
		// TODO if we have no lock file, do...other stuff
		return memoMismatch, hasMissingPkgs, nil
	}

	// While the network churns on ListVersions() requests, statically analyze
	// code from the current project.
	ptree, err := pkgtree.ListPackages(p.AbsRoot, string(p.ImportRoot))
	if err != nil {
		return memoMismatch, hasMissingPkgs, errors.Errorf("analysis of local packages failed: %v", err)
	}

	// Set up a solver in order to check the InputHash.
	params := gps.SolveParameters{
		ProjectAnalyzer: dep.Analyzer{},
		RootDir:         p.AbsRoot,
		RootPackageTree: ptree,
		Manifest:        p.Manifest,
		// Locks aren't a part of the input hash check, so we can omit it.
	}
	if loggers.Verbose {
		params.TraceLogger = loggers.Err
	}

	s, err := gps.Prepare(params, sm)
	if err != nil {
		return memoMismatch, hasMissingPkgs, errors.Errorf("could not set up solver for input hashing: %s", err)
	}

	cm := collectConstraints(ptree, p, sm)

	// Get the project list and sort it so that the printed output users see is
	// deterministically ordered. (This may be superfluous if the lock is always
	// written in alpha order, but it doesn't hurt to double down.)
	slp := p.Lock.Projects()
	sort.Sort(dep.SortedLockedProjects(slp))

	if bytes.Equal(s.HashInputs(), p.Lock.SolveMeta.InputsDigest) {
		// If these are equal, we're guaranteed that the lock is a transitively
		// complete picture of all deps. That eliminates the need for at least
		// some checks.

		out.BasicHeader()

		for _, proj := range slp {
			bs := BasicStatus{
				ProjectRoot:  string(proj.Ident().ProjectRoot),
				PackageCount: len(proj.Packages()),
			}

			// Get children only for specific outputers
			// in order to avoid slower status process
			switch out.(type) {
			case *dotOutput:
				ptr, err := sm.ListPackages(proj.Ident(), proj.Version())

				if err != nil {
					return memoMismatch, hasMissingPkgs, fmt.Errorf("analysis of %s package failed: %v", proj.Ident().ProjectRoot, err)
				}

				prm, _ := ptr.ToReachMap(true, false, false, nil)
				bs.Children = prm.FlattenFn(paths.IsStandardImportPath)
			}

			// Split apart the version from the lock into its constituent parts
			switch tv := proj.Version().(type) {
			case gps.UnpairedVersion:
				bs.Version = tv
			case gps.Revision:
				bs.Revision = tv
			case gps.PairedVersion:
				bs.Version = tv.Unpair()
				bs.Revision = tv.Underlying()
			}

			// Check if the manifest has an override for this project. If so,
			// set that as the constraint.
			if pp, has := p.Manifest.Ovr[proj.Ident().ProjectRoot]; has && pp.Constraint != nil {
				// TODO note somehow that it's overridden
				bs.Constraint = pp.Constraint
			} else {
				bs.Constraint = gps.Any()
				for _, c := range cm[bs.ProjectRoot] {
					bs.Constraint = c.Intersect(bs.Constraint)
				}
			}

			// Only if we have a non-rev and non-plain version do/can we display
			// anything wrt the version's updateability.
			if bs.Version != nil && bs.Version.Type() != gps.IsVersion {
				c, has := p.Manifest.Constraints[proj.Ident().ProjectRoot]
				if !has {
					c.Constraint = gps.Any()
				}
				// TODO: This constraint is only the constraint imposed by the
				// current project, not by any transitive deps. As a result,
				// transitive project deps will always show "any" here.
				bs.Constraint = c.Constraint

				vl, err := sm.ListVersions(proj.Ident())
				if err == nil {
					gps.SortPairedForUpgrade(vl)

					for _, v := range vl {
						// Because we've sorted the version list for
						// upgrade, the first version we encounter that
						// matches our constraint will be what we want.
						if c.Constraint.Matches(v) {
							bs.Latest = v.Underlying()
							break
						}
					}
				}
			}

			out.BasicLine(&bs)
		}
		out.BasicFooter()

		return memoMismatch, hasMissingPkgs, nil
	}

	// Hash digest mismatch may indicate that some deps are no longer
	// needed, some are missing, or that some constraints or source
	// locations have changed.
	//
	// It's possible for digests to not match, but still have a correct
	// lock.
	memoMismatch = true
	rm, _ := ptree.ToReachMap(true, true, false, nil)

	external := rm.FlattenFn(paths.IsStandardImportPath)
	roots := make(map[gps.ProjectRoot][]string, len(external))

	type fail struct {
		ex  string
		err error
	}
	var errs []fail
	for _, e := range external {
		root, err := sm.DeduceProjectRoot(e)
		if err != nil {
			errs = append(errs, fail{
				ex:  e,
				err: err,
			})
			continue
		}

		roots[root] = append(roots[root], e)
	}

	if len(errs) != 0 {
		// TODO this is just a fix quick so staticcheck doesn't complain.
		// Visually reconciling failure to deduce project roots with the rest of
		// the mismatch output is a larger problem.
		loggers.Err.Printf("Failed to deduce project roots for import paths:\n")
		for _, fail := range errs {
			loggers.Err.Printf("\t%s: %s\n", fail.ex, fail.err.Error())
		}

		return memoMismatch, hasMissingPkgs, errors.New("address issues with undeducable import paths to get more status information.")
	}

	out.MissingHeader()

outer:
	for root, pkgs := range roots {
		// TODO also handle the case where the project is present, but there
		// are items missing from just the package list
		for _, lp := range slp {
			if lp.Ident().ProjectRoot == root {
				continue outer
			}
		}

		hasMissingPkgs = true
		out.MissingLine(&MissingStatus{ProjectRoot: string(root), MissingPackages: pkgs})
	}
	out.MissingFooter()

	return memoMismatch, hasMissingPkgs, nil
}

func formatVersion(v gps.Version) string {
	if v == nil {
		return ""
	}
	switch v.Type() {
	case gps.IsBranch:
		return "branch " + v.String()
	case gps.IsRevision:
		r := v.String()
		if len(r) > 7 {
			r = r[:7]
		}
		return r
	}
	return v.String()
}

func collectConstraints(ptree pkgtree.PackageTree, p *dep.Project, sm gps.SourceManager) map[string][]gps.Constraint {
	// TODO
	return map[string][]gps.Constraint{}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"github.com/rkoesters/goblin/lib"
	"log"
	"os"
)

var (
	pflag = flag.Bool("p", false, "create parent directories")
	mode  = flag.String("m", "0777", "permissions for creating directory")
)

func main() {
	lib.Usage = "[-p] [-m mode] dirname..."
	lib.InitFlag()
	lib.InitLog()
	flag.Parse()

	perm, err := lib.ParsePerm(*mode)
	if err != nil {
		log.Fatal(err)
	}

	for _, d := range flag.Args() {
		if *pflag {
			err = os.MkdirAll(d, perm)
		} else {
			err = os.Mkdir(d, perm)
		}

		if err != nil {
			log.Fatal(err)
		}
	}
}
<commit_msg>Updated cmd/mkdir for mango-doc.<commit_after>// Create directories.
package main

import (
	"flag"
	"github.com/rkoesters/goblin/lib"
	"log"
	"os"
)

var (
	pflag = flag.Bool("p", false, "create parent directories")
	mode  = flag.String("m", "0777", "permissions for creating directory")
)

// Usage: %name %flags dirname...
func main() {
	lib.Usage = "[-p] [-m mode] dirname..."
	lib.InitFlag()
	lib.InitLog()
	flag.Parse()

	perm, err := lib.ParsePerm(*mode)
	if err != nil {
		log.Fatal(err)
	}

	for _, d := range flag.Args() {
		if *pflag {
			err = os.MkdirAll(d, perm)
		} else {
			err = os.Mkdir(d, perm)
		}

		if err != nil {
			log.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Marc-Antoine Ruel. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

// panic crashes in various ways.
//
// It is a tool to help test pp.
package main

// To install, run:
//   go install github.com/maruel/panicparse/cmd/panic
//   panic -help
//   panic str |& pp
//
// Some panics require the race detector with -race:
//   go install -race github.com/maruel/panicparse/cmd/panic
//   panic race |& pp
//
// To add a new panic stack signature, add it to types type below, keeping the
// list ordered by name. If you need utility functions, add it in the section
// below. That's it!

import (
	"fmt"
	"io"
	"os"
	"runtime"
	"sort"
	"sync"
	"time"

	"github.com/maruel/panicparse/cmd/panic/internal"
)

// Utility functions.

func panicint(i int) {
	panic(i)
}

func panicstr(a string) {
	panic(a)
}

func panicslicestr(a []string) {
	panic(a)
}

func panicArgsElided(a, b, c, d, e, f, g, h, i, j, k int) {
	panic(a)
}

func recurse(i int) {
	if i > 0 {
		recurse(i - 1)
		return
	}
	panic(42)
}

//

// types is all the supported types of panics.
//
// Keep the list sorted.
//
// TODO(maruel): Figure out a way to reliably trigger "(scan)" output:
// - disable automatic GC with runtime.SetGCPercent(-1)
// - a goroutine with a large number of items in the stack
// - large heap to make the scanning process slow enough
// - trigger a manual GC with go runtime.GC()
// - panic in the meantime
// This would still not be deterministic.
//
// TODO(maruel): Figure out a way to reliably trigger sleep output.
var types = map[string]struct {
	desc string
	f    func()
}{
	"args_elided": {
		"too many args in stack line, causing the call arguments to be elided",
		func() {
			panicArgsElided(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)
		},
	},

	"chan_receive": {
		"goroutine blocked on <-c",
		func() {
			c := make(chan bool)
			go func() {
				<-c
				<-c
			}()
			c <- true
			panic(42)
		},
	},

	"chan_send": {
		"goroutine blocked on c<-",
		func() {
			c := make(chan bool)
			go func() {
				c <- true
				c <- true
			}()
			<-c
			panic(42)
		},
	},

	"goroutine_1": {
		"panic in one goroutine",
		func() {
			go func() {
				panicint(42)
			}()
			time.Sleep(time.Minute)
		},
	},

	"goroutine_100": {
		"start 100 goroutines before panicking",
		func() {
			var wg sync.WaitGroup
			for i := 0; i < 100; i++ {
				wg.Add(1)
				go func() {
					wg.Done()
					time.Sleep(time.Minute)
				}()
			}
			wg.Wait()
			panicint(42)
		},
	},

	"goroutine_dedupe_pointers": {
		"start 100 goroutines with different pointers before panicking",
		func() {
			var wg sync.WaitGroup
			for i := 0; i < 100; i++ {
				wg.Add(1)
				go func(b *int) {
					wg.Done()
					time.Sleep(time.Minute)
				}(new(int))
			}
			wg.Wait()
			panicint(42)
		},
	},

	"int": {
		"panic(42)",
		func() {
			panicint(42)
		},
	},

	"locked": {
		"thread locked goroutine via runtime.LockOSThread()",
		func() {
			runtime.LockOSThread()
			panic(42)
		},
	},

	"other": {
		"panics with other package in the call stack, with both exported and unexpected functions",
		func() {
			internal.Callback(func() {
				panic("allo")
			})
		},
	},

	"race": {
		"will cause a crash by -race detector",
		panicRace,
	},

	"stack_cut_off": {
		"too many call lines in traceback, causing higher up calls to missing",
		func() {
			// Observed limit is 99.
			// src/runtime/runtime2.go:const _TracebackMaxFrames = 100
			recurse(100)
		},
	},

	"simple": {
		// This is not used for real, here for documentation.
		"skip the map for a shorter stack trace",
		func() {},
	},

	"slice_str": {
		"panic([]string{\"allo\"}) with cap=2",
		func() {
			a := make([]string, 1, 2)
			a[0] = "allo"
			panicslicestr(a)
		},
	},

	"stdlib": {
		"panics with stdlib in the call stack, with both exported and unexpected functions",
		func() {
			a := []string{"a", "b"}
			sort.Slice(a, func(i, j int) bool {
				panic("allo")
			})
		},
	},

	"stdlib_and_other": {
		"panics with both other and stdlib packages in the call stack",
		func() {
			a := []string{"a", "b"}
			sort.Slice(a, func(i, j int) bool {
				internal.Callback(func() {
					panic("allo")
				})
				return false
			})
		},
	},

	"str": {
		"panic(\"allo\")",
		func() {
			panicstr("allo")
		},
	},
}

//

func main() {
	if len(os.Args) == 2 {
		n := os.Args[1]
		if f, ok := types[n]; ok {
			fmt.Printf("GOTRACEBACK=%s\n", os.Getenv("GOTRACEBACK"))
			if n == "simple" {
				// Since the map lookup creates another call stack entry, add a one-off
				// "simple" panic style to test the very minimal case.
				// types["simple"].f is never called.
				panic("simple")
			}
			f.f()
		}
		fmt.Fprintf(os.Stderr, "unknown panic style %q\n", n)
		os.Exit(1)
	}
	usage()
}

func usage() {
	t := `usage: panic <way>

This tool is meant to be used with pp to test different parsing scenarios and
ensure output on different version of the Go toolchain can be successfully
parsed.

Set GOTRACEBACK before running this tool to see how it affects the panic output.

Select the way to panic:
`
	io.WriteString(os.Stderr, t)
	names := make([]string, 0, len(types))
	m := 0
	for n := range types {
		names = append(names, n)
		if i := len(n); i > m {
			m = i
		}
	}
	sort.Strings(names)
	for _, n := range names {
		fmt.Fprintf(os.Stderr, "- %-*s  %s\n", m, n, types[n].desc)
	}
	os.Exit(2)
}
<commit_msg>panic: add asleep<commit_after>// Copyright 2017 Marc-Antoine Ruel. All rights reserved.
// Use of this source code is governed under the Apache License, Version 2.0
// that can be found in the LICENSE file.

// panic crashes in various ways.
//
// It is a tool to help test pp.
package main

// To install, run:
//   go install github.com/maruel/panicparse/cmd/panic
//   panic -help
//   panic str |& pp
//
// Some panics require the race detector with -race:
//   go install -race github.com/maruel/panicparse/cmd/panic
//   panic race |& pp
//
// To add a new panic stack signature, add it to types type below, keeping the
// list ordered by name. If you need utility functions, add it in the section
// below. That's it!

import (
	"fmt"
	"io"
	"os"
	"runtime"
	"sort"
	"sync"
	"time"

	"github.com/maruel/panicparse/cmd/panic/internal"
)

// Utility functions.

func panicint(i int) {
	panic(i)
}

func panicstr(a string) {
	panic(a)
}

func panicslicestr(a []string) {
	panic(a)
}

func panicArgsElided(a, b, c, d, e, f, g, h, i, j, k int) {
	panic(a)
}

func recurse(i int) {
	if i > 0 {
		recurse(i - 1)
		return
	}
	panic(42)
}

//

// types is all the supported types of panics.
//
// Keep the list sorted.
//
// TODO(maruel): Figure out a way to reliably trigger "(scan)" output:
// - disable automatic GC with runtime.SetGCPercent(-1)
// - a goroutine with a large number of items in the stack
// - large heap to make the scanning process slow enough
// - trigger a manual GC with go runtime.GC()
// - panic in the meantime
// This would still not be deterministic.
//
// TODO(maruel): Figure out a way to reliably trigger sleep output.
var types = map[string]struct {
	desc string
	f    func()
}{
	"args_elided": {
		"too many args in stack line, causing the call arguments to be elided",
		func() {
			panicArgsElided(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11)
		},
	},

	"chan_receive": {
		"goroutine blocked on <-c",
		func() {
			c := make(chan bool)
			go func() {
				<-c
				<-c
			}()
			c <- true
			panic(42)
		},
	},

	"chan_send": {
		"goroutine blocked on c<-",
		func() {
			c := make(chan bool)
			go func() {
				c <- true
				c <- true
			}()
			<-c
			panic(42)
		},
	},

	"goroutine_1": {
		"panic in one goroutine",
		func() {
			go func() {
				panicint(42)
			}()
			time.Sleep(time.Minute)
		},
	},

	"goroutine_100": {
		"start 100 goroutines before panicking",
		func() {
			var wg sync.WaitGroup
			for i := 0; i < 100; i++ {
				wg.Add(1)
				go func() {
					wg.Done()
					time.Sleep(time.Minute)
				}()
			}
			wg.Wait()
			panicint(42)
		},
	},

	"goroutine_dedupe_pointers": {
		"start 100 goroutines with different pointers before panicking",
		func() {
			var wg sync.WaitGroup
			for i := 0; i < 100; i++ {
				wg.Add(1)
				go func(b *int) {
					wg.Done()
					time.Sleep(time.Minute)
				}(new(int))
			}
			wg.Wait()
			panicint(42)
		},
	},

	"int": {
		"panic(42)",
		func() {
			panicint(42)
		},
	},

	"locked": {
		"thread locked goroutine via runtime.LockOSThread()",
		func() {
			runtime.LockOSThread()
			panic(42)
		},
	},

	"other": {
		"panics with other package in the call stack, with both exported and unexpected functions",
		func() {
			internal.Callback(func() {
				panic("allo")
			})
		},
	},

	"asleep": {
		"panics with 'all goroutines are asleep - deadlock'",
		func() {
			// When built with the race detector, this hangs!? Without, it's panics as expected.
			// Repro:
			//   go install -race github.com/maruel/panicparse/cmd/panic; panic asleep
			//
			// TODO(maruel): File a bug.
			// See checkdead() in src/runtime/proc.go.
			var mu sync.Mutex
			mu.Lock()
			mu.Lock()
		},
	},

	"race": {
		"cause a crash by race detector",
		panicRace,
	},

	"stack_cut_off": {
		"too many call lines in traceback, causing higher up calls to missing",
		func() {
			// Observed limit is 99.
			// src/runtime/runtime2.go:const _TracebackMaxFrames = 100
			recurse(100)
		},
	},

	"simple": {
		// This is not used for real, here for documentation.
		"skip the map for a shorter stack trace",
		func() {},
	},

	"slice_str": {
		"panic([]string{\"allo\"}) with cap=2",
		func() {
			a := make([]string, 1, 2)
			a[0] = "allo"
			panicslicestr(a)
		},
	},

	"stdlib": {
		"panics with stdlib in the call stack, with both exported and unexpected functions",
		func() {
			a := []string{"a", "b"}
			sort.Slice(a, func(i, j int) bool {
				panic("allo")
			})
		},
	},

	"stdlib_and_other": {
		"panics with both other and stdlib packages in the call stack",
		func() {
			a := []string{"a", "b"}
			sort.Slice(a, func(i, j int) bool {
				internal.Callback(func() {
					panic("allo")
				})
				return false
			})
		},
	},

	"str": {
		"panic(\"allo\")",
		func() {
			panicstr("allo")
		},
	},
}

//

func main() {
	if len(os.Args) == 2 {
		n := os.Args[1]
		if f, ok := types[n]; ok {
			fmt.Printf("GOTRACEBACK=%s\n", os.Getenv("GOTRACEBACK"))
			if n == "simple" {
				// Since the map lookup creates another call stack entry, add a one-off
				// "simple" panic style to test the very minimal case.
				// types["simple"].f is never called.
				panic("simple")
			}
			f.f()
		}
		fmt.Fprintf(os.Stderr, "unknown panic style %q\n", n)
		os.Exit(1)
	}
	usage()
}

func usage() {
	t := `usage: panic <way>

This tool is meant to be used with pp to test different parsing scenarios and
ensure output on different version of the Go toolchain can be successfully
parsed.

Set GOTRACEBACK before running this tool to see how it affects the panic output.

Select the way to panic:
`
	io.WriteString(os.Stderr, t)
	names := make([]string, 0, len(types))
	m := 0
	for n := range types {
		names = append(names, n)
		if i := len(n); i > m {
			m = i
		}
	}
	sort.Strings(names)
	for _, n := range names {
		fmt.Fprintf(os.Stderr, "- %-*s  %s\n", m, n, types[n].desc)
	}
	os.Exit(2)
}
<|endoftext|>