<commit_before>package build

import (
	"fmt"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"neon/util"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"unicode/utf8"
)

type Build struct {
	File    string
	Dir     string
	Here    string
	Name    string
	Default []string
	Doc     string
	Context *Context
	Targets map[string]*Target
	Debug   bool
}

func NewBuild(file string, debug bool) (*Build, error) {
	build := &Build{}
	build.Debug = debug
	build.Log("Loading build file '%s'", file)
	source, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, fmt.Errorf("loading build file '%s': %v", file, err)
	}
	build.Log("Parsing build file")
	var object util.Object
	err = yaml.Unmarshal(source, &object)
	if err != nil {
		return nil, fmt.Errorf("build must be a YAML map with string keys")
	}
	build.Log("Build structure: %#v", object)
	build.Log("Reading build first level fields")
	err = object.CheckFields([]string{"name", "default", "doc", "properties",
		"environment", "targets"})
	if err != nil {
		return nil, err
	}
	str, err := object.GetString("name")
	if err == nil {
		build.Name = str
	}
	list, err := object.GetListStringsOrString("default")
	if err == nil {
		build.Default = list
	}
	str, err = object.GetString("doc")
	if err == nil {
		build.Doc = str
	}
	path, err := filepath.Abs(file)
	if err != nil {
		return nil, fmt.Errorf("getting build file path: %v", err)
	}
	build.File = path
	build.Dir = filepath.Dir(path)
	here, err := os.Getwd()
	if err != nil {
		return nil, fmt.Errorf("getting current directory: %v", err)
	}
	build.Here = here
	properties, err := object.GetObject("properties")
	if err != nil {
		if err.Error() == "field 'properties' not found" {
			properties = make(map[string]interface{})
		} else {
			return nil, err
		}
	}
	environment, err := object.GetObject("environment")
	if err != nil {
		if err.Error() == "field 'environment' not found" {
			environment = make(map[string]interface{})
		} else {
			return nil, err
		}
	}
	context, err := NewContext(build, properties, environment)
	if err != nil {
		return nil, err
	}
	build.Context = context
	targets, err := object.GetObject("targets")
	if err != nil {
		if err.Error() == "field 'targets' not found" {
			properties = make(map[string]interface{})
		} else {
			return nil, err
		}
	}
	build.Targets = make(map[string]*Target)
	for name, _ := range targets {
		object, err := targets.GetObject(name)
		if err != nil {
			return nil, err
		}
		target, err := NewTarget(build, name, object)
		if err != nil {
			return nil, err
		}
		build.Targets[name] = target
	}
	return build, nil
}

func (build *Build) Run(targets []string) error {
	if len(targets) == 0 {
		if len(build.Default) == 0 {
			return fmt.Errorf("no default target")
		}
		return build.Run(build.Default)
	} else {
		for _, name := range targets {
			target, ok := build.Targets[name]
			if !ok {
				return fmt.Errorf("target '%s' not found", name)
			}
			err := target.Run()
			if err != nil {
				return err
			}
		}
		return nil
	}
}

func (build *Build) Help() error {
	newLine := false
	// print build documentation
	if build.Doc != "" {
		fmt.Println(build.Doc)
		newLine = true
	}
	// print build properties
	length := 0
	for _, name := range build.Context.Properties {
		if utf8.RuneCountInString(name) > length {
			length = utf8.RuneCountInString(name)
		}
	}
	if len(build.Context.Properties) > 0 {
		if newLine {
			fmt.Println()
		}
		fmt.Println("Properties:")
		for _, name := range build.Context.Properties {
			value, err := build.Context.GetProperty(name)
			if err != nil {
				return fmt.Errorf("getting property '%s': %v", name, err)
			}
			valueStr, err := PropertyToString(value, true)
			if err != nil {
				return fmt.Errorf("formatting property '%s': %v", name, err)
			}
			util.PrintTargetHelp(name, valueStr, []string{}, length)
		}
		newLine = true
	}
	// print build environment
	length = 0
	var names []string
	for name, _ := range build.Context.Environment {
		if utf8.RuneCountInString(name) > length {
			length = utf8.RuneCountInString(name)
		}
		names = append(names, name)
	}
	sort.Strings(names)
	if len(build.Context.Environment) > 0 {
		if newLine {
			fmt.Println()
		}
		fmt.Println("Environment:")
		for _, name := range names {
			value := "\"" + build.Context.Environment[name] + "\""
			util.PrintTargetHelp(name, value, []string{}, length)
		}
		newLine = true
	}
	// print targets documentation
	length = 0
	var targets []string
	for name, _ := range build.Targets {
		if utf8.RuneCountInString(name) > length {
			length = utf8.RuneCountInString(name)
		}
		targets = append(targets, name)
	}
	sort.Strings(targets)
	if len(targets) > 0 {
		if newLine {
			fmt.Println()
		}
		fmt.Println("Targets:")
		for _, name := range targets {
			target := build.Targets[name]
			util.PrintTargetHelp(name, target.Doc, target.Depends, length)
		}
	}
	return nil
}

func (build *Build) PrintTargets() {
	var targets []string
	for name, _ := range build.Targets {
		targets = append(targets, name)
	}
	sort.Strings(targets)
	fmt.Println(strings.Join(targets, " "))
}

func (build *Build) PrintTasks() {
	var tasks []string
	for name, _ := range TaskMap {
		tasks = append(tasks, name)
	}
	sort.Strings(tasks)
	fmt.Println(strings.Join(tasks, " "))
}

func (build *Build) PrintHelpTask(task string) {
	for name, descriptor := range TaskMap {
		if name == task {
			fmt.Println(descriptor.Help)
			return
		}
	}
	fmt.Printf("Task '%s' was not found\n", task)
}

func (build *Build) PrintBuiltins() {
	var builtins []string
	for name, _ := range BuiltinMap {
		builtins = append(builtins, name)
	}
	sort.Strings(builtins)
	fmt.Println(strings.Join(builtins, " "))
}

func (build *Build) PrintHelpBuiltin(builtin string) {
	descriptor, ok := BuiltinMap[builtin]
	if !ok {
		fmt.Printf("Builtin '%s' was not found\n", builtin)
	} else {
		fmt.Println(descriptor.Help)
	}
}

func (build *Build) Log(message string, args ...interface{}) {
	if build.Debug {
		fmt.Println(fmt.Sprintf(message, args...))
	}
}
<commit_msg>Improved code<commit_after>package build

import (
	"fmt"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"neon/util"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"unicode/utf8"
)

type Build struct {
	File    string
	Dir     string
	Here    string
	Name    string
	Default []string
	Doc     string
	Context *Context
	Targets map[string]*Target
	Debug   bool
}

func NewBuild(file string, debug bool) (*Build, error) {
	build := &Build{}
	build.Debug = debug
	build.Log("Loading build file '%s'", file)
	source, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, fmt.Errorf("loading build file '%s': %v", file, err)
	}
	build.Log("Parsing build file")
	var object util.Object
	err = yaml.Unmarshal(source, &object)
	if err != nil {
		return nil, fmt.Errorf("build must be a YAML map with string keys")
	}
	build.Log("Build structure: %#v", object)
	build.Log("Reading build first level fields")
	err = object.CheckFields([]string{"name", "default", "doc", "properties",
		"environment", "targets"})
	if err != nil {
		return nil, err
	}
	str, err := object.GetString("name")
	if err == nil {
		build.Name = str
	}
	list, err := object.GetListStringsOrString("default")
	if err == nil {
		build.Default = list
	}
	str, err = object.GetString("doc")
	if err == nil {
		build.Doc = str
	}
	path, err := filepath.Abs(file)
	if err != nil {
		return nil, fmt.Errorf("getting build file path: %v", err)
	}
	build.File = path
	build.Dir = filepath.Dir(path)
	here, err := os.Getwd()
	if err != nil {
		return nil, fmt.Errorf("getting current directory: %v", err)
	}
	build.Here = here
	properties, err := object.GetObject("properties")
	if err != nil {
		if err.Error() == "field 'properties' not found" {
			properties = make(map[string]interface{})
		} else {
			return nil, err
		}
	}
	environment, err := object.GetObject("environment")
	if err != nil {
		if err.Error() == "field 'environment' not found" {
			environment = make(map[string]interface{})
		} else {
			return nil, err
		}
	}
	context, err := NewContext(build, properties, environment)
	if err != nil {
		return nil, err
	}
	build.Context = context
	targets, err := object.GetObject("targets")
	if err != nil {
		if err.Error() == "field 'targets' not found" {
			properties = make(map[string]interface{})
		} else {
			return nil, err
		}
	}
	build.Targets = make(map[string]*Target)
	for name, _ := range targets {
		object, err := targets.GetObject(name)
		if err != nil {
			return nil, err
		}
		target, err := NewTarget(build, name, object)
		if err != nil {
			return nil, err
		}
		build.Targets[name] = target
	}
	return build, nil
}

func (build *Build) Run(targets []string) error {
	if len(targets) == 0 {
		if len(build.Default) == 0 {
			return fmt.Errorf("no default target")
		}
		return build.Run(build.Default)
	} else {
		for _, name := range targets {
			target, ok := build.Targets[name]
			if !ok {
				return fmt.Errorf("target '%s' not found", name)
			}
			err := target.Run()
			if err != nil {
				return err
			}
		}
		return nil
	}
}

func (build *Build) Help() error {
	newLine := false
	// print build documentation
	if build.Doc != "" {
		fmt.Println(build.Doc)
		newLine = true
	}
	// print build properties
	length := 0
	for _, name := range build.Context.Properties {
		if utf8.RuneCountInString(name) > length {
			length = utf8.RuneCountInString(name)
		}
	}
	if len(build.Context.Properties) > 0 {
		if newLine {
			fmt.Println()
		}
		fmt.Println("Properties:")
		for _, name := range build.Context.Properties {
			value, err := build.Context.GetProperty(name)
			if err != nil {
				return fmt.Errorf("getting property '%s': %v", name, err)
			}
			valueStr, err := PropertyToString(value, true)
			if err != nil {
				return fmt.Errorf("formatting property '%s': %v", name, err)
			}
			util.PrintTargetHelp(name, valueStr, []string{}, length)
		}
		newLine = true
	}
	// print build environment
	length = 0
	var names []string
	for name, _ := range build.Context.Environment {
		if utf8.RuneCountInString(name) > length {
			length = utf8.RuneCountInString(name)
		}
		names = append(names, name)
	}
	sort.Strings(names)
	if len(build.Context.Environment) > 0 {
		if newLine {
			fmt.Println()
		}
		fmt.Println("Environment:")
		for _, name := range names {
			value := "\"" + build.Context.Environment[name] + "\""
			util.PrintTargetHelp(name, value, []string{}, length)
		}
		newLine = true
	}
	// print targets documentation
	length = 0
	var targets []string
	for name, _ := range build.Targets {
		if utf8.RuneCountInString(name) > length {
			length = utf8.RuneCountInString(name)
		}
		targets = append(targets, name)
	}
	sort.Strings(targets)
	if len(targets) > 0 {
		if newLine {
			fmt.Println()
		}
		fmt.Println("Targets:")
		for _, name := range targets {
			target := build.Targets[name]
			util.PrintTargetHelp(name, target.Doc, target.Depends, length)
		}
	}
	return nil
}

func (build *Build) PrintTargets() {
	var targets []string
	for name, _ := range build.Targets {
		targets = append(targets, name)
	}
	sort.Strings(targets)
	fmt.Println(strings.Join(targets, " "))
}

func (build *Build) PrintTasks() {
	var tasks []string
	for name, _ := range TaskMap {
		tasks = append(tasks, name)
	}
	sort.Strings(tasks)
	fmt.Println(strings.Join(tasks, " "))
}

func (build *Build) PrintHelpTask(task string) {
	descriptor, found := TaskMap["task"]
	if found {
		fmt.Println(descriptor.Help)
	} else {
		fmt.Printf("Task '%s' was not found\n", task)
	}
}

func (build *Build) PrintBuiltins() {
	var builtins []string
	for name, _ := range BuiltinMap {
		builtins = append(builtins, name)
	}
	sort.Strings(builtins)
	fmt.Println(strings.Join(builtins, " "))
}

func (build *Build) PrintHelpBuiltin(builtin string) {
	descriptor, found := BuiltinMap[builtin]
	if found {
		fmt.Println(descriptor.Help)
	} else {
		fmt.Printf("Builtin '%s' was not found\n", builtin)
	}
}

func (build *Build) Log(message string, args ...interface{}) {
	if build.Debug {
		fmt.Println(fmt.Sprintf(message, args...))
	}
}
<|endoftext|>


<commit_before>package core

import (
	cm "github.com/tendermint/tendermint/consensus"
	cstypes "github.com/tendermint/tendermint/consensus/types"
	ctypes "github.com/tendermint/tendermint/rpc/core/types"
	"github.com/tendermint/tendermint/types"
)

// Get the validator set at the given block height.
// If no height is provided, it will fetch the current validator set.
//
// ```shell
// curl 'localhost:46657/validators'
// ```
//
// ```go
// client := client.NewHTTP("tcp://0.0.0.0:46657", "/websocket")
// state, err := client.Validators()
// ```
//
// > The above command returns JSON structured like this:
//
// ```json
// {
// 	"error": "",
// 	"result": {
// 		"validators": [
// 			{
// 				"accum": 0,
// 				"voting_power": 10,
// 				"pub_key": {
// 					"data": "68DFDA7E50F82946E7E8546BED37944A422CD1B831E70DF66BA3B8430593944D",
// 					"type": "ed25519"
// 				},
// 				"address": "E89A51D60F68385E09E716D353373B11F8FACD62"
// 			}
// 		],
// 		"block_height": 5241
// 	},
// 	"id": "",
// 	"jsonrpc": "2.0"
// }
// ```
func Validators(heightPtr *int) (*ctypes.ResultValidators, error) {
	if heightPtr == nil {
		blockHeight, validators := consensusState.GetValidators()
		return &ctypes.ResultValidators{blockHeight, validators}, nil
	}

	height := *heightPtr
	state := consensusState.GetState()
	validators, err := state.LoadValidators(height)
	if err != nil {
		return nil, err
	}
	return &ctypes.ResultValidators{height, validators.Validators}, nil
}

// Dump consensus state.
//
// ```shell
// curl 'localhost:46657/dump_consensus_state'
// ```
//
// ```go
// client := client.NewHTTP("tcp://0.0.0.0:46657", "/websocket")
// state, err := client.DumpConsensusState()
// ```
//
// > The above command returns JSON structured like this:
//
// ```json
// {
// 	"error": "",
// 	"result": {
// 		"peer_round_states": [],
// 		"round_state": "RoundState{\n  H:3537 R:0 S:RoundStepNewHeight\n  StartTime:     2017-05-31 12:32:31.178653883 +0000 UTC\n  CommitTime:    2017-05-31 12:32:30.178653883 +0000 UTC\n  Validators:    ValidatorSet{\n      Proposer: Validator{E89A51D60F68385E09E716D353373B11F8FACD62 {PubKeyEd25519{68DFDA7E50F82946E7E8546BED37944A422CD1B831E70DF66BA3B8430593944D}} VP:10 A:0}\n      Validators:\n        Validator{E89A51D60F68385E09E716D353373B11F8FACD62 {PubKeyEd25519{68DFDA7E50F82946E7E8546BED37944A422CD1B831E70DF66BA3B8430593944D}} VP:10 A:0}\n    }\n  Proposal:      <nil>\n  ProposalBlock: nil-PartSet nil-Block\n  LockedRound:   0\n  LockedBlock:   nil-PartSet nil-Block\n  Votes:         HeightVoteSet{H:3537 R:0~0\n      VoteSet{H:3537 R:0 T:1 +2/3:<nil> BA{1:_} map[]}\n      VoteSet{H:3537 R:0 T:2 +2/3:<nil> BA{1:_} map[]}\n    }\n  LastCommit: VoteSet{H:3536 R:0 T:2 +2/3:B7F988FBCDC68F9320E346EECAA76E32F6054654:1:673BE7C01F74 BA{1:X} map[]}\n  LastValidators:    ValidatorSet{\n      Proposer: Validator{E89A51D60F68385E09E716D353373B11F8FACD62 {PubKeyEd25519{68DFDA7E50F82946E7E8546BED37944A422CD1B831E70DF66BA3B8430593944D}} VP:10 A:0}\n      Validators:\n        Validator{E89A51D60F68385E09E716D353373B11F8FACD62 {PubKeyEd25519{68DFDA7E50F82946E7E8546BED37944A422CD1B831E70DF66BA3B8430593944D}} VP:10 A:0}\n    }\n}"
// 	},
// 	"id": "",
// 	"jsonrpc": "2.0"
// }
// ```
func DumpConsensusState() (*ctypes.ResultDumpConsensusState, error) {
	peerRoundStates := make(map[string]*cstypes.PeerRoundState)
	for _, peer := range p2pSwitch.Peers().List() {
		// TODO: clean this up?
		peerState := peer.Get(types.PeerStateKey).(*cm.PeerState)
		peerRoundState := peerState.GetRoundState()
		peerRoundStates[peer.Key()] = peerRoundState
	}
	return &ctypes.ResultDumpConsensusState{consensusState.GetRoundState(), peerRoundStates}, nil
}
<commit_msg>remove a stale comment<commit_after>package core

import (
	cm "github.com/tendermint/tendermint/consensus"
	cstypes "github.com/tendermint/tendermint/consensus/types"
	ctypes "github.com/tendermint/tendermint/rpc/core/types"
	"github.com/tendermint/tendermint/types"
)

// Get the validator set at the given block height.
// If no height is provided, it will fetch the current validator set.
//
// ```shell
// curl 'localhost:46657/validators'
// ```
//
// ```go
// client := client.NewHTTP("tcp://0.0.0.0:46657", "/websocket")
// state, err := client.Validators()
// ```
//
// > The above command returns JSON structured like this:
//
// ```json
// {
// 	"error": "",
// 	"result": {
// 		"validators": [
// 			{
// 				"accum": 0,
// 				"voting_power": 10,
// 				"pub_key": {
// 					"data": "68DFDA7E50F82946E7E8546BED37944A422CD1B831E70DF66BA3B8430593944D",
// 					"type": "ed25519"
// 				},
// 				"address": "E89A51D60F68385E09E716D353373B11F8FACD62"
// 			}
// 		],
// 		"block_height": 5241
// 	},
// 	"id": "",
// 	"jsonrpc": "2.0"
// }
// ```
func Validators(heightPtr *int) (*ctypes.ResultValidators, error) {
	if heightPtr == nil {
		blockHeight, validators := consensusState.GetValidators()
		return &ctypes.ResultValidators{blockHeight, validators}, nil
	}

	height := *heightPtr
	state := consensusState.GetState()
	validators, err := state.LoadValidators(height)
	if err != nil {
		return nil, err
	}
	return &ctypes.ResultValidators{height, validators.Validators}, nil
}

// Dump consensus state.
//
// ```shell
// curl 'localhost:46657/dump_consensus_state'
// ```
//
// ```go
// client := client.NewHTTP("tcp://0.0.0.0:46657", "/websocket")
// state, err := client.DumpConsensusState()
// ```
//
// > The above command returns JSON structured like this:
//
// ```json
// {
// 	"error": "",
// 	"result": {
// 		"peer_round_states": [],
// 		"round_state": "RoundState{\n  H:3537 R:0 S:RoundStepNewHeight\n  StartTime:     2017-05-31 12:32:31.178653883 +0000 UTC\n  CommitTime:    2017-05-31 12:32:30.178653883 +0000 UTC\n  Validators:    ValidatorSet{\n      Proposer: Validator{E89A51D60F68385E09E716D353373B11F8FACD62 {PubKeyEd25519{68DFDA7E50F82946E7E8546BED37944A422CD1B831E70DF66BA3B8430593944D}} VP:10 A:0}\n      Validators:\n        Validator{E89A51D60F68385E09E716D353373B11F8FACD62 {PubKeyEd25519{68DFDA7E50F82946E7E8546BED37944A422CD1B831E70DF66BA3B8430593944D}} VP:10 A:0}\n    }\n  Proposal:      <nil>\n  ProposalBlock: nil-PartSet nil-Block\n  LockedRound:   0\n  LockedBlock:   nil-PartSet nil-Block\n  Votes:         HeightVoteSet{H:3537 R:0~0\n      VoteSet{H:3537 R:0 T:1 +2/3:<nil> BA{1:_} map[]}\n      VoteSet{H:3537 R:0 T:2 +2/3:<nil> BA{1:_} map[]}\n    }\n  LastCommit: VoteSet{H:3536 R:0 T:2 +2/3:B7F988FBCDC68F9320E346EECAA76E32F6054654:1:673BE7C01F74 BA{1:X} map[]}\n  LastValidators:    ValidatorSet{\n      Proposer: Validator{E89A51D60F68385E09E716D353373B11F8FACD62 {PubKeyEd25519{68DFDA7E50F82946E7E8546BED37944A422CD1B831E70DF66BA3B8430593944D}} VP:10 A:0}\n      Validators:\n        Validator{E89A51D60F68385E09E716D353373B11F8FACD62 {PubKeyEd25519{68DFDA7E50F82946E7E8546BED37944A422CD1B831E70DF66BA3B8430593944D}} VP:10 A:0}\n    }\n}"
// 	},
// 	"id": "",
// 	"jsonrpc": "2.0"
// }
// ```
func DumpConsensusState() (*ctypes.ResultDumpConsensusState, error) {
	peerRoundStates := make(map[string]*cstypes.PeerRoundState)
	for _, peer := range p2pSwitch.Peers().List() {
		peerState := peer.Get(types.PeerStateKey).(*cm.PeerState)
		peerRoundState := peerState.GetRoundState()
		peerRoundStates[peer.Key()] = peerRoundState
	}
	return &ctypes.ResultDumpConsensusState{consensusState.GetRoundState(), peerRoundStates}, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// Cluster contains information about a cluster in a cluster registry.
// +k8s:openapi-gen=x-kubernetes-print-columns:custom-columns=NAME:.metadata.name,CIDR:.spec.kubernetesApiEndpoints.serverEndpoints[].clientCIDR,SERVER:.spec.kubernetesApiEndpoints.serverEndpoints[].serverAddress,CREATION TIME:.metadata.creationTimestamp
// +resource:path=clusters
type Cluster struct {
	metav1.TypeMeta `json:",inline"`
	// Standard object's metadata.
	// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`

	// Spec is the specification of the cluster. This may or may not be
	// reconciled by an active controller.
	// +optional
	Spec ClusterSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"`

	// Status is the status of the cluster.
	// +optional
	Status ClusterStatus `json:"status" protobuf:"bytes,3,opt,name=status"`
}

// ClusterSpec contains the specification of a cluster.
type ClusterSpec struct {
	// KubernetesAPIEndpoints represents the endpoints of the API server for this
	// cluster.
	// +optional
	KubernetesAPIEndpoints KubernetesAPIEndpoints `json:"kubernetesApiEndpoints,omitempty" protobuf:"bytes,1,opt,name=kubernetesApiEndpoints"`

	// AuthInfo contains public information that can be used to authenticate
	// to and authorize with this cluster. It is not meant to store private
	// information (e.g., tokens or client certificates) and cluster registry
	// implementations are not expected to provide hardened storage for
	// secrets.
	// +optional
	AuthInfo AuthInfo `json:"authInfo,omitempty" protobuf:"bytes,2,opt,name=authInfo"`
}

// ClusterStatus contains the status of a cluster.
type ClusterStatus struct {
	// Conditions contains the different condition statuses for this cluster.
	Conditions []ClusterCondition `json:"conditions,omitempty" protobuf:"bytes,1,rep,name=conditions"`

	// TODO https://github.com/kubernetes/cluster-registry/issues/28
}

// KubernetesAPIEndpoints represents the endpoints for one and only one
// Kubernetes API server.
type KubernetesAPIEndpoints struct {
	// ServerEndpoints specifies the address(es) of the Kubernetes API server’s
	// network identity or identities.
	// +optional
	ServerEndpoints []ServerAddressByClientCIDR `json:"serverEndpoints,omitempty" protobuf:"bytes,1,rep,name=serverEndpoints"`

	// CABundle contains the certificate authority information.
	// +optional
	CABundle []byte `json:"caBundle,omitempty" protobuf:"bytes,2,opt,name=caBundle"`
}

// ServerAddressByClientCIDR helps clients determine the server address that
// they should use, depending on the ClientCIDR that they match.
type ServerAddressByClientCIDR struct {
	// The CIDR with which clients can match their IP to figure out if they should
	// use the corresponding server address.
	// +optional
	ClientCIDR string `json:"clientCIDR,omitempty" protobuf:"bytes,1,opt,name=clientCIDR"`
	// Address of this server, suitable for a client that matches the above CIDR.
	// This can be a hostname, hostname:port, IP or IP:port.
	// +optional
	ServerAddress string `json:"serverAddress,omitempty" protobuf:"bytes,2,opt,name=serverAddress"`
}

// AuthInfo holds public information that describes how a client can get
// credentials to access the cluster. For example, OAuth2 client registration
// endpoints and supported flows, or Kerberos servers locations.
//
// It should not hold any private or sensitive information.
type AuthInfo struct {
	// AuthProviders is a list of configurations for auth providers.
	// +optional
	Providers []AuthProviderConfig `json:"providers,omitempty" protobuf:"bytes,1,rep,name=providers"`
}

// AuthProviderConfig contains the information necessary for a client to
// authenticate to a Kubernetes API server. It is modeled after
// k8s.io/client-go/tools/clientcmd/api/v1.AuthProviderConfig.
type AuthProviderConfig struct {
	// Name is the name of this configuration.
	// +optional
	Name string `json:"name,omitempty" protobuf:"bytes,1,opt,name=name"`

	// Type contains type information about this auth provider. Clients of the
	// cluster registry should use this field to differentiate between different
	// kinds of authentication providers.
	// +optional
	Type AuthProviderType `json:"type,omitempty" protobuf:"bytes,2,opt,name=type"`

	// Config is a map of values that contains the information necessary for a
	// client to determine how to authenticate to a Kubernetes API server.
	// +optional
	Config map[string]string `json:"config,omitempty" protobuf:"bytes,3,rep,name=config"`
}

// AuthProviderType contains metadata about the auth provider. It should be used
// by clients to differentiate between different kinds of auth providers, and to
// select a relevant provider for the client's configuration. For example, a
// controller would look for a provider type that denotes a service account
// that it should use to access the cluster, whereas a user would look for a
// provider type that denotes an authentication system from which they should
// request a token.
type AuthProviderType struct {
	// Name is the name of the auth provider.
	// +optional
	Name string `json:"name,omitempty" protobuf:"bytes,1,opt,name=name"`
}

// ClusterConditionType marks the kind of cluster condition being reported.
type ClusterConditionType string

const (
	// ClusterOK means that the cluster is "OK".
	//
	// Since the cluster registry does not have a standard status controller, the
	// meaning of this condition is defined by the environment in which the
	// cluster is running. It is expected to mean that the cluster is reachable by
	// a controller that is reporting on its status, and that the cluster is ready
	// to have workloads scheduled.
	ClusterOK ClusterConditionType = "OK"
)

// ClusterCondition contains condition information for a cluster.
type ClusterCondition struct {
	// Type is the type of the cluster condition.
	Type ClusterConditionType `json:"type" protobuf:"bytes,1,opt,name=type,casttype=ClusterConditionType"`

	// Status is the status of the condition. One of True, False, Unknown.
	Status v1.ConditionStatus `json:"status" protobuf:"bytes,2,opt,name=status,casttype=ConditionStatus"`

	// LastHeartbeatTime is the last time this condition was updated.
	// +optional
	LastHeartbeatTime metav1.Time `json:"lastHeartbeatTime,omitempty" protobuf:"bytes,3,opt,name=lastHeartbeatTime"`

	// LastTransitionTime is the last time the condition changed from one status to another.
	// +optional
	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty" protobuf:"bytes,4,opt,name=lastTransitionTime"`

	// Reason is a (brief) reason for the condition's last status change.
	// +optional
	Reason string `json:"reason,omitempty" protobuf:"bytes,5,opt,name=reason"`

	// Message is a human-readable message indicating details about the last status change.
	// +optional
	Message string `json:"message,omitempty" protobuf:"bytes,6,opt,name=message"`
}
<commit_msg>Update the AuthInfo field to use object references for user and controller auth.<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// Cluster contains information about a cluster in a cluster registry.
// +k8s:openapi-gen=x-kubernetes-print-columns:custom-columns=NAME:.metadata.name,CIDR:.spec.kubernetesApiEndpoints.serverEndpoints[].clientCIDR,SERVER:.spec.kubernetesApiEndpoints.serverEndpoints[].serverAddress,CREATION TIME:.metadata.creationTimestamp
// +resource:path=clusters
type Cluster struct {
	metav1.TypeMeta `json:",inline"`
	// Standard object's metadata.
	// More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata
	// +optional
	metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`

	// Spec is the specification of the cluster. This may or may not be
	// reconciled by an active controller.
	// +optional
	Spec ClusterSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"`

	// Status is the status of the cluster.
	// +optional
	Status ClusterStatus `json:"status" protobuf:"bytes,3,opt,name=status"`
}

// ClusterSpec contains the specification of a cluster.
type ClusterSpec struct {
	// KubernetesAPIEndpoints represents the endpoints of the API server for this
	// cluster.
	// +optional
	KubernetesAPIEndpoints KubernetesAPIEndpoints `json:"kubernetesApiEndpoints,omitempty" protobuf:"bytes,1,opt,name=kubernetesApiEndpoints"`

	// AuthInfo contains public information that can be used to authenticate
	// to and authorize with this cluster. It is not meant to store private
	// information (e.g., tokens or client certificates) and cluster registry
	// implementations are not expected to provide hardened storage for
	// secrets.
	// +optional
	AuthInfo AuthInfo `json:"authInfo,omitempty" protobuf:"bytes,2,opt,name=authInfo"`
}

// ClusterStatus contains the status of a cluster.
type ClusterStatus struct {
	// Conditions contains the different condition statuses for this cluster.
	Conditions []ClusterCondition `json:"conditions,omitempty" protobuf:"bytes,1,rep,name=conditions"`

	// TODO https://github.com/kubernetes/cluster-registry/issues/28
}

// KubernetesAPIEndpoints represents the endpoints for one and only one
// Kubernetes API server.
type KubernetesAPIEndpoints struct {
	// ServerEndpoints specifies the address(es) of the Kubernetes API server’s
	// network identity or identities.
	// +optional
	ServerEndpoints []ServerAddressByClientCIDR `json:"serverEndpoints,omitempty" protobuf:"bytes,1,rep,name=serverEndpoints"`

	// CABundle contains the certificate authority information.
	// +optional
	CABundle []byte `json:"caBundle,omitempty" protobuf:"bytes,2,opt,name=caBundle"`
}

// ServerAddressByClientCIDR helps clients determine the server address that
// they should use, depending on the ClientCIDR that they match.
type ServerAddressByClientCIDR struct {
	// The CIDR with which clients can match their IP to figure out if they should
	// use the corresponding server address.
	// +optional
	ClientCIDR string `json:"clientCIDR,omitempty" protobuf:"bytes,1,opt,name=clientCIDR"`
	// Address of this server, suitable for a client that matches the above CIDR.
	// This can be a hostname, hostname:port, IP or IP:port.
	// +optional
	ServerAddress string `json:"serverAddress,omitempty" protobuf:"bytes,2,opt,name=serverAddress"`
}

// AuthInfo holds information that describes how a client can get
// credentials to access the cluster. For example, OAuth2 client registration
// endpoints and supported flows, or Kerberos server locations.
type AuthInfo struct {
	// UserAuthInfo references an object that contains implementation-specific
	// details about how a user should authenticate against this cluster.
	// +optional
	UserAuthInfo *v1.ObjectReference `json:"userAuthInfo,omitempty" protobuf:"bytes,1,opt,name=userAuthInfo"`

	// ControllerAuthInfo references an object that contains
	// implementation-specific details about how a controller should authenticate.
	// A simple use case for this would be to reference a secret in another
	// namespace that stores a bearer token that can be used to authenticate
	// against this cluster's API server.
	ControllerAuthInfo *v1.ObjectReference `json:"controllerAuthInfo,omitempty" protobuf:"bytes,2,opt,name=controllerAuthInfo"`
}

// ClusterConditionType marks the kind of cluster condition being reported.
type ClusterConditionType string

const (
	// ClusterOK means that the cluster is "OK".
	//
	// Since the cluster registry does not have a standard status controller, the
	// meaning of this condition is defined by the environment in which the
	// cluster is running. It is expected to mean that the cluster is reachable by
	// a controller that is reporting on its status, and that the cluster is ready
	// to have workloads scheduled.
	ClusterOK ClusterConditionType = "OK"
)

// ClusterCondition contains condition information for a cluster.
type ClusterCondition struct {
	// Type is the type of the cluster condition.
	Type ClusterConditionType `json:"type" protobuf:"bytes,1,opt,name=type,casttype=ClusterConditionType"`

	// Status is the status of the condition. One of True, False, Unknown.
	Status v1.ConditionStatus `json:"status" protobuf:"bytes,2,opt,name=status,casttype=ConditionStatus"`

	// LastHeartbeatTime is the last time this condition was updated.
	// +optional
	LastHeartbeatTime metav1.Time `json:"lastHeartbeatTime,omitempty" protobuf:"bytes,3,opt,name=lastHeartbeatTime"`

	// LastTransitionTime is the last time the condition changed from one status to another.
	// +optional
	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty" protobuf:"bytes,4,opt,name=lastTransitionTime"`

	// Reason is a (brief) reason for the condition's last status change.
	// +optional
	Reason string `json:"reason,omitempty" protobuf:"bytes,5,opt,name=reason"`

	// Message is a human-readable message indicating details about the last status change.
	// +optional
	Message string `json:"message,omitempty" protobuf:"bytes,6,opt,name=message"`
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1beta1

const (
	// Healthy means that the device is healthy
	Healthy = "Healthy"
	// Unhealthy means that the device is unhealthy
	Unhealthy = "Unhealthy"

	// Version means current version of the API supported by kubelet
	Version = "v1beta1"
	// DevicePluginPath is the folder the Device Plugin is expecting sockets to be on
	// Only privileged pods have access to this path
	// Note: Placeholder until we find a "standard path"
	DevicePluginPath = "/var/lib/kubelet/device-plugins/"
	// KubeletSocket is the path of the Kubelet registry socket
	KubeletSocket = DevicePluginPath + "kubelet.sock"
	// KubeletPreStartContainerRPCTimeoutInSecs is the timeout duration in secs for PreStartContainer RPC
	KubeletPreStartContainerRPCTimeoutInSecs = 30
)

// SupportedVersions provides a list of supported version
var SupportedVersions = [...]string{"v1beta1"}
<commit_msg>Port deviceManager in windows container manager<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1beta1

const (
	// Healthy means that the device is healthy
	Healthy = "Healthy"
	// Unhealthy means that the device is unhealthy
	Unhealthy = "Unhealthy"

	// Version means current version of the API supported by kubelet
	Version = "v1beta1"
	// DevicePluginPath is the folder the Device Plugin is expecting sockets to be on
	// Only privileged pods have access to this path
	// Note: Placeholder until we find a "standard path"
	DevicePluginPath = "/var/lib/kubelet/device-plugins/"
	// KubeletSocket is the path of the Kubelet registry socket
	KubeletSocket = DevicePluginPath + "kubelet.sock"

	// DevicePluginPathWindows Avoid failed to run Kubelet: bad socketPath,
	// must be an absolute path: /var/lib/kubelet/device-plugins/kubelet.sock
	// https://github.com/kubernetes/kubernetes/issues/93262
	// https://github.com/kubernetes/kubernetes/pull/93285#discussion_r458140701
	DevicePluginPathWindows = "\\var\\lib\\kubelet\\device-plugins\\"
	// KubeletSocketWindows is the path of the Kubelet registry socket on windows
	KubeletSocketWindows = DevicePluginPathWindows + "kubelet.sock"

	// KubeletPreStartContainerRPCTimeoutInSecs is the timeout duration in secs for PreStartContainer RPC
	// Timeout duration in secs for PreStartContainer RPC
	KubeletPreStartContainerRPCTimeoutInSecs = 30
)

// SupportedVersions provides a list of supported version
var SupportedVersions = [...]string{"v1beta1"}
<|endoftext|>


<commit_before>package github

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/sirupsen/logrus"
	"github.com/tomnomnom/linkheader"

	"github.com/rancher/types/apis/management.cattle.io/v3"
)

const (
	gheAPI                = "/api/v3"
	githubAPI             = "https://api.github.com"
	githubDefaultHostName = "https://github.com"
)

//GClient implements a httpclient for github
type GClient struct {
	httpClient *http.Client
}

func (g *GClient) getAccessToken(code string, config *v3.GithubConfig) (string, error) {

	form := url.Values{}
	form.Add("client_id", config.ClientID)
	form.Add("client_secret", config.ClientSecret)
	form.Add("code", code)

	url := g.getURL("TOKEN", config)

	resp, err := g.postToGithub(url, form)
	if err != nil {
		logrus.Errorf("Github getAccessToken: GET url %v received error from github, err: %v", url, err)
		return "", err
	}
	defer resp.Body.Close()

	// Decode the response
	var respMap map[string]interface{}
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logrus.Errorf("Github getAccessToken: received error reading response body, err: %v", err)
		return "", err
	}

	if err := json.Unmarshal(b, &respMap); err != nil {
		logrus.Errorf("Github getAccessToken: received error unmarshalling response body, err: %v", err)
		return "", err
	}

	if respMap["error"] != nil {
		desc := respMap["error_description"]
		logrus.Errorf("Received Error from github %v, description from github %v", respMap["error"], desc)
		return "", fmt.Errorf("Received Error from github %v, description from github %v", respMap["error"], desc)
	}

	acessToken, ok := respMap["access_token"].(string)
	if !ok {
		return "", fmt.Errorf("Received Error reading accessToken from response %v", respMap)
	}
	return acessToken, nil
}

func (g *GClient) getUser(githubAccessToken string, config *v3.GithubConfig) (Account, error) {

	url := g.getURL("USER_INFO", config)
	resp, err := g.getFromGithub(githubAccessToken, url)
	if err != nil {
		logrus.Errorf("Github getGithubUser: GET url %v received error from github, err: %v", url, err)
		return Account{}, err
	}
	defer resp.Body.Close()
	var githubAcct Account

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logrus.Errorf("Github getGithubUser: error reading response, err: %v", err)
		return Account{}, err
	}

	if err := json.Unmarshal(b, &githubAcct); err != nil {
		logrus.Errorf("Github getGithubUser: error unmarshalling response, err: %v", err)
		return Account{}, err
	}

	return githubAcct, nil
}

func (g *GClient) getOrgs(githubAccessToken string, config *v3.GithubConfig) ([]Account, error) {
	var orgs []Account

	url := g.getURL("ORG_INFO", config)
	responses, err := g.paginateGithub(githubAccessToken, url)
	if err != nil {
		logrus.Errorf("Github getGithubOrgs: GET url %v received error from github, err: %v", url, err)
		return orgs, err
	}

	for _, response := range responses {
		defer response.Body.Close()
		var orgObjs []Account
		b, err := ioutil.ReadAll(response.Body)
		if err != nil {
			logrus.Errorf("Github getGithubOrgs: error reading the response from github, err: %v", err)
			return orgs, err
		}
		if err := json.Unmarshal(b, &orgObjs); err != nil {
			logrus.Errorf("Github getGithubOrgs: received error unmarshalling org array, err: %v", err)
			return orgs, err
		}
		for _, orgObj := range orgObjs {
			orgs = append(orgs, orgObj)
		}
	}

	return orgs, nil
}

func (g *GClient) getTeams(githubAccessToken string, config *v3.GithubConfig) ([]Account, error) {
	var teams []Account

	url := g.getURL("TEAMS", config)
	responses, err := g.paginateGithub(githubAccessToken, url)
	if err != nil {
		logrus.Errorf("Github getGithubTeams: GET url %v received error from github, err: %v", url, err)
		return teams, err
	}
	for _, response := range responses {
		defer response.Body.Close()
		teamObjs, err := g.getTeamInfo(response, config)

		if err != nil {
			logrus.Errorf("Github getGithubTeams: received error unmarshalling teams array, err: %v", err)
			return teams, err
		}
		for _, teamObj := range teamObjs {
			teams = append(teams, teamObj)
		}

	}
	return teams, nil
}

func (g *GClient) getTeamInfo(response *http.Response, config *v3.GithubConfig) ([]Account, error) {
	var teams []Account
	b, err := ioutil.ReadAll(response.Body)
	if err != nil {
		logrus.Errorf("Github getTeamInfo: error reading the response from github, err: %v", err)
		return teams, err
	}
	var teamObjs []Team
	if err := json.Unmarshal(b, &teamObjs); err != nil {
		logrus.Errorf("Github getTeamInfo: received error unmarshalling team array, err: %v", err)
		return teams, err
	}

	url := g.getURL("TEAM_PROFILE", config)
	for _, team := range teamObjs {
		teamAcct := Account{}
		team.toGithubAccount(url, &teamAcct)
		teams = append(teams, teamAcct)
	}

	return teams, nil
}

func (g *GClient) getTeamByID(id string, githubAccessToken string, config *v3.GithubConfig) (Account, error) {
	var teamAcct Account

	url := g.getURL("TEAM", config) + id
	response, err := g.getFromGithub(githubAccessToken, url)
	if err != nil {
		logrus.Errorf("Github getTeamByID: GET url %v received error from github, err: %v", url, err)
		return teamAcct, err
	}
	b, err := ioutil.ReadAll(response.Body)
	if err != nil {
		logrus.Errorf("Github getTeamByID: error reading the response from github, err: %v", err)
		return teamAcct, err
	}
	var teamObj Team
	if err := json.Unmarshal(b, &teamObj); err != nil {
		logrus.Errorf("Github getTeamByID: received error unmarshalling team array, err: %v", err)
		return teamAcct, err
	}
	url = g.getURL("TEAM_PROFILE", config)
	teamObj.toGithubAccount(url, &teamAcct)

	return teamAcct, nil
}

func (g *GClient) paginateGithub(githubAccessToken string, url string) ([]*http.Response, error) {
	var responses []*http.Response

	response, err := g.getFromGithub(githubAccessToken, url)
	if err != nil {
		return responses, err
	}
	responses = append(responses, response)
	nextURL := g.nextGithubPage(response)
	for nextURL != "" {
		response, err = g.getFromGithub(githubAccessToken, nextURL)
		if err != nil {
			return responses, err
		}
		responses = append(responses, response)
		nextURL = g.nextGithubPage(response)
	}

	return responses, nil
}

func (g *GClient) nextGithubPage(response *http.Response) string {
	header := response.Header.Get("link")

	if header != "" {
		links := linkheader.Parse(header)
		for _, link := range links {
			if link.Rel == "next" {
				return link.URL
			}
		}
	}

	return ""
}

func (g *GClient) searchUsers(searchTerm, searchType string, githubAccessToken string, config *v3.GithubConfig) ([]Account, error) {
	if searchType == "group" {
		searchType = orgType
	}

	search := searchTerm
	if searchType != "" {
		search += "+type:" + searchType
	}
	search = URLEncoded(search)
	url := g.getURL("USER_SEARCH", config) + search

	resp, err := g.getFromGithub(githubAccessToken, url)
	if err != nil {
		// no match on search returns an error. do not log
		return nil, nil
	}
	defer resp.Body.Close()

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	result := &searchResult{}
	if err := json.Unmarshal(b, result); err != nil {
		return nil, err
	}

	return result.Items, nil
}

func (g *GClient) getOrgByName(org string, githubAccessToken string, config *v3.GithubConfig) (Account, error) {
	org = URLEncoded(org)
	url := g.getURL("ORGS", config) + org

	resp, err := g.getFromGithub(githubAccessToken, url)
	if err != nil {
		logrus.Debugf("Github getGithubOrgByName: GET url %v received error from github, err: %v", url, err)
		return Account{}, err
	}
	defer resp.Body.Close()
	var githubAcct Account

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logrus.Errorf("Github getGithubOrgByName: error reading response, err: %v", err)
		return Account{}, err
	}

	if err := json.Unmarshal(b, &githubAcct); err != nil {
		logrus.Errorf("Github getGithubOrgByName: error unmarshalling response, err: %v", err)
		return Account{}, err
	}

	return githubAcct, nil
}

func (g *GClient) getUserOrgByID(id string, githubAccessToken string, config *v3.GithubConfig) (Account, error) {
	url := g.getURL("USER_INFO", config) + "/" + id

	resp, err := g.getFromGithub(githubAccessToken, url)
	if err != nil {
		logrus.Errorf("Github getUserOrgById: GET url %v received error from github, err: %v", url, err)
		return Account{}, err
	}
	defer resp.Body.Close()
	var githubAcct Account

	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logrus.Errorf("Github getUserOrgById: error reading response, err: %v", err)
		return Account{}, err
	}

	if err := json.Unmarshal(b, &githubAcct); err != nil {
		logrus.Errorf("Github getUserOrgById: error unmarshalling response, err: %v", err)
		return Account{}, err
	}

	return githubAcct, nil
}

//URLEncoded encodes the string
func URLEncoded(str string) string {
	u, err := url.Parse(str)
	if err != nil {
		logrus.Errorf("Error encoding the url: %s, error: %v", str, err)
		return str
	}
	return u.String()
}

func (g *GClient) postToGithub(url string, form url.Values) (*http.Response, error) {
	req, err := http.NewRequest("POST", url, strings.NewReader(form.Encode()))
	if err != nil {
		logrus.Error(err)
	}
	req.PostForm = form
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Add("Accept", "application/json")
	resp, err := g.httpClient.Do(req)
	if err != nil {
		logrus.Errorf("Received error from github: %v", err)
		return resp, err
	}
	// Check the status code
	switch resp.StatusCode {
	case 200:
	case 201:
	default:
		var body bytes.Buffer
		io.Copy(&body, resp.Body)
		return resp, fmt.Errorf("Request failed, got status code: %d. Response: %s",
			resp.StatusCode, body.Bytes())
	}
	return resp, nil
}

func (g *GClient) getFromGithub(githubAccessToken string, url string) (*http.Response, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		logrus.Error(err)
	}
	req.Header.Add("Authorization", "token "+githubAccessToken)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36)")
	resp, err := g.httpClient.Do(req)
	if err != nil {
		logrus.Errorf("Received error from github: %v", err)
		return resp, err
	}
	// Check the status code
	switch resp.StatusCode {
	case 200:
	case 201:
	default:
		var body bytes.Buffer
		io.Copy(&body, resp.Body)
		return resp, fmt.Errorf("Request failed, got status code: %d. Response: %s",
			resp.StatusCode, body.Bytes())
	}
	return resp, nil
}

func (g *GClient) getURL(endpoint string, config *v3.GithubConfig) string {

	var hostName, apiEndpoint, toReturn string

	if config.Hostname != "" {
		scheme := "http://"
		if config.TLS {
			scheme = "https://"
		}
		hostName = scheme + config.Hostname
		if hostName == githubDefaultHostName {
			apiEndpoint = githubAPI
		} else {
			apiEndpoint = scheme + config.Hostname + gheAPI
		}
	} else {
		hostName = githubDefaultHostName
		apiEndpoint = githubAPI
	}

	switch endpoint {
	case "API":
		toReturn = apiEndpoint
	case "TOKEN":
		toReturn = hostName + "/login/oauth/access_token"
	case "USERS":
		toReturn = apiEndpoint + "/users/"
	case "ORGS":
		toReturn = apiEndpoint + "/orgs/"
	case "USER_INFO":
		toReturn = apiEndpoint + "/user"
	case "ORG_INFO":
		toReturn = apiEndpoint + "/user/orgs?per_page=1"
	case "USER_PICTURE":
		toReturn = "https://avatars.githubusercontent.com/u/" + endpoint + "?v=3&s=72"
	case "USER_SEARCH":
		toReturn = apiEndpoint + "/search/users?q="
	case "TEAM":
		toReturn = apiEndpoint + "/teams/"
	case "TEAMS":
		toReturn = apiEndpoint + "/user/teams?per_page=100"
	case "TEAM_PROFILE":
		toReturn = hostName + "/orgs/%s/teams/%s"
	default:
		toReturn = apiEndpoint
	}

	return toReturn
}
<commit_msg>Close response body in github client<commit_after>package github

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"

	"github.com/sirupsen/logrus"
	"github.com/tomnomnom/linkheader"

	"github.com/rancher/types/apis/management.cattle.io/v3"
)

const (
	gheAPI                = "/api/v3"
	githubAPI             = "https://api.github.com"
	githubDefaultHostName = "https://github.com"
)

//GClient implements a httpclient for github
type GClient struct {
	httpClient *http.Client
}

func (g *GClient) getAccessToken(code string, config *v3.GithubConfig) (string, error) {

	form := url.Values{}
	form.Add("client_id", config.ClientID)
	form.Add("client_secret", config.ClientSecret)
	form.Add("code", code)

	url := g.getURL("TOKEN", config)

	b, err := g.postToGithub(url, form)
	if err != nil {
		logrus.Errorf("Github getAccessToken: GET url %v received error from github, err: %v", url, err)
		return "", err
	}

	// Decode the response
	var respMap map[string]interface{}

	if err := json.Unmarshal(b, &respMap); err != nil {
		logrus.Errorf("Github getAccessToken: received error unmarshalling response body, err: %v", err)
		return "", err
	}

	if respMap["error"] != nil {
		desc := respMap["error_description"]
		logrus.Errorf("Received Error from github %v, description from github %v", respMap["error"], desc)
		return "", fmt.Errorf("Received Error from github %v, description from github %v", respMap["error"], desc)
	}

	acessToken, ok := respMap["access_token"].(string)
	if !ok {
		return "", fmt.Errorf("Received Error reading accessToken from response %v", respMap)
	}
	return acessToken, nil
}

func (g *GClient) getUser(githubAccessToken string, config *v3.GithubConfig) (Account, error) {

	url := g.getURL("USER_INFO", config)
	b, _, err := g.getFromGithub(githubAccessToken, url)
	if err != nil {
		logrus.Errorf("Github getGithubUser: GET url %v received error from github, err: %v", url, err)
		return Account{}, err
	}
	var githubAcct Account

	if err := json.Unmarshal(b, &githubAcct); err != nil {
		logrus.Errorf("Github getGithubUser: error unmarshalling response, err: %v", err)
		return Account{}, err
	}

	return githubAcct, nil
}

func (g *GClient) getOrgs(githubAccessToken string, config *v3.GithubConfig) ([]Account, error) {
	var orgs []Account

	url := g.getURL("ORG_INFO", config)
	responses, err := g.paginateGithub(githubAccessToken, url)
	if err != nil {
		logrus.Errorf("Github getGithubOrgs: GET url %v received error from github, err: %v", url, err)
		return orgs, err
	}

	for _, b := range responses {
		var orgObjs []Account
		if err := json.Unmarshal(b, &orgObjs); err != nil {
			logrus.Errorf("Github getGithubOrgs: received error unmarshalling org array, err: %v", err)
			return nil, err
		}
		orgs = append(orgs, orgObjs...)
	}

	return orgs, nil
}

func (g *GClient) getTeams(githubAccessToken string, config *v3.GithubConfig) ([]Account, error) {
	var teams []Account

	url := g.getURL("TEAMS", config)
	responses, err := g.paginateGithub(githubAccessToken, url)
	if err != nil {
		logrus.Errorf("Github getGithubTeams: GET url %v received error from github, err: %v", url, err)
		return teams, err
	}
	for _, response := range responses {
		teamObjs, err := g.getTeamInfo(response, config)

		if err != nil {
			logrus.Errorf("Github getGithubTeams: received error unmarshalling teams array, err: %v", err)
			return teams, err
		}
		for _, teamObj := range teamObjs {
			teams = append(teams, teamObj)
		}

	}
	return teams, nil
}

func (g *GClient) getTeamInfo(b []byte, config *v3.GithubConfig) ([]Account, error) {
	var teams []Account
	var teamObjs []Team
	if err := json.Unmarshal(b, &teamObjs); err != nil {
		logrus.Errorf("Github getTeamInfo: received error unmarshalling team array, err: %v", err)
		return teams, err
	}

	url := g.getURL("TEAM_PROFILE", config)
	for _, team := range teamObjs {
		teamAcct := Account{}
		team.toGithubAccount(url, &teamAcct)
		teams = append(teams, teamAcct)
	}

	return teams, nil
}

func (g *GClient) getTeamByID(id string, githubAccessToken string, config *v3.GithubConfig) (Account, error) {
	var teamAcct Account

	url := g.getURL("TEAM", config) + id
	b, _, err := g.getFromGithub(githubAccessToken, url)
	if err != nil {
		logrus.Errorf("Github getTeamByID: GET url %v received error from github, err: %v", url, err)
		return teamAcct, err
	}
	var teamObj Team
	if err := json.Unmarshal(b, &teamObj); err != nil {
		logrus.Errorf("Github getTeamByID: received error unmarshalling team array, err: %v", err)
		return teamAcct, err
	}
	url = g.getURL("TEAM_PROFILE", config)
	teamObj.toGithubAccount(url, &teamAcct)

	return teamAcct, nil
}

func (g *GClient) paginateGithub(githubAccessToken string, url string) ([][]byte, error) {
	var responses [][]byte
	var err error
	var response []byte
	nextURL := url
	for nextURL != "" {
		response, nextURL, err = g.getFromGithub(githubAccessToken, nextURL)
		if err != nil {
			return nil, err
		}
		responses = append(responses, response)
	}

	return responses, nil
}

func (g *GClient) nextGithubPage(response *http.Response) string {
	header := response.Header.Get("link")

	if header != "" {
		links := linkheader.Parse(header)
		for _, link := range links {
			if link.Rel == "next" {
				return link.URL
			}
		}
	}

	return ""
}

func (g *GClient) searchUsers(searchTerm, searchType string, githubAccessToken string, config *v3.GithubConfig) ([]Account, error) {
	if searchType == "group" {
		searchType = orgType
	}

	search := searchTerm
	if searchType != "" {
		search += "+type:" + searchType
	}
	search = URLEncoded(search)
	url := g.getURL("USER_SEARCH", config) + search

	b, _, err := g.getFromGithub(githubAccessToken, url)
	if err != nil {
		// no match on search returns an error. do not log
		return nil, nil
	}

	result := &searchResult{}
	if err := json.Unmarshal(b, result); err != nil {
		return nil, err
	}

	return result.Items, nil
}

func (g *GClient) getOrgByName(org string, githubAccessToken string, config *v3.GithubConfig) (Account, error) {
	org = URLEncoded(org)
	url := g.getURL("ORGS", config) + org

	b, _, err := g.getFromGithub(githubAccessToken, url)
	if err != nil {
		logrus.Debugf("Github getGithubOrgByName: GET url %v received error from github, err: %v", url, err)
		return Account{}, err
	}
	var githubAcct Account
	if err := json.Unmarshal(b, &githubAcct); err != nil {
		logrus.Errorf("Github getGithubOrgByName: error unmarshalling response, err: %v", err)
		return Account{}, err
	}

	return githubAcct, nil
}

func (g *GClient) getUserOrgByID(id string, githubAccessToken string, config *v3.GithubConfig) (Account, error) {
	url := g.getURL("USER_INFO", config) + "/" + id

	b, _, err := g.getFromGithub(githubAccessToken, url)
	if err != nil {
		logrus.Errorf("Github getUserOrgById: GET url %v received error from github, err: %v", url, err)
		return Account{}, err
	}
	var githubAcct Account

	if err := json.Unmarshal(b, &githubAcct); err != nil {
		logrus.Errorf("Github getUserOrgById: error unmarshalling response, err: %v", err)
		return Account{}, err
	}

	return githubAcct, nil
}

//URLEncoded encodes the string
func URLEncoded(str string) string {
	u, err := url.Parse(str)
	if err != nil {
		logrus.Errorf("Error encoding the url: %s, error: %v", str, err)
		return str
	}
	return u.String()
}

func (g *GClient) postToGithub(url string, form url.Values) ([]byte, error) {
	req, err := http.NewRequest("POST", url, strings.NewReader(form.Encode()))
	if err != nil {
		logrus.Error(err)
	}
	req.PostForm = form
	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Add("Accept", "application/json")
	resp, err := g.httpClient.Do(req)
	if err != nil {
		logrus.Errorf("Received error from github: %v", err)
		return nil, err
	}

	defer resp.Body.Close()
	// Check the status code
	switch resp.StatusCode {
	case 200:
	case 201:
	default:
		var body bytes.Buffer
		io.Copy(&body, resp.Body)
		return nil, fmt.Errorf("Request failed, got status code: %d. Response: %s",
			resp.StatusCode, body.Bytes())
	}
	return ioutil.ReadAll(resp.Body)
}

func (g *GClient) getFromGithub(githubAccessToken string, url string) ([]byte, string, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, "", err
	}
	req.Header.Add("Authorization", "token "+githubAccessToken)
	req.Header.Add("Accept", "application/json")
	req.Header.Add("user-agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36)")
	resp, err := g.httpClient.Do(req)
	if err != nil {
		logrus.Errorf("Received error from github: %v", err)
		return nil, "", err
	}
	defer resp.Body.Close()
	// Check the status code
	switch resp.StatusCode {
	case 200:
	case 201:
	default:
		var body bytes.Buffer
		io.Copy(&body, resp.Body)
		return nil, "", fmt.Errorf("request failed, got status code: %d. Response: %s",
			resp.StatusCode, body.Bytes())
	}

	nextURL := g.nextGithubPage(resp)
	b, err := ioutil.ReadAll(resp.Body)
	return b, nextURL, err
}

func (g *GClient) getURL(endpoint string, config *v3.GithubConfig) string {

	var hostName, apiEndpoint, toReturn string

	if config.Hostname != "" {
		scheme := "http://"
		if config.TLS {
			scheme = "https://"
		}
		hostName = scheme + config.Hostname
		if hostName == githubDefaultHostName {
			apiEndpoint = githubAPI
		} else {
			apiEndpoint = scheme + config.Hostname + gheAPI
		}
	} else {
		hostName = githubDefaultHostName
		apiEndpoint = githubAPI
	}

	switch endpoint {
	case "API":
		toReturn = apiEndpoint
	case "TOKEN":
		toReturn = hostName + "/login/oauth/access_token"
	case "USERS":
		toReturn = apiEndpoint + "/users/"
	case "ORGS":
		toReturn = apiEndpoint + "/orgs/"
	case "USER_INFO":
		toReturn = apiEndpoint + "/user"
	case "ORG_INFO":
		toReturn = apiEndpoint + "/user/orgs?per_page=1"
	case "USER_PICTURE":
		toReturn = "https://avatars.githubusercontent.com/u/" + endpoint + "?v=3&s=72"
	case "USER_SEARCH":
		toReturn = apiEndpoint + "/search/users?q="
	case "TEAM":
		toReturn = apiEndpoint + "/teams/"
	case "TEAMS":
		toReturn = apiEndpoint + "/user/teams?per_page=100"
	case "TEAM_PROFILE":
		toReturn = hostName + "/orgs/%s/teams/%s"
	default:
		toReturn = apiEndpoint
	}

	return toReturn
}
<|endoftext|>


<commit_before>// Copyright 2018 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package controller provides a Kubernetes controller for a TFJob resource.
package controller

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"

	tfv1alpha2 "github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1alpha2"
	"github.com/kubeflow/tf-operator/pkg/generator"
)

// TFConfig is a struct representing the distributed TensorFlow config.
// This struct is turned into an environment variable TF_CONFIG
// which is used by TensorFlow processes to configure themselves.
// https://cloud.google.com/ml-engine/docs/trainer-considerations#use_tf_config
type TFConfig struct {
	// Cluster represents a TensorFlow ClusterSpec.
	// See: https://www.tensorflow.org/api_docs/python/tf/train/ClusterSpec
	Cluster ClusterSpec `json:"cluster"`
	Task    TaskSpec    `json:"task"`
}

// ClusterSpec represents a cluster TensorFlow specification.
// https://www.tensorflow.org/deploy/distributed#create_a_tftrainclusterspec_to_describe_the_cluster
// It is a map from job names to network addresses.
type ClusterSpec map[string][]string

// TaskSpec is the specification for a task (PS or worker) of the TFJob.
type TaskSpec struct {
	Type  string `json:"type"`
	Index int    `json:"index"`
}

// genTFConfig will generate the environment variable TF_CONFIG
// {
//     "cluster": {
//         "ps": ["ps1:2222", "ps2:2222"],
//         "worker": ["worker1:2222", "worker2:2222", "worker3:2222"]
//     },
//     "task": {
//         "type": "ps",
//         "index": 1
//         },
//     }
// }
func genTFConfigJSONStr(tfjob *tfv1alpha2.TFJob, rtype, index string) (string, error) {
	// Configure the TFCONFIG environment variable.
	i, _ := strconv.ParseInt(index, 0, 32)

	cluster, err := genClusterSpec(tfjob)
	if err != nil {
		return "", err
	}

	tfConfig := TFConfig{
		Cluster: cluster,
		Task: TaskSpec{
			Type:  rtype,
			Index: int(i),
		},
	}

	tfConfigJSONStr, err := json.Marshal(tfConfig)
	if err != nil {
		return "", err
	}

	return string(tfConfigJSONStr), nil
}

// genClusterSpec will generate ClusterSpec.
func genClusterSpec(tfjob *tfv1alpha2.TFJob) (ClusterSpec, error) {
	clusterSpec := make(ClusterSpec)

	for rtype, spec := range tfjob.Spec.TFReplicaSpecs {
		if rtype == tfv1alpha2.TFReplicaTypeEval {
			// https://www.tensorflow.org/api_docs/python/tf/estimator/RunConfig
			// evaluator is not part of training cluster
			continue
		}
		rt := strings.ToLower(string(rtype))
		replicaNames := make([]string, 0, *spec.Replicas)

		port, err := generator.GetPortFromTFJob(tfjob, rtype)
		if err != nil {
			return nil, err
		}
		for i := int32(0); i < *spec.Replicas; i++ {
			host := fmt.Sprintf("%s:%d", generator.GenDNSRecord(tfjob.Name, rt, fmt.Sprintf("%d", i), tfjob.ObjectMeta.Namespace), port)
			replicaNames = append(replicaNames, host)
		}

		clusterSpec[rt] = replicaNames
	}

	return clusterSpec, nil
}
<commit_msg>chang comment with more related link (#679)<commit_after>// Copyright 2018 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package controller provides a Kubernetes controller for a TFJob resource.
package controller

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"

	tfv1alpha2 "github.com/kubeflow/tf-operator/pkg/apis/tensorflow/v1alpha2"
	"github.com/kubeflow/tf-operator/pkg/generator"
)

// TFConfig is a struct representing the distributed TensorFlow config.
// This struct is turned into an environment variable TF_CONFIG
// which is used by TensorFlow processes to configure themselves.
// https://www.tensorflow.org/api_docs/python/tf/estimator/RunConfig#methods
// https://cloud.google.com/ml-engine/docs/tensorflow/distributed-training-details
type TFConfig struct {
	// Cluster represents a TensorFlow ClusterSpec.
	// See: https://www.tensorflow.org/api_docs/python/tf/train/ClusterSpec
	Cluster ClusterSpec `json:"cluster"`
	Task    TaskSpec    `json:"task"`
}

// ClusterSpec represents a cluster TensorFlow specification.
// https://www.tensorflow.org/deploy/distributed#create_a_tftrainclusterspec_to_describe_the_cluster
// It is a map from job names to network addresses.
type ClusterSpec map[string][]string

// TaskSpec is the specification for a task (PS or worker) of the TFJob.
type TaskSpec struct {
	Type  string `json:"type"`
	Index int    `json:"index"`
}

// genTFConfig will generate the environment variable TF_CONFIG
// {
//     "cluster": {
//         "ps": ["ps1:2222", "ps2:2222"],
//         "worker": ["worker1:2222", "worker2:2222", "worker3:2222"]
//     },
//     "task": {
//         "type": "ps",
//         "index": 1
//         },
//     }
// }
func genTFConfigJSONStr(tfjob *tfv1alpha2.TFJob, rtype, index string) (string, error) {
	// Configure the TFCONFIG environment variable.
	i, _ := strconv.ParseInt(index, 0, 32)

	cluster, err := genClusterSpec(tfjob)
	if err != nil {
		return "", err
	}

	tfConfig := TFConfig{
		Cluster: cluster,
		Task: TaskSpec{
			Type:  rtype,
			Index: int(i),
		},
	}

	tfConfigJSONStr, err := json.Marshal(tfConfig)
	if err != nil {
		return "", err
	}

	return string(tfConfigJSONStr), nil
}

// genClusterSpec will generate ClusterSpec.
func genClusterSpec(tfjob *tfv1alpha2.TFJob) (ClusterSpec, error) {
	clusterSpec := make(ClusterSpec)

	for rtype, spec := range tfjob.Spec.TFReplicaSpecs {
		if rtype == tfv1alpha2.TFReplicaTypeEval {
			// https://www.tensorflow.org/api_docs/python/tf/estimator/RunConfig
			// evaluator is not part of training cluster
			continue
		}
		rt := strings.ToLower(string(rtype))
		replicaNames := make([]string, 0, *spec.Replicas)

		port, err := generator.GetPortFromTFJob(tfjob, rtype)
		if err != nil {
			return nil, err
		}
		for i := int32(0); i < *spec.Replicas; i++ {
			host := fmt.Sprintf("%s:%d", generator.GenDNSRecord(tfjob.Name, rt, fmt.Sprintf("%d", i), tfjob.ObjectMeta.Namespace), port)
			replicaNames = append(replicaNames, host)
		}

		clusterSpec[rt] = replicaNames
	}

	return clusterSpec, nil
}
<|endoftext|>


<commit_before>package common

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/ghodss/yaml"
	"github.com/pkg/errors"
	"github.com/rancher/rancher/pkg/jailer"
	"github.com/rancher/rancher/pkg/namespace"
	v3 "github.com/rancher/types/apis/project.cattle.io/v3"
	"github.com/sirupsen/logrus"
)

const (
	base            = 32768
	end             = 61000
	tillerName      = "rancher-tiller"
	HelmV2          = "rancher-helm"
	HelmV3          = "helm_v3"
	forceUpgradeStr = "--force"
)

type HelmPath struct {
	// /opt/jail/<app-name>
	FullPath string
	// /
	InJailPath string
	// /opt/jail/<app-name>/.kubeconfig
	KubeConfigFull string
	// /.kubeconfig
	KubeConfigInJail string
	// /opt/jail/<app-name>/<app-sub>
	AppDirFull string
	// /<app-sub>
	AppDirInJail string
	// /opt/jail/<app-name>/kustomize.sh
	KustomizeFull string
	// /kustomize.sh
	KustomizeInJail string
}

//Labels that need added for kustomization
type Label struct {
	AppLabel string `json:"io.cattle.field/appId"`
}

//Marshal kustomization settings into YAML
type Kustomization struct {
	CommonLabel Label    `json:"commonLabels"`
	Resrouces   []string `json:"resources"`
}

func ParseExternalID(externalID string) (string, string, error) {
	templateVersionNamespace, catalog, _, template, version, err := SplitExternalID(externalID)
	if err != nil {
		return "", "", err
	}
	return strings.Join([]string{catalog, template, version}, "-"), templateVersionNamespace, nil
}

func SplitExternalID(externalID string) (string, string, string, string, string, error) {
	var templateVersionNamespace, catalog string
	values, err := url.Parse(externalID)
	if err != nil {
		return "", "", "", "", "", err
	}
	catalogWithNamespace := values.Query().Get("catalog")
	catalogType := values.Query().Get("type")
	template := values.Query().Get("template")
	version := values.Query().Get("version")
	split := strings.SplitN(catalogWithNamespace, "/", 2)
	if len(split) == 2 {
		templateVersionNamespace = split[0]
		catalog = split[1]
	}
	//pre-upgrade setups will have global catalogs, where externalId field on templateversions won't have namespace.
	// since these are global catalogs, we can default to global namespace
	if templateVersionNamespace == "" {
		templateVersionNamespace = namespace.GlobalNamespace
		catalog = catalogWithNamespace
	}
	return templateVersionNamespace, catalog, catalogType, template, version, nil
}

// StartTiller start tiller server and return the listening address of the grpc address
func StartTiller(context context.Context, tempDirs *HelmPath, port, namespace string) error {
	probePort := GenerateRandomPort()
	cmd := exec.Command(tillerName, "--listen", ":"+port, "--probe-listen", ":"+probePort)
	cmd.Env = []string{fmt.Sprintf("%s=%s", "KUBECONFIG", tempDirs.KubeConfigInJail), fmt.Sprintf("%s=%s", "TILLER_NAMESPACE", namespace), fmt.Sprintf("%s=%s", "TILLER_HISTORY_MAX", "10")}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	cmd, err := JailCommand(cmd, tempDirs.FullPath)
	if err != nil {
		return err
	}

	if err := cmd.Start(); err != nil {
		return err
	}
	defer cmd.Wait()
	select {
	case <-context.Done():
		logrus.Debug("Stopping Tiller")
		return cmd.Process.Kill()
	}
}

func GenerateRandomPort() string {
	s1 := rand.NewSource(time.Now().UnixNano())
	r1 := rand.New(s1)
	for {
		port := base + r1.Intn(end-base+1)
		ln, err := net.Listen("tcp", ":"+strconv.Itoa(port))
		if err != nil {
			continue
		}
		ln.Close()
		return strconv.Itoa(port)
	}
}

func InstallCharts(tempDirs *HelmPath, port string, obj *v3.App) error {
	logrus.Debugf("InstallCharts - helm path info %+v\n", tempDirs)
	extraArgs := GetExtraArgs(obj)
	timeoutArgs := getTimeoutArgs(obj)
	setValues, err := GenerateAnswerSetValues(obj, tempDirs, extraArgs)
	if err != nil {
		return err
	}
	commands := make([]string, 0)
	if IsHelm3(obj.Status.HelmVersion) {
		err = createKustomizeFiles(tempDirs, obj.Name)
		if err != nil {
			return err
		}
		logrus.Infof("Installing chart using helm version: %s", HelmV3)
		commands = append([]string{"upgrade", "--install", obj.Name, "--namespace", obj.Spec.TargetNamespace, "--kubeconfig", tempDirs.KubeConfigInJail, "--post-renderer", tempDirs.KustomizeInJail}, timeoutArgs...)
	} else {
		logrus.Infof("Installing chart using helm version: %s", HelmV2)
		commands = append([]string{"upgrade", "--install", "--namespace", obj.Spec.TargetNamespace, obj.Name}, timeoutArgs...)
	}
	commands = append(commands, setValues...)
	commands = append(commands, tempDirs.AppDirInJail)

	if v3.AppConditionForceUpgrade.IsUnknown(obj) {
		commands = append(commands, forceUpgradeStr)
		// don't leave force recreate on the object
		v3.AppConditionForceUpgrade.True(obj)
	}
	var cmd *exec.Cmd
	// switch userTriggeredAction back
	v3.AppConditionUserTriggeredAction.Unknown(obj)
	if IsHelm3(obj.Status.HelmVersion) {
		cmd = exec.Command(HelmV3, commands...)
	} else {
		cmd = exec.Command(HelmV2, commands...)
	}
	cmd.Env = []string{fmt.Sprintf("%s=%s", "HELM_HOST", "127.0.0.1:"+port), fmt.Sprintf("%s=%s", "CATTLE_KUSTOMIZE_YAML", tempDirs.InJailPath)}
	stderrBuf := &bytes.Buffer{}
	cmd.Stdout = os.Stdout
	cmd.Stderr = stderrBuf
	cmd, err = JailCommand(cmd, tempDirs.FullPath)
	if err != nil {
		return err
	}
	if err := cmd.Start(); err != nil {
		return errors.Errorf("failed to install app %s. %s", obj.Name, stderrBuf.String())
	}
	if err := cmd.Wait(); err != nil {
		// if the first install failed, the second install would have error message like `has no deployed releases`, then the
		// original error is masked. We need to filter out the message and always return the last one if error matches this pattern
		if strings.Contains(stderrBuf.String(), "has no deployed releases") {
			if !v3.AppConditionForceUpgrade.IsUnknown(obj) {
				v3.AppConditionForceUpgrade.Unknown(obj)
			}
			return errors.New(v3.AppConditionInstalled.GetMessage(obj))
		}
		return errors.Errorf("failed to install app %s. %s", obj.Name, stderrBuf.String())
	}
	return nil
}

// getTimeoutArgs generates the appropriate arguments for helm depending on the app's timeout and wait fields
func getTimeoutArgs(app *v3.App) []string {
	var timeoutArgs []string

	if app.Spec.Wait {
		timeoutArgs = append(timeoutArgs, "--wait")
	}
	if app.Status.HelmVersion == HelmV3 {
		timeoutArgs = append(timeoutArgs, "--timeout", fmt.Sprintf("%ss", strconv.Itoa(app.Spec.Timeout)))
	} else {
		timeoutArgs = append(timeoutArgs, "--timeout", strconv.Itoa(app.Spec.Timeout))
	}
	return timeoutArgs
}

func GenerateAnswerSetValues(app *v3.App, tempDir *HelmPath, extraArgs map[string]string) ([]string, error) {
	setValues := []string{}
	// a user-supplied values file will overridden default values.yaml
	if app.Spec.ValuesYaml != "" {
		custom := "custom-values.yaml"
		valuesYaml := filepath.Join(tempDir.FullPath, custom)
		if err := ioutil.WriteFile(valuesYaml, []byte(app.Spec.ValuesYaml), 0755); err != nil {
			return setValues, err
		}
		setValues = append(setValues, "--values", filepath.Join(tempDir.InJailPath, custom))
	}
	// `--set` values will overridden the user-supplied values.yaml file
	if app.Spec.Answers != nil || extraArgs != nil {
		answers := app.Spec.Answers
		var values = []string{}
		for k, v := range answers {
			if _, ok := extraArgs[k]; ok {
				continue
			}
			// helm will only accept escaped commas in values
			escapedValue := fmt.Sprintf("%s=%s", k, escapeCommas(v))
			values = append(values, escapedValue)
		}
		for k, v := range extraArgs {
			escapedValue := fmt.Sprintf("%s=%s", k, escapeCommas(v))
			values = append(values, escapedValue)
		}
		setValues = append(setValues, "--set", strings.Join(values, ","))
	}
	return setValues, nil
}

func DeleteCharts(tempDirs *HelmPath, port string, obj *v3.App) error {
	var cmd *exec.Cmd
	if IsHelm3(obj.Status.HelmVersion) {
		logrus.Infof("Deleting chart using helm version: %s", HelmV3)
		cmd = exec.Command(HelmV3, "uninstall", obj.Name, "--kubeconfig", tempDirs.KubeConfigInJail, "--namespace", obj.Spec.TargetNamespace)
	} else {
		logrus.Infof("Deleting chart using helm version: %s", HelmV2)
		cmd = exec.Command(HelmV2, "delete", "--purge", obj.Name)
	}
	cmd.Env = []string{fmt.Sprintf("%s=%s", "HELM_HOST", "127.0.0.1:"+port)}
	cmd, err := JailCommand(cmd, tempDirs.FullPath)
	if err != nil {
		return err
	}
	combinedOutput, err := cmd.CombinedOutput()
	if err != nil && combinedOutput != nil && strings.Contains(string(combinedOutput), fmt.Sprintf("Error: release: \"%s\" not found", obj.Name)) {
		return nil
	}
	return errors.New(string(combinedOutput))
}

func JailCommand(cmd *exec.Cmd, jailPath string) (*exec.Cmd, error) {
	if os.Getenv("CATTLE_DEV_MODE") != "" {
		return cmd, nil
	}

	cred, err := jailer.GetUserCred()
	if err != nil {
		return nil, errors.WithMessage(err, "get user cred error")
	}

	cmd.SysProcAttr = &syscall.SysProcAttr{}
	cmd.SysProcAttr.Credential = cred
	cmd.SysProcAttr.Chroot = jailPath
	cmd.Env = jailer.WhitelistEnvvars(cmd.Env)
	cmd.Env = append(cmd.Env, "PWD=/")
	cmd.Dir = "/"
	return cmd, nil
}

// escapeCommas will escape the commas in a string, unless helm would identify it as a list
func escapeCommas(value string) string {
	if len(value) == 0 {
		return value
	}
	// as far as helm is concerned, an answer starting with '{' is a list
	// commas in lists are not escaped for users as they are also used as delimiters
	if strings.HasPrefix(value, "{") {
		return value
	}
	return strings.Replace(value, ",", "\\,", -1)
}

// generates the kustomization yaml that is used by the helm 3 post rendered to
// add the app labels needed to track the workloads with the app deployment
func createKustomizeFiles(tempDirs *HelmPath, labelValue string) error {
	var resources []string
	resources = append(resources, "all.yaml")
	k := Kustomization{Label{labelValue}, resources}
	y, err := yaml.Marshal(k)
	if err != nil {
		return err
	}
	kpath := filepath.Join(tempDirs.FullPath, "/kustomization.yaml")
	err = ioutil.WriteFile(kpath, y, 0644)
	if err != nil {
		return err
	}
	logrus.Debugf("successfully created kustomization.yaml in %s", kpath)

	// set file link to ensure kustomize.sh works in dev move without having to copy
	// the file to the "injail" location
	if os.Getenv("CATTLE_DEV_MODE") != "" {
		kpath = filepath.Join(tempDirs.InJailPath, "/kustomize.sh")
		err = os.Link("./package/kustomize.sh", kpath)
		if err != nil {
			err = os.Link(filepath.Join(os.Getenv("DAPPER_SOURCE"), "package/kustomize.sh"), kpath)
			if err != nil {
				return err
			}
		}
		logrus.Debugf("successfully linked kustomize.sh to %s", kpath)
	}
	return nil
}

func IsHelm3(helmName string) bool {
	if helmName == HelmV3 {
		return true
	}
	return false
}
<commit_msg>Add max history helm3<commit_after>package common

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/ghodss/yaml"
	"github.com/pkg/errors"
	"github.com/rancher/rancher/pkg/jailer"
	"github.com/rancher/rancher/pkg/namespace"
	v3 "github.com/rancher/types/apis/project.cattle.io/v3"
	"github.com/sirupsen/logrus"
)

const (
	base            = 32768
	end             = 61000
	tillerName      = "rancher-tiller"
	HelmV2          = "rancher-helm"
	HelmV3          = "helm_v3"
	forceUpgradeStr = "--force"
	maxHistory      = "10"
)

type HelmPath struct {
	// /opt/jail/<app-name>
	FullPath string
	// /
	InJailPath string
	// /opt/jail/<app-name>/.kubeconfig
	KubeConfigFull string
	// /.kubeconfig
	KubeConfigInJail string
	// /opt/jail/<app-name>/<app-sub>
	AppDirFull string
	// /<app-sub>
	AppDirInJail string
	// /opt/jail/<app-name>/kustomize.sh
	KustomizeFull string
	// /kustomize.sh
	KustomizeInJail string
}

//Labels that need added for kustomization
type Label struct {
	AppLabel string `json:"io.cattle.field/appId"`
}

//Marshal kustomization settings into YAML
type Kustomization struct {
	CommonLabel Label    `json:"commonLabels"`
	Resrouces   []string `json:"resources"`
}

func ParseExternalID(externalID string) (string, string, error) {
	templateVersionNamespace, catalog, _, template, version, err := SplitExternalID(externalID)
	if err != nil {
		return "", "", err
	}
	return strings.Join([]string{catalog, template, version}, "-"), templateVersionNamespace, nil
}

func SplitExternalID(externalID string) (string, string, string, string, string, error) {
	var templateVersionNamespace, catalog string
	values, err := url.Parse(externalID)
	if err != nil {
		return "", "", "", "", "", err
	}
	catalogWithNamespace := values.Query().Get("catalog")
	catalogType := values.Query().Get("type")
	template := values.Query().Get("template")
	version := values.Query().Get("version")
	split := strings.SplitN(catalogWithNamespace, "/", 2)
	if len(split) == 2 {
		templateVersionNamespace = split[0]
		catalog = split[1]
	}
	//pre-upgrade setups will have global catalogs, where externalId field on templateversions won't have namespace.
	// since these are global catalogs, we can default to global namespace
	if templateVersionNamespace == "" {
		templateVersionNamespace = namespace.GlobalNamespace
		catalog = catalogWithNamespace
	}
	return templateVersionNamespace, catalog, catalogType, template, version, nil
}

// StartTiller start tiller server and return the listening address of the grpc address
func StartTiller(context context.Context, tempDirs *HelmPath, port, namespace string) error {
	probePort := GenerateRandomPort()
	cmd := exec.Command(tillerName, "--listen", ":"+port, "--probe-listen", ":"+probePort)
	cmd.Env = []string{fmt.Sprintf("%s=%s", "KUBECONFIG", tempDirs.KubeConfigInJail), fmt.Sprintf("%s=%s", "TILLER_NAMESPACE", namespace), fmt.Sprintf("%s=%s", "TILLER_HISTORY_MAX", maxHistory)}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	cmd, err := JailCommand(cmd, tempDirs.FullPath)
	if err != nil {
		return err
	}

	if err := cmd.Start(); err != nil {
		return err
	}
	defer cmd.Wait()
	select {
	case <-context.Done():
		logrus.Debug("Stopping Tiller")
		return cmd.Process.Kill()
	}
}

func GenerateRandomPort() string {
	s1 := rand.NewSource(time.Now().UnixNano())
	r1 := rand.New(s1)
	for {
		port := base + r1.Intn(end-base+1)
		ln, err := net.Listen("tcp", ":"+strconv.Itoa(port))
		if err != nil {
			continue
		}
		ln.Close()
		return strconv.Itoa(port)
	}
}

func InstallCharts(tempDirs *HelmPath, port string, obj *v3.App) error {
	logrus.Debugf("InstallCharts - helm path info %+v\n", tempDirs)
	extraArgs := GetExtraArgs(obj)
	timeoutArgs := getTimeoutArgs(obj)
	setValues, err := GenerateAnswerSetValues(obj, tempDirs, extraArgs)
	if err != nil {
		return err
	}
	commands := make([]string, 0)
	if IsHelm3(obj.Status.HelmVersion) {
		err = createKustomizeFiles(tempDirs, obj.Name)
		if err != nil {
			return err
		}
		logrus.Infof("Installing chart using helm version: %s", HelmV3)
		commands = append([]string{"upgrade", "--install", obj.Name, "--namespace", obj.Spec.TargetNamespace, "--kubeconfig", tempDirs.KubeConfigInJail, "--post-renderer", tempDirs.KustomizeInJail, "--history-max", maxHistory}, timeoutArgs...)
	} else {
		logrus.Infof("Installing chart using helm version: %s", HelmV2)
		commands = append([]string{"upgrade", "--install", "--namespace", obj.Spec.TargetNamespace, obj.Name}, timeoutArgs...)
	}
	commands = append(commands, setValues...)
	commands = append(commands, tempDirs.AppDirInJail)

	if v3.AppConditionForceUpgrade.IsUnknown(obj) {
		commands = append(commands, forceUpgradeStr)
		// don't leave force recreate on the object
		v3.AppConditionForceUpgrade.True(obj)
	}
	var cmd *exec.Cmd
	// switch userTriggeredAction back
	v3.AppConditionUserTriggeredAction.Unknown(obj)
	if IsHelm3(obj.Status.HelmVersion) {
		cmd = exec.Command(HelmV3, commands...)
	} else {
		cmd = exec.Command(HelmV2, commands...)
	}
	cmd.Env = []string{fmt.Sprintf("%s=%s", "HELM_HOST", "127.0.0.1:"+port), fmt.Sprintf("%s=%s", "CATTLE_KUSTOMIZE_YAML", tempDirs.InJailPath)}
	stderrBuf := &bytes.Buffer{}
	cmd.Stdout = os.Stdout
	cmd.Stderr = stderrBuf
	cmd, err = JailCommand(cmd, tempDirs.FullPath)
	if err != nil {
		return err
	}
	if err := cmd.Start(); err != nil {
		return errors.Errorf("failed to install app %s. %s", obj.Name, stderrBuf.String())
	}
	if err := cmd.Wait(); err != nil {
		// if the first install failed, the second install would have error message like `has no deployed releases`, then the
		// original error is masked. We need to filter out the message and always return the last one if error matches this pattern
		if strings.Contains(stderrBuf.String(), "has no deployed releases") {
			if !v3.AppConditionForceUpgrade.IsUnknown(obj) {
				v3.AppConditionForceUpgrade.Unknown(obj)
			}
			return errors.New(v3.AppConditionInstalled.GetMessage(obj))
		}
		return errors.Errorf("failed to install app %s. %s", obj.Name, stderrBuf.String())
	}
	return nil
}

// getTimeoutArgs generates the appropriate arguments for helm depending on the app's timeout and wait fields
func getTimeoutArgs(app *v3.App) []string {
	var timeoutArgs []string

	if app.Spec.Wait {
		timeoutArgs = append(timeoutArgs, "--wait")
	}
	if app.Status.HelmVersion == HelmV3 {
		timeoutArgs = append(timeoutArgs, "--timeout", fmt.Sprintf("%ss", strconv.Itoa(app.Spec.Timeout)))
	} else {
		timeoutArgs = append(timeoutArgs, "--timeout", strconv.Itoa(app.Spec.Timeout))
	}
	return timeoutArgs
}

func GenerateAnswerSetValues(app *v3.App, tempDir *HelmPath, extraArgs map[string]string) ([]string, error) {
	setValues := []string{}
	// a user-supplied values file will overridden default values.yaml
	if app.Spec.ValuesYaml != "" {
		custom := "custom-values.yaml"
		valuesYaml := filepath.Join(tempDir.FullPath, custom)
		if err := ioutil.WriteFile(valuesYaml, []byte(app.Spec.ValuesYaml), 0755); err != nil {
			return setValues, err
		}
		setValues = append(setValues, "--values", filepath.Join(tempDir.InJailPath, custom))
	}
	// `--set` values will overridden the user-supplied values.yaml file
	if app.Spec.Answers != nil || extraArgs != nil {
		answers := app.Spec.Answers
		var values = []string{}
		for k, v := range answers {
			if _, ok := extraArgs[k]; ok {
				continue
			}
			// helm will only accept escaped commas in values
			escapedValue := fmt.Sprintf("%s=%s", k, escapeCommas(v))
			values = append(values, escapedValue)
		}
		for k, v := range extraArgs {
			escapedValue := fmt.Sprintf("%s=%s", k, escapeCommas(v))
			values = append(values, escapedValue)
		}
		setValues = append(setValues, "--set", strings.Join(values, ","))
	}
	return setValues, nil
}

func DeleteCharts(tempDirs *HelmPath, port string, obj *v3.App) error {
	var cmd *exec.Cmd
	if IsHelm3(obj.Status.HelmVersion) {
		logrus.Infof("Deleting chart using helm version: %s", HelmV3)
		cmd = exec.Command(HelmV3, "uninstall", obj.Name, "--kubeconfig", tempDirs.KubeConfigInJail, "--namespace", obj.Spec.TargetNamespace)
	} else {
		logrus.Infof("Deleting chart using helm version: %s", HelmV2)
		cmd = exec.Command(HelmV2, "delete", "--purge", obj.Name)
	}
	cmd.Env = []string{fmt.Sprintf("%s=%s", "HELM_HOST", "127.0.0.1:"+port)}
	cmd, err := JailCommand(cmd, tempDirs.FullPath)
	if err != nil {
		return err
	}
	combinedOutput, err := cmd.CombinedOutput()
	if err != nil && combinedOutput != nil && strings.Contains(string(combinedOutput), fmt.Sprintf("Error: release: \"%s\" not found", obj.Name)) {
		return nil
	}
	return errors.New(string(combinedOutput))
}

func JailCommand(cmd *exec.Cmd, jailPath string) (*exec.Cmd, error) {
	if os.Getenv("CATTLE_DEV_MODE") != "" {
		return cmd, nil
	}

	cred, err := jailer.GetUserCred()
	if err != nil {
		return nil, errors.WithMessage(err, "get user cred error")
	}

	cmd.SysProcAttr = &syscall.SysProcAttr{}
	cmd.SysProcAttr.Credential = cred
	cmd.SysProcAttr.Chroot = jailPath
	cmd.Env = jailer.WhitelistEnvvars(cmd.Env)
	cmd.Env = append(cmd.Env, "PWD=/")
	cmd.Dir = "/"
	return cmd, nil
}

// escapeCommas will escape the commas in a string, unless helm would identify it as a list
func escapeCommas(value string) string {
	if len(value) == 0 {
		return value
	}
	// as far as helm is concerned, an answer starting with '{' is a list
	// commas in lists are not escaped for users as they are also used as delimiters
	if strings.HasPrefix(value, "{") {
		return value
	}
	return strings.Replace(value, ",", "\\,", -1)
}

// generates the kustomization yaml that is used by the helm 3 post rendered to
// add the app labels needed to track the workloads with the app deployment
func createKustomizeFiles(tempDirs *HelmPath, labelValue string) error {
	var resources []string
	resources = append(resources, "all.yaml")
	k := Kustomization{Label{labelValue}, resources}
	y, err := yaml.Marshal(k)
	if err != nil {
		return err
	}
	kpath := filepath.Join(tempDirs.FullPath, "/kustomization.yaml")
	err = ioutil.WriteFile(kpath, y, 0644)
	if err != nil {
		return err
	}
	logrus.Debugf("successfully created kustomization.yaml in %s", kpath)

	// set file link to ensure kustomize.sh works in dev move without having to copy
	// the file to the "injail" location
	if os.Getenv("CATTLE_DEV_MODE") != "" {
		kpath = filepath.Join(tempDirs.InJailPath, "/kustomize.sh")
		err = os.Link("./package/kustomize.sh", kpath)
		if err != nil {
			err = os.Link(filepath.Join(os.Getenv("DAPPER_SOURCE"), "package/kustomize.sh"), kpath)
			if err != nil {
				return err
			}
		}
		logrus.Debugf("successfully linked kustomize.sh to %s", kpath)
	}
	return nil
}

func IsHelm3(helmName string) bool {
	if helmName == HelmV3 {
		return true
	}
	return false
}
<|endoftext|>