<commit_before>package app

import (
	"sort"

	"github.com/golang/glog"
)

// Resolver is an interface for resolving provided input to component matches.
// A Resolver should return ErrMultipleMatches when more than one result can
// be constructed as a match. It should also set the score to 0.0 if this is a
// perfect match, and to higher values the less adequate the match is.
type Resolver interface {
	Resolve(value string) (*ComponentMatch, error)
}

// Searcher is responsible for performing a search based on the given terms and return
// all results found as component matches. Notice they can even return zero or multiple
// matches, meaning they will never return ErrNoMatch or ErrMultipleMatches and any error
// returned is an actual error. The component match score can be used to determine how
// precise a given match is, where 0.0 is an exact match.
type Searcher interface {
	Search(terms ...string) (ComponentMatches, error)
}

// WeightedResolver is a resolver identified as exact or not, depending on its weight
type WeightedResolver struct {
	Searcher
	Weight float32
}

// PerfectMatchWeightedResolver returns only matches from resolvers that are identified as exact
// (weight 0.0), and only matches from those resolvers that qualify as exact (score = 0.0). If no
// perfect matches exist, an ErrMultipleMatches is returned indicating the remaining candidate(s).
// Note that this method may resolve ErrMultipleMatches with a single match, indicating an error
// (no perfect match) but with only one candidate.
type PerfectMatchWeightedResolver []WeightedResolver

// Resolve resolves the provided input and returns only exact matches
func (r PerfectMatchWeightedResolver) Resolve(value string) (*ComponentMatch, error) {
	imperfect := ScoredComponentMatches{}
	group := []WeightedResolver{}
	for i, resolver := range r {
		if len(group) == 0 || resolver.Weight == group[0].Weight {
			group = append(group, resolver)
			if i != len(r)-1 && r[i+1].Weight == group[0].Weight {
				continue
			}
		}
		exact, inexact, err := resolveExact(WeightedResolvers(group), value)
		switch {
		case exact != nil:
			if exact.Score == 0.0 {
				return exact, nil
			}
			if resolver.Weight != 0.0 {
				exact.Score = resolver.Weight * exact.Score
			}
			imperfect = append(imperfect, exact)
		case len(inexact) > 0:
			sort.Sort(ScoredComponentMatches(inexact))
			if inexact[0].Score == 0.0 && (len(inexact) == 1 || inexact[1].Score != 0.0) {
				return inexact[0], nil
			}
			for _, m := range inexact {
				if resolver.Weight != 0.0 {
					m.Score = resolver.Weight * m.Score
				}
				imperfect = append(imperfect, m)
			}
		case err != nil:
			glog.V(2).Infof("Error from resolver: %v\n", err)
		}
		group = nil
	}
	switch len(imperfect) {
	case 0:
		return nil, ErrNoMatch{value: value}
	case 1:
		return imperfect[0], nil
	default:
		sort.Sort(imperfect)
		if imperfect[0].Score < imperfect[1].Score {
			return imperfect[0], nil
		}
		return nil, ErrMultipleMatches{value, imperfect}
	}
}

// WeightedResolvers is a set of weighted resolvers
type WeightedResolvers []WeightedResolver

// Resolve resolves the provided input and returns both exact and inexact matches
func (r WeightedResolvers) Resolve(value string) (*ComponentMatch, error) {
	candidates := []*ComponentMatch{}
	errs := []error{}
	for _, resolver := range r {
		exact, inexact, err := searchExact(resolver.Searcher, value)
		switch {
		case exact != nil:
			candidates = append(candidates, exact)
		case len(inexact) > 0:
			candidates = append(candidates, inexact...)
		case err != nil:
			errs = append(errs, err)
		}
	}
	if len(errs) != 0 {
		glog.V(2).Infof("Errors occurred during resolution: %#v", errs)
	}
	switch len(candidates) {
	case 0:
		return nil, ErrNoMatch{value: value}
	case 1:
		return candidates[0], nil
	default:
		return nil, ErrMultipleMatches{value, candidates}
	}
}

// FirstMatchResolver simply takes the first search result returned by the
// searcher it holds and resolves it to that match. An ErrMultipleMatches will
// never happen given it will just take the first result, but a ErrNoMatch can
// happen if the searcher returns no matches.
type FirstMatchResolver struct {
	Searcher Searcher
}

// Resolve resolves as the first match returned by the Searcher
func (r FirstMatchResolver) Resolve(value string) (*ComponentMatch, error) {
	matches, err := r.Searcher.Search(value)
	if err != nil {
		return nil, err
	}
	if len(matches) == 0 {
		return nil, ErrNoMatch{value: value}
	}
	return matches[0], nil
}

// HighestScoreResolver takes search result returned by the searcher it holds
// and resolves it to the highest scored match present. An ErrMultipleMatches
// will never happen given it will just take the best scored result, but a
// ErrNoMatch can happen if the searcher returns no matches.
type HighestScoreResolver struct {
	Searcher Searcher
}

// Resolve resolves as the first highest scored match returned by the Searcher
func (r HighestScoreResolver) Resolve(value string) (*ComponentMatch, error) {
	matches, err := r.Searcher.Search(value)
	if err != nil {
		return nil, err
	}
	if len(matches) == 0 {
		return nil, ErrNoMatch{value: value}
	}
	sort.Sort(ScoredComponentMatches(matches))
	return matches[0], nil
}

// HighestUniqueScoreResolver takes search result returned by the searcher it
// holds and resolves it to the highest scored match present. If more than one
// match exists with that same score, returns an ErrMultipleMatches. A ErrNoMatch
// can happen if the searcher returns no matches.
type HighestUniqueScoreResolver struct {
	Searcher Searcher
}

// Resolve resolves as the highest scored match returned by the Searcher, and
// guarantees the match is unique (the only match with that given score)
func (r HighestUniqueScoreResolver) Resolve(value string) (*ComponentMatch, error) {
	matches, err := r.Searcher.Search(value)
	if err != nil {
		return nil, err
	}
	sort.Sort(ScoredComponentMatches(matches))
	switch len(matches) {
	case 0:
		return nil, ErrNoMatch{value: value}
	case 1:
		return matches[0], nil
	default:
		if matches[0].Score == matches[1].Score {
			return nil, ErrMultipleMatches{value, matches}
		}
		return matches[0], nil
	}
}

// UniqueExactOrInexactMatchResolver takes search result returned by the searcher
// it holds. Returns the single exact match present, if more that one exact match
// is present, returns a ErrMultipleMatches. If no exact match is present, try with
// inexact ones, which must also be unique otherwise ErrMultipleMatches. A ErrNoMatch
// can happen if the searcher returns no exact or inexact matches.
type UniqueExactOrInexactMatchResolver struct {
	Searcher Searcher
}

// Resolve resolves as the single exact or inexact match present
func (r UniqueExactOrInexactMatchResolver) Resolve(value string) (*ComponentMatch, error) {
	matches, err := r.Searcher.Search(value)
	if err != nil {
		return nil, err
	}
	sort.Sort(ScoredComponentMatches(matches))

	exact := matches.Exact()
	switch len(exact) {
	case 0:
		inexact := matches.Inexact()
		switch len(inexact) {
		case 0:
			return nil, ErrNoMatch{value: value}
		case 1:
			return inexact[0], nil
		default:
			return nil, ErrMultipleMatches{value, exact}
		}
	case 1:
		return exact[0], nil
	default:
		return nil, ErrMultipleMatches{value, exact}
	}
}

// MultiSimpleSearcher is a set of searchers
type MultiSimpleSearcher []Searcher

// Search searches using all searchers it holds
func (s MultiSimpleSearcher) Search(terms ...string) (ComponentMatches, error) {
	componentMatches := ComponentMatches{}
	for _, searcher := range s {
		matches, err := searcher.Search(terms...)
		if err != nil {
			glog.V(2).Infof("Error occurred during search: %#v", err)
			continue
		}
		componentMatches = append(componentMatches, matches...)
	}
	sort.Sort(ScoredComponentMatches(componentMatches))
	return componentMatches, nil
}

// WeightedSearcher is a searcher identified as exact or not, depending on its weight
type WeightedSearcher struct {
	Searcher
	Weight float32
}

// MultiWeightedSearcher is a set of weighted searchers where lower weight has higher
// priority in search results
type MultiWeightedSearcher []WeightedSearcher

// Search searches using all searchers it holds and score according to searcher height
func (s MultiWeightedSearcher) Search(terms ...string) (ComponentMatches, error) {
	componentMatches := ComponentMatches{}
	for _, searcher := range s {
		matches, err := searcher.Search(terms...)
		if err != nil {
			glog.V(2).Infof("Error occurred during search: %#v", err)
			continue
		}
		for _, match := range matches {
			match.Score += searcher.Weight
			componentMatches = append(componentMatches, match)
		}
	}
	sort.Sort(ScoredComponentMatches(componentMatches))
	return componentMatches, nil
}

func resolveExact(resolver Resolver, value string) (exact *ComponentMatch, inexact []*ComponentMatch, err error) {
	match, err := resolver.Resolve(value)
	if err != nil {
		switch t := err.(type) {
		case ErrNoMatch:
			return nil, nil, nil
		case ErrMultipleMatches:
			return nil, t.Matches, nil
		default:
			return nil, nil, err
		}
	}
	return match, nil, nil
}

func searchExact(searcher Searcher, value string) (exact *ComponentMatch, inexact []*ComponentMatch, err error) {
	matches, err := searcher.Search(value)
	if err != nil {
		return nil, nil, err
	}

	exactMatches := matches.Exact()
	inexactMatches := matches.Inexact()

	switch len(exactMatches) {
	case 0:
		return nil, inexactMatches, nil
	case 1:
		return exactMatches[0], inexactMatches, nil
	default:
		return nil, nil, ErrMultipleMatches{value, exactMatches}
	}
}
<commit_msg>Show corrent error message if passed json template is invalid<commit_after>package app

import (
	"sort"

	"github.com/golang/glog"
)

// Resolver is an interface for resolving provided input to component matches.
// A Resolver should return ErrMultipleMatches when more than one result can
// be constructed as a match. It should also set the score to 0.0 if this is a
// perfect match, and to higher values the less adequate the match is.
type Resolver interface {
	Resolve(value string) (*ComponentMatch, error)
}

// Searcher is responsible for performing a search based on the given terms and return
// all results found as component matches. Notice they can even return zero or multiple
// matches, meaning they will never return ErrNoMatch or ErrMultipleMatches and any error
// returned is an actual error. The component match score can be used to determine how
// precise a given match is, where 0.0 is an exact match.
type Searcher interface {
	Search(terms ...string) (ComponentMatches, error)
}

// WeightedResolver is a resolver identified as exact or not, depending on its weight
type WeightedResolver struct {
	Searcher
	Weight float32
}

// PerfectMatchWeightedResolver returns only matches from resolvers that are identified as exact
// (weight 0.0), and only matches from those resolvers that qualify as exact (score = 0.0). If no
// perfect matches exist, an ErrMultipleMatches is returned indicating the remaining candidate(s).
// Note that this method may resolve ErrMultipleMatches with a single match, indicating an error
// (no perfect match) but with only one candidate.
type PerfectMatchWeightedResolver []WeightedResolver

// Resolve resolves the provided input and returns only exact matches
func (r PerfectMatchWeightedResolver) Resolve(value string) (*ComponentMatch, error) {
	imperfect := ScoredComponentMatches{}
	group := []WeightedResolver{}
	for i, resolver := range r {
		if len(group) == 0 || resolver.Weight == group[0].Weight {
			group = append(group, resolver)
			if i != len(r)-1 && r[i+1].Weight == group[0].Weight {
				continue
			}
		}
		exact, inexact, err := resolveExact(WeightedResolvers(group), value)
		switch {
		case exact != nil:
			if exact.Score == 0.0 {
				return exact, nil
			}
			if resolver.Weight != 0.0 {
				exact.Score = resolver.Weight * exact.Score
			}
			imperfect = append(imperfect, exact)
		case len(inexact) > 0:
			sort.Sort(ScoredComponentMatches(inexact))
			if inexact[0].Score == 0.0 && (len(inexact) == 1 || inexact[1].Score != 0.0) {
				return inexact[0], nil
			}
			for _, m := range inexact {
				if resolver.Weight != 0.0 {
					m.Score = resolver.Weight * m.Score
				}
				imperfect = append(imperfect, m)
			}
		case err != nil:
			glog.V(2).Infof("Error from resolver: %v\n", err)
		}
		group = nil
	}
	switch len(imperfect) {
	case 0:
		// If value is a file and there is a TemplateFileSearcher in one of the resolvers
		// and trying to use it gives an error, use this error instead of ErrNoMatch.
		// E.g., calling `oc new-app template.json` where template.json is a file
		// with invalid JSON, it's better to return the JSON syntax error than a more
		// generic message.
		if isFile(value) {
			for _, resolver := range r {
				if _, ok := resolver.Searcher.(*TemplateFileSearcher); ok {
					if _, err := resolver.Search(value); err != nil {
						return nil, err
					}
				}
			}
		}
		return nil, ErrNoMatch{value: value}
	case 1:
		return imperfect[0], nil
	default:
		sort.Sort(imperfect)
		if imperfect[0].Score < imperfect[1].Score {
			return imperfect[0], nil
		}
		return nil, ErrMultipleMatches{value, imperfect}
	}
}

// WeightedResolvers is a set of weighted resolvers
type WeightedResolvers []WeightedResolver

// Resolve resolves the provided input and returns both exact and inexact matches
func (r WeightedResolvers) Resolve(value string) (*ComponentMatch, error) {
	candidates := []*ComponentMatch{}
	errs := []error{}
	for _, resolver := range r {
		exact, inexact, err := searchExact(resolver.Searcher, value)
		switch {
		case exact != nil:
			candidates = append(candidates, exact)
		case len(inexact) > 0:
			candidates = append(candidates, inexact...)
		case err != nil:
			errs = append(errs, err)
		}
	}
	if len(errs) != 0 {
		glog.V(2).Infof("Errors occurred during resolution: %#v", errs)
	}
	switch len(candidates) {
	case 0:
		return nil, ErrNoMatch{value: value}
	case 1:
		return candidates[0], nil
	default:
		return nil, ErrMultipleMatches{value, candidates}
	}
}

// FirstMatchResolver simply takes the first search result returned by the
// searcher it holds and resolves it to that match. An ErrMultipleMatches will
// never happen given it will just take the first result, but a ErrNoMatch can
// happen if the searcher returns no matches.
type FirstMatchResolver struct {
	Searcher Searcher
}

// Resolve resolves as the first match returned by the Searcher
func (r FirstMatchResolver) Resolve(value string) (*ComponentMatch, error) {
	matches, err := r.Searcher.Search(value)
	if err != nil {
		return nil, err
	}
	if len(matches) == 0 {
		return nil, ErrNoMatch{value: value}
	}
	return matches[0], nil
}

// HighestScoreResolver takes search result returned by the searcher it holds
// and resolves it to the highest scored match present. An ErrMultipleMatches
// will never happen given it will just take the best scored result, but a
// ErrNoMatch can happen if the searcher returns no matches.
type HighestScoreResolver struct {
	Searcher Searcher
}

// Resolve resolves as the first highest scored match returned by the Searcher
func (r HighestScoreResolver) Resolve(value string) (*ComponentMatch, error) {
	matches, err := r.Searcher.Search(value)
	if err != nil {
		return nil, err
	}
	if len(matches) == 0 {
		return nil, ErrNoMatch{value: value}
	}
	sort.Sort(ScoredComponentMatches(matches))
	return matches[0], nil
}

// HighestUniqueScoreResolver takes search result returned by the searcher it
// holds and resolves it to the highest scored match present. If more than one
// match exists with that same score, returns an ErrMultipleMatches. A ErrNoMatch
// can happen if the searcher returns no matches.
type HighestUniqueScoreResolver struct {
	Searcher Searcher
}

// Resolve resolves as the highest scored match returned by the Searcher, and
// guarantees the match is unique (the only match with that given score)
func (r HighestUniqueScoreResolver) Resolve(value string) (*ComponentMatch, error) {
	matches, err := r.Searcher.Search(value)
	if err != nil {
		return nil, err
	}
	sort.Sort(ScoredComponentMatches(matches))
	switch len(matches) {
	case 0:
		return nil, ErrNoMatch{value: value}
	case 1:
		return matches[0], nil
	default:
		if matches[0].Score == matches[1].Score {
			return nil, ErrMultipleMatches{value, matches}
		}
		return matches[0], nil
	}
}

// UniqueExactOrInexactMatchResolver takes search result returned by the searcher
// it holds. Returns the single exact match present, if more that one exact match
// is present, returns a ErrMultipleMatches. If no exact match is present, try with
// inexact ones, which must also be unique otherwise ErrMultipleMatches. A ErrNoMatch
// can happen if the searcher returns no exact or inexact matches.
type UniqueExactOrInexactMatchResolver struct {
	Searcher Searcher
}

// Resolve resolves as the single exact or inexact match present
func (r UniqueExactOrInexactMatchResolver) Resolve(value string) (*ComponentMatch, error) {
	matches, err := r.Searcher.Search(value)
	if err != nil {
		return nil, err
	}
	sort.Sort(ScoredComponentMatches(matches))

	exact := matches.Exact()
	switch len(exact) {
	case 0:
		inexact := matches.Inexact()
		switch len(inexact) {
		case 0:
			return nil, ErrNoMatch{value: value}
		case 1:
			return inexact[0], nil
		default:
			return nil, ErrMultipleMatches{value, exact}
		}
	case 1:
		return exact[0], nil
	default:
		return nil, ErrMultipleMatches{value, exact}
	}
}

// MultiSimpleSearcher is a set of searchers
type MultiSimpleSearcher []Searcher

// Search searches using all searchers it holds
func (s MultiSimpleSearcher) Search(terms ...string) (ComponentMatches, error) {
	componentMatches := ComponentMatches{}
	for _, searcher := range s {
		matches, err := searcher.Search(terms...)
		if err != nil {
			glog.V(2).Infof("Error occurred during search: %#v", err)
			continue
		}
		componentMatches = append(componentMatches, matches...)
	}
	sort.Sort(ScoredComponentMatches(componentMatches))
	return componentMatches, nil
}

// WeightedSearcher is a searcher identified as exact or not, depending on its weight
type WeightedSearcher struct {
	Searcher
	Weight float32
}

// MultiWeightedSearcher is a set of weighted searchers where lower weight has higher
// priority in search results
type MultiWeightedSearcher []WeightedSearcher

// Search searches using all searchers it holds and score according to searcher height
func (s MultiWeightedSearcher) Search(terms ...string) (ComponentMatches, error) {
	componentMatches := ComponentMatches{}
	for _, searcher := range s {
		matches, err := searcher.Search(terms...)
		if err != nil {
			glog.V(2).Infof("Error occurred during search: %#v", err)
			continue
		}
		for _, match := range matches {
			match.Score += searcher.Weight
			componentMatches = append(componentMatches, match)
		}
	}
	sort.Sort(ScoredComponentMatches(componentMatches))
	return componentMatches, nil
}

func resolveExact(resolver Resolver, value string) (exact *ComponentMatch, inexact []*ComponentMatch, err error) {
	match, err := resolver.Resolve(value)
	if err != nil {
		switch t := err.(type) {
		case ErrNoMatch:
			return nil, nil, nil
		case ErrMultipleMatches:
			return nil, t.Matches, nil
		default:
			return nil, nil, err
		}
	}
	return match, nil, nil
}

func searchExact(searcher Searcher, value string) (exact *ComponentMatch, inexact []*ComponentMatch, err error) {
	matches, err := searcher.Search(value)
	if err != nil {
		return nil, nil, err
	}

	exactMatches := matches.Exact()
	inexactMatches := matches.Inexact()

	switch len(exactMatches) {
	case 0:
		return nil, inexactMatches, nil
	case 1:
		return exactMatches[0], inexactMatches, nil
	default:
		return nil, nil, ErrMultipleMatches{value, exactMatches}
	}
}
<|endoftext|>


<commit_before>package jenkins

import (
	"fmt"
	"regexp"
	"strings"

	images "github.com/rancher/rancher/pkg/image"
	"github.com/rancher/rancher/pkg/pipeline/utils"
	"github.com/rancher/rancher/pkg/ref"
	"github.com/rancher/rancher/pkg/settings"
	mv3 "github.com/rancher/types/apis/management.cattle.io/v3"
	v3 "github.com/rancher/types/apis/project.cattle.io/v3"
	"gopkg.in/yaml.v2"
	v1 "k8s.io/api/core/v1"
)

func (c *jenkinsPipelineConverter) getStepContainer(stageOrdinal int, stepOrdinal int) (v1.Container, error) {
	stage := c.execution.Spec.PipelineConfig.Stages[stageOrdinal]
	step := &stage.Steps[stepOrdinal]

	container := v1.Container{
		Name:    fmt.Sprintf("step-%d-%d", stageOrdinal, stepOrdinal),
		TTY:     true,
		Command: []string{"cat"},
		Env:     []v1.EnvVar{},
	}
	if step.SourceCodeConfig != nil {
		if err := c.configCloneStepContainer(&container, step); err != nil {
			return container, err
		}
	} else if step.RunScriptConfig != nil {
		c.configRunScriptStepContainer(&container, step)
	} else if step.PublishImageConfig != nil {
		c.configPublishStepContainer(&container, step)
	} else if step.ApplyYamlConfig != nil {
		if err := c.configApplyYamlStepContainer(&container, step, stageOrdinal); err != nil {
			return container, err
		}
	} else if step.PublishCatalogConfig != nil {
		if err := c.configPublishCatalogContainer(&container, step); err != nil {
			return container, err
		}
	} else if step.ApplyAppConfig != nil {
		if err := c.configApplyAppContainer(&container, step); err != nil {
			return container, err
		}
	}

	//common step configurations
	for k, v := range utils.GetEnvVarMap(c.execution) {
		container.Env = append(container.Env, v1.EnvVar{Name: k, Value: v})
	}
	for k, v := range step.Env {
		container.Env = append(container.Env, v1.EnvVar{Name: k, Value: v})
	}
	if c.execution.Spec.Event != utils.WebhookEventPullRequest {
		//expose no secrets on pull_request events
		for _, e := range step.EnvFrom {
			envName := e.SourceKey
			if e.TargetKey != "" {
				envName = e.TargetKey
			}
			container.Env = append(container.Env, v1.EnvVar{
				Name: envName,
				ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{
					LocalObjectReference: v1.LocalObjectReference{
						Name: e.SourceName,
					},
					Key: e.SourceKey,
				}}})
		}
	}
	if step.Privileged {
		container.SecurityContext = &v1.SecurityContext{Privileged: &step.Privileged}
	}
	err := injectSetpContainerResources(&container, step)
	return container, err
}

func (c *jenkinsPipelineConverter) getJenkinsStepCommand(stageOrdinal int, stepOrdinal int) string {
	stage := c.execution.Spec.PipelineConfig.Stages[stageOrdinal]
	step := &stage.Steps[stepOrdinal]
	command := ""

	if !utils.MatchAll(stage.When, c.execution) || !utils.MatchAll(step.When, c.execution) {
		stepName := fmt.Sprintf("step-%d-%d", stageOrdinal, stepOrdinal)
		command = fmt.Sprintf(markSkipScript, stepName)
	} else if step.SourceCodeConfig != nil {
		command = fmt.Sprintf("checkout([$class: 'GitSCM', branches: [[name: 'local/temp']], userRemoteConfigs: [[url: '%s', refspec: '+%s:refs/remotes/local/temp', credentialsId: '%s']]])",
			c.execution.Spec.RepositoryURL, c.execution.Spec.Ref, c.execution.Name)
	} else if step.RunScriptConfig != nil {
		command = fmt.Sprintf(`sh ''' %s '''`, step.RunScriptConfig.ShellScript)
	} else if step.PublishImageConfig != nil {
		command = `sh '''/usr/local/bin/dockerd-entrypoint.sh /bin/drone-docker'''`
	} else if step.ApplyYamlConfig != nil {
		command = `sh ''' kube-apply '''`
	} else if step.PublishCatalogConfig != nil {
		command = `sh ''' publish-catalog '''`
	} else if step.ApplyAppConfig != nil {
		command = `sh ''' apply-app '''`
	}
	return command
}

func (c *jenkinsPipelineConverter) getAgentContainer() (v1.Container, error) {
	container := v1.Container{
		Name:  utils.JenkinsAgentContainerName,
		Image: images.Resolve(mv3.ToolsSystemImages.PipelineSystemImages.JenkinsJnlp),
		Args:  []string{"$(JENKINS_SECRET)", "$(JENKINS_NAME)"},
	}
	cloneContainer, err := c.getStepContainer(0, 0)
	if err != nil {
		return container, err
	}
	container.Env = append(container.Env, cloneContainer.Env...)
	container.EnvFrom = append(container.EnvFrom, cloneContainer.EnvFrom...)
	err = c.injectAgentResources(&container)
	return container, err
}

func (c *jenkinsPipelineConverter) configCloneStepContainer(container *v1.Container, step *v3.Step) error {
	container.Image = images.Resolve(mv3.ToolsSystemImages.PipelineSystemImages.AlpineGit)
	return injectResources(container, utils.PipelineToolsCPULimitDefault, utils.PipelineToolsCPURequestDefault, utils.PipelineToolsMemoryLimitDefault, utils.PipelineToolsMemoryRequestDefault)
}

func (c *jenkinsPipelineConverter) configRunScriptStepContainer(container *v1.Container, step *v3.Step) {
	container.Image = step.RunScriptConfig.Image
}

func (c *jenkinsPipelineConverter) configPublishStepContainer(container *v1.Container, step *v3.Step) {
	ns := utils.GetPipelineCommonName(c.execution.Spec.ProjectName)
	config := step.PublishImageConfig
	m := utils.GetEnvVarMap(c.execution)
	config.Tag = substituteEnvVar(m, config.Tag)

	registry, repo, tag := utils.SplitImageTag(config.Tag)

	if config.PushRemote {
		registry = config.Registry
	} else {
		_, projectID := ref.Parse(c.execution.Spec.ProjectName)
		registry = fmt.Sprintf("%s.%s-pipeline", utils.LocalRegistry, projectID)
	}

	reg, _ := regexp.Compile("[^a-zA-Z0-9]+")
	processedRegistry := strings.ToLower(reg.ReplaceAllString(registry, ""))
	secretName := fmt.Sprintf("%s-%s", c.execution.Namespace, processedRegistry)
	secretUserKey := utils.PublishSecretUserKey
	secretPwKey := utils.PublishSecretPwKey
	if !config.PushRemote {
		//use local registry credential
		secretName = utils.PipelineSecretName
		secretUserKey = utils.PipelineSecretUserKey
		secretPwKey = utils.PipelineSecretTokenKey
	}
	pluginRepo := fmt.Sprintf("%s/%s", registry, repo)
	if registry == utils.DefaultRegistry {
		//the `plugins/docker` image fails when setting DOCKER_REGISTRY to index.docker.io
		registry = ""
	}

	container.Image = images.Resolve(mv3.ToolsSystemImages.PipelineSystemImages.PluginsDocker)
	publishEnv := map[string]string{
		"DOCKER_REGISTRY":   registry,
		"PLUGIN_REPO":       pluginRepo,
		"PLUGIN_TAG":        tag,
		"PLUGIN_DOCKERFILE": config.DockerfilePath,
		"PLUGIN_CONTEXT":    config.BuildContext,
	}
	for k, v := range publishEnv {
		container.Env = append(container.Env, v1.EnvVar{Name: k, Value: v})
	}
	container.Env = append(container.Env, v1.EnvVar{
		Name: "DOCKER_USERNAME",
		ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{
			LocalObjectReference: v1.LocalObjectReference{
				Name: secretName,
			},
			Key: secretUserKey,
		}}})
	container.Env = append(container.Env, v1.EnvVar{
		Name: "DOCKER_PASSWORD",
		ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{
			LocalObjectReference: v1.LocalObjectReference{
				Name: secretName,
			},
			Key: secretPwKey,
		}}})
	privileged := true
	container.SecurityContext = &v1.SecurityContext{Privileged: &privileged}
	container.VolumeMounts = []v1.VolumeMount{
		{
			Name:      utils.RegistryCrtVolumeName,
			MountPath: fmt.Sprintf("/etc/docker/certs.d/docker-registry.%s", ns),
			ReadOnly:  true,
		},
	}
}

func (c *jenkinsPipelineConverter) configApplyYamlStepContainer(container *v1.Container, step *v3.Step, stageOrdinal int) error {
	config := step.ApplyYamlConfig
	container.Image = images.Resolve(mv3.ToolsSystemImages.PipelineSystemImages.KubeApply)

	applyEnv := map[string]string{
		"YAML_PATH":    config.Path,
		"YAML_CONTENT": config.Content,
		"NAMESPACE":    config.Namespace,
	}

	//for deploy step, get registry & image variable from a previous publish step
	var registry, imageRepo string
StageLoop:
	for i := stageOrdinal; i >= 0; i-- {
		stage := c.execution.Spec.PipelineConfig.Stages[i]
		for j := len(stage.Steps) - 1; j >= 0; j-- {
			step := stage.Steps[j]
			if step.PublishImageConfig != nil {
				config := step.PublishImageConfig
				if config.PushRemote {
					registry = step.PublishImageConfig.Registry
				}
				_, imageRepo, _ = utils.SplitImageTag(step.PublishImageConfig.Tag)
				break StageLoop
			}
		}
	}

	applyEnv[utils.EnvRegistry] = registry
	applyEnv[utils.EnvImageRepo] = imageRepo

	for k, v := range applyEnv {
		container.Env = append(container.Env, v1.EnvVar{Name: k, Value: v})
	}
	return injectResources(container, utils.PipelineToolsCPULimitDefault, utils.PipelineToolsCPURequestDefault, utils.PipelineToolsMemoryLimitDefault, utils.PipelineToolsMemoryRequestDefault)
}

func (c *jenkinsPipelineConverter) configPublishCatalogContainer(container *v1.Container, step *v3.Step) error {
	if c.opts.gitCaCerts != "" {
		c.injectGitCaCertToContainer(container)
	}
	config := step.PublishCatalogConfig
	container.Image = images.Resolve(mv3.ToolsSystemImages.PipelineSystemImages.KubeApply)
	envs := map[string]string{
		"CATALOG_PATH":          config.Path,
		"CATALOG_TEMPLATE_NAME": config.CatalogTemplate,
		"VERSION":               config.Version,
		"GIT_AUTHOR":            config.GitAuthor,
		"GIT_EMAIL":             config.GitEmail,
		"GIT_URL":               config.GitURL,
		"GIT_BRANCH":            config.GitBranch,
	}
	for k, v := range envs {
		container.Env = append(container.Env, v1.EnvVar{Name: k, Value: v})
	}
	return injectResources(container, utils.PipelineToolsCPULimitDefault, utils.PipelineToolsCPURequestDefault, utils.PipelineToolsMemoryLimitDefault, utils.PipelineToolsMemoryRequestDefault)
}

func (c *jenkinsPipelineConverter) configApplyAppContainer(container *v1.Container, step *v3.Step) error {
	config := step.ApplyAppConfig
	container.Image = images.Resolve(mv3.ToolsSystemImages.PipelineSystemImages.KubeApply)
	answerBytes, _ := yaml.Marshal(config.Answers)
	envs := map[string]string{
		"APP_NAME":              config.Name,
		"ANSWERS":               string(answerBytes),
		"CATALOG_TEMPLATE_NAME": config.CatalogTemplate,
		"VERSION":               config.Version,
		"TARGET_NAMESPACE":      config.TargetNamespace,
		"RANCHER_URL":           settings.ServerURL.Get(),
	}
	for k, v := range envs {
		container.Env = append(container.Env, v1.EnvVar{Name: k, Value: v})
	}
	container.Env = append(container.Env, v1.EnvVar{
		Name: utils.PipelineSecretAPITokenKey,
		ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{
			LocalObjectReference: v1.LocalObjectReference{
				Name: utils.PipelineAPIKeySecretName,
			},
			Key: utils.PipelineSecretAPITokenKey,
		}}})
	return injectResources(container, utils.PipelineToolsCPULimitDefault, utils.PipelineToolsCPURequestDefault, utils.PipelineToolsMemoryLimitDefault, utils.PipelineToolsMemoryRequestDefault)
}
<commit_msg>Support custom envvar substitution in pipeline publishcatalog steps<commit_after>package jenkins

import (
	"fmt"
	"regexp"
	"strings"

	images "github.com/rancher/rancher/pkg/image"
	"github.com/rancher/rancher/pkg/pipeline/utils"
	"github.com/rancher/rancher/pkg/ref"
	"github.com/rancher/rancher/pkg/settings"
	mv3 "github.com/rancher/types/apis/management.cattle.io/v3"
	v3 "github.com/rancher/types/apis/project.cattle.io/v3"
	"gopkg.in/yaml.v2"
	v1 "k8s.io/api/core/v1"
)

func (c *jenkinsPipelineConverter) getStepContainer(stageOrdinal int, stepOrdinal int) (v1.Container, error) {
	stage := c.execution.Spec.PipelineConfig.Stages[stageOrdinal]
	step := &stage.Steps[stepOrdinal]

	container := v1.Container{
		Name:    fmt.Sprintf("step-%d-%d", stageOrdinal, stepOrdinal),
		TTY:     true,
		Command: []string{"cat"},
		Env:     []v1.EnvVar{},
	}
	if step.SourceCodeConfig != nil {
		if err := c.configCloneStepContainer(&container, step); err != nil {
			return container, err
		}
	} else if step.RunScriptConfig != nil {
		c.configRunScriptStepContainer(&container, step)
	} else if step.PublishImageConfig != nil {
		c.configPublishStepContainer(&container, step)
	} else if step.ApplyYamlConfig != nil {
		if err := c.configApplyYamlStepContainer(&container, step, stageOrdinal); err != nil {
			return container, err
		}
	} else if step.PublishCatalogConfig != nil {
		if err := c.configPublishCatalogContainer(&container, step); err != nil {
			return container, err
		}
	} else if step.ApplyAppConfig != nil {
		if err := c.configApplyAppContainer(&container, step); err != nil {
			return container, err
		}
	}

	//common step configurations
	for k, v := range utils.GetEnvVarMap(c.execution) {
		container.Env = append(container.Env, v1.EnvVar{Name: k, Value: v})
	}
	for k, v := range step.Env {
		container.Env = append(container.Env, v1.EnvVar{Name: k, Value: v})
	}
	if c.execution.Spec.Event != utils.WebhookEventPullRequest {
		//expose no secrets on pull_request events
		for _, e := range step.EnvFrom {
			envName := e.SourceKey
			if e.TargetKey != "" {
				envName = e.TargetKey
			}
			container.Env = append(container.Env, v1.EnvVar{
				Name: envName,
				ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{
					LocalObjectReference: v1.LocalObjectReference{
						Name: e.SourceName,
					},
					Key: e.SourceKey,
				}}})
		}
	}
	if step.Privileged {
		container.SecurityContext = &v1.SecurityContext{Privileged: &step.Privileged}
	}
	err := injectSetpContainerResources(&container, step)
	return container, err
}

func (c *jenkinsPipelineConverter) getJenkinsStepCommand(stageOrdinal int, stepOrdinal int) string {
	stage := c.execution.Spec.PipelineConfig.Stages[stageOrdinal]
	step := &stage.Steps[stepOrdinal]
	command := ""

	if !utils.MatchAll(stage.When, c.execution) || !utils.MatchAll(step.When, c.execution) {
		stepName := fmt.Sprintf("step-%d-%d", stageOrdinal, stepOrdinal)
		command = fmt.Sprintf(markSkipScript, stepName)
	} else if step.SourceCodeConfig != nil {
		command = fmt.Sprintf("checkout([$class: 'GitSCM', branches: [[name: 'local/temp']], userRemoteConfigs: [[url: '%s', refspec: '+%s:refs/remotes/local/temp', credentialsId: '%s']]])",
			c.execution.Spec.RepositoryURL, c.execution.Spec.Ref, c.execution.Name)
	} else if step.RunScriptConfig != nil {
		command = fmt.Sprintf(`sh ''' %s '''`, step.RunScriptConfig.ShellScript)
	} else if step.PublishImageConfig != nil {
		command = `sh '''/usr/local/bin/dockerd-entrypoint.sh /bin/drone-docker'''`
	} else if step.ApplyYamlConfig != nil {
		command = `sh ''' kube-apply '''`
	} else if step.PublishCatalogConfig != nil {
		command = `sh ''' publish-catalog '''`
	} else if step.ApplyAppConfig != nil {
		command = `sh ''' apply-app '''`
	}
	return command
}

func (c *jenkinsPipelineConverter) getAgentContainer() (v1.Container, error) {
	container := v1.Container{
		Name:  utils.JenkinsAgentContainerName,
		Image: images.Resolve(mv3.ToolsSystemImages.PipelineSystemImages.JenkinsJnlp),
		Args:  []string{"$(JENKINS_SECRET)", "$(JENKINS_NAME)"},
	}
	cloneContainer, err := c.getStepContainer(0, 0)
	if err != nil {
		return container, err
	}
	container.Env = append(container.Env, cloneContainer.Env...)
	container.EnvFrom = append(container.EnvFrom, cloneContainer.EnvFrom...)
	err = c.injectAgentResources(&container)
	return container, err
}

func (c *jenkinsPipelineConverter) configCloneStepContainer(container *v1.Container, step *v3.Step) error {
	container.Image = images.Resolve(mv3.ToolsSystemImages.PipelineSystemImages.AlpineGit)
	return injectResources(container, utils.PipelineToolsCPULimitDefault, utils.PipelineToolsCPURequestDefault, utils.PipelineToolsMemoryLimitDefault, utils.PipelineToolsMemoryRequestDefault)
}

func (c *jenkinsPipelineConverter) configRunScriptStepContainer(container *v1.Container, step *v3.Step) {
	container.Image = step.RunScriptConfig.Image
}

func (c *jenkinsPipelineConverter) configPublishStepContainer(container *v1.Container, step *v3.Step) {
	ns := utils.GetPipelineCommonName(c.execution.Spec.ProjectName)
	config := step.PublishImageConfig
	m := utils.GetEnvVarMap(c.execution)
	config.Tag = substituteEnvVar(m, config.Tag)

	registry, repo, tag := utils.SplitImageTag(config.Tag)

	if config.PushRemote {
		registry = config.Registry
	} else {
		_, projectID := ref.Parse(c.execution.Spec.ProjectName)
		registry = fmt.Sprintf("%s.%s-pipeline", utils.LocalRegistry, projectID)
	}

	reg, _ := regexp.Compile("[^a-zA-Z0-9]+")
	processedRegistry := strings.ToLower(reg.ReplaceAllString(registry, ""))
	secretName := fmt.Sprintf("%s-%s", c.execution.Namespace, processedRegistry)
	secretUserKey := utils.PublishSecretUserKey
	secretPwKey := utils.PublishSecretPwKey
	if !config.PushRemote {
		//use local registry credential
		secretName = utils.PipelineSecretName
		secretUserKey = utils.PipelineSecretUserKey
		secretPwKey = utils.PipelineSecretTokenKey
	}
	pluginRepo := fmt.Sprintf("%s/%s", registry, repo)
	if registry == utils.DefaultRegistry {
		//the `plugins/docker` image fails when setting DOCKER_REGISTRY to index.docker.io
		registry = ""
	}

	container.Image = images.Resolve(mv3.ToolsSystemImages.PipelineSystemImages.PluginsDocker)
	publishEnv := map[string]string{
		"DOCKER_REGISTRY":   registry,
		"PLUGIN_REPO":       pluginRepo,
		"PLUGIN_TAG":        tag,
		"PLUGIN_DOCKERFILE": config.DockerfilePath,
		"PLUGIN_CONTEXT":    config.BuildContext,
	}
	for k, v := range publishEnv {
		container.Env = append(container.Env, v1.EnvVar{Name: k, Value: v})
	}
	container.Env = append(container.Env, v1.EnvVar{
		Name: "DOCKER_USERNAME",
		ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{
			LocalObjectReference: v1.LocalObjectReference{
				Name: secretName,
			},
			Key: secretUserKey,
		}}})
	container.Env = append(container.Env, v1.EnvVar{
		Name: "DOCKER_PASSWORD",
		ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{
			LocalObjectReference: v1.LocalObjectReference{
				Name: secretName,
			},
			Key: secretPwKey,
		}}})
	privileged := true
	container.SecurityContext = &v1.SecurityContext{Privileged: &privileged}
	container.VolumeMounts = []v1.VolumeMount{
		{
			Name:      utils.RegistryCrtVolumeName,
			MountPath: fmt.Sprintf("/etc/docker/certs.d/docker-registry.%s", ns),
			ReadOnly:  true,
		},
	}
}

func (c *jenkinsPipelineConverter) configApplyYamlStepContainer(container *v1.Container, step *v3.Step, stageOrdinal int) error {
	config := step.ApplyYamlConfig
	container.Image = images.Resolve(mv3.ToolsSystemImages.PipelineSystemImages.KubeApply)

	applyEnv := map[string]string{
		"YAML_PATH":    config.Path,
		"YAML_CONTENT": config.Content,
		"NAMESPACE":    config.Namespace,
	}

	//for deploy step, get registry & image variable from a previous publish step
	var registry, imageRepo string
StageLoop:
	for i := stageOrdinal; i >= 0; i-- {
		stage := c.execution.Spec.PipelineConfig.Stages[i]
		for j := len(stage.Steps) - 1; j >= 0; j-- {
			step := stage.Steps[j]
			if step.PublishImageConfig != nil {
				config := step.PublishImageConfig
				if config.PushRemote {
					registry = step.PublishImageConfig.Registry
				}
				_, imageRepo, _ = utils.SplitImageTag(step.PublishImageConfig.Tag)
				break StageLoop
			}
		}
	}

	applyEnv[utils.EnvRegistry] = registry
	applyEnv[utils.EnvImageRepo] = imageRepo

	for k, v := range applyEnv {
		container.Env = append(container.Env, v1.EnvVar{Name: k, Value: v})
	}
	return injectResources(container, utils.PipelineToolsCPULimitDefault, utils.PipelineToolsCPURequestDefault, utils.PipelineToolsMemoryLimitDefault, utils.PipelineToolsMemoryRequestDefault)
}

func (c *jenkinsPipelineConverter) configPublishCatalogContainer(container *v1.Container, step *v3.Step) error {
	if c.opts.gitCaCerts != "" {
		c.injectGitCaCertToContainer(container)
	}
	config := step.PublishCatalogConfig
	container.Image = images.Resolve(mv3.ToolsSystemImages.PipelineSystemImages.KubeApply)
	envs := map[string]string{
		"CATALOG_PATH":          config.Path,
		"CATALOG_TEMPLATE_NAME": config.CatalogTemplate,
		"VERSION":               config.Version,
		"GIT_AUTHOR":            config.GitAuthor,
		"GIT_EMAIL":             config.GitEmail,
		"GIT_URL":               config.GitURL,
		"GIT_BRANCH":            config.GitBranch,
	}
	for k, v := range envs {
		container.Env = append(container.Env, v1.EnvVar{Name: k, Value: v})
	}
	var customEnvs []string
	for k := range step.Env {
		customEnvs = append(customEnvs, k)
	}
	container.Env = append(container.Env, v1.EnvVar{Name: "CICD_SUBSTITUTE_VARS", Value: strings.Join(customEnvs, ",")})
	return injectResources(container, utils.PipelineToolsCPULimitDefault, utils.PipelineToolsCPURequestDefault, utils.PipelineToolsMemoryLimitDefault, utils.PipelineToolsMemoryRequestDefault)
}

func (c *jenkinsPipelineConverter) configApplyAppContainer(container *v1.Container, step *v3.Step) error {
	config := step.ApplyAppConfig
	container.Image = images.Resolve(mv3.ToolsSystemImages.PipelineSystemImages.KubeApply)
	answerBytes, _ := yaml.Marshal(config.Answers)
	envs := map[string]string{
		"APP_NAME":              config.Name,
		"ANSWERS":               string(answerBytes),
		"CATALOG_TEMPLATE_NAME": config.CatalogTemplate,
		"VERSION":               config.Version,
		"TARGET_NAMESPACE":      config.TargetNamespace,
		"RANCHER_URL":           settings.ServerURL.Get(),
	}
	for k, v := range envs {
		container.Env = append(container.Env, v1.EnvVar{Name: k, Value: v})
	}
	container.Env = append(container.Env, v1.EnvVar{
		Name: utils.PipelineSecretAPITokenKey,
		ValueFrom: &v1.EnvVarSource{SecretKeyRef: &v1.SecretKeySelector{
			LocalObjectReference: v1.LocalObjectReference{
				Name: utils.PipelineAPIKeySecretName,
			},
			Key: utils.PipelineSecretAPITokenKey,
		}}})
	return injectResources(container, utils.PipelineToolsCPULimitDefault, utils.PipelineToolsCPURequestDefault, utils.PipelineToolsMemoryLimitDefault, utils.PipelineToolsMemoryRequestDefault)
}
<|endoftext|>


<commit_before><commit_msg>remove unused package<commit_after><|endoftext|>


<commit_before>package gocb

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
)

// ClusterManager provides methods for performing cluster management operations.
type ClusterManager struct {
	hosts    []string
	username string
	password string
	httpCli  *http.Client
}

// BucketType specifies the kind of bucket
type BucketType int

const (
	// Couchbase indicates a Couchbase bucket type.
	Couchbase = BucketType(0)

	// Memcached indicates a Memcached bucket type.
	Memcached = BucketType(1)
)

type bucketDataIn struct {
	Name         string `json:"name"`
	BucketType   string `json:"bucketType"`
	AuthType     string `json:"authType"`
	SaslPassword string `json:"saslPassword"`
	Quota        struct {
		Ram    int `json:"ram"`
		RawRam int `json:"rawRAM"`
	} `json:"quota"`
	ReplicaNumber int  `json:"replicaNumber"`
	ReplicaIndex  bool `json:"replicaIndex"`
	Controllers   struct {
		Flush string `json:"flush"`
	} `json:"controllers"`
}

// BucketSettings holds information about the settings for a bucket.
type BucketSettings struct {
	FlushEnabled  bool
	IndexReplicas bool
	Name          string
	Password      string
	Quota         int
	Replicas      int
	Type          BucketType
}

func (cm *ClusterManager) getMgmtEp() string {
	return cm.hosts[rand.Intn(len(cm.hosts))]
}

func (cm *ClusterManager) mgmtRequest(method, uri string, contentType string, body io.Reader) (*http.Response, error) {
	if contentType == "" && body != nil {
		panic("Content-type must be specified for non-null body.")
	}

	reqUri := cm.getMgmtEp() + uri
	req, err := http.NewRequest(method, reqUri, body)
	if contentType != "" {
		req.Header.Add("Content-Type", contentType)
	}
	if err != nil {
		return nil, err
	}

	req.SetBasicAuth(cm.username, cm.password)
	return cm.httpCli.Do(req)
}

func bucketDataInToSettings(bucketData *bucketDataIn) *BucketSettings {
	settings := &BucketSettings{
		FlushEnabled:  bucketData.Controllers.Flush != "",
		IndexReplicas: bucketData.ReplicaIndex,
		Name:          bucketData.Name,
		Password:      bucketData.SaslPassword,
		Quota:         bucketData.Quota.Ram,
		Replicas:      bucketData.ReplicaNumber,
	}
	if bucketData.BucketType == "membase" {
		settings.Type = Couchbase
	} else if bucketData.BucketType == "memcached" {
		settings.Type = Memcached
	} else {
		panic("Unrecognized bucket type string.")
	}
	if bucketData.AuthType != "sasl" {
		settings.Password = ""
	}
	return settings
}

// GetBuckets returns a list of all active buckets on the cluster.
func (cm *ClusterManager) GetBuckets() ([]*BucketSettings, error) {
	resp, err := cm.mgmtRequest("GET", "/pools/default/buckets", "", nil)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
		err = resp.Body.Close()
		if err != nil {
			logDebugf("Failed to close socket (%s)", err)
		}
		return nil, clientError{string(data)}
	}

	var bucketsData []*bucketDataIn
	jsonDec := json.NewDecoder(resp.Body)
	err = jsonDec.Decode(&bucketsData)
	if err != nil {
		return nil, err
	}

	var buckets []*BucketSettings
	for _, bucketData := range bucketsData {
		buckets = append(buckets, bucketDataInToSettings(bucketData))
	}

	return buckets, nil
}

// InsertBucket creates a new bucket on the cluster.
func (cm *ClusterManager) InsertBucket(settings *BucketSettings) error {
	posts := url.Values{}
	posts.Add("name", settings.Name)
	if settings.Type == Couchbase {
		posts.Add("bucketType", "couchbase")
	} else if settings.Type == Memcached {
		posts.Add("bucketType", "memcached")
	} else {
		panic("Unrecognized bucket type.")
	}
	if settings.FlushEnabled {
		posts.Add("flushEnabled", "1")
	} else {
		posts.Add("flushEnabled", "0")
	}
	posts.Add("replicaNumber", fmt.Sprintf("%d", settings.Replicas))
	posts.Add("authType", "sasl")
	posts.Add("saslPassword", settings.Password)
	posts.Add("ramQuotaMB", fmt.Sprintf("%d", settings.Quota))
	posts.Add("proxyPort", "11210")

	data := []byte(posts.Encode())
	resp, err := cm.mgmtRequest("POST", "/pools/default/buckets", "application/x-www-form-urlencoded", bytes.NewReader(data))
	if err != nil {
		return nil
	}

	if resp.StatusCode != 202 {
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		err = resp.Body.Close()
		if err != nil {
			logDebugf("Failed to close socket (%s)", err)
		}
		return clientError{string(data)}
	}

	return nil
}

// UpdateBucket will update the settings for a specific bucket on the cluster.
func (cm *ClusterManager) UpdateBucket(settings *BucketSettings) error {
	// Cluster-side, updates are the same as creates.
	return cm.InsertBucket(settings)
}

// RemoveBucket will delete a bucket from the cluster by name.
func (cm *ClusterManager) RemoveBucket(name string) error {
	reqUri := fmt.Sprintf("/pools/default/buckets/%s", name)

	resp, err := cm.mgmtRequest("DELETE", reqUri, "", nil)
	if err != nil {
		return err
	}

	if resp.StatusCode != 200 {
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		err = resp.Body.Close()
		if err != nil {
			logDebugf("Failed to close socket (%s)", err)
		}
		return clientError{string(data)}
	}

	return nil
}

// UserRole represents a role for a particular user on the server.
type UserRole struct {
	Role       string
	BucketName string
}

// User represents a user which was retrieved from the server.
type User struct {
	Id    string
	Name  string
	Type  string
	Roles []UserRole
}

// UserSettings represents a user during user creation.
type UserSettings struct {
	Name     string
	Password string
	Roles    []UserRole
}

type userRoleJson struct {
	Role       string `json:"role"`
	BucketName string `json:"bucket_name"`
}

type userJson struct {
	Id    string         `json:"id"`
	Name  string         `json:"name"`
	Type  string         `json:"type"`
	Roles []userRoleJson `json:"roles"`
}

type userSettingsJson struct {
	Name     string         `json:"name"`
	Password string         `json:"password"`
	Roles    []userRoleJson `json:"roles"`
}

// GetUsers returns a list of all users on the cluster.
func (cm *ClusterManager) GetUsers() ([]*User, error) {
	resp, err := cm.mgmtRequest("GET", "/settings/rbac/users", "", nil)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
		err = resp.Body.Close()
		if err != nil {
			logDebugf("Failed to close socket (%s)", err)
		}
		return nil, clientError{string(data)}
	}

	var usersData []*userJson
	jsonDec := json.NewDecoder(resp.Body)
	err = jsonDec.Decode(&usersData)
	if err != nil {
		return nil, err
	}

	var users []*User
	for _, userData := range usersData {
		var user User
		user.Id = userData.Id
		user.Name = userData.Name
		user.Type = userData.Type
		for _, roleData := range userData.Roles {
			user.Roles = append(user.Roles, UserRole{
				Role:       roleData.Role,
				BucketName: roleData.BucketName,
			})
		}
		users = append(users, &user)
	}

	return users, nil
}

// UpsertUser updates a built-in RBAC user on the cluster.
func (cm *ClusterManager) UpsertUser(name string, settings *UserSettings) error {
	var reqRoleStrs []string
	for _, roleData := range settings.Roles {
		reqRoleStrs = append(reqRoleStrs, fmt.Sprintf("%s[%s]", roleData.Role, roleData.BucketName))
	}

	reqForm := make(url.Values)
	reqForm.Add("name", settings.Name)
	reqForm.Add("password", settings.Password)
	reqForm.Add("roles", strings.Join(reqRoleStrs, ","))

	uri := fmt.Sprintf("/settings/rbac/users/builtin/%s", name)
	reqBody := bytes.NewReader([]byte(reqForm.Encode()))
	resp, err := cm.mgmtRequest("PUT", uri, "application/x-www-form-urlencoded", reqBody)
	if err != nil {
		return err
	}

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		err = resp.Body.Close()
		if err != nil {
			logDebugf("Failed to close socket (%s)", err)
		}
		return clientError{string(data)}
	}

	return nil
}

// RemoveUser removes a built-in RBAC user on the cluster.
func (cm *ClusterManager) RemoveUser(name string) error {
	uri := fmt.Sprintf("/settings/rbac/users/builtin/%s", name)
	resp, err := cm.mgmtRequest("DELETE", uri, "", nil)
	if err != nil {
		return err
	}

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		err = resp.Body.Close()
		if err != nil {
			logDebugf("Failed to close socket (%s)", err)
		}
		return clientError{string(data)}
	}

	return nil
}
<commit_msg>InsertBucket should return err instead nil.<commit_after>package gocb

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"strings"
)

// ClusterManager provides methods for performing cluster management operations.
type ClusterManager struct {
	hosts    []string
	username string
	password string
	httpCli  *http.Client
}

// BucketType specifies the kind of bucket
type BucketType int

const (
	// Couchbase indicates a Couchbase bucket type.
	Couchbase = BucketType(0)

	// Memcached indicates a Memcached bucket type.
	Memcached = BucketType(1)
)

type bucketDataIn struct {
	Name         string `json:"name"`
	BucketType   string `json:"bucketType"`
	AuthType     string `json:"authType"`
	SaslPassword string `json:"saslPassword"`
	Quota        struct {
		Ram    int `json:"ram"`
		RawRam int `json:"rawRAM"`
	} `json:"quota"`
	ReplicaNumber int  `json:"replicaNumber"`
	ReplicaIndex  bool `json:"replicaIndex"`
	Controllers   struct {
		Flush string `json:"flush"`
	} `json:"controllers"`
}

// BucketSettings holds information about the settings for a bucket.
type BucketSettings struct {
	FlushEnabled  bool
	IndexReplicas bool
	Name          string
	Password      string
	Quota         int
	Replicas      int
	Type          BucketType
}

func (cm *ClusterManager) getMgmtEp() string {
	return cm.hosts[rand.Intn(len(cm.hosts))]
}

func (cm *ClusterManager) mgmtRequest(method, uri string, contentType string, body io.Reader) (*http.Response, error) {
	if contentType == "" && body != nil {
		panic("Content-type must be specified for non-null body.")
	}

	reqUri := cm.getMgmtEp() + uri
	req, err := http.NewRequest(method, reqUri, body)
	if contentType != "" {
		req.Header.Add("Content-Type", contentType)
	}
	if err != nil {
		return nil, err
	}

	req.SetBasicAuth(cm.username, cm.password)
	return cm.httpCli.Do(req)
}

func bucketDataInToSettings(bucketData *bucketDataIn) *BucketSettings {
	settings := &BucketSettings{
		FlushEnabled:  bucketData.Controllers.Flush != "",
		IndexReplicas: bucketData.ReplicaIndex,
		Name:          bucketData.Name,
		Password:      bucketData.SaslPassword,
		Quota:         bucketData.Quota.Ram,
		Replicas:      bucketData.ReplicaNumber,
	}
	if bucketData.BucketType == "membase" {
		settings.Type = Couchbase
	} else if bucketData.BucketType == "memcached" {
		settings.Type = Memcached
	} else {
		panic("Unrecognized bucket type string.")
	}
	if bucketData.AuthType != "sasl" {
		settings.Password = ""
	}
	return settings
}

// GetBuckets returns a list of all active buckets on the cluster.
func (cm *ClusterManager) GetBuckets() ([]*BucketSettings, error) {
	resp, err := cm.mgmtRequest("GET", "/pools/default/buckets", "", nil)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != 200 {
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
		err = resp.Body.Close()
		if err != nil {
			logDebugf("Failed to close socket (%s)", err)
		}
		return nil, clientError{string(data)}
	}

	var bucketsData []*bucketDataIn
	jsonDec := json.NewDecoder(resp.Body)
	err = jsonDec.Decode(&bucketsData)
	if err != nil {
		return nil, err
	}

	var buckets []*BucketSettings
	for _, bucketData := range bucketsData {
		buckets = append(buckets, bucketDataInToSettings(bucketData))
	}

	return buckets, nil
}

// InsertBucket creates a new bucket on the cluster.
func (cm *ClusterManager) InsertBucket(settings *BucketSettings) error {
	posts := url.Values{}
	posts.Add("name", settings.Name)
	if settings.Type == Couchbase {
		posts.Add("bucketType", "couchbase")
	} else if settings.Type == Memcached {
		posts.Add("bucketType", "memcached")
	} else {
		panic("Unrecognized bucket type.")
	}
	if settings.FlushEnabled {
		posts.Add("flushEnabled", "1")
	} else {
		posts.Add("flushEnabled", "0")
	}
	posts.Add("replicaNumber", fmt.Sprintf("%d", settings.Replicas))
	posts.Add("authType", "sasl")
	posts.Add("saslPassword", settings.Password)
	posts.Add("ramQuotaMB", fmt.Sprintf("%d", settings.Quota))
	posts.Add("proxyPort", "11210")

	data := []byte(posts.Encode())
	resp, err := cm.mgmtRequest("POST", "/pools/default/buckets", "application/x-www-form-urlencoded", bytes.NewReader(data))
	if err != nil {
		return err
	}

	if resp.StatusCode != 202 {
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		err = resp.Body.Close()
		if err != nil {
			logDebugf("Failed to close socket (%s)", err)
		}
		return clientError{string(data)}
	}

	return nil
}

// UpdateBucket will update the settings for a specific bucket on the cluster.
func (cm *ClusterManager) UpdateBucket(settings *BucketSettings) error {
	// Cluster-side, updates are the same as creates.
	return cm.InsertBucket(settings)
}

// RemoveBucket will delete a bucket from the cluster by name.
func (cm *ClusterManager) RemoveBucket(name string) error {
	reqUri := fmt.Sprintf("/pools/default/buckets/%s", name)

	resp, err := cm.mgmtRequest("DELETE", reqUri, "", nil)
	if err != nil {
		return err
	}

	if resp.StatusCode != 200 {
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		err = resp.Body.Close()
		if err != nil {
			logDebugf("Failed to close socket (%s)", err)
		}
		return clientError{string(data)}
	}

	return nil
}

// UserRole represents a role for a particular user on the server.
type UserRole struct {
	Role       string
	BucketName string
}

// User represents a user which was retrieved from the server.
type User struct {
	Id    string
	Name  string
	Type  string
	Roles []UserRole
}

// UserSettings represents a user during user creation.
type UserSettings struct {
	Name     string
	Password string
	Roles    []UserRole
}

type userRoleJson struct {
	Role       string `json:"role"`
	BucketName string `json:"bucket_name"`
}

type userJson struct {
	Id    string         `json:"id"`
	Name  string         `json:"name"`
	Type  string         `json:"type"`
	Roles []userRoleJson `json:"roles"`
}

type userSettingsJson struct {
	Name     string         `json:"name"`
	Password string         `json:"password"`
	Roles    []userRoleJson `json:"roles"`
}

// GetUsers returns a list of all users on the cluster.
func (cm *ClusterManager) GetUsers() ([]*User, error) {
	resp, err := cm.mgmtRequest("GET", "/settings/rbac/users", "", nil)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, err
		}
		err = resp.Body.Close()
		if err != nil {
			logDebugf("Failed to close socket (%s)", err)
		}
		return nil, clientError{string(data)}
	}

	var usersData []*userJson
	jsonDec := json.NewDecoder(resp.Body)
	err = jsonDec.Decode(&usersData)
	if err != nil {
		return nil, err
	}

	var users []*User
	for _, userData := range usersData {
		var user User
		user.Id = userData.Id
		user.Name = userData.Name
		user.Type = userData.Type
		for _, roleData := range userData.Roles {
			user.Roles = append(user.Roles, UserRole{
				Role:       roleData.Role,
				BucketName: roleData.BucketName,
			})
		}
		users = append(users, &user)
	}

	return users, nil
}

// UpsertUser updates a built-in RBAC user on the cluster.
func (cm *ClusterManager) UpsertUser(name string, settings *UserSettings) error {
	var reqRoleStrs []string
	for _, roleData := range settings.Roles {
		reqRoleStrs = append(reqRoleStrs, fmt.Sprintf("%s[%s]", roleData.Role, roleData.BucketName))
	}

	reqForm := make(url.Values)
	reqForm.Add("name", settings.Name)
	reqForm.Add("password", settings.Password)
	reqForm.Add("roles", strings.Join(reqRoleStrs, ","))

	uri := fmt.Sprintf("/settings/rbac/users/builtin/%s", name)
	reqBody := bytes.NewReader([]byte(reqForm.Encode()))
	resp, err := cm.mgmtRequest("PUT", uri, "application/x-www-form-urlencoded", reqBody)
	if err != nil {
		return err
	}

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		err = resp.Body.Close()
		if err != nil {
			logDebugf("Failed to close socket (%s)", err)
		}
		return clientError{string(data)}
	}

	return nil
}

// RemoveUser removes a built-in RBAC user on the cluster.
func (cm *ClusterManager) RemoveUser(name string) error {
	uri := fmt.Sprintf("/settings/rbac/users/builtin/%s", name)
	resp, err := cm.mgmtRequest("DELETE", uri, "", nil)
	if err != nil {
		return err
	}

	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
		data, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		err = resp.Body.Close()
		if err != nil {
			logDebugf("Failed to close socket (%s)", err)
		}
		return clientError{string(data)}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"runtime"

	"github.com/google/gapid/core/os/file"
	"github.com/google/gapid/core/os/shell"
)

func env(cfg Config) *shell.Env {
	path := []string{}

	if runtime.GOOS == "windows" {
		if !cfg.MSYS2Path.IsEmpty() {
			path = append(path,
				cfg.MSYS2Path.Join("usr/bin").System(),     // Required for sh.exe and other unixy tools.
				cfg.MSYS2Path.Join("mingw64/bin").System(), // Required to pick up DLLs
			)
		}

		// Add windows and system32 to path
		cmd, err := file.FindExecutable("cmd.exe")
		if err != nil {
			panic("Couldn't find cmd.exe on PATH")
		}
		system32 := cmd.Parent()
		windows := system32.Parent()
		path = append(path, system32.System(), windows.System())
		path = append(path, exePaths("node.exe", "adb.exe")...)
	} else {
		path = append(path, exePaths(
			"sh", "uname", "sed", "clang", "gcc", "node", "adb",
		)...)
	}

	path = append(path,
		cfg.bin().System(),
		cfg.JavaHome.Join("bin").System(),
	)

	env := shell.CloneEnv()
	env.Unset("PATH")
	env.AddPathEnd("PATH", path...)
	return env
}

func exePaths(exes ...string) []string {
	path := []string{}
	added := map[file.Path]bool{}
	for _, name := range exes {
		exe, err := file.FindExecutable(name)
		if err != nil {
			continue
		}
		dir := exe.Parent()
		if !added[dir] {
			path = append(path, dir.System())
			added[dir] = true
		}
	}
	return path
}
<commit_msg>do: Search ${AndroidSDKRoot}/platform-tools as well as PATH for exes.<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"runtime"

	"github.com/google/gapid/core/os/file"
	"github.com/google/gapid/core/os/shell"
)

func env(cfg Config) *shell.Env {
	path := []string{}

	if runtime.GOOS == "windows" {
		if !cfg.MSYS2Path.IsEmpty() {
			path = append(path,
				cfg.MSYS2Path.Join("usr/bin").System(),     // Required for sh.exe and other unixy tools.
				cfg.MSYS2Path.Join("mingw64/bin").System(), // Required to pick up DLLs
			)
		}

		// Add windows and system32 to path
		cmd, err := file.FindExecutable("cmd.exe")
		if err != nil {
			panic("Couldn't find cmd.exe on PATH")
		}
		system32 := cmd.Parent()
		windows := system32.Parent()
		path = append(path, system32.System(), windows.System())
		path = append(path, exePaths(cfg, "node.exe", "adb.exe")...)
	} else {
		path = append(path, exePaths(cfg,
			"sh", "uname", "sed", "clang", "gcc", "node", "adb",
		)...)
	}

	path = append(path,
		cfg.bin().System(),
		cfg.JavaHome.Join("bin").System(),
	)

	env := shell.CloneEnv()
	env.Unset("PATH")
	env.AddPathEnd("PATH", path...)
	return env
}

func exePaths(cfg Config, exes ...string) []string {
	path := []string{}
	added := map[file.Path]bool{}
	for _, name := range exes {
		// First search PATH
		exe, err := file.FindExecutable(name)
		if err != nil {
			// Not found on PATH, try ${AndroidSDKRoot}/platform-tools.
			exe, err = file.FindExecutable(cfg.AndroidSDKRoot.Join("platform-tools", name).System())
			if err != nil {
				continue
			}
		}
		dir := exe.Parent()
		if !added[dir] {
			path = append(path, dir.System())
			added[dir] = true
		}
	}
	return path
}
<|endoftext|>


<commit_before>package cmd

import (
	"errors"
	"fmt"
	"strconv"
	"time"

	"github.com/spf13/cobra"
	"net/http"
)

type stressRequest interface{}

type finishedStress struct{}

type workerDone struct{}

type requestStat struct {
	duration int64 //nanoseconds
}
type requestStatSummary struct {
	avgDuration int64 //nanoseconds
	maxDuration int64 //nanoseconds
	minDuration int64 //nanoseconds
}

//flags
var (
	numTests    int
	timeout     int
	concurrency int
)

func init() {
	RootCmd.AddCommand(stressCmd)

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// stressCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// stressCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")

	stressCmd.Flags().IntVarP(&numTests, "num", "n", 100, "Number of requests to make")
	stressCmd.Flags().IntVarP(&concurrency, "concurrent", "c", 1, "Number of multiple requests to make")
	stressCmd.Flags().IntVarP(&timeout, "timeout", "t", 0, "Maximum seconds to wait for response. 0 means unlimited")
}

// stressCmd represents the stress command
var stressCmd = &cobra.Command{
	Use:   "stress http[s]://hostname[:port]/path",
	Short: "Run predefined load of requests",
	Long:  `Run predefined load of requests`,
	RunE:  RunStress,
}

func RunStress(cmd *cobra.Command, args []string) error {
	//checks
	if len(args) != 1 {
		return errors.New("needs URL")
	}
	if numTests <= 0 {
		return errors.New("number of requests must be one or more")
	}
	if concurrency <= 0 {
		return errors.New("concurrency must be one or more")
	}
	if timeout < 0 {
		return errors.New("timeout must be zero or more")
	}
	if concurrency > numTests {
		return errors.New("concurrency must be higher than number of requests")
	}

	url := args[0]

	fmt.Println("Stress testing " + url + "...")

	//setup the queue of requests
	requestChan := make(chan stressRequest, numTests+concurrency)
	for i := 0; i < numTests; i++ {
		//TODO optimize by not creating a new http request each time since it's the same thing
		req, err := http.NewRequest("GET", url, nil)
		if err != nil {
			return errors.New("failed to create request: " + err.Error())
		}
		requestChan <- req
	}
	for i := 0; i < concurrency; i++ {
		requestChan <- finishedStress{}
	}

	workerDoneChan := make(chan workerDone)   //workers use this to indicate they are done
	requestStatChan := make(chan requestStat) //workers communicate each requests' info

	//workers
	totalStartTime := time.Now()
	for i := 0; i < concurrency; i++ {
		go func() {
			client := &http.Client{Timeout: time.Duration(timeout) * time.Second}
			for {
				select {
				case req := <-requestChan:
					switch req.(type) {
					case *http.Request:
						//run the acutal request
						reqStartTime := time.Now()
						_, err := client.Do(req.(*http.Request))
						reqEndTime := time.Now()
						if err != nil {
							fmt.Errorf(err.Error()) //TODO handle this further up
						}
						reqTimeNs := (reqEndTime.UnixNano() - reqStartTime.UnixNano())
						fmt.Printf("request took %dms\n", reqTimeNs/1000000)
						requestStatChan <- requestStat{duration: reqTimeNs}
					case finishedStress:
						workerDoneChan <- workerDone{}
						return
					}
				}
			}
		}()
	}

	allRequestStats := make([]requestStat, numTests)
	requestsCompleteCount := 0
	workersDoneCount := 0
	//wait for all workers to finish
	for {
		select {
		case <-workerDoneChan:
			workersDoneCount++
			if workersDoneCount == concurrency {
				//all workers are done
				totalEndTime := time.Now()

				reqStats := createRequestsStats(allRequestStats)
				totalTimeNs := totalEndTime.UnixNano() - totalStartTime.UnixNano()
				fmt.Println(createTextSummary(reqStats, totalTimeNs))
				return nil
			}
		case requestStat := <-requestStatChan:
			allRequestStats[requestsCompleteCount] = requestStat
			requestsCompleteCount++
		}
	}
}

func createRequestsStats(requestStats []requestStat) requestStatSummary {
	if len(requestStats) == 0 {
		return requestStatSummary{}
	}

	summary := requestStatSummary{maxDuration: requestStats[0].duration, minDuration: requestStats[0].duration}
	var totalDurations int64
	totalDurations = 0
	for i := 0; i < len(requestStats); i++ {
		if requestStats[i].duration > summary.maxDuration {
			summary.maxDuration = requestStats[i].duration
		}
		if requestStats[i].duration < summary.minDuration {
			summary.minDuration = requestStats[i].duration
		}
		totalDurations += requestStats[i].duration
	}
	summary.avgDuration = totalDurations / int64(len(requestStats))
	return summary
}

func createTextSummary(reqStatSummary requestStatSummary, totalTimeNs int64) string {
	summary := "\n"
	summary = summary + "Average:    " + strconv.Itoa(int(reqStatSummary.avgDuration/1000000)) + "ms\n"
	summary = summary + "Max:        " + strconv.Itoa(int(reqStatSummary.maxDuration/1000000)) + "ms\n"
	summary = summary + "Min:        " + strconv.Itoa(int(reqStatSummary.minDuration/1000000)) + "ms\n"
	summary = summary + "Total Time: " + strconv.Itoa(int(totalTimeNs/1000000)) + "ms"
	return summary
}
<commit_msg>Adding flag for HTTP request method type/verb<commit_after>package cmd

import (
	"errors"
	"fmt"
	"strconv"
	"time"

	"github.com/spf13/cobra"
	"net/http"
)

type stressRequest interface{}

type finishedStress struct{}

type workerDone struct{}

type requestStat struct {
	duration int64 //nanoseconds
}
type requestStatSummary struct {
	avgDuration int64 //nanoseconds
	maxDuration int64 //nanoseconds
	minDuration int64 //nanoseconds
}

//flags
var (
	numTests      int
	timeout       int
	concurrency   int
	requestMethod string
)

func init() {
	RootCmd.AddCommand(stressCmd)

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// stressCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// stressCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")

	stressCmd.Flags().IntVarP(&numTests, "num", "n", 100, "Number of requests to make")
	stressCmd.Flags().IntVarP(&concurrency, "concurrent", "c", 1, "Number of multiple requests to make")
	stressCmd.Flags().IntVarP(&timeout, "timeout", "t", 0, "Maximum seconds to wait for response. 0 means unlimited")
	stressCmd.Flags().StringVarP(&requestMethod, "requestMethod", "X", "GET", "Request type. GET, HEAD, POST, PUT, etc.")
}

// stressCmd represents the stress command
var stressCmd = &cobra.Command{
	Use:   "stress http[s]://hostname[:port]/path",
	Short: "Run predefined load of requests",
	Long:  `Run predefined load of requests`,
	RunE:  RunStress,
}

func RunStress(cmd *cobra.Command, args []string) error {
	//checks
	if len(args) != 1 {
		return errors.New("needs URL")
	}
	if numTests <= 0 {
		return errors.New("number of requests must be one or more")
	}
	if concurrency <= 0 {
		return errors.New("concurrency must be one or more")
	}
	if timeout < 0 {
		return errors.New("timeout must be zero or more")
	}
	if concurrency > numTests {
		return errors.New("concurrency must be higher than number of requests")
	}

	url := args[0]

	fmt.Println("Stress testing " + url + "...")

	//setup the queue of requests
	requestChan := make(chan stressRequest, numTests+concurrency)
	for i := 0; i < numTests; i++ {
		//TODO optimize by not creating a new http request each time since it's the same thing
		req, err := http.NewRequest(requestMethod, url, nil)
		if err != nil {
			return errors.New("failed to create request: " + err.Error())
		}
		requestChan <- req
	}
	for i := 0; i < concurrency; i++ {
		requestChan <- finishedStress{}
	}

	workerDoneChan := make(chan workerDone)   //workers use this to indicate they are done
	requestStatChan := make(chan requestStat) //workers communicate each requests' info

	//workers
	totalStartTime := time.Now()
	for i := 0; i < concurrency; i++ {
		go func() {
			client := &http.Client{Timeout: time.Duration(timeout) * time.Second}
			for {
				select {
				case req := <-requestChan:
					switch req.(type) {
					case *http.Request:
						//run the acutal request
						reqStartTime := time.Now()
						_, err := client.Do(req.(*http.Request))
						reqEndTime := time.Now()
						if err != nil {
							fmt.Errorf(err.Error()) //TODO handle this further up
						}
						reqTimeNs := (reqEndTime.UnixNano() - reqStartTime.UnixNano())
						fmt.Printf("request took %dms\n", reqTimeNs/1000000)
						requestStatChan <- requestStat{duration: reqTimeNs}
					case finishedStress:
						workerDoneChan <- workerDone{}
						return
					}
				}
			}
		}()
	}

	allRequestStats := make([]requestStat, numTests)
	requestsCompleteCount := 0
	workersDoneCount := 0
	//wait for all workers to finish
	for {
		select {
		case <-workerDoneChan:
			workersDoneCount++
			if workersDoneCount == concurrency {
				//all workers are done
				totalEndTime := time.Now()

				reqStats := createRequestsStats(allRequestStats)
				totalTimeNs := totalEndTime.UnixNano() - totalStartTime.UnixNano()
				fmt.Println(createTextSummary(reqStats, totalTimeNs))
				return nil
			}
		case requestStat := <-requestStatChan:
			allRequestStats[requestsCompleteCount] = requestStat
			requestsCompleteCount++
		}
	}
}

func createRequestsStats(requestStats []requestStat) requestStatSummary {
	if len(requestStats) == 0 {
		return requestStatSummary{}
	}

	summary := requestStatSummary{maxDuration: requestStats[0].duration, minDuration: requestStats[0].duration}
	var totalDurations int64
	totalDurations = 0
	for i := 0; i < len(requestStats); i++ {
		if requestStats[i].duration > summary.maxDuration {
			summary.maxDuration = requestStats[i].duration
		}
		if requestStats[i].duration < summary.minDuration {
			summary.minDuration = requestStats[i].duration
		}
		totalDurations += requestStats[i].duration
	}
	summary.avgDuration = totalDurations / int64(len(requestStats))
	return summary
}

func createTextSummary(reqStatSummary requestStatSummary, totalTimeNs int64) string {
	summary := "\n"
	summary = summary + "Average:    " + strconv.Itoa(int(reqStatSummary.avgDuration/1000000)) + "ms\n"
	summary = summary + "Max:        " + strconv.Itoa(int(reqStatSummary.maxDuration/1000000)) + "ms\n"
	summary = summary + "Min:        " + strconv.Itoa(int(reqStatSummary.minDuration/1000000)) + "ms\n"
	summary = summary + "Total Time: " + strconv.Itoa(int(totalTimeNs/1000000)) + "ms"
	return summary
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

// `direnv stdlib`
var CmdStdlib = &Cmd{
	Name:    "stdlib",
	Desc:    "Outputs the stdlib that is available in the .envrc",
	Private: true,
	Fn: func(env Env, args []string) (err error) {
		var config *Config
		if config, err = LoadConfig(env); err != nil {
			return
		}

		fmt.Printf(STDLIB, config.SelfPath)
		return
	},
}

const STDLIB = `
# These are the commands available in an .envrc context
set -e
DIRENV_PATH="%s"

# Determines if "something" is availabe as a command
#
# Usage: has something
has() {
	type "$1" &>/dev/null
}

# Usage: expand_path ./rel/path [RELATIVE_TO]
# RELATIVE_TO is $PWD by default
expand_path() {
	"$DIRENV_PATH" expand_path "$@"
}

# Loads a .env in the current environment
#
# Usage: dotenv
dotenv() {
	eval "$("$DIRENV_PATH" dotenv "$@")"
}

# Usage: user_rel_path /Users/you/some_path => ~/some_path
user_rel_path() {
	local path="${1#-}"

	if [ -z "$path" ]; then return; fi

	if [ -n "$HOME" ]; then
		local rel_path="${path#$HOME}"
		if [ "$rel_path" != "$path" ]; then
			path="~${rel_path}"
		fi
	fi

	echo $path
}

# Usage: find_up FILENAME
find_up() {
	(
		cd "$(pwd -P 2>/dev/null)"
		while true; do
			if [ -f "$1" ]; then
				echo $PWD/$1
				return 0
			fi
			if [ "$PWD" = "/" ] || [ "$PWD" = "//" ]; then
				return 1
			fi
			cd ..
		done
	)
}

# Inherit another .envrc
#
# Usage: source_env <FILE_OR_DIR_PATH>
source_env() {
	local rcfile="$1"
	local rcpath="${1/#\~/$HOME}"
	if ! [ -f "$rcpath" ]; then
		rcfile="$rcfile/.envrc"
		rcpath="$rcpath/.envrc"
	fi
	echo "direnv: loading $rcfile"
	pushd "$(dirname "$rcpath")" > /dev/null
	. "./$(basename "$rcpath")"
	popd > /dev/null
}

# Inherits the first .envrc (or given FILENAME) it finds in the path
#
# Usage: source_up [FILENAME]
source_up() {
	local file="$1"
	if [ -z "$file" ]; then
		file=".envrc"
	fi
	local path="$(cd .. && find_up "$file")"
	if [ -n "$path" ]; then
		source_env "$(user_rel_path "$path")"
	fi
}

# Safer PATH handling
#
# Usage: PATH_add PATH
# Example: PATH_add bin
PATH_add() {
	export PATH="$(expand_path "$1"):$PATH"
}

# Safer path handling
#
# Usage: path_add VARNAME PATH
# Example: path_add LD_LIBRARY_PATH ./lib
path_add() {
	local old_paths="${!1}"
	local path="$(expand_path "$2")"

	if [ -z "$old_paths" ]; then
		old_paths="$path"
	else
		old_paths="$path:$old_paths"
	fi

	export $1="$old_paths"
}

#
# Usage: load_prefix PATH
load_prefix() {
	local path="$(expand_path "$1")"
	path_add CPATH "$path/include"
	path_add LD_LIBRARY_PATH "$path/lib"
	path_add LIBRARY_PATH "$path/lib"
	path_add MANPATH "$path/man"
	path_add MANPATH "$path/share/man"
	path_add PATH "$path/bin"
	path_add PKG_CONFIG_PATH "$path/lib/pkgconfig"
}

# Usage: layout TYPE
layout() {
	eval "layout_$1"
}

# Usage: layout ruby
layout_ruby() {
	# TODO: ruby_version should be the ABI version
	local ruby_version="$(ruby -e"puts (defined?(RUBY_ENGINE) ? RUBY_ENGINE : 'ruby') + '-' + RUBY_VERSION")"

	export GEM_HOME="$PWD/.direnv/${ruby_version}"
	export BUNDLE_BIN="$PWD/.direnv/bin"

	PATH_add ".direnv/${ruby_version}/bin"
	PATH_add ".direnv/bin"
}

# Usage: layout python
layout_python() {
	if ! [ -d .direnv/virtualenv ]; then
		virtualenv --no-site-packages --distribute .direnv/virtualenv
		virtualenv --relocatable .direnv/virtualenv
	fi
	source .direnv/virtualenv/bin/activate
}

# Usage: layout node
layout_node() {
	PATH_add node_modules/.bin
}

# This folder contains a <program-name>/<version> structure
use_prefix=/usr/local/Cellar
set_use_prefix() {
	use_prefix="$1"
}

# Usage: use PROGRAM_NAME VERSION
# Example: use ruby 1.9.3
use() {
	local cmd="$1"
	if has use_$cmd ; then
		echo "Using $@"
		shift
		use_$cmd "$@"
		return $?
	fi

	local path="$use_prefix/$1/$2"
	if [ -d "$path" ]; then
		echo "Using $1 v$2"
		load_prefix "$path"
		return
	fi

	echo "* Unable to load $path"
	return 1
}

# Usage: use rbenv
use_rbenv() {
	eval "$(rbenv init -)"
}

# Sources rvm on first call. Should work like the rvm command-line.
rvm() {
	unset rvm
	if [ -n "${rvm_scripts_path:-}" ]; then
		source "${rvm_scripts_path}/rvm"
	elif [ -n "${rvm_path:-}" ]; then
		source "${rvm_path}/scripts/rvm"
	else
		source "$HOME/.rvm/scripts/rvm"
	fi
	rvm "$@"
}

## Load the global ~/.direnvrc if present
if [ -f "$HOME/.direnvrc" ]; then
	source_env "~/.direnvrc" >&2
fi
`
<commit_msg>Remove the set_use_prefix() command in the stdlib<commit_after>package main

import (
	"fmt"
)

// `direnv stdlib`
var CmdStdlib = &Cmd{
	Name:    "stdlib",
	Desc:    "Outputs the stdlib that is available in the .envrc",
	Private: true,
	Fn: func(env Env, args []string) (err error) {
		var config *Config
		if config, err = LoadConfig(env); err != nil {
			return
		}

		fmt.Printf(STDLIB, config.SelfPath)
		return
	},
}

const STDLIB = `
# These are the commands available in an .envrc context
set -e
DIRENV_PATH="%s"

# Determines if "something" is availabe as a command
#
# Usage: has something
has() {
	type "$1" &>/dev/null
}

# Usage: expand_path ./rel/path [RELATIVE_TO]
# RELATIVE_TO is $PWD by default
expand_path() {
	"$DIRENV_PATH" expand_path "$@"
}

# Loads a .env in the current environment
#
# Usage: dotenv
dotenv() {
	eval "$("$DIRENV_PATH" dotenv "$@")"
}

# Usage: user_rel_path /Users/you/some_path => ~/some_path
user_rel_path() {
	local path="${1#-}"

	if [ -z "$path" ]; then return; fi

	if [ -n "$HOME" ]; then
		local rel_path="${path#$HOME}"
		if [ "$rel_path" != "$path" ]; then
			path="~${rel_path}"
		fi
	fi

	echo $path
}

# Usage: find_up FILENAME
find_up() {
	(
		cd "$(pwd -P 2>/dev/null)"
		while true; do
			if [ -f "$1" ]; then
				echo $PWD/$1
				return 0
			fi
			if [ "$PWD" = "/" ] || [ "$PWD" = "//" ]; then
				return 1
			fi
			cd ..
		done
	)
}

# Inherit another .envrc
#
# Usage: source_env <FILE_OR_DIR_PATH>
source_env() {
	local rcfile="$1"
	local rcpath="${1/#\~/$HOME}"
	if ! [ -f "$rcpath" ]; then
		rcfile="$rcfile/.envrc"
		rcpath="$rcpath/.envrc"
	fi
	echo "direnv: loading $rcfile"
	pushd "$(dirname "$rcpath")" > /dev/null
	. "./$(basename "$rcpath")"
	popd > /dev/null
}

# Inherits the first .envrc (or given FILENAME) it finds in the path
#
# Usage: source_up [FILENAME]
source_up() {
	local file="$1"
	if [ -z "$file" ]; then
		file=".envrc"
	fi
	local path="$(cd .. && find_up "$file")"
	if [ -n "$path" ]; then
		source_env "$(user_rel_path "$path")"
	fi
}

# Safer PATH handling
#
# Usage: PATH_add PATH
# Example: PATH_add bin
PATH_add() {
	export PATH="$(expand_path "$1"):$PATH"
}

# Safer path handling
#
# Usage: path_add VARNAME PATH
# Example: path_add LD_LIBRARY_PATH ./lib
path_add() {
	local old_paths="${!1}"
	local path="$(expand_path "$2")"

	if [ -z "$old_paths" ]; then
		old_paths="$path"
	else
		old_paths="$path:$old_paths"
	fi

	export $1="$old_paths"
}

#
# Usage: load_prefix PATH
load_prefix() {
	local path="$(expand_path "$1")"
	path_add CPATH "$path/include"
	path_add LD_LIBRARY_PATH "$path/lib"
	path_add LIBRARY_PATH "$path/lib"
	path_add MANPATH "$path/man"
	path_add MANPATH "$path/share/man"
	path_add PATH "$path/bin"
	path_add PKG_CONFIG_PATH "$path/lib/pkgconfig"
}

# Pre-programmed project layout. Add your own in your ~/.direnvrc.
#
# Usage: layout TYPE
layout() {
	eval "layout_$1"
}

# Usage: layout ruby
layout_ruby() {
	# TODO: ruby_version should be the ABI version
	local ruby_version="$(ruby -e"puts (defined?(RUBY_ENGINE) ? RUBY_ENGINE : 'ruby') + '-' + RUBY_VERSION")"

	export GEM_HOME="$PWD/.direnv/${ruby_version}"
	export BUNDLE_BIN="$PWD/.direnv/bin"

	PATH_add ".direnv/${ruby_version}/bin"
	PATH_add ".direnv/bin"
}

# Usage: layout python
layout_python() {
	if ! [ -d .direnv/virtualenv ]; then
		virtualenv --no-site-packages --distribute .direnv/virtualenv
		virtualenv --relocatable .direnv/virtualenv
	fi
	source .direnv/virtualenv/bin/activate
}

# Usage: layout node
layout_node() {
	PATH_add node_modules/.bin
}

# Intended to load external dependencies into the environment.
#
# Usage: use PROGRAM_NAME VERSION
# Example: use ruby 1.9.3
use() {
	local cmd="$1"
	echo "Using $@"
	shift
	use_$cmd "$@"
}

# Usage: use rbenv
use_rbenv() {
	eval "$(rbenv init -)"
}

# Sources rvm on first call. Should work like the rvm command-line.
rvm() {
	unset rvm
	if [ -n "${rvm_scripts_path:-}" ]; then
		source "${rvm_scripts_path}/rvm"
	elif [ -n "${rvm_path:-}" ]; then
		source "${rvm_path}/scripts/rvm"
	else
		source "$HOME/.rvm/scripts/rvm"
	fi
	rvm "$@"
}

## Load the global ~/.direnvrc if present
if [ -f "$HOME/.direnvrc" ]; then
	source_env "~/.direnvrc" >&2
fi
`
<|endoftext|>


<commit_before>// Copyright (c) 2017 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Romana CNI plugin configures kubernetes pods on Romana network.
package main

import (
	"encoding/json"
	"fmt"
	"net"
	"os"
	"runtime"
	"syscall"

	"github.com/romana/core/pkg/cni/kubernetes"

	"github.com/containernetworking/cni/pkg/ip"
	"github.com/containernetworking/cni/pkg/ns"
	"github.com/containernetworking/cni/pkg/skel"
	"github.com/containernetworking/cni/pkg/types"
	"github.com/containernetworking/cni/pkg/types/current"
	"github.com/containernetworking/cni/pkg/version"
	util "github.com/romana/core/pkg/cni"
	log "github.com/romana/rlog"
	"github.com/vishvananda/netlink"
)

func init() {
	// This ensures that main runs only on main thread (thread group leader).
	// since namespace ops (unshare, setns) are done for a single thread, we
	// must ensure that the goroutine does not jump from OS thread to thread
	runtime.LockOSThread()
}

// cmdAdd is a callback functions that gets called by skel.PluginMain
// in response to ADD method.
func cmdAdd(args *skel.CmdArgs) error {
	var err error
	// netConf stores Romana related config
	// that comes form stdin.
	netConf, _, _ := loadConf(args.StdinData)
	cniVersion := netConf.CNIVersion
	log.Debugf("Loaded netConf %v", netConf)

	// LoadArgs parses kubernetes related parameters from CNI
	// environment variables.
	k8sargs := kubernetes.K8sArgs{}
	err = types.LoadArgs(args.Args, &k8sargs)
	if err != nil {
		return fmt.Errorf("Failed to types.LoadArgs, err=(%s)", err)
	}
	log.Debugf("Loaded Kubernetes args %v", k8sargs)

	// Retrieves additional information about the pod
	pod, err := kubernetes.GetPodDescription(k8sargs, netConf.KubernetesConfig)
	if err != nil {
		return err
	}

	// Deferring deallocation before allocating ip address,
	// deallocation will be called on any return unless
	// flag set to false.
	var deallocateOnExit = true
	defer func() {
		if deallocateOnExit {
			deallocator, err := util.NewRomanaAddressManager(util.DefaultProvider)

			// don't want to panic here
			if netConf != nil && err == nil {
				log.Errorf("Deallocating IP on exit, something went wrong")
				_ = deallocator.Deallocate(*netConf, pod.Name)
			}
		}
	}()

	// Allocating ip address.
	allocator, err := util.NewRomanaAddressManager(util.DefaultProvider)
	if err != nil {
		return err
	}
	podAddress, err := allocator.Allocate(*netConf, util.RomanaAllocatorPodDescription{
		Name:        pod.Name,
		Hostname:    netConf.RomanaHostName,
		Namespace:   pod.Namespace,
		Labels:      pod.Labels,
		Annotations: pod.Annotations,
	})
	if err != nil {
		return err
	}

	// Networking setup
	gwAddr, err := GetRomanaGwAddr()
	if err != nil {
		return fmt.Errorf("Failed to detect ipv4 address on romana-gw interface, err=(%s)", err)
	}

	netns, err := ns.GetNS(args.Netns)
	if err != nil {
		return fmt.Errorf("failed to open netns %q: %v", args.Netns, err)
	}
	defer netns.Close()

	// Magic variables for callback.
	contIface := &current.Interface{}
	hostIface := &current.Interface{}
	ifName := "eth0"
	mtu := 1500 //TODO for stas, make configurable
	_, defaultNet, _ := net.ParseCIDR("0.0.0.0/0")

	// And this is a callback inside the callback, it sets up networking
	// withing a pod namespace, nice thing it save us from shellouts
	// but still, callback within a callback.
	err = netns.Do(func(hostNS ns.NetNS) error {
		// Creates veth interfacces.
		hostVeth, containerVeth, err := ip.SetupVeth(ifName, mtu, hostNS)
		if err != nil {
			return err
		}

		// transportNet is a romana-gw cidr turned into romana-gw.IP/32
		transportNet := net.IPNet{IP: gwAddr.IP, Mask: net.IPMask([]byte{0xff, 0xff, 0xff, 0xff})}
		transportRoute := netlink.Route{
			LinkIndex: containerVeth.Index,
			Dst:       &transportNet,
		}

		// sets up transport route to allow installing default route
		err = netlink.RouteAdd(&transportRoute)
		if err != nil {
			return fmt.Errorf("route add error=(%s)", err)
		}

		// default route for the pod
		defaultRoute := netlink.Route{
			Dst:       defaultNet,
			LinkIndex: containerVeth.Index,
		}
		err = netlink.RouteAdd(&defaultRoute)
		if err != nil {
			return fmt.Errorf("route add default error=(%s)", err)
		}

		containerVethLink, err := netlink.LinkByIndex(containerVeth.Index)
		if err != nil {
			return fmt.Errorf("failed to discover container veth, err=(%s)", err)
		}

		podIP, err := netlink.ParseAddr(podAddress.String())
		if err != nil {
			return fmt.Errorf("netlink failed to parse address %s, err=(%s)", podAddress, err)
		}

		err = netlink.AddrAdd(containerVethLink, podIP)
		if err != nil {
			return fmt.Errorf("failed to add ip address %s to the interface %s, err=(%s)", podIP, containerVeth.Name, err)
		}

		contIface.Name = containerVeth.Name
		contIface.Mac = containerVeth.HardwareAddr.String()
		contIface.Sandbox = netns.Path()
		hostIface.Name = hostVeth.Name
		return nil
	})
	if err != nil {
		return fmt.Errorf("Failed to create veth interfaces in namespace %v, err=(%s)", netns, err)
	}

	// Return route.
	err = AddEndpointRoute(hostIface.Name, podAddress)
	if err != nil {
		return fmt.Errorf("Failed to setup return route to %s via interface %s, err=(%s)", podAddress, hostIface.Name, err)
	}

	result := &current.Result{
		IPs: []*current.IPConfig{
			&current.IPConfig{
				Version:   "4",
				Address:   *podAddress,
				Interface: 0,
			},
		},
	}

	result.Interfaces = []*current.Interface{hostIface}

	deallocateOnExit = false
	return types.PrintResult(result, cniVersion)
}

// cmdDel is a callback functions that gets called by skel.PluginMain
// in response to DEL method.
func cmdDel(args *skel.CmdArgs) error {
	var err error
	// netConf stores Romana related config
	// that comes form stdin.
	netConf, _, _ := loadConf(args.StdinData)

	// LoadArgs parses kubernetes related parameters from CNI
	// environment variables.
	k8sargs := kubernetes.K8sArgs{}
	err = types.LoadArgs(args.Args, &k8sargs)
	if err != nil {
		return err
	}

	deallocator, err := util.NewRomanaAddressManager(util.DefaultProvider)
	if err != nil {
		return err
	}
	err = deallocator.Deallocate(*netConf, k8sargs.MakePodName())
	if err != nil {
		return fmt.Errorf("Failed to tear down pod network for %s, err=(%s)", k8sargs.MakePodName(), err)
	}

	return nil
}

// GetRomanaGwAddr detects ip address assigned to romana-gw interface.
func GetRomanaGwAddr() (*net.IPNet, error) {
	const gwIface = "romana-gw"
	romanaGw, err := netlink.LinkByName(gwIface)
	if err != nil {
		return nil, err
	}

	addr, err := netlink.AddrList(romanaGw, syscall.AF_INET)
	if err != nil {
		return nil, err
	}

	if len(addr) != 1 {
		return nil, fmt.Errorf("Expected exactly 1 ipv4 address on romana-gw interface, found %d", len(addr))
	}

	return addr[0].IPNet, nil
}

// AddEndpointRoute adds return /32 route from host to pod.
func AddEndpointRoute(ifaceName string, ip *net.IPNet) error {
	veth, err := netlink.LinkByName(ifaceName)
	if err != nil {
		return err
	}

	returnRoute := netlink.Route{
		Dst:       ip,
		LinkIndex: veth.Attrs().Index,
	}

	err = netlink.RouteAdd(&returnRoute)

	return nil
}

// loadConf initializes romana config from stdin.
func loadConf(bytes []byte) (*util.NetConf, string, error) {
	n := &util.NetConf{}
	if err := json.Unmarshal(bytes, n); err != nil {
		return nil, "", fmt.Errorf("failed to load netconf: %s", err)
	}

	// TODO for stas
	// verify config here
	if n.RomanaHostName == "" {
		hostname, err := os.Hostname()
		if err != nil {
			return nil, "", fmt.Errorf("failed to load netconf: %s", err)
		}

		n.RomanaHostName = hostname
	}

	return n, n.CNIVersion, nil
}

func main() {
	skel.PluginMain(cmdAdd, cmdDel, version.All)
}
<commit_msg>Stop loadConf function from returning unused values<commit_after>// Copyright (c) 2017 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Romana CNI plugin configures kubernetes pods on Romana network.
package main

import (
	"encoding/json"
	"fmt"
	"net"
	"os"
	"runtime"
	"syscall"

	"github.com/romana/core/pkg/cni/kubernetes"

	"github.com/containernetworking/cni/pkg/ip"
	"github.com/containernetworking/cni/pkg/ns"
	"github.com/containernetworking/cni/pkg/skel"
	"github.com/containernetworking/cni/pkg/types"
	"github.com/containernetworking/cni/pkg/types/current"
	"github.com/containernetworking/cni/pkg/version"
	util "github.com/romana/core/pkg/cni"
	log "github.com/romana/rlog"
	"github.com/vishvananda/netlink"
)

func init() {
	// This ensures that main runs only on main thread (thread group leader).
	// since namespace ops (unshare, setns) are done for a single thread, we
	// must ensure that the goroutine does not jump from OS thread to thread
	runtime.LockOSThread()
}

// cmdAdd is a callback functions that gets called by skel.PluginMain
// in response to ADD method.
func cmdAdd(args *skel.CmdArgs) error {
	var err error
	// netConf stores Romana related config
	// that comes form stdin.
	netConf, err := loadConf(args.StdinData)
	if err != nil {
		return err
	}
	cniVersion := netConf.CNIVersion
	log.Debugf("Loaded netConf %v", netConf)

	// LoadArgs parses kubernetes related parameters from CNI
	// environment variables.
	k8sargs := kubernetes.K8sArgs{}
	err = types.LoadArgs(args.Args, &k8sargs)
	if err != nil {
		return fmt.Errorf("Failed to types.LoadArgs, err=(%s)", err)
	}
	log.Debugf("Loaded Kubernetes args %v", k8sargs)

	// Retrieves additional information about the pod
	pod, err := kubernetes.GetPodDescription(k8sargs, netConf.KubernetesConfig)
	if err != nil {
		return err
	}

	// Deferring deallocation before allocating ip address,
	// deallocation will be called on any return unless
	// flag set to false.
	var deallocateOnExit = true
	defer func() {
		if deallocateOnExit {
			deallocator, err := util.NewRomanaAddressManager(util.DefaultProvider)

			// don't want to panic here
			if netConf != nil && err == nil {
				log.Errorf("Deallocating IP on exit, something went wrong")
				_ = deallocator.Deallocate(*netConf, pod.Name)
			}
		}
	}()

	// Allocating ip address.
	allocator, err := util.NewRomanaAddressManager(util.DefaultProvider)
	if err != nil {
		return err
	}
	podAddress, err := allocator.Allocate(*netConf, util.RomanaAllocatorPodDescription{
		Name:        pod.Name,
		Hostname:    netConf.RomanaHostName,
		Namespace:   pod.Namespace,
		Labels:      pod.Labels,
		Annotations: pod.Annotations,
	})
	if err != nil {
		return err
	}

	// Networking setup
	gwAddr, err := GetRomanaGwAddr()
	if err != nil {
		return fmt.Errorf("Failed to detect ipv4 address on romana-gw interface, err=(%s)", err)
	}

	netns, err := ns.GetNS(args.Netns)
	if err != nil {
		return fmt.Errorf("failed to open netns %q: %v", args.Netns, err)
	}
	defer netns.Close()

	// Magic variables for callback.
	contIface := &current.Interface{}
	hostIface := &current.Interface{}
	ifName := "eth0"
	mtu := 1500 //TODO for stas, make configurable
	_, defaultNet, _ := net.ParseCIDR("0.0.0.0/0")

	// And this is a callback inside the callback, it sets up networking
	// withing a pod namespace, nice thing it save us from shellouts
	// but still, callback within a callback.
	err = netns.Do(func(hostNS ns.NetNS) error {
		// Creates veth interfacces.
		hostVeth, containerVeth, err := ip.SetupVeth(ifName, mtu, hostNS)
		if err != nil {
			return err
		}

		// transportNet is a romana-gw cidr turned into romana-gw.IP/32
		transportNet := net.IPNet{IP: gwAddr.IP, Mask: net.IPMask([]byte{0xff, 0xff, 0xff, 0xff})}
		transportRoute := netlink.Route{
			LinkIndex: containerVeth.Index,
			Dst:       &transportNet,
		}

		// sets up transport route to allow installing default route
		err = netlink.RouteAdd(&transportRoute)
		if err != nil {
			return fmt.Errorf("route add error=(%s)", err)
		}

		// default route for the pod
		defaultRoute := netlink.Route{
			Dst:       defaultNet,
			LinkIndex: containerVeth.Index,
		}
		err = netlink.RouteAdd(&defaultRoute)
		if err != nil {
			return fmt.Errorf("route add default error=(%s)", err)
		}

		containerVethLink, err := netlink.LinkByIndex(containerVeth.Index)
		if err != nil {
			return fmt.Errorf("failed to discover container veth, err=(%s)", err)
		}

		podIP, err := netlink.ParseAddr(podAddress.String())
		if err != nil {
			return fmt.Errorf("netlink failed to parse address %s, err=(%s)", podAddress, err)
		}

		err = netlink.AddrAdd(containerVethLink, podIP)
		if err != nil {
			return fmt.Errorf("failed to add ip address %s to the interface %s, err=(%s)", podIP, containerVeth.Name, err)
		}

		contIface.Name = containerVeth.Name
		contIface.Mac = containerVeth.HardwareAddr.String()
		contIface.Sandbox = netns.Path()
		hostIface.Name = hostVeth.Name
		return nil
	})
	if err != nil {
		return fmt.Errorf("Failed to create veth interfaces in namespace %v, err=(%s)", netns, err)
	}

	// Return route.
	err = AddEndpointRoute(hostIface.Name, podAddress)
	if err != nil {
		return fmt.Errorf("Failed to setup return route to %s via interface %s, err=(%s)", podAddress, hostIface.Name, err)
	}

	result := &current.Result{
		IPs: []*current.IPConfig{
			&current.IPConfig{
				Version:   "4",
				Address:   *podAddress,
				Interface: 0,
			},
		},
	}

	result.Interfaces = []*current.Interface{hostIface}

	deallocateOnExit = false
	return types.PrintResult(result, cniVersion)
}

// cmdDel is a callback functions that gets called by skel.PluginMain
// in response to DEL method.
func cmdDel(args *skel.CmdArgs) error {
	var err error
	// netConf stores Romana related config
	// that comes form stdin.
	netConf, err := loadConf(args.StdinData)
	if err != nil {
		return err
	}

	// LoadArgs parses kubernetes related parameters from CNI
	// environment variables.
	k8sargs := kubernetes.K8sArgs{}
	err = types.LoadArgs(args.Args, &k8sargs)
	if err != nil {
		return err
	}

	deallocator, err := util.NewRomanaAddressManager(util.DefaultProvider)
	if err != nil {
		return err
	}
	err = deallocator.Deallocate(*netConf, k8sargs.MakePodName())
	if err != nil {
		return fmt.Errorf("Failed to tear down pod network for %s, err=(%s)", k8sargs.MakePodName(), err)
	}

	return nil
}

// GetRomanaGwAddr detects ip address assigned to romana-gw interface.
func GetRomanaGwAddr() (*net.IPNet, error) {
	const gwIface = "romana-gw"
	romanaGw, err := netlink.LinkByName(gwIface)
	if err != nil {
		return nil, err
	}

	addr, err := netlink.AddrList(romanaGw, syscall.AF_INET)
	if err != nil {
		return nil, err
	}

	if len(addr) != 1 {
		return nil, fmt.Errorf("Expected exactly 1 ipv4 address on romana-gw interface, found %d", len(addr))
	}

	return addr[0].IPNet, nil
}

// AddEndpointRoute adds return /32 route from host to pod.
func AddEndpointRoute(ifaceName string, ip *net.IPNet) error {
	veth, err := netlink.LinkByName(ifaceName)
	if err != nil {
		return err
	}

	returnRoute := netlink.Route{
		Dst:       ip,
		LinkIndex: veth.Attrs().Index,
	}

	err = netlink.RouteAdd(&returnRoute)

	return nil
}

// loadConf initializes romana config from stdin.
func loadConf(bytes []byte) (*util.NetConf, error) {
	n := &util.NetConf{}
	if err := json.Unmarshal(bytes, n); err != nil {
		return nil, fmt.Errorf("failed to load netconf: %s", err)
	}

	// TODO for stas
	// verify config here
	if n.RomanaHostName == "" {
		hostname, err := os.Hostname()
		if err != nil {
			return nil, fmt.Errorf("failed to load netconf: %s", err)
		}

		n.RomanaHostName = hostname
	}

	return n, nil
}

func main() {
	skel.PluginMain(cmdAdd, cmdDel, version.All)
}
<|endoftext|>