<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sync

import (
	"context"
	"fmt"
	"os/exec"
	"path/filepath"

	"github.com/bmatcuk/doublestar"
	"github.com/sirupsen/logrus"

	"strings"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/watch"
	"github.com/google/go-containerregistry/pkg/name"
	"github.com/pkg/errors"
	v1 "k8s.io/api/core/v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	lenDigest = 71
)

var (
	// WorkingDir is here for testing
	WorkingDir = retrieveWorkingDir
)

type Syncer interface {
	Sync(context.Context, *Item) error
}

type Item struct {
	Image  string
	Copy   map[string]string
	Delete map[string]string
}

func NewItem(a *latest.Artifact, e watch.Events, builds []build.Artifact) (*Item, error) {
	// If there are no changes, short circuit and don't sync anything
	if !e.HasChanged() || len(a.Sync) == 0 {
		return nil, nil
	}

	tag := latestTag(a.ImageName, builds)
	if tag == "" {
		return nil, fmt.Errorf("could not find latest tag for image %s in builds: %v", a.ImageName, builds)
	}

	wd, err := WorkingDir(a.ImageName, tag)
	if err != nil {
		return nil, errors.Wrapf(err, "retrieving working dir for %s", tag)
	}

	toCopy, err := intersect(a.Workspace, a.Sync, append(e.Added, e.Modified...), wd)
	if err != nil {
		return nil, errors.Wrap(err, "intersecting sync map and added, modified files")
	}

	toDelete, err := intersect(a.Workspace, a.Sync, e.Deleted, wd)
	if err != nil {
		return nil, errors.Wrap(err, "intersecting sync map and deleted files")
	}

	// Something went wrong, don't sync, rebuild.
	if toCopy == nil || toDelete == nil {
		return nil, nil
	}

	return &Item{
		Image:  tag,
		Copy:   toCopy,
		Delete: toDelete,
	}, nil
}

func retrieveWorkingDir(image, tagged string) (string, error) {
	fullyQualifedImage := stripTagIfDigestPresent(image, tagged)
	cf, err := docker.RetrieveRemoteConfig(fullyQualifedImage)
	if err != nil {
		return "", errors.Wrap(err, "retrieving remote config")

	}
	if cf.Config.WorkingDir == "" {
		return "/", nil
	}
	return cf.Config.WorkingDir, nil
}

// stripTagIfDigestPresent removes the tag from the image if there is a tag and a digest
func stripTagIfDigestPresent(image, tagged string) string {
	// try to parse the reference, return image if it works
	_, err := name.ParseReference(tagged, name.WeakValidation)
	if err == nil {
		return image
	}
	// strip out the tag
	digestIndex := strings.Index(tagged, "sha256:")
	if digestIndex == -1 {
		return image
	}
	digest := tagged[digestIndex : digestIndex+lenDigest]
	return fmt.Sprintf("%s@%s", image, digest)
}

func latestTag(image string, builds []build.Artifact) string {
	for _, build := range builds {
		if build.ImageName == image {
			return build.Tag
		}
	}
	return ""
}

func intersect(context string, syncMap map[string]string, files []string, workingDir string) (map[string]string, error) {
	ret := map[string]string{}

	for _, f := range files {
		relPath, err := filepath.Rel(context, f)
		if err != nil {
			return nil, errors.Wrapf(err, "changed file %s can't be found relative to context %s", f, context)
		}
		var matches bool
		for p, dst := range syncMap {
			match, err := doublestar.PathMatch(filepath.FromSlash(p), relPath)
			if err != nil {
				return nil, errors.Wrapf(err, "pattern error for %s", relPath)
			}
			if match {
				if filepath.IsAbs(dst) {
					dst = filepath.ToSlash(filepath.Join(dst, filepath.Base(relPath)))
				} else {
					dst = filepath.ToSlash(filepath.Join(workingDir, dst, filepath.Base(relPath)))
				}
				// Every file must match at least one sync pattern, if not we'll have to
				// skip the entire sync
				matches = true
				ret[f] = dst
			}
		}
		if !matches {
			logrus.Infof("Changed file %s does not match any sync pattern. Skipping sync", relPath)
			return nil, nil
		}

	}
	return ret, nil
}

func Perform(ctx context.Context, image string, files map[string]string, cmdFn func(context.Context, v1.Pod, v1.Container, map[string]string) []*exec.Cmd, namespaces []string) error {
	if len(files) == 0 {
		return nil
	}

	client, err := kubernetes.Client()
	if err != nil {
		return errors.Wrap(err, "getting k8s client")
	}

	numSynced := 0
	for _, ns := range namespaces {
		pods, err := client.CoreV1().Pods(ns).List(meta_v1.ListOptions{})
		if err != nil {
			return errors.Wrap(err, "getting pods for namespace "+ns)
		}

		for _, p := range pods.Items {
			for _, c := range p.Spec.Containers {
				if c.Image != image {
					continue
				}

				cmds := cmdFn(ctx, p, c, files)
				for _, cmd := range cmds {
					if err := util.RunCmd(cmd); err != nil {
						return err
					}
					numSynced++
				}
			}
		}
	}

	if numSynced == 0 {
		return errors.New("didn't sync any files")
	}

	return nil
}
<commit_msg>unit test error<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sync

import (
	"context"
	"fmt"
	"os/exec"
	"path/filepath"

	"github.com/bmatcuk/doublestar"
	"github.com/sirupsen/logrus"

	"strings"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/watch"
	"github.com/google/go-containerregistry/pkg/name"
	"github.com/pkg/errors"
	v1 "k8s.io/api/core/v1"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	lenDigest = 71
)

var (
	// WorkingDir is here for testing
	WorkingDir = retrieveWorkingDir
)

type Syncer interface {
	Sync(context.Context, *Item) error
}

type Item struct {
	Image  string
	Copy   map[string]string
	Delete map[string]string
}

func NewItem(a *latest.Artifact, e watch.Events, builds []build.Artifact) (*Item, error) {
	// If there are no changes, short circuit and don't sync anything
	if !e.HasChanged() || len(a.Sync) == 0 {
		return nil, nil
	}

	tag := latestTag(a.ImageName, builds)
	if tag == "" {
		return nil, fmt.Errorf("could not find latest tag for image %s in builds: %v", a.ImageName, builds)
	}

	wd, err := WorkingDir(a.ImageName, tag)
	if err != nil {
		return nil, errors.Wrapf(err, "retrieving working dir for %s", tag)
	}

	toCopy, err := intersect(a.Workspace, a.Sync, append(e.Added, e.Modified...), wd)
	if err != nil {
		return nil, errors.Wrap(err, "intersecting sync map and added, modified files")
	}

	toDelete, err := intersect(a.Workspace, a.Sync, e.Deleted, wd)
	if err != nil {
		return nil, errors.Wrap(err, "intersecting sync map and deleted files")
	}

	// Something went wrong, don't sync, rebuild.
	if toCopy == nil || toDelete == nil {
		return nil, nil
	}

	return &Item{
		Image:  tag,
		Copy:   toCopy,
		Delete: toDelete,
	}, nil
}

func retrieveWorkingDir(image, tagged string) (string, error) {
	fullyQualifedImage := stripTagIfDigestPresent(image, tagged)
	cf, err := docker.RetrieveRemoteConfig(fullyQualifedImage)
	if err != nil {
		return "", errors.Wrap(err, "retrieving remote config")

	}
	if cf.Config.WorkingDir == "" {
		return "/", nil
	}
	return cf.Config.WorkingDir, nil
}

// stripTagIfDigestPresent removes the tag from the image if there is a tag and a digest
func stripTagIfDigestPresent(image, tagged string) string {
	// try to parse the reference, return image if it works
	_, err := name.ParseReference(tagged, name.WeakValidation)
	if err == nil {
		return tagged
	}
	// strip out the tag
	digestIndex := strings.Index(tagged, "sha256:")
	if digestIndex == -1 {
		return tagged
	}
	digest := tagged[digestIndex : digestIndex+lenDigest]
	return fmt.Sprintf("%s@%s", image, digest)
}

func latestTag(image string, builds []build.Artifact) string {
	for _, build := range builds {
		if build.ImageName == image {
			return build.Tag
		}
	}
	return ""
}

func intersect(context string, syncMap map[string]string, files []string, workingDir string) (map[string]string, error) {
	ret := map[string]string{}

	for _, f := range files {
		relPath, err := filepath.Rel(context, f)
		if err != nil {
			return nil, errors.Wrapf(err, "changed file %s can't be found relative to context %s", f, context)
		}
		var matches bool
		for p, dst := range syncMap {
			match, err := doublestar.PathMatch(filepath.FromSlash(p), relPath)
			if err != nil {
				return nil, errors.Wrapf(err, "pattern error for %s", relPath)
			}
			if match {
				if filepath.IsAbs(dst) {
					dst = filepath.ToSlash(filepath.Join(dst, filepath.Base(relPath)))
				} else {
					dst = filepath.ToSlash(filepath.Join(workingDir, dst, filepath.Base(relPath)))
				}
				// Every file must match at least one sync pattern, if not we'll have to
				// skip the entire sync
				matches = true
				ret[f] = dst
			}
		}
		if !matches {
			logrus.Infof("Changed file %s does not match any sync pattern. Skipping sync", relPath)
			return nil, nil
		}

	}
	return ret, nil
}

func Perform(ctx context.Context, image string, files map[string]string, cmdFn func(context.Context, v1.Pod, v1.Container, map[string]string) []*exec.Cmd, namespaces []string) error {
	if len(files) == 0 {
		return nil
	}

	client, err := kubernetes.Client()
	if err != nil {
		return errors.Wrap(err, "getting k8s client")
	}

	numSynced := 0
	for _, ns := range namespaces {
		pods, err := client.CoreV1().Pods(ns).List(meta_v1.ListOptions{})
		if err != nil {
			return errors.Wrap(err, "getting pods for namespace "+ns)
		}

		for _, p := range pods.Items {
			for _, c := range p.Spec.Containers {
				if c.Image != image {
					continue
				}

				cmds := cmdFn(ctx, p, c, files)
				for _, cmd := range cmds {
					if err := util.RunCmd(cmd); err != nil {
						return err
					}
					numSynced++
				}
			}
		}
	}

	if numSynced == 0 {
		return errors.New("didn't sync any files")
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"time"

	contracts "github.com/estafette/estafette-ci-contracts"
	manifest "github.com/estafette/estafette-ci-manifest"
	"github.com/rs/zerolog/log"
)

// PipelineRunner is the interface for running the pipeline steps
type PipelineRunner interface {
	runStage(string, map[string]string, manifest.EstafetteStage) (estafetteStageRunResult, error)
	runStages([]*manifest.EstafetteStage, string, map[string]string) (estafetteRunStagesResult, error)
}

type pipelineRunnerImpl struct {
	envvarHelper  EnvvarHelper
	whenEvaluator WhenEvaluator
	dockerRunner  DockerRunner
	ciServer      string
}

// NewPipelineRunner returns a new PipelineRunner
func NewPipelineRunner(envvarHelper EnvvarHelper, whenEvaluator WhenEvaluator, dockerRunner DockerRunner) PipelineRunner {
	return &pipelineRunnerImpl{
		envvarHelper:  envvarHelper,
		whenEvaluator: whenEvaluator,
		dockerRunner:  dockerRunner,
		ciServer:      os.Getenv("ESTAFETTE_CI_SERVER"),
	}
}

type estafetteStageRunResult struct {
	Stage               manifest.EstafetteStage
	RunIndex            int
	IsDockerImagePulled bool
	DockerImageSize     int64
	DockerPullDuration  time.Duration
	DockerPullError     error
	DockerRunDuration   time.Duration
	DockerRunError      error
	OtherError          error
	ExitCode            int64
	Status              string
	LogLines            []contracts.BuildLogLine
}

// Errors combines the different type of errors that occurred during this pipeline stage
func (result *estafetteStageRunResult) Errors() (errors []error) {

	if result.DockerPullError != nil {
		errors = append(errors, result.DockerPullError)
	}

	if result.DockerRunError != nil {
		errors = append(errors, result.DockerRunError)
	}

	if result.OtherError != nil {
		errors = append(errors, result.OtherError)
	}

	return errors
}

// HasErrors indicates whether any errors happened in this pipeline stage
func (result *estafetteStageRunResult) HasErrors() bool {

	errors := result.Errors()

	return len(errors) > 0
}

func (pr *pipelineRunnerImpl) runStage(dir string, envvars map[string]string, p manifest.EstafetteStage) (result estafetteStageRunResult, err error) {

	result.Stage = p
	result.LogLines = make([]contracts.BuildLogLine, 0)

	log.Info().Msgf("[%v] Starting pipeline '%v'", p.Name, p.Name)

	result.IsDockerImagePulled = pr.dockerRunner.isDockerImagePulled(p)

	if !result.IsDockerImagePulled {

		// pull docker image
		dockerPullStart := time.Now()
		result.DockerPullError = pr.dockerRunner.runDockerPull(p)
		result.DockerPullDuration = time.Since(dockerPullStart)
		if result.DockerPullError != nil {
			return result, result.DockerPullError
		}

		// set docker image size
		size, err := pr.dockerRunner.getDockerImageSize(p)
		if err != nil {
			return result, err
		}
		result.DockerImageSize = size

	}

	if pr.ciServer != "gocd" {
		imageLogLine := contracts.TailLogLine{
			Step:         p.Name,
			Image:        getBuildLogStepDockerImage(result),
			AutoInjected: &result.Stage.AutoInjected,
		}

		// log as json, to be tailed when looking at live logs from gui
		log.Info().Interface("tailLogLine", imageLogLine).Msg("")
	}

	// run commands in docker container
	dockerRunStart := time.Now()
	result.LogLines, result.ExitCode, result.DockerRunError = pr.dockerRunner.runDockerRun(dir, envvars, p)
	result.DockerRunDuration = time.Since(dockerRunStart)
	if result.DockerRunError != nil {
		return result, result.DockerRunError
	}

	if pr.ciServer != "gocd" {
		imageLogLine := contracts.TailLogLine{
			Step:     p.Name,
			Duration: &result.DockerRunDuration,
			ExitCode: &result.ExitCode,
			Status:   &result.Status,
		}

		// log as json, to be tailed when looking at live logs from gui
		log.Info().Interface("tailLogLine", imageLogLine).Msg("")
	}

	log.Info().Msgf("[%v] Finished pipeline '%v' successfully", p.Name, p.Name)

	return
}

type estafetteRunStagesResult struct {
	StageResults []estafetteStageRunResult
}

// Errors combines the different type of errors that occurred during this pipeline stage
func (result *estafetteRunStagesResult) Errors() (errors []error) {

	for _, pr := range result.StageResults {
		if pr.HasErrors() {
			errors = append(errors, pr.Errors()...)
		}
	}

	return errors
}

// HasErrors indicates whether any errors happened in this pipeline stages
func (result *estafetteRunStagesResult) HasErrors() bool {

	errors := result.Errors()

	return len(errors) > 0
}

func (pr *pipelineRunnerImpl) runStages(stages []*manifest.EstafetteStage, dir string, envvars map[string]string) (result estafetteRunStagesResult, err error) {

	// set default build status if not set
	err = pr.envvarHelper.initBuildStatus()
	if err != nil {
		return
	}

	if len(stages) == 0 {
		return result, fmt.Errorf("Manifest has no stages, failing the build")
	}

	for _, p := range stages {

		whenEvaluationResult, err := pr.whenEvaluator.evaluate(p.Name, p.When, pr.whenEvaluator.getParameters())
		if err != nil {
			return result, err
		}

		if whenEvaluationResult {

			runIndex := 0
			for runIndex <= p.Retries {

				r, err := pr.runStage(dir, envvars, *p)
				if err != nil {

					// add error to log lines
					r.RunIndex = runIndex
					r.LogLines = append(r.LogLines, contracts.BuildLogLine{
						Timestamp:  time.Now().UTC(),
						StreamType: "stderr",
						Text:       err.Error(),
					})

					// set 'failed' build status
					if runIndex == p.Retries {
						pr.envvarHelper.setEstafetteEnv("ESTAFETTE_BUILD_STATUS", "failed")
						envvars[pr.envvarHelper.getEstafetteEnvvarName("ESTAFETTE_BUILD_STATUS")] = "failed"
					}

					r.Status = "FAILED"
					r.OtherError = err

					result.StageResults = append(result.StageResults, r)

					runIndex++

					continue
				}

				// set 'succeeded' build status
				r.Status = "SUCCEEDED"

				result.StageResults = append(result.StageResults, r)

				break
			}

		} else {

			// if an error has happened in one of the previous steps or the when expression evaluates to false we still want to render the following steps in the result table
			r := estafetteStageRunResult{
				Stage:  *p,
				Status: "SKIPPED",
			}

			result.StageResults = append(result.StageResults, r)

			continue
		}
	}

	return
}
<commit_msg>rename variables for tailing logs<commit_after>package main

import (
	"fmt"
	"os"
	"time"

	contracts "github.com/estafette/estafette-ci-contracts"
	manifest "github.com/estafette/estafette-ci-manifest"
	"github.com/rs/zerolog/log"
)

// PipelineRunner is the interface for running the pipeline steps
type PipelineRunner interface {
	runStage(string, map[string]string, manifest.EstafetteStage) (estafetteStageRunResult, error)
	runStages([]*manifest.EstafetteStage, string, map[string]string) (estafetteRunStagesResult, error)
}

type pipelineRunnerImpl struct {
	envvarHelper  EnvvarHelper
	whenEvaluator WhenEvaluator
	dockerRunner  DockerRunner
	ciServer      string
}

// NewPipelineRunner returns a new PipelineRunner
func NewPipelineRunner(envvarHelper EnvvarHelper, whenEvaluator WhenEvaluator, dockerRunner DockerRunner) PipelineRunner {
	return &pipelineRunnerImpl{
		envvarHelper:  envvarHelper,
		whenEvaluator: whenEvaluator,
		dockerRunner:  dockerRunner,
		ciServer:      os.Getenv("ESTAFETTE_CI_SERVER"),
	}
}

type estafetteStageRunResult struct {
	Stage               manifest.EstafetteStage
	RunIndex            int
	IsDockerImagePulled bool
	DockerImageSize     int64
	DockerPullDuration  time.Duration
	DockerPullError     error
	DockerRunDuration   time.Duration
	DockerRunError      error
	OtherError          error
	ExitCode            int64
	Status              string
	LogLines            []contracts.BuildLogLine
}

// Errors combines the different type of errors that occurred during this pipeline stage
func (result *estafetteStageRunResult) Errors() (errors []error) {

	if result.DockerPullError != nil {
		errors = append(errors, result.DockerPullError)
	}

	if result.DockerRunError != nil {
		errors = append(errors, result.DockerRunError)
	}

	if result.OtherError != nil {
		errors = append(errors, result.OtherError)
	}

	return errors
}

// HasErrors indicates whether any errors happened in this pipeline stage
func (result *estafetteStageRunResult) HasErrors() bool {

	errors := result.Errors()

	return len(errors) > 0
}

func (pr *pipelineRunnerImpl) runStage(dir string, envvars map[string]string, p manifest.EstafetteStage) (result estafetteStageRunResult, err error) {

	result.Stage = p
	result.LogLines = make([]contracts.BuildLogLine, 0)

	log.Info().Msgf("[%v] Starting pipeline '%v'", p.Name, p.Name)

	result.IsDockerImagePulled = pr.dockerRunner.isDockerImagePulled(p)

	if !result.IsDockerImagePulled {

		// pull docker image
		dockerPullStart := time.Now()
		result.DockerPullError = pr.dockerRunner.runDockerPull(p)
		result.DockerPullDuration = time.Since(dockerPullStart)
		if result.DockerPullError != nil {
			return result, result.DockerPullError
		}

		// set docker image size
		size, err := pr.dockerRunner.getDockerImageSize(p)
		if err != nil {
			return result, err
		}
		result.DockerImageSize = size

	}

	if pr.ciServer != "gocd" {
		tailLogLine := contracts.TailLogLine{
			Step:         p.Name,
			Image:        getBuildLogStepDockerImage(result),
			AutoInjected: &result.Stage.AutoInjected,
		}

		// log as json, to be tailed when looking at live logs from gui
		log.Info().Interface("tailLogLine", tailLogLine).Msg("")
	}

	// run commands in docker container
	dockerRunStart := time.Now()
	result.LogLines, result.ExitCode, result.DockerRunError = pr.dockerRunner.runDockerRun(dir, envvars, p)
	result.DockerRunDuration = time.Since(dockerRunStart)
	if result.DockerRunError != nil {
		return result, result.DockerRunError
	}

	if pr.ciServer != "gocd" {
		tailLogLine := contracts.TailLogLine{
			Step:     p.Name,
			Duration: &result.DockerRunDuration,
			ExitCode: &result.ExitCode,
			Status:   &result.Status,
		}

		// log as json, to be tailed when looking at live logs from gui
		log.Info().Interface("tailLogLine", tailLogLine).Msg("")
	}

	log.Info().Msgf("[%v] Finished pipeline '%v' successfully", p.Name, p.Name)

	return
}

type estafetteRunStagesResult struct {
	StageResults []estafetteStageRunResult
}

// Errors combines the different type of errors that occurred during this pipeline stage
func (result *estafetteRunStagesResult) Errors() (errors []error) {

	for _, pr := range result.StageResults {
		if pr.HasErrors() {
			errors = append(errors, pr.Errors()...)
		}
	}

	return errors
}

// HasErrors indicates whether any errors happened in this pipeline stages
func (result *estafetteRunStagesResult) HasErrors() bool {

	errors := result.Errors()

	return len(errors) > 0
}

func (pr *pipelineRunnerImpl) runStages(stages []*manifest.EstafetteStage, dir string, envvars map[string]string) (result estafetteRunStagesResult, err error) {

	// set default build status if not set
	err = pr.envvarHelper.initBuildStatus()
	if err != nil {
		return
	}

	if len(stages) == 0 {
		return result, fmt.Errorf("Manifest has no stages, failing the build")
	}

	for _, p := range stages {

		whenEvaluationResult, err := pr.whenEvaluator.evaluate(p.Name, p.When, pr.whenEvaluator.getParameters())
		if err != nil {
			return result, err
		}

		if whenEvaluationResult {

			runIndex := 0
			for runIndex <= p.Retries {

				r, err := pr.runStage(dir, envvars, *p)
				if err != nil {

					// add error to log lines
					r.RunIndex = runIndex
					r.LogLines = append(r.LogLines, contracts.BuildLogLine{
						Timestamp:  time.Now().UTC(),
						StreamType: "stderr",
						Text:       err.Error(),
					})

					// set 'failed' build status
					if runIndex == p.Retries {
						pr.envvarHelper.setEstafetteEnv("ESTAFETTE_BUILD_STATUS", "failed")
						envvars[pr.envvarHelper.getEstafetteEnvvarName("ESTAFETTE_BUILD_STATUS")] = "failed"
					}

					r.Status = "FAILED"
					r.OtherError = err

					result.StageResults = append(result.StageResults, r)

					runIndex++

					continue
				}

				// set 'succeeded' build status
				r.Status = "SUCCEEDED"

				result.StageResults = append(result.StageResults, r)

				break
			}

		} else {

			// if an error has happened in one of the previous steps or the when expression evaluates to false we still want to render the following steps in the result table
			r := estafetteStageRunResult{
				Stage:  *p,
				Status: "SKIPPED",
			}

			result.StageResults = append(result.StageResults, r)

			continue
		}
	}

	return
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"context"
	"io"
	"os"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	runcontext "github.com/GoogleContainerTools/skaffold/pkg/skaffold/runner/context"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/test/structure"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"

	"github.com/pkg/errors"
)

// NewTester parses the provided test cases from the Skaffold config,
// and returns a Tester instance with all the necessary test runners
// to run all specified tests.
func NewTester(ctx *runcontext.RunContext) (Tester, error) {
	// TODO(nkubala): copied this from runner.getDeployer(), this should be moved somewhere else
	cwd, err := os.Getwd()
	if err != nil {
		return nil, errors.Wrap(err, "finding current directory")
	}

	return FullTester{
		testCases:  ctx.Cfg.Test,
		workingDir: ctx.WorkingDir,
	}, nil
}

// TestDependencies returns the watch dependencies to the runner.
func (t FullTester) TestDependencies() ([]string, error) {
	var deps []string

	for _, test := range t.testCases {
		if test.StructureTests == nil {
			continue
		}

		files, err := util.ExpandPathsGlob(t.workingDir, test.StructureTests)
		if err != nil {
			return nil, errors.Wrap(err, "expanding test file paths")
		}

		deps = append(deps, files...)
	}

	return deps, nil
}

// Test is the top level testing execution call. It serves as the
// entrypoint to all individual tests.
func (t FullTester) Test(ctx context.Context, out io.Writer, bRes []build.Artifact) error {
	for _, test := range t.testCases {
		if err := t.runStructureTests(ctx, out, bRes, test); err != nil {
			return errors.Wrap(err, "running structure tests")
		}
	}

	return nil
}

func (t FullTester) runStructureTests(ctx context.Context, out io.Writer, bRes []build.Artifact, testCase *latest.TestCase) error {
	if len(testCase.StructureTests) == 0 {
		return nil
	}

	files, err := util.ExpandPathsGlob(t.workingDir, testCase.StructureTests)
	if err != nil {
		return errors.Wrap(err, "expanding test file paths")
	}

	runner := structure.NewRunner(files)
	fqn := resolveArtifactImageTag(testCase.ImageName, bRes)

	return runner.Test(ctx, out, fqn)
}

func resolveArtifactImageTag(imageName string, bRes []build.Artifact) string {
	for _, res := range bRes {
		if imageName == res.ImageName {
			return res.Tag
		}
	}

	return imageName
}
<commit_msg>fix typo<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"context"
	"io"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	runcontext "github.com/GoogleContainerTools/skaffold/pkg/skaffold/runner/context"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/test/structure"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"

	"github.com/pkg/errors"
)

// NewTester parses the provided test cases from the Skaffold config,
// and returns a Tester instance with all the necessary test runners
// to run all specified tests.
func NewTester(ctx *runcontext.RunContext) (Tester, error) {
	return FullTester{
		testCases:  ctx.Cfg.Test,
		workingDir: ctx.WorkingDir,
	}, nil
}

// TestDependencies returns the watch dependencies to the runner.
func (t FullTester) TestDependencies() ([]string, error) {
	var deps []string

	for _, test := range t.testCases {
		if test.StructureTests == nil {
			continue
		}

		files, err := util.ExpandPathsGlob(t.workingDir, test.StructureTests)
		if err != nil {
			return nil, errors.Wrap(err, "expanding test file paths")
		}

		deps = append(deps, files...)
	}

	return deps, nil
}

// Test is the top level testing execution call. It serves as the
// entrypoint to all individual tests.
func (t FullTester) Test(ctx context.Context, out io.Writer, bRes []build.Artifact) error {
	for _, test := range t.testCases {
		if err := t.runStructureTests(ctx, out, bRes, test); err != nil {
			return errors.Wrap(err, "running structure tests")
		}
	}

	return nil
}

func (t FullTester) runStructureTests(ctx context.Context, out io.Writer, bRes []build.Artifact, testCase *latest.TestCase) error {
	if len(testCase.StructureTests) == 0 {
		return nil
	}

	files, err := util.ExpandPathsGlob(t.workingDir, testCase.StructureTests)
	if err != nil {
		return errors.Wrap(err, "expanding test file paths")
	}

	runner := structure.NewRunner(files)
	fqn := resolveArtifactImageTag(testCase.ImageName, bRes)

	return runner.Test(ctx, out, fqn)
}

func resolveArtifactImageTag(imageName string, bRes []build.Artifact) string {
	for _, res := range bRes {
		if imageName == res.ImageName {
			return res.Tag
		}
	}

	return imageName
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package gce

import (
	"net/http"
	"net/mail"
	"time"

	"code.google.com/p/goauth2/oauth"
	"code.google.com/p/goauth2/oauth/jwt"
	"code.google.com/p/google-api-go-client/compute/v1"
	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/utils"

	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/network"
	"github.com/juju/juju/provider/common"
)

const (
	driverScopes = "https://www.googleapis.com/auth/compute " +
		"https://www.googleapis.com/auth/devstorage.full_control"

	tokenURL = "https://accounts.google.com/o/oauth2/token"

	authURL = "https://accounts.google.com/o/oauth2/auth"

	partialMachineType = "zones/%s/machineTypes/%s"

	diskTypeScratch    = "SCRATCH"
	diskTypePersistent = "PERSISTENT"
	diskModeRW         = "READ_WRITE"
	diskModeRO         = "READ_ONLY"

	statusDone         = "DONE"
	statusDown         = "DOWN"
	statusPending      = "PENDING"
	statusProvisioning = "PROVISIONING"
	statusRunning      = "RUNNING"
	statusStaging      = "STAGING"
	statusStopped      = "STOPPED"
	statusStopping     = "STOPPING"
	statusTerminated   = "TERMINATED"
	statusUp           = "UP"

	operationTimeout = 60 // seconds

	// minDiskSize is the minimum/default size (in megabytes) for GCE
	// disks. GCE does not currently have a minimum disk size.
	minDiskSize int64 = 0
)

var (
	logger = loggo.GetLogger("juju.provider.gce")

	errNotImplemented = errors.NotImplementedf("gce provider")

	operationAttempts = utils.AttemptStrategy{
		Total: operationTimeout * time.Second,
		Delay: 10 * time.Second,
	}

	signedImageDataOnly = false
)

type gceAuth struct {
	clientID    string
	clientEmail string
	privateKey  []byte
}

func (ga gceAuth) validate() error {
	if ga.clientID == "" {
		return &config.InvalidConfigValue{Key: osEnvClientID}
	}
	if ga.clientEmail == "" {
		return &config.InvalidConfigValue{Key: osEnvClientEmail}
	} else if _, err := mail.ParseAddress(ga.clientEmail); err != nil {
		err = errors.Trace(err)
		return &config.InvalidConfigValue{osEnvClientEmail, ga.clientEmail, err}
	}
	if len(ga.privateKey) == 0 {
		return &config.InvalidConfigValue{Key: osEnvPrivateKey}
	}
	return nil
}

func (ga gceAuth) newTransport() (*oauth.Transport, error) {
	token, err := newToken(ga, driverScopes)
	if err != nil {
		return nil, errors.Trace(err)
	}

	transport := oauth.Transport{
		Config: &oauth.Config{
			ClientId: ga.clientID,
			Scope:    driverScopes,
			TokenURL: tokenURL,
			AuthURL:  authURL,
		},
		Token: token,
	}
	return &transport, nil
}

var newToken = func(auth gceAuth, scopes string) (*oauth.Token, error) {
	jtok := jwt.NewToken(auth.clientEmail, scopes, auth.privateKey)
	jtok.ClaimSet.Aud = tokenURL

	token, err := jtok.Assert(&http.Client{})
	if err != nil {
		msg := "retrieving auth token for %s"
		return nil, errors.Annotatef(err, msg, auth.clientEmail)
	}
	return token, nil
}

func (ga *gceAuth) newConnection() (*compute.Service, error) {
	transport, err := ga.newTransport()
	if err != nil {
		return nil, errors.Trace(err)
	}
	service, err := newService(transport)
	return service, errors.Trace(err)
}

var newService = func(transport *oauth.Transport) (*compute.Service, error) {
	return compute.New(transport.Client())
}

type gceConnection struct {
	*compute.Service

	region    string
	projectID string
}

func (gce *gceConnection) validate() error {
	if gce.projectID == "" {
		return &config.InvalidConfigValue{Key: osEnvProjectID}
	}
	return nil
}

func (gce *gceConnection) connect(auth gceAuth) error {
	if gce.Service != nil {
		return errors.New("connect() failed (already connected)")
	}

	service, err := auth.newConnection()
	if err != nil {
		return errors.Trace(err)
	}

	gce.Service = service
	return nil
}

func (gce *gceConnection) verifyCredentials() error {
	call := gce.Projects.Get(gce.projectID)
	if _, err := call.Do(); err != nil {
		return errors.Trace(err)
	}
	return nil
}

func (gce *gceConnection) waitOperation(operation *compute.Operation) error {
	opID := operation.ClientOperationId

	for a := operationAttempts.Start(); a.Next(); {
		var err error
		if operation.Status == statusDone {
			return nil
		}
		call := gce.GlobalOperations.Get(gce.projectID, opID)
		operation, err = call.Do()
		if err != nil {
			return errors.Annotate(err, "waiting for operation to complete")
		}
	}
	if operation.Status == statusDone {
		return nil
	}

	msg := "timed out after %d seconds waiting for GCE operation to finish"
	return errors.Errorf(msg, operationTimeout)
}

func (gce *gceConnection) instance(zone, id string) (*compute.Instance, error) {
	call := gce.Instances.Get(gce.projectID, zone, id)
	inst, err := call.Do()
	return inst, errors.Trace(err)
}

func (gce *gceConnection) newInstance(inst *compute.Instance, machineType string, zones []string) error {
	for _, zoneName := range zones {
		inst.MachineType = resolveMachineType(zoneName, machineType)
		call := gce.Instances.Insert(
			gce.projectID,
			zoneName,
			inst,
		)
		operation, err := call.Do()
		if err != nil {
			// XXX Handle zone-is-full error.
			return errors.Annotate(err, "sending new instance request")
		}
		if err := gce.waitOperation(operation); err != nil {
			// TODO(ericsnow) Handle zone-is-full error here?
			return errors.Trace(err)
		}

		// Get the instance here.
		updated, err := gce.instance(zoneName, inst.Name)
		if err != nil {
			return errors.Trace(err)
		}
		*inst = *updated
		// Success!
		return nil
	}
	return errors.Errorf("not able to provision in any zone")
}

func (gce *gceConnection) instances(env environs.Environ) ([]*compute.Instance, error) {
	// env won't be nil.
	prefix := common.MachineFullName(env, "")

	// TODO(ericsnow) MaxResults arg defaults to 500... (call.MaxResults()).
	call := gce.Instances.AggregatedList(gce.projectID)
	call = call.Filter("name eq " + prefix + ".*")
	// TODO(ericsnow) If we can use multiple filters, filter on status here.
	raw, err := call.Do()
	if err != nil {
		return nil, errors.Trace(err)
	}

	var results []*compute.Instance
	for _, item := range raw.Items {
		for _, inst := range item.Instances {
			results = append(results, inst)
		}
	}
	return results, nil
}

func (gce *gceConnection) availabilityZones(region string) ([]*compute.Zone, error) {
	//TODO(wwtizel3) support paging requests if we receive a truncated result.
	call := gce.Zones.List(gce.projectID)
	if region != "" {
		call = call.Filter("name eq " + region + "-")
	}
	raw, err := call.Do()
	if err != nil {
		return nil, errors.Trace(err)
	}
	return raw.Items, nil
}

func (gce *gceConnection) removeInstance(id, zone string) error {
	call := gce.Instances.Delete(gce.projectID, zone, id)
	operation, err := call.Do()
	if err != nil {
		return errors.Trace(err)
	}
	if err := gce.waitOperation(operation); err != nil {
		return errors.Trace(err)
	}
	return nil
}

func (gce *gceConnection) removeInstances(env environs.Environ, ids ...string) error {
	if len(ids) == 0 {
		return nil
	}

	instances, err := gce.instances(env)
	if err != nil {
		return errors.Annotatef(err, "while removing instances %v", ids)
	}

	var failed []string
	for _, instID := range ids {
		for _, inst := range instances {
			if inst.Name == instID {
				if err := gce.removeInstance(instID, inst.Zone); err != nil {
					failed = append(failed, instID)
					logger.Errorf("while removing instance %q: %v", instID, err)
				}
				break
			}
		}
	}
	if len(failed) != 0 {
		return errors.Errorf("some instance removals failed: %v", failed)
	}
	return nil
}

func (gce *gceConnection) firewall(name string) (*compute.Firewall, error) {
	call := gce.Firewalls.Get(gce.projectID, name)
	firewall, err := call.Do()
	if err != nil {
		return nil, errors.Annotate(err, "while getting firewall from GCE")
	}
	return firewall, nil
}

func (gce *gceConnection) setFirewall(name string, firewall *compute.Firewall) error {
	var err error
	var operation *compute.Operation
	if firewall == nil {
		call := gce.Firewalls.Delete(gce.projectID, name)
		operation, err = call.Do()
		if err != nil {
			return errors.Trace(err)
		}
	} else if name == "" {
		call := gce.Firewalls.Insert(gce.projectID, firewall)
		operation, err = call.Do()
		if err != nil {
			return errors.Trace(err)
		}
	} else {
		call := gce.Firewalls.Update(gce.projectID, name, firewall)
		operation, err = call.Do()
		if err != nil {
			return errors.Trace(err)
		}
	}
	if err := gce.waitOperation(operation); err != nil {
		return errors.Trace(err)
	}
	return nil
}

func filterInstances(instances []*compute.Instance, statuses ...string) []*compute.Instance {
	// TODO(ericsnow) Filter in-place?
	// TODO(ericsnow) Also filter metadata (or tags)? While highly
	// unlikely (due to our choice of instance ID), it is possible that
	// the filter in gce.instances() results in a false positive.
	// An additional filter on the metadata would address that
	// possibility.
	var results []*compute.Instance
	for _, inst := range instances {
		if !checkInstStatus(inst, statuses...) {
			continue
		}
		results = append(results, inst)
	}
	return results
}

func checkInstStatus(inst *compute.Instance, statuses ...string) bool {
	for _, status := range statuses {
		if inst.Status == status {
			return true
		}
	}
	return false
}

type diskSpec struct {
	// sizeHint is the requested disk size in Gigabytes.
	sizeHint int64
	imageURL string
	boot     bool
	scratch  bool
	readonly bool
}

func (ds *diskSpec) size() int64 {
	size := minDiskSize
	if ds.sizeHint >= minDiskSize {
		size = ds.sizeHint
	}
	return size
}

func (ds *diskSpec) newAttached() *compute.AttachedDisk {
	diskType := diskTypePersistent // The default.
	if ds.scratch {
		diskType = diskTypeScratch
	}
	mode := diskModeRW // The default.
	if ds.readonly {
		mode = diskModeRO
	}

	disk := compute.AttachedDisk{
		Type: diskType,
		Boot: ds.boot,
		Mode: mode,
		InitializeParams: &compute.AttachedDiskInitializeParams{
			// DiskName (defaults to instance name)
			DiskSizeGb: ds.size(),
			// DiskType (defaults to pd-standard, pd-ssd, local-ssd)
			SourceImage: ds.imageURL,
		},
		// Interface (defaults to SCSI)
		// DeviceName (GCE sets this, persistent disk only)
	}
	return &disk
}

// firewallSpec expands a port range set in to compute.FirewallAllowed
// and returns a compute.Firewall for the provided name.
func firewallSpec(name string, ps network.PortSet) *compute.Firewall {
	firewall := compute.Firewall{
		// Allowed is set below.
		// Description is not set.
		Name: name,
		// TODO(ericsnow) Does Network need to be set?
		// Network: "",
		// SourceRanges is not set.
		// SourceTags is not set.
		// TargetTags is not set.
	}

	for _, protocol := range ps.Protocols() {
		allowed := compute.FirewallAllowed{
			IPProtocol: protocol,
			Ports:      ps.PortStrings(protocol),
		}
		firewall.Allowed = append(firewall.Allowed, &allowed)
	}
	return &firewall
}
<commit_msg>provider/gce: Handle errors better (try next zone) in newInstance.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package gce

import (
	"net/http"
	"net/mail"
	"time"

	"code.google.com/p/goauth2/oauth"
	"code.google.com/p/goauth2/oauth/jwt"
	"code.google.com/p/google-api-go-client/compute/v1"
	"github.com/juju/errors"
	"github.com/juju/loggo"
	"github.com/juju/utils"

	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/network"
	"github.com/juju/juju/provider/common"
)

const (
	driverScopes = "https://www.googleapis.com/auth/compute " +
		"https://www.googleapis.com/auth/devstorage.full_control"

	tokenURL = "https://accounts.google.com/o/oauth2/token"

	authURL = "https://accounts.google.com/o/oauth2/auth"

	partialMachineType = "zones/%s/machineTypes/%s"

	diskTypeScratch    = "SCRATCH"
	diskTypePersistent = "PERSISTENT"
	diskModeRW         = "READ_WRITE"
	diskModeRO         = "READ_ONLY"

	statusDone         = "DONE"
	statusDown         = "DOWN"
	statusPending      = "PENDING"
	statusProvisioning = "PROVISIONING"
	statusRunning      = "RUNNING"
	statusStaging      = "STAGING"
	statusStopped      = "STOPPED"
	statusStopping     = "STOPPING"
	statusTerminated   = "TERMINATED"
	statusUp           = "UP"

	operationTimeout = 60 // seconds

	// minDiskSize is the minimum/default size (in megabytes) for GCE
	// disks. GCE does not currently have a minimum disk size.
	minDiskSize int64 = 0
)

var (
	logger = loggo.GetLogger("juju.provider.gce")

	errNotImplemented = errors.NotImplementedf("gce provider")

	operationAttempts = utils.AttemptStrategy{
		Total: operationTimeout * time.Second,
		Delay: 10 * time.Second,
	}

	signedImageDataOnly = false
)

type gceAuth struct {
	clientID    string
	clientEmail string
	privateKey  []byte
}

func (ga gceAuth) validate() error {
	if ga.clientID == "" {
		return &config.InvalidConfigValue{Key: osEnvClientID}
	}
	if ga.clientEmail == "" {
		return &config.InvalidConfigValue{Key: osEnvClientEmail}
	} else if _, err := mail.ParseAddress(ga.clientEmail); err != nil {
		err = errors.Trace(err)
		return &config.InvalidConfigValue{osEnvClientEmail, ga.clientEmail, err}
	}
	if len(ga.privateKey) == 0 {
		return &config.InvalidConfigValue{Key: osEnvPrivateKey}
	}
	return nil
}

func (ga gceAuth) newTransport() (*oauth.Transport, error) {
	token, err := newToken(ga, driverScopes)
	if err != nil {
		return nil, errors.Trace(err)
	}

	transport := oauth.Transport{
		Config: &oauth.Config{
			ClientId: ga.clientID,
			Scope:    driverScopes,
			TokenURL: tokenURL,
			AuthURL:  authURL,
		},
		Token: token,
	}
	return &transport, nil
}

var newToken = func(auth gceAuth, scopes string) (*oauth.Token, error) {
	jtok := jwt.NewToken(auth.clientEmail, scopes, auth.privateKey)
	jtok.ClaimSet.Aud = tokenURL

	token, err := jtok.Assert(&http.Client{})
	if err != nil {
		msg := "retrieving auth token for %s"
		return nil, errors.Annotatef(err, msg, auth.clientEmail)
	}
	return token, nil
}

func (ga *gceAuth) newConnection() (*compute.Service, error) {
	transport, err := ga.newTransport()
	if err != nil {
		return nil, errors.Trace(err)
	}
	service, err := newService(transport)
	return service, errors.Trace(err)
}

var newService = func(transport *oauth.Transport) (*compute.Service, error) {
	return compute.New(transport.Client())
}

type gceConnection struct {
	*compute.Service

	region    string
	projectID string
}

func (gce *gceConnection) validate() error {
	if gce.projectID == "" {
		return &config.InvalidConfigValue{Key: osEnvProjectID}
	}
	return nil
}

func (gce *gceConnection) connect(auth gceAuth) error {
	if gce.Service != nil {
		return errors.New("connect() failed (already connected)")
	}

	service, err := auth.newConnection()
	if err != nil {
		return errors.Trace(err)
	}

	gce.Service = service
	return nil
}

func (gce *gceConnection) verifyCredentials() error {
	call := gce.Projects.Get(gce.projectID)
	if _, err := call.Do(); err != nil {
		return errors.Trace(err)
	}
	return nil
}

func (gce *gceConnection) waitOperation(operation *compute.Operation) error {
	opID := operation.ClientOperationId

	logger.Infof("GCE operation %q, waiting...", opID)
	for a := operationAttempts.Start(); a.Next(); {
		var err error
		if operation.Status == statusDone {
			break
		}
		call := gce.GlobalOperations.Get(gce.projectID, opID)
		operation, err = call.Do()
		if err != nil {
			return errors.Annotate(err, "waiting for operation to complete")
		}
	}
	if operation.Status != statusDone {
		msg := "timed out after %d seconds waiting for GCE operation to finish"
		return errors.Errorf(msg, operationTimeout)
	}
	if operation.Error != nil {
		for _, err := range operation.Error.Errors {
			logger.Errorf("GCE operation failed: (%s) %s", err.Code, err.Message)
		}
		return errors.Errorf("GCE operation %q failed", opID)
	}

	logger.Infof("GCE operation %q finished", opID)
	return nil
}

func (gce *gceConnection) instance(zone, id string) (*compute.Instance, error) {
	call := gce.Instances.Get(gce.projectID, zone, id)
	inst, err := call.Do()
	return inst, errors.Trace(err)
}

func (gce *gceConnection) newInstance(inst *compute.Instance, machineType string, zones []string) error {
	for _, zoneName := range zones {
		inst.MachineType = resolveMachineType(zoneName, machineType)
		call := gce.Instances.Insert(
			gce.projectID,
			zoneName,
			inst,
		)
		operation, err := call.Do()
		if err != nil {
			// We are guaranteed the insert failed at the point.
			return errors.Annotate(err, "sending new instance request")
		}
		waitErr := gce.waitOperation(operation)

		// Check if the instance was created.
		updated, err := gce.instance(zoneName, inst.Name)
		if err != nil {
			if waitErr == nil {
				return errors.Trace(err)
			}
			// Try the next zone.
			logger.Errorf("failed to get new instance in zone %q: %v", zoneName, waitErr)
			continue
		}

		// Success!
		*inst = *updated
		return nil
	}
	return errors.Errorf("not able to provision in any zone")
}

func (gce *gceConnection) instances(env environs.Environ) ([]*compute.Instance, error) {
	// env won't be nil.
	prefix := common.MachineFullName(env, "")

	// TODO(ericsnow) MaxResults arg defaults to 500... (call.MaxResults()).
	call := gce.Instances.AggregatedList(gce.projectID)
	call = call.Filter("name eq " + prefix + ".*")
	// TODO(ericsnow) If we can use multiple filters, filter on status here.
	raw, err := call.Do()
	if err != nil {
		return nil, errors.Trace(err)
	}

	var results []*compute.Instance
	for _, item := range raw.Items {
		for _, inst := range item.Instances {
			results = append(results, inst)
		}
	}
	return results, nil
}

func (gce *gceConnection) availabilityZones(region string) ([]*compute.Zone, error) {
	//TODO(wwtizel3) support paging requests if we receive a truncated result.
	call := gce.Zones.List(gce.projectID)
	if region != "" {
		call = call.Filter("name eq " + region + "-")
	}
	raw, err := call.Do()
	if err != nil {
		return nil, errors.Trace(err)
	}
	return raw.Items, nil
}

func (gce *gceConnection) removeInstance(id, zone string) error {
	call := gce.Instances.Delete(gce.projectID, zone, id)
	operation, err := call.Do()
	if err != nil {
		return errors.Trace(err)
	}
	if err := gce.waitOperation(operation); err != nil {
		return errors.Trace(err)
	}
	return nil
}

func (gce *gceConnection) removeInstances(env environs.Environ, ids ...string) error {
	if len(ids) == 0 {
		return nil
	}

	instances, err := gce.instances(env)
	if err != nil {
		return errors.Annotatef(err, "while removing instances %v", ids)
	}

	var failed []string
	for _, instID := range ids {
		for _, inst := range instances {
			if inst.Name == instID {
				if err := gce.removeInstance(instID, inst.Zone); err != nil {
					failed = append(failed, instID)
					logger.Errorf("while removing instance %q: %v", instID, err)
				}
				break
			}
		}
	}
	if len(failed) != 0 {
		return errors.Errorf("some instance removals failed: %v", failed)
	}
	return nil
}

func (gce *gceConnection) firewall(name string) (*compute.Firewall, error) {
	call := gce.Firewalls.Get(gce.projectID, name)
	firewall, err := call.Do()
	if err != nil {
		return nil, errors.Annotate(err, "while getting firewall from GCE")
	}
	return firewall, nil
}

func (gce *gceConnection) setFirewall(name string, firewall *compute.Firewall) error {
	var err error
	var operation *compute.Operation
	if firewall == nil {
		call := gce.Firewalls.Delete(gce.projectID, name)
		operation, err = call.Do()
		if err != nil {
			return errors.Trace(err)
		}
	} else if name == "" {
		call := gce.Firewalls.Insert(gce.projectID, firewall)
		operation, err = call.Do()
		if err != nil {
			return errors.Trace(err)
		}
	} else {
		call := gce.Firewalls.Update(gce.projectID, name, firewall)
		operation, err = call.Do()
		if err != nil {
			return errors.Trace(err)
		}
	}
	if err := gce.waitOperation(operation); err != nil {
		return errors.Trace(err)
	}
	return nil
}

func filterInstances(instances []*compute.Instance, statuses ...string) []*compute.Instance {
	// TODO(ericsnow) Filter in-place?
	// TODO(ericsnow) Also filter metadata (or tags)? While highly
	// unlikely (due to our choice of instance ID), it is possible that
	// the filter in gce.instances() results in a false positive.
	// An additional filter on the metadata would address that
	// possibility.
	var results []*compute.Instance
	for _, inst := range instances {
		if !checkInstStatus(inst, statuses...) {
			continue
		}
		results = append(results, inst)
	}
	return results
}

func checkInstStatus(inst *compute.Instance, statuses ...string) bool {
	for _, status := range statuses {
		if inst.Status == status {
			return true
		}
	}
	return false
}

type diskSpec struct {
	// sizeHint is the requested disk size in Gigabytes.
	sizeHint int64
	imageURL string
	boot     bool
	scratch  bool
	readonly bool
}

func (ds *diskSpec) size() int64 {
	size := minDiskSize
	if ds.sizeHint >= minDiskSize {
		size = ds.sizeHint
	}
	return size
}

func (ds *diskSpec) newAttached() *compute.AttachedDisk {
	diskType := diskTypePersistent // The default.
	if ds.scratch {
		diskType = diskTypeScratch
	}
	mode := diskModeRW // The default.
	if ds.readonly {
		mode = diskModeRO
	}

	disk := compute.AttachedDisk{
		Type: diskType,
		Boot: ds.boot,
		Mode: mode,
		InitializeParams: &compute.AttachedDiskInitializeParams{
			// DiskName (defaults to instance name)
			DiskSizeGb: ds.size(),
			// DiskType (defaults to pd-standard, pd-ssd, local-ssd)
			SourceImage: ds.imageURL,
		},
		// Interface (defaults to SCSI)
		// DeviceName (GCE sets this, persistent disk only)
	}
	return &disk
}

// firewallSpec expands a port range set in to compute.FirewallAllowed
// and returns a compute.Firewall for the provided name.
func firewallSpec(name string, ps network.PortSet) *compute.Firewall {
	firewall := compute.Firewall{
		// Allowed is set below.
		// Description is not set.
		Name: name,
		// TODO(ericsnow) Does Network need to be set?
		// Network: "",
		// SourceRanges is not set.
		// SourceTags is not set.
		// TargetTags is not set.
	}

	for _, protocol := range ps.Protocols() {
		allowed := compute.FirewallAllowed{
			IPProtocol: protocol,
			Ports:      ps.PortStrings(protocol),
		}
		firewall.Allowed = append(firewall.Allowed, &allowed)
	}
	return &firewall
}
<|endoftext|>


<commit_before>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aws

import (
	"bytes"
	"context"
	"fmt"
	"os"
	"sort"
	"strings"

	"github.com/GoogleCloudPlatform/terraformer/terraformutils"
	"github.com/aws/aws-sdk-go-v2/service/ec2"
	"github.com/aws/aws-sdk-go-v2/service/ec2/types"
	"github.com/hashicorp/terraform/flatmap"
	"gonum.org/v1/gonum/graph"
	simplegraph "gonum.org/v1/gonum/graph/simple"
	"gonum.org/v1/gonum/graph/topo"
)

var SgAllowEmptyValues = []string{"tags."}

type void struct{}

var member void

type SecurityGenerator struct {
	AWSService
}

type ByGroupPair []types.UserIdGroupPair

func (b ByGroupPair) Len() int      { return len(b) }
func (b ByGroupPair) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b ByGroupPair) Less(i, j int) bool {
	if b[i].GroupId != nil && b[j].GroupId != nil {
		return *b[i].GroupId < *b[j].GroupId
	}
	if b[i].GroupName != nil && b[j].GroupName != nil {
		return *b[i].GroupName < *b[j].GroupName
	}

	panic("mismatched security group rules, may be a terraform bug")
}

func (SecurityGenerator) createResources(securityGroups []types.SecurityGroup) []terraformutils.Resource {
	var sgIDsToMoveOut []string
	_, shouldSplitRules := os.LookupEnv("SPLIT_SG_RULES")
	if shouldSplitRules {
		for _, sg := range securityGroups {
			sgIDsToMoveOut = append(sgIDsToMoveOut, *sg.GroupId)
		}
	} else {
		sgIDsToMoveOut = findSgsToMoveOut(securityGroups)
	}

	var resources []terraformutils.Resource
	for _, sg := range securityGroups {
		if sg.VpcId == nil {
			continue
		}
		ruleAttributes := map[string]interface{}{}
		// we must move out all of the rules - https://github.com/hashicorp/terraform/issues/11011#issuecomment-283076580
		for _, groupIDToMoveOut := range sgIDsToMoveOut {
			if groupIDToMoveOut == *sg.GroupId {
				ruleAttributes["clearRules"] = true
				for _, rule := range sg.IpPermissions {
					resources = processRule(rule, "ingress", sg, resources)
				}
				for _, rule := range sg.IpPermissionsEgress {
					resources = processRule(rule, "egress", sg, resources)
				}
			}
		}

		resources = append(resources, terraformutils.NewResource(
			StringValue(sg.GroupId),
			strings.Trim(StringValue(sg.GroupName)+"_"+StringValue(sg.GroupId), " "),
			"aws_security_group",
			"aws",
			map[string]string{},
			SgAllowEmptyValues,
			ruleAttributes))
	}
	return resources
}

func processRule(rule types.IpPermission, ruleType string, sg types.SecurityGroup, resources []terraformutils.Resource) []terraformutils.Resource {
	if rule.UserIdGroupPairs != nil && len(rule.UserIdGroupPairs) > 0 {
		if len(rule.IpRanges) > 0 { // we must unwind coupled CIDR IPv4 range + security group rules
			attributes := baseRuleAttributes(ruleType, rule, sg)
			resources = append(resources, terraformutils.NewResource(
				permissionID(*sg.GroupId, ruleType, "", rule),
				permissionID(*sg.GroupId, ruleType, "", rule),
				"aws_security_group_rule",
				"aws",
				flatmap.Flatten(attributes),
				SgAllowEmptyValues,
				map[string]interface{}{}))
		}
		if len(rule.Ipv6Ranges) > 0 { // we must unwind coupled CIDR IPv6 range + security group rules
			attributes := baseRuleAttributes(ruleType, rule, sg)
			resources = append(resources, terraformutils.NewResource(
				permissionID(*sg.GroupId, ruleType, "", rule),
				permissionID(*sg.GroupId, ruleType, "", rule),
				"aws_security_group_rule",
				"aws",
				flatmap.Flatten(attributes),
				SgAllowEmptyValues,
				map[string]interface{}{}))
		}
		for _, groupPair := range rule.UserIdGroupPairs {
			attributes := baseRuleAttributes(ruleType, rule, sg)
			delete(attributes, "cidr_blocks")
			delete(attributes, "ipv6_cidr_blocks")
			if *groupPair.GroupId == *sg.GroupId { // Solution to C1
				attributes["self"] = true
			} else {
				attributes["source_security_group_id"] = *groupPair.GroupId
			}

			resources = append(resources, terraformutils.NewResource(
				permissionID(*sg.GroupId, ruleType, *groupPair.GroupId, rule),
				permissionID(*sg.GroupId, ruleType, *groupPair.GroupId, rule),
				"aws_security_group_rule",
				"aws",
				flatmap.Flatten(attributes),
				SgAllowEmptyValues,
				map[string]interface{}{}))
		}
	} else {
		attributes := baseRuleAttributes(ruleType, rule, sg)
		resources = append(resources, terraformutils.NewResource(
			permissionID(*sg.GroupId, ruleType, "", rule),
			permissionID(*sg.GroupId, ruleType, "", rule),
			"aws_security_group_rule",
			"aws",
			flatmap.Flatten(attributes),
			SgAllowEmptyValues,
			map[string]interface{}{}))
	}
	return resources
}

func baseRuleAttributes(ruleType string, rule types.IpPermission, sg types.SecurityGroup) map[string]interface{} {
	attributes := map[string]interface{}{
		"type":              ruleType,
		"cidr_blocks":       ipRange(rule),
		"ipv6_cidr_blocks":  ip6Range(rule),
		"prefix_list_ids":   prefixes(rule),
		"from_port":         fromPort(rule),
		"protocol":          *rule.IpProtocol,
		"security_group_id": *sg.GroupId,
		"to_port":           toPort(rule),
	}
	return attributes
}

// Let's try to find all cycles by applying Johnson's method on the directed graph
// We cannot build a line graph and move out only rules because of hashicorp/terraform#11011
func findSgsToMoveOut(securityGroups []types.SecurityGroup) []string {
	// Vertexes are security groups, edges are rules. The task is to find correct set of rule definitions, so that we
	// won't have cycles
	sourceGraph := simplegraph.NewDirectedGraph()
	idToSg := make(map[int]types.SecurityGroup)
	sgToIdx := make(map[string]int64)
	for idx, sg := range securityGroups {
		idToSg[idx] = sg
		sgToIdx[StringValue(sg.GroupId)] = int64(idx)
		sourceGraph.AddNode(sourceGraph.NewNode())
	}
	for idx, sg := range securityGroups {
		for _, rule := range sg.IpPermissions {
			pairs := rule.UserIdGroupPairs
			for _, pair := range pairs {
				if pair.GroupId != nil {
					fromNode := sourceGraph.Node(int64(idx))
					toNode := sourceGraph.Node(sgToIdx[StringValue(pair.GroupId)])
					if fromNode.ID() != toNode.ID() {
						sourceGraph.SetEdge(sourceGraph.NewEdge(fromNode, toNode))
					}
				}
			}
		}
	}

	cyclesInLineGraph := topo.DirectedCyclesIn(sourceGraph) // C1 cycles won't be found but Terraform solves that issue
	resultingSet := make(map[string]void)

	for _, v := range cyclesInLineGraph {
		if elementAlreadyFound(resultingSet, v, idToSg) {
			continue
		}

		// Try to move out node with lowest number of rules
		group := idToSg[int(v[0].ID())]
		for _, vi := range v {
			viGroup := idToSg[int(vi.ID())]
			if len(viGroup.IpPermissions) < len(group.IpPermissions) {
				group = viGroup
			}
		}

		resultingSet[*group.GroupId] = member
	}

	result := make([]string, len(resultingSet))
	i := 0
	for k := range resultingSet {
		result[i] = k
		i++
	}

	return result
}

func elementAlreadyFound(resultingSet map[string]void, v []graph.Node, idToSg map[int]types.SecurityGroup) bool {
	for k := range resultingSet {
		for _, vi := range v {
			viGroupID := *idToSg[int(vi.ID())].GroupId
			if k == viGroupID {
				return true
			}
		}
	}
	return false
}

func (g *SecurityGenerator) InitResources() error {
	config, err := g.generateConfig()
	if err != nil {
		return err
	}
	svc := ec2.NewFromConfig(config)
	p := ec2.NewDescribeSecurityGroupsPaginator(svc, &ec2.DescribeSecurityGroupsInput{})
	var resourcesToFilter []types.SecurityGroup
	for p.HasMorePages() {
		page, err := p.NextPage(context.TODO())
		if err != nil {
			return err
		}
		resourcesToFilter = append(resourcesToFilter, page.SecurityGroups...)
	}
	sort.Slice(resourcesToFilter, func(i, j int) bool {
		return *resourcesToFilter[i].GroupId < *resourcesToFilter[j].GroupId
	})
	g.Resources = g.createResources(resourcesToFilter)

	return nil
}

func (g *SecurityGenerator) PostConvertHook() error {
	for _, resource := range g.Resources {
		if resource.InstanceInfo.Type == "aws_security_group_rule" {
			if resource.Item["self"] == "true" {
				delete(resource.Item, "source_security_group_id")
			}
		} else if resource.InstanceInfo.Type == "aws_security_group" {
			if resource.Item["clearRules"] == true {
				delete(resource.Item, "ingress")
				delete(resource.Item, "egress")
				delete(resource.Item, "clearRules")
				continue
			}

			if val, ok := resource.Item["ingress"]; ok {
				g.sortRules(val.([]interface{}))
			}
			if val, ok := resource.Item["egress"]; ok {
				g.sortRules(val.([]interface{}))
			}
		}
	}
	return nil
}

func (g *SecurityGenerator) sortRules(rules []interface{}) {
	for _, rule := range rules {
		ruleMap := rule.(map[string]interface{})
		g.sortIfExist("cidr_blocks", ruleMap)
		g.sortIfExist("ipv6_cidr_blocks", ruleMap)
		g.sortIfExist("security_groups", ruleMap)
	}
	sort.Slice(rules, func(i, j int) bool {
		return fmt.Sprintf("%v", rules[i]) < fmt.Sprintf("%v", rules[j])
	})
}

func (g *SecurityGenerator) sortIfExist(attribute string, ruleMap map[string]interface{}) {
	if val, ok := ruleMap[attribute]; ok {
		sort.Slice(val.([]interface{}), func(i, j int) bool {
			return val.([]interface{})[i].(string) < val.([]interface{})[j].(string)
		})
	}
}

func permissionID(sgID, ruleType, groupID string, ip types.IpPermission) string {
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf("%s_%s_%s_%d_%d_", sgID, ruleType, *ip.IpProtocol, fromPort(ip), toPort(ip)))

	if len(ip.IpRanges) > 0 {
		s := make([]string, len(ip.IpRanges))
		for i, r := range ip.IpRanges {
			s[i] = *r.CidrIp
		}
		sort.Strings(s)

		for _, v := range s {
			buf.WriteString(fmt.Sprintf("%s_", v))
		}
	}

	if len(ip.Ipv6Ranges) > 0 {
		s := make([]string, len(ip.Ipv6Ranges))
		for i, r := range ip.Ipv6Ranges {
			s[i] = *r.CidrIpv6
		}
		sort.Strings(s)

		for _, v := range s {
			buf.WriteString(fmt.Sprintf("%s_", v))
		}
	}

	if len(ip.PrefixListIds) > 0 {
		s := make([]string, len(ip.PrefixListIds))
		for i, pl := range ip.PrefixListIds {
			s[i] = *pl.PrefixListId
		}
		sort.Strings(s)

		for _, v := range s {
			buf.WriteString(fmt.Sprintf("%s_", v))
		}
	}

	if groupID != "" {
		buf.WriteString(fmt.Sprintf("%s_", groupID))
	}

	idPreformatted := buf.String()
	return idPreformatted[:len(idPreformatted)-1]
}

func fromPort(ip types.IpPermission) int {
	switch {
	case *ip.IpProtocol == "icmp":
		return -1
	case *ip.FromPort > 0:
		return int(*ip.FromPort)
	default:
		return 0
	}
}

func toPort(ip types.IpPermission) int {
	switch {
	case *ip.IpProtocol == "icmp":
		return -1
	case *ip.ToPort > 0:
		return int(*ip.ToPort)
	default:
		return 65536
	}
}

func ipRange(rule types.IpPermission) []string {
	result := make([]string, len(rule.IpRanges))
	for idx, rule := range rule.IpRanges {
		result[idx] = *rule.CidrIp
	}
	return result
}

func ip6Range(rule types.IpPermission) []string {
	result := make([]string, len(rule.Ipv6Ranges))
	for idx, rule := range rule.Ipv6Ranges {
		result[idx] = *rule.CidrIpv6
	}
	return result
}

func prefixes(rule types.IpPermission) []string {
	result := make([]string, len(rule.PrefixListIds))
	for idx, rule := range rule.PrefixListIds {
		result[idx] = *rule.PrefixListId
	}
	return result
}
<commit_msg>fix nil port error with FromPort and ToPort for aws provider (#1486)<commit_after>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aws

import (
	"bytes"
	"context"
	"fmt"
	"os"
	"sort"
	"strings"

	"github.com/GoogleCloudPlatform/terraformer/terraformutils"
	"github.com/aws/aws-sdk-go-v2/service/ec2"
	"github.com/aws/aws-sdk-go-v2/service/ec2/types"
	"github.com/hashicorp/terraform/flatmap"
	"gonum.org/v1/gonum/graph"
	simplegraph "gonum.org/v1/gonum/graph/simple"
	"gonum.org/v1/gonum/graph/topo"
)

var SgAllowEmptyValues = []string{"tags."}

type void struct{}

var member void

type SecurityGenerator struct {
	AWSService
}

type ByGroupPair []types.UserIdGroupPair

func (b ByGroupPair) Len() int      { return len(b) }
func (b ByGroupPair) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b ByGroupPair) Less(i, j int) bool {
	if b[i].GroupId != nil && b[j].GroupId != nil {
		return *b[i].GroupId < *b[j].GroupId
	}
	if b[i].GroupName != nil && b[j].GroupName != nil {
		return *b[i].GroupName < *b[j].GroupName
	}

	panic("mismatched security group rules, may be a terraform bug")
}

func (SecurityGenerator) createResources(securityGroups []types.SecurityGroup) []terraformutils.Resource {
	var sgIDsToMoveOut []string
	_, shouldSplitRules := os.LookupEnv("SPLIT_SG_RULES")
	if shouldSplitRules {
		for _, sg := range securityGroups {
			sgIDsToMoveOut = append(sgIDsToMoveOut, *sg.GroupId)
		}
	} else {
		sgIDsToMoveOut = findSgsToMoveOut(securityGroups)
	}

	var resources []terraformutils.Resource
	for _, sg := range securityGroups {
		if sg.VpcId == nil {
			continue
		}
		ruleAttributes := map[string]interface{}{}
		// we must move out all of the rules - https://github.com/hashicorp/terraform/issues/11011#issuecomment-283076580
		for _, groupIDToMoveOut := range sgIDsToMoveOut {
			if groupIDToMoveOut == *sg.GroupId {
				ruleAttributes["clearRules"] = true
				for _, rule := range sg.IpPermissions {
					resources = processRule(rule, "ingress", sg, resources)
				}
				for _, rule := range sg.IpPermissionsEgress {
					resources = processRule(rule, "egress", sg, resources)
				}
			}
		}

		resources = append(resources, terraformutils.NewResource(
			StringValue(sg.GroupId),
			strings.Trim(StringValue(sg.GroupName)+"_"+StringValue(sg.GroupId), " "),
			"aws_security_group",
			"aws",
			map[string]string{},
			SgAllowEmptyValues,
			ruleAttributes))
	}
	return resources
}

func processRule(rule types.IpPermission, ruleType string, sg types.SecurityGroup, resources []terraformutils.Resource) []terraformutils.Resource {
	if rule.UserIdGroupPairs != nil && len(rule.UserIdGroupPairs) > 0 {
		if len(rule.IpRanges) > 0 { // we must unwind coupled CIDR IPv4 range + security group rules
			attributes := baseRuleAttributes(ruleType, rule, sg)
			resources = append(resources, terraformutils.NewResource(
				permissionID(*sg.GroupId, ruleType, "", rule),
				permissionID(*sg.GroupId, ruleType, "", rule),
				"aws_security_group_rule",
				"aws",
				flatmap.Flatten(attributes),
				SgAllowEmptyValues,
				map[string]interface{}{}))
		}
		if len(rule.Ipv6Ranges) > 0 { // we must unwind coupled CIDR IPv6 range + security group rules
			attributes := baseRuleAttributes(ruleType, rule, sg)
			resources = append(resources, terraformutils.NewResource(
				permissionID(*sg.GroupId, ruleType, "", rule),
				permissionID(*sg.GroupId, ruleType, "", rule),
				"aws_security_group_rule",
				"aws",
				flatmap.Flatten(attributes),
				SgAllowEmptyValues,
				map[string]interface{}{}))
		}
		for _, groupPair := range rule.UserIdGroupPairs {
			attributes := baseRuleAttributes(ruleType, rule, sg)
			delete(attributes, "cidr_blocks")
			delete(attributes, "ipv6_cidr_blocks")
			if *groupPair.GroupId == *sg.GroupId { // Solution to C1
				attributes["self"] = true
			} else {
				attributes["source_security_group_id"] = *groupPair.GroupId
			}

			resources = append(resources, terraformutils.NewResource(
				permissionID(*sg.GroupId, ruleType, *groupPair.GroupId, rule),
				permissionID(*sg.GroupId, ruleType, *groupPair.GroupId, rule),
				"aws_security_group_rule",
				"aws",
				flatmap.Flatten(attributes),
				SgAllowEmptyValues,
				map[string]interface{}{}))
		}
	} else {
		attributes := baseRuleAttributes(ruleType, rule, sg)
		resources = append(resources, terraformutils.NewResource(
			permissionID(*sg.GroupId, ruleType, "", rule),
			permissionID(*sg.GroupId, ruleType, "", rule),
			"aws_security_group_rule",
			"aws",
			flatmap.Flatten(attributes),
			SgAllowEmptyValues,
			map[string]interface{}{}))
	}
	return resources
}

func baseRuleAttributes(ruleType string, rule types.IpPermission, sg types.SecurityGroup) map[string]interface{} {
	attributes := map[string]interface{}{
		"type":              ruleType,
		"cidr_blocks":       ipRange(rule),
		"ipv6_cidr_blocks":  ip6Range(rule),
		"prefix_list_ids":   prefixes(rule),
		"from_port":         fromPort(rule),
		"protocol":          *rule.IpProtocol,
		"security_group_id": *sg.GroupId,
		"to_port":           toPort(rule),
	}
	return attributes
}

// Let's try to find all cycles by applying Johnson's method on the directed graph
// We cannot build a line graph and move out only rules because of hashicorp/terraform#11011
func findSgsToMoveOut(securityGroups []types.SecurityGroup) []string {
	// Vertexes are security groups, edges are rules. The task is to find correct set of rule definitions, so that we
	// won't have cycles
	sourceGraph := simplegraph.NewDirectedGraph()
	idToSg := make(map[int]types.SecurityGroup)
	sgToIdx := make(map[string]int64)
	for idx, sg := range securityGroups {
		idToSg[idx] = sg
		sgToIdx[StringValue(sg.GroupId)] = int64(idx)
		sourceGraph.AddNode(sourceGraph.NewNode())
	}
	for idx, sg := range securityGroups {
		for _, rule := range sg.IpPermissions {
			pairs := rule.UserIdGroupPairs
			for _, pair := range pairs {
				if pair.GroupId != nil {
					fromNode := sourceGraph.Node(int64(idx))
					toNode := sourceGraph.Node(sgToIdx[StringValue(pair.GroupId)])
					if fromNode.ID() != toNode.ID() {
						sourceGraph.SetEdge(sourceGraph.NewEdge(fromNode, toNode))
					}
				}
			}
		}
	}

	cyclesInLineGraph := topo.DirectedCyclesIn(sourceGraph) // C1 cycles won't be found but Terraform solves that issue
	resultingSet := make(map[string]void)

	for _, v := range cyclesInLineGraph {
		if elementAlreadyFound(resultingSet, v, idToSg) {
			continue
		}

		// Try to move out node with lowest number of rules
		group := idToSg[int(v[0].ID())]
		for _, vi := range v {
			viGroup := idToSg[int(vi.ID())]
			if len(viGroup.IpPermissions) < len(group.IpPermissions) {
				group = viGroup
			}
		}

		resultingSet[*group.GroupId] = member
	}

	result := make([]string, len(resultingSet))
	i := 0
	for k := range resultingSet {
		result[i] = k
		i++
	}

	return result
}

func elementAlreadyFound(resultingSet map[string]void, v []graph.Node, idToSg map[int]types.SecurityGroup) bool {
	for k := range resultingSet {
		for _, vi := range v {
			viGroupID := *idToSg[int(vi.ID())].GroupId
			if k == viGroupID {
				return true
			}
		}
	}
	return false
}

func (g *SecurityGenerator) InitResources() error {
	config, err := g.generateConfig()
	if err != nil {
		return err
	}
	svc := ec2.NewFromConfig(config)
	p := ec2.NewDescribeSecurityGroupsPaginator(svc, &ec2.DescribeSecurityGroupsInput{})
	var resourcesToFilter []types.SecurityGroup
	for p.HasMorePages() {
		page, err := p.NextPage(context.TODO())
		if err != nil {
			return err
		}
		resourcesToFilter = append(resourcesToFilter, page.SecurityGroups...)
	}
	sort.Slice(resourcesToFilter, func(i, j int) bool {
		return *resourcesToFilter[i].GroupId < *resourcesToFilter[j].GroupId
	})
	g.Resources = g.createResources(resourcesToFilter)

	return nil
}

func (g *SecurityGenerator) PostConvertHook() error {
	for _, resource := range g.Resources {
		if resource.InstanceInfo.Type == "aws_security_group_rule" {
			if resource.Item["self"] == "true" {
				delete(resource.Item, "source_security_group_id")
			}
		} else if resource.InstanceInfo.Type == "aws_security_group" {
			if resource.Item["clearRules"] == true {
				delete(resource.Item, "ingress")
				delete(resource.Item, "egress")
				delete(resource.Item, "clearRules")
				continue
			}

			if val, ok := resource.Item["ingress"]; ok {
				g.sortRules(val.([]interface{}))
			}
			if val, ok := resource.Item["egress"]; ok {
				g.sortRules(val.([]interface{}))
			}
		}
	}
	return nil
}

func (g *SecurityGenerator) sortRules(rules []interface{}) {
	for _, rule := range rules {
		ruleMap := rule.(map[string]interface{})
		g.sortIfExist("cidr_blocks", ruleMap)
		g.sortIfExist("ipv6_cidr_blocks", ruleMap)
		g.sortIfExist("security_groups", ruleMap)
	}
	sort.Slice(rules, func(i, j int) bool {
		return fmt.Sprintf("%v", rules[i]) < fmt.Sprintf("%v", rules[j])
	})
}

func (g *SecurityGenerator) sortIfExist(attribute string, ruleMap map[string]interface{}) {
	if val, ok := ruleMap[attribute]; ok {
		sort.Slice(val.([]interface{}), func(i, j int) bool {
			return val.([]interface{})[i].(string) < val.([]interface{})[j].(string)
		})
	}
}

func permissionID(sgID, ruleType, groupID string, ip types.IpPermission) string {
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf("%s_%s_%s_%d_%d_", sgID, ruleType, *ip.IpProtocol, fromPort(ip), toPort(ip)))

	if len(ip.IpRanges) > 0 {
		s := make([]string, len(ip.IpRanges))
		for i, r := range ip.IpRanges {
			s[i] = *r.CidrIp
		}
		sort.Strings(s)

		for _, v := range s {
			buf.WriteString(fmt.Sprintf("%s_", v))
		}
	}

	if len(ip.Ipv6Ranges) > 0 {
		s := make([]string, len(ip.Ipv6Ranges))
		for i, r := range ip.Ipv6Ranges {
			s[i] = *r.CidrIpv6
		}
		sort.Strings(s)

		for _, v := range s {
			buf.WriteString(fmt.Sprintf("%s_", v))
		}
	}

	if len(ip.PrefixListIds) > 0 {
		s := make([]string, len(ip.PrefixListIds))
		for i, pl := range ip.PrefixListIds {
			s[i] = *pl.PrefixListId
		}
		sort.Strings(s)

		for _, v := range s {
			buf.WriteString(fmt.Sprintf("%s_", v))
		}
	}

	if groupID != "" {
		buf.WriteString(fmt.Sprintf("%s_", groupID))
	}

	idPreformatted := buf.String()
	return idPreformatted[:len(idPreformatted)-1]
}

func fromPort(ip types.IpPermission) int {
	switch {
	case *ip.IpProtocol == "icmp":
		return -1
	case ip.FromPort != nil && *ip.FromPort > 0:
		return int(*ip.FromPort)
	default:
		return 0
	}
}

func toPort(ip types.IpPermission) int {
	switch {
	case *ip.IpProtocol == "icmp":
		return -1
	case ip.ToPort != nil && *ip.ToPort > 0:
		return int(*ip.ToPort)
	default:
		return 65536
	}
}

func ipRange(rule types.IpPermission) []string {
	result := make([]string, len(rule.IpRanges))
	for idx, rule := range rule.IpRanges {
		result[idx] = *rule.CidrIp
	}
	return result
}

func ip6Range(rule types.IpPermission) []string {
	result := make([]string, len(rule.Ipv6Ranges))
	for idx, rule := range rule.Ipv6Ranges {
		result[idx] = *rule.CidrIpv6
	}
	return result
}

func prefixes(rule types.IpPermission) []string {
	result := make([]string, len(rule.PrefixListIds))
	for idx, rule := range rule.PrefixListIds {
		result[idx] = *rule.PrefixListId
	}
	return result
}
<|endoftext|>


<commit_before>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"crypto/sha1"
	"io/ioutil"
	"sync"
	"time"

	"github.com/golang/glog"

	"istio.io/mixer/pkg/adapter"
	"istio.io/mixer/pkg/attribute"
	"istio.io/mixer/pkg/config/descriptor"
	pb "istio.io/mixer/pkg/config/proto"
	"istio.io/mixer/pkg/expr"
)

// Resolver resolves configuration to a list of combined configs.
type Resolver interface {
	// Resolve resolves configuration to a list of combined configs.
	Resolve(bag attribute.Bag, aspectSet AspectSet) ([]*pb.Combined, error)
}

// ChangeListener listens for config change notifications.
type ChangeListener interface {
	ConfigChange(cfg Resolver, df descriptor.Finder)
}

// Manager represents the config Manager.
// It is responsible for fetching and receiving configuration changes.
// It applies validated changes to the registered config change listeners.
// api.Handler listens for config changes.
type Manager struct {
	eval             expr.Evaluator
	aspectFinder     AspectValidatorFinder
	builderFinder    BuilderValidatorFinder
	descriptorFinder descriptor.Finder
	findAspects      AdapterToAspectMapper
	loopDelay        time.Duration
	globalConfig     string
	serviceConfig    string

	cl      []ChangeListener
	closing chan bool
	scSHA   [sha1.Size]byte
	gcSHA   [sha1.Size]byte

	sync.RWMutex
	lastError error
}

// NewManager returns a config.Manager.
// Eval validates and evaluates selectors.
// It is also used downstream for attribute mapping.
// AspectFinder finds aspect validator given aspect 'Kind'.
// BuilderFinder finds builder validator given builder 'Impl'.
// LoopDelay determines how often configuration is updated.
// The following fields will be eventually replaced by a
// repository location. At present we use GlobalConfig and ServiceConfig
// as command line input parameters.
// GlobalConfig specifies the location of Global Config.
// ServiceConfig specifies the location of Service config.
func NewManager(eval expr.Evaluator, aspectFinder AspectValidatorFinder, builderFinder BuilderValidatorFinder,
	findAspects AdapterToAspectMapper, globalConfig string, serviceConfig string, loopDelay time.Duration) *Manager {
	m := &Manager{
		eval:          eval,
		aspectFinder:  aspectFinder,
		builderFinder: builderFinder,
		findAspects:   findAspects,
		loopDelay:     loopDelay,
		globalConfig:  globalConfig,
		serviceConfig: serviceConfig,
		closing:       make(chan bool),
	}
	return m
}

// Register makes the ConfigManager aware of a ConfigChangeListener.
func (c *Manager) Register(cc ChangeListener) {
	c.cl = append(c.cl, cc)
}

func read(fname string) ([sha1.Size]byte, string, error) {
	var data []byte
	var err error
	if data, err = ioutil.ReadFile(fname); err != nil {
		return [sha1.Size]byte{}, "", err
	}
	return sha1.Sum(data), string(data[:]), nil
}

// fetch config and return runtime if a new one is available.
func (c *Manager) fetch() (*Runtime, descriptor.Finder, error) {
	var vd *Validated
	var cerr *adapter.ConfigErrors

	gcSHA, gc, err2 := read(c.globalConfig)
	if err2 != nil {
		return nil, nil, err2
	}

	scSHA, sc, err1 := read(c.serviceConfig)
	if err1 != nil {
		return nil, nil, err1
	}

	if gcSHA == c.gcSHA && scSHA == c.scSHA {
		return nil, nil, nil
	}

	v := NewValidator(c.aspectFinder, c.builderFinder, c.findAspects, true, c.eval)
	if vd, cerr = v.Validate(sc, gc); cerr != nil {
		return nil, nil, cerr
	}

	c.descriptorFinder = descriptor.NewFinder(v.validated.globalConfig)

	c.gcSHA = gcSHA
	c.scSHA = scSHA
	return NewRuntime(vd, c.eval), c.descriptorFinder, nil
}

// fetchAndNotify fetches a new config and notifies listeners if something has changed
func (c *Manager) fetchAndNotify() error {
	rt, df, err := c.fetch()
	if err != nil {
		c.Lock()
		c.lastError = err
		c.Unlock()
		return err
	}
	if rt == nil {
		return nil
	}

	glog.Infof("Installing new config from %s sha=%x ", c.serviceConfig, c.scSHA)
	for _, cl := range c.cl {
		cl.ConfigChange(rt, df)
	}
	return nil
}

// LastError returns last error encountered by the manager while processing config.
func (c *Manager) LastError() (err error) {
	c.RLock()
	err = c.lastError
	c.RUnlock()
	return err
}

// Close stops the config manager go routine.
func (c *Manager) Close() { close(c.closing) }

func (c *Manager) loop() {
	ticker := time.NewTicker(c.loopDelay)
	defer ticker.Stop()
	done := false
	for !done {
		select {
		case <-ticker.C:
			err := c.fetchAndNotify()
			if err != nil {
				glog.Warning(err)
			}
		case <-c.closing:
			done = true
		}
	}
}

// Start watching for configuration changes and handle updates.
func (c *Manager) Start() {
	err := c.fetchAndNotify()
	// We make an attempt to synchronously fetch and notify configuration
	// If it is not successful, we will continue to watch for changes.
	go c.loop()
	if err != nil {
		glog.Warning("Unable to process config: ", err)
	}
}
<commit_msg>Simplify config manager background goroutine logic. (#467)<commit_after>// Copyright 2017 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"crypto/sha1"
	"io/ioutil"
	"sync"
	"time"

	"github.com/golang/glog"

	"istio.io/mixer/pkg/adapter"
	"istio.io/mixer/pkg/attribute"
	"istio.io/mixer/pkg/config/descriptor"
	pb "istio.io/mixer/pkg/config/proto"
	"istio.io/mixer/pkg/expr"
)

// Resolver resolves configuration to a list of combined configs.
type Resolver interface {
	// Resolve resolves configuration to a list of combined configs.
	Resolve(bag attribute.Bag, aspectSet AspectSet) ([]*pb.Combined, error)
}

// ChangeListener listens for config change notifications.
type ChangeListener interface {
	ConfigChange(cfg Resolver, df descriptor.Finder)
}

// Manager represents the config Manager.
// It is responsible for fetching and receiving configuration changes.
// It applies validated changes to the registered config change listeners.
// api.Handler listens for config changes.
type Manager struct {
	eval             expr.Evaluator
	aspectFinder     AspectValidatorFinder
	builderFinder    BuilderValidatorFinder
	descriptorFinder descriptor.Finder
	findAspects      AdapterToAspectMapper
	loopDelay        time.Duration
	globalConfig     string
	serviceConfig    string
	ticker           *time.Ticker

	cl    []ChangeListener
	scSHA [sha1.Size]byte
	gcSHA [sha1.Size]byte

	sync.RWMutex
	lastError error
}

// NewManager returns a config.Manager.
// Eval validates and evaluates selectors.
// It is also used downstream for attribute mapping.
// AspectFinder finds aspect validator given aspect 'Kind'.
// BuilderFinder finds builder validator given builder 'Impl'.
// LoopDelay determines how often configuration is updated.
// The following fields will be eventually replaced by a
// repository location. At present we use GlobalConfig and ServiceConfig
// as command line input parameters.
// GlobalConfig specifies the location of Global Config.
// ServiceConfig specifies the location of Service config.
func NewManager(eval expr.Evaluator, aspectFinder AspectValidatorFinder, builderFinder BuilderValidatorFinder,
	findAspects AdapterToAspectMapper, globalConfig string, serviceConfig string, loopDelay time.Duration) *Manager {
	m := &Manager{
		eval:          eval,
		aspectFinder:  aspectFinder,
		builderFinder: builderFinder,
		findAspects:   findAspects,
		loopDelay:     loopDelay,
		globalConfig:  globalConfig,
		serviceConfig: serviceConfig,
	}
	return m
}

// Register makes the ConfigManager aware of a ConfigChangeListener.
func (c *Manager) Register(cc ChangeListener) {
	c.cl = append(c.cl, cc)
}

func read(fname string) ([sha1.Size]byte, string, error) {
	var data []byte
	var err error
	if data, err = ioutil.ReadFile(fname); err != nil {
		return [sha1.Size]byte{}, "", err
	}
	return sha1.Sum(data), string(data[:]), nil
}

// fetch config and return runtime if a new one is available.
func (c *Manager) fetch() (*Runtime, descriptor.Finder, error) {
	var vd *Validated
	var cerr *adapter.ConfigErrors

	gcSHA, gc, err2 := read(c.globalConfig)
	if err2 != nil {
		return nil, nil, err2
	}

	scSHA, sc, err1 := read(c.serviceConfig)
	if err1 != nil {
		return nil, nil, err1
	}

	if gcSHA == c.gcSHA && scSHA == c.scSHA {
		return nil, nil, nil
	}

	v := NewValidator(c.aspectFinder, c.builderFinder, c.findAspects, true, c.eval)
	if vd, cerr = v.Validate(sc, gc); cerr != nil {
		return nil, nil, cerr
	}

	c.descriptorFinder = descriptor.NewFinder(v.validated.globalConfig)

	c.gcSHA = gcSHA
	c.scSHA = scSHA
	return NewRuntime(vd, c.eval), c.descriptorFinder, nil
}

// fetchAndNotify fetches a new config and notifies listeners if something has changed
func (c *Manager) fetchAndNotify() error {
	rt, df, err := c.fetch()
	if err != nil {
		c.Lock()
		c.lastError = err
		c.Unlock()
		return err
	}
	if rt == nil {
		return nil
	}

	glog.Infof("Installing new config from %s sha=%x ", c.serviceConfig, c.scSHA)
	for _, cl := range c.cl {
		cl.ConfigChange(rt, df)
	}
	return nil
}

// LastError returns last error encountered by the manager while processing config.
func (c *Manager) LastError() (err error) {
	c.RLock()
	err = c.lastError
	c.RUnlock()
	return err
}

// Close stops the config manager go routine.
func (c *Manager) Close() {
	if c.ticker != nil {
		c.ticker.Stop()
	}
}

func (c *Manager) loop() {
	for range c.ticker.C {
		err := c.fetchAndNotify()
		if err != nil {
			glog.Warning(err)
		}
	}
}

// Start watching for configuration changes and handle updates.
func (c *Manager) Start() {
	err := c.fetchAndNotify()
	// We make an attempt to synchronously fetch and notify configuration
	// If it is not successful, we will continue to watch for changes.
	c.ticker = time.NewTicker(c.loopDelay)
	go c.loop()
	if err != nil {
		glog.Warning("Unable to process config: ", err)
	}
}
<|endoftext|>


<commit_before>package model

import (
	"net/http"
	"strconv"

	"github.com/skygeario/skygear-server/pkg/core/config"
)

type KeyType int

const (
	// NoAccessKey means no correct access key
	NoAccessKey KeyType = iota
	// APIAccessKey means request is using api key
	APIAccessKey
	// MasterAccessKey means request is using master key
	MasterAccessKey
)

func header(i interface{}) http.Header {
	switch i.(type) {
	case *http.Request:
		return (i.(*http.Request)).Header
	case http.ResponseWriter:
		return (i.(http.ResponseWriter)).Header()
	default:
		panic("Invalid type")
	}
}

func GetAccessKeyType(i interface{}) KeyType {
	ktv, err := strconv.Atoi(header(i).Get("X-Skygear-AccessKeyType"))
	if err != nil {
		return NoAccessKey
	}

	return KeyType(ktv)
}

func SetAccessKeyType(i interface{}, kt KeyType) {
	header(i).Set("X-Skygear-AccessKeyType", strconv.Itoa(int(kt)))
}

func GetAPIKey(i interface{}) string {
	return header(i).Get("X-Skygear-APIKey")
}

func CheckAccessKeyType(config config.TenantConfiguration, apiKey string) KeyType {
	if apiKey == config.APIKey {
		return APIAccessKey
	}

	if apiKey == config.MasterKey {
		return MasterAccessKey
	}

	return NoAccessKey
}
<commit_msg>Fix api key in http request header<commit_after>package model

import (
	"net/http"
	"strconv"

	"github.com/skygeario/skygear-server/pkg/core/config"
)

type KeyType int

const (
	// NoAccessKey means no correct access key
	NoAccessKey KeyType = iota
	// APIAccessKey means request is using api key
	APIAccessKey
	// MasterAccessKey means request is using master key
	MasterAccessKey
)

func header(i interface{}) http.Header {
	switch i.(type) {
	case *http.Request:
		return (i.(*http.Request)).Header
	case http.ResponseWriter:
		return (i.(http.ResponseWriter)).Header()
	default:
		panic("Invalid type")
	}
}

func GetAccessKeyType(i interface{}) KeyType {
	ktv, err := strconv.Atoi(header(i).Get("X-Skygear-AccessKeyType"))
	if err != nil {
		return NoAccessKey
	}

	return KeyType(ktv)
}

func SetAccessKeyType(i interface{}, kt KeyType) {
	header(i).Set("X-Skygear-AccessKeyType", strconv.Itoa(int(kt)))
}

func GetAPIKey(i interface{}) string {
	return header(i).Get("X-Skygear-Api-Key")
}

func CheckAccessKeyType(config config.TenantConfiguration, apiKey string) KeyType {
	if apiKey == config.APIKey {
		return APIAccessKey
	}

	if apiKey == config.MasterKey {
		return MasterAccessKey
	}

	return NoAccessKey
}
<|endoftext|>


<commit_before>package dialer

import (
	"fmt"
	"net"
	"time"

	"github.com/rancher/rancher/pkg/encryptedstore"
	"github.com/rancher/rancher/pkg/nodeconfig"
	"github.com/rancher/rancher/pkg/remotedialer"
	"github.com/rancher/rancher/pkg/tunnelserver"
	"github.com/rancher/types/apis/management.cattle.io/v3"
	"github.com/rancher/types/config"
	"github.com/rancher/types/config/dialer"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/tools/cache"
)

func NewFactory(apiContext *config.ScaledContext) (dialer.Factory, error) {
	authorizer := tunnelserver.NewAuthorizer(apiContext)
	tunneler := tunnelserver.NewTunnelServer(apiContext, authorizer)

	secretStore, err := nodeconfig.NewStore(apiContext.Core.Namespaces(""), apiContext.K8sClient.CoreV1())
	if err != nil {
		return nil, err
	}

	apiContext.Management.Nodes("local").Controller().Informer().AddIndexers(cache.Indexers{
		nodeAccessIndexer: nodeIndexer,
	})

	return &Factory{
		clusterLister:       apiContext.Management.Clusters("").Controller().Lister(),
		localNodeController: apiContext.Management.Nodes("local").Controller(),
		nodeLister:          apiContext.Management.Nodes("").Controller().Lister(),
		TunnelServer:        tunneler,
		TunnelAuthorizer:    authorizer,
		store:               secretStore,
	}, nil
}

type Factory struct {
	localNodeController v3.NodeController
	nodeLister          v3.NodeLister
	clusterLister       v3.ClusterLister
	TunnelServer        *remotedialer.Server
	TunnelAuthorizer    *tunnelserver.Authorizer
	store               *encryptedstore.GenericEncryptedStore
}

func (f *Factory) ClusterDialer(clusterName string) (dialer.Dialer, error) {
	return func(network, address string) (net.Conn, error) {
		d, err := f.clusterDialer(clusterName)
		if err != nil {
			return nil, err
		}
		return d(network, address)
	}, nil
}

func (f *Factory) clusterDialer(clusterName string) (dialer.Dialer, error) {
	cluster, err := f.clusterLister.Get("", clusterName)
	if err != nil {
		return nil, err
	}

	if f.TunnelServer.HasSession(cluster.Name) {
		return f.TunnelServer.Dialer(cluster.Name, 15*time.Second), nil
	}

	nodes, err := f.nodeLister.List(cluster.Name, labels.Everything())
	if err != nil {
		return nil, err
	}

	for _, node := range nodes {
		if node.DeletionTimestamp == nil && v3.NodeConditionProvisioned.IsTrue(node) {
			return f.NodeDialer(clusterName, node.Name)
		}
	}

	return net.Dial, nil
}

func (f *Factory) DockerDialer(clusterName, machineName string) (dialer.Dialer, error) {
	machine, err := f.nodeLister.Get(clusterName, machineName)
	if err != nil {
		return nil, err
	}

	if f.TunnelServer.HasSession(machine.Name) {
		d := f.TunnelServer.Dialer(machine.Name, 15*time.Second)
		return func(string, string) (net.Conn, error) {
			return d("unix", "/var/run/docker.sock")
		}, nil
	}

	if machine.Spec.CustomConfig != nil && machine.Spec.CustomConfig.Address != "" && machine.Spec.CustomConfig.SSHKey != "" {
		return f.sshDialer(machine)
	}

	if machine.Spec.NodeTemplateName != "" {
		return f.tlsDialer(machine)
	}

	return nil, fmt.Errorf("can not build dailer to %s:%s", clusterName, machineName)
}

func (f *Factory) NodeDialer(clusterName, machineName string) (dialer.Dialer, error) {
	return func(network, address string) (net.Conn, error) {
		d, err := f.nodeDialer(clusterName, machineName)
		if err != nil {
			return nil, err
		}
		return d(network, address)
	}, nil
}

func (f *Factory) nodeDialer(clusterName, machineName string) (dialer.Dialer, error) {
	machine, err := f.nodeLister.Get(clusterName, machineName)
	if err != nil {
		return nil, err
	}

	if f.TunnelServer.HasSession(machine.Name) {
		d := f.TunnelServer.Dialer(machine.Name, 15*time.Second)
		return dialer.Dialer(d), nil
	}

	if machine.Spec.CustomConfig != nil && machine.Spec.CustomConfig.Address != "" && machine.Spec.CustomConfig.SSHKey != "" {
		return f.sshLocalDialer(machine)
	}

	if machine.Spec.NodeTemplateName != "" {
		return f.sshLocalDialer(machine)
	}

	return nil, fmt.Errorf("can not build dailer to %s:%s", clusterName, machineName)
}
<commit_msg>Don't error on first bad dialer<commit_after>package dialer

import (
	"fmt"
	"net"
	"time"

	"github.com/rancher/rancher/pkg/encryptedstore"
	"github.com/rancher/rancher/pkg/nodeconfig"
	"github.com/rancher/rancher/pkg/remotedialer"
	"github.com/rancher/rancher/pkg/tunnelserver"
	"github.com/rancher/types/apis/management.cattle.io/v3"
	"github.com/rancher/types/config"
	"github.com/rancher/types/config/dialer"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/client-go/tools/cache"
)

func NewFactory(apiContext *config.ScaledContext) (dialer.Factory, error) {
	authorizer := tunnelserver.NewAuthorizer(apiContext)
	tunneler := tunnelserver.NewTunnelServer(apiContext, authorizer)

	secretStore, err := nodeconfig.NewStore(apiContext.Core.Namespaces(""), apiContext.K8sClient.CoreV1())
	if err != nil {
		return nil, err
	}

	apiContext.Management.Nodes("local").Controller().Informer().AddIndexers(cache.Indexers{
		nodeAccessIndexer: nodeIndexer,
	})

	return &Factory{
		clusterLister:       apiContext.Management.Clusters("").Controller().Lister(),
		localNodeController: apiContext.Management.Nodes("local").Controller(),
		nodeLister:          apiContext.Management.Nodes("").Controller().Lister(),
		TunnelServer:        tunneler,
		TunnelAuthorizer:    authorizer,
		store:               secretStore,
	}, nil
}

type Factory struct {
	localNodeController v3.NodeController
	nodeLister          v3.NodeLister
	clusterLister       v3.ClusterLister
	TunnelServer        *remotedialer.Server
	TunnelAuthorizer    *tunnelserver.Authorizer
	store               *encryptedstore.GenericEncryptedStore
}

func (f *Factory) ClusterDialer(clusterName string) (dialer.Dialer, error) {
	return func(network, address string) (net.Conn, error) {
		d, err := f.clusterDialer(clusterName)
		if err != nil {
			return nil, err
		}
		return d(network, address)
	}, nil
}

func (f *Factory) clusterDialer(clusterName string) (dialer.Dialer, error) {
	cluster, err := f.clusterLister.Get("", clusterName)
	if err != nil {
		return nil, err
	}

	if f.TunnelServer.HasSession(cluster.Name) {
		return f.TunnelServer.Dialer(cluster.Name, 15*time.Second), nil
	}

	nodes, err := f.nodeLister.List(cluster.Name, labels.Everything())
	if err != nil {
		return nil, err
	}

	for _, node := range nodes {
		if node.DeletionTimestamp == nil && v3.NodeConditionProvisioned.IsTrue(node) {
			if nodeDialer, err := f.nodeDialer(clusterName, node.Name); err == nil {
				return nodeDialer, err
			}
		}
	}

	return net.Dial, nil
}

func (f *Factory) DockerDialer(clusterName, machineName string) (dialer.Dialer, error) {
	machine, err := f.nodeLister.Get(clusterName, machineName)
	if err != nil {
		return nil, err
	}

	if f.TunnelServer.HasSession(machine.Name) {
		d := f.TunnelServer.Dialer(machine.Name, 15*time.Second)
		return func(string, string) (net.Conn, error) {
			return d("unix", "/var/run/docker.sock")
		}, nil
	}

	if machine.Spec.CustomConfig != nil && machine.Spec.CustomConfig.Address != "" && machine.Spec.CustomConfig.SSHKey != "" {
		return f.sshDialer(machine)
	}

	if machine.Spec.NodeTemplateName != "" {
		return f.tlsDialer(machine)
	}

	return nil, fmt.Errorf("can not build dialer to %s:%s", clusterName, machineName)
}

func (f *Factory) NodeDialer(clusterName, machineName string) (dialer.Dialer, error) {
	return func(network, address string) (net.Conn, error) {
		d, err := f.nodeDialer(clusterName, machineName)
		if err != nil {
			return nil, err
		}
		return d(network, address)
	}, nil
}

func (f *Factory) nodeDialer(clusterName, machineName string) (dialer.Dialer, error) {
	machine, err := f.nodeLister.Get(clusterName, machineName)
	if err != nil {
		return nil, err
	}

	if f.TunnelServer.HasSession(machine.Name) {
		d := f.TunnelServer.Dialer(machine.Name, 15*time.Second)
		return dialer.Dialer(d), nil
	}

	if machine.Spec.CustomConfig != nil && machine.Spec.CustomConfig.Address != "" && machine.Spec.CustomConfig.SSHKey != "" {
		return f.sshLocalDialer(machine)
	}

	if machine.Spec.NodeTemplateName != "" {
		return f.sshLocalDialer(machine)
	}

	return nil, fmt.Errorf("can not build dialer to %s:%s", clusterName, machineName)
}
<|endoftext|>