<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package to keep track of API Versions that should be registered in api.Scheme.
package registered

import (
	"os"
	"strings"

	"github.com/golang/glog"
	apiutil "k8s.io/kubernetes/pkg/api/util"
)

// List of registered API versions.
// The list is in the order of most preferred to the least.
var RegisteredVersions []string

func init() {
	// TODO: caesarxuchao: rename this variable to validGroupVersions
	validAPIVersions := map[string]bool{
		"v1":                       true,
		"extensions/v1beta1":       true,
		"componentconfig/v1alpha1": true,
		"metrics/v1alpha1":         true,
	}

	// The default list of supported api versions, in order of most preferred to the least.
	defaultSupportedVersions := "v1,extensions/v1beta1,componentconfig/v1alpha1"
	// Env var KUBE_API_VERSIONS is a comma separated list of API versions that should be registered in the scheme.
	// The versions should be in the order of most preferred to the least.
	supportedVersions := os.Getenv("KUBE_API_VERSIONS")
	if supportedVersions == "" {
		supportedVersions = defaultSupportedVersions
	}
	versions := strings.Split(supportedVersions, ",")
	for _, version := range versions {
		// Verify that the version is valid.
		valid, ok := validAPIVersions[version]
		if !ok || !valid {
			// Not a valid API version.
			glog.Fatalf("invalid api version: %s in KUBE_API_VERSIONS: %s. List of valid API versions: %v",
				version, os.Getenv("KUBE_API_VERSIONS"), validAPIVersions)
		}
		RegisteredVersions = append(RegisteredVersions, version)
	}
}

// Returns true if the given api version is one of the registered api versions.
func IsRegisteredAPIVersion(version string) bool {
	for _, apiVersion := range RegisteredVersions {
		if apiVersion == version {
			return true
		}
	}
	return false
}

// GroupVersionsForGroup returns the registered versions of a group in the form
// of "group/version".
func GroupVersionsForGroup(group string) []string {
	ret := []string{}
	for _, v := range RegisteredVersions {
		if apiutil.GetGroup(v) == group {
			ret = append(ret, v)
		}
	}
	return ret
}
<commit_msg>  Kill TODO, rename var<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package to keep track of API Versions that should be registered in api.Scheme.
package registered

import (
	"os"
	"strings"

	"github.com/golang/glog"
	apiutil "k8s.io/kubernetes/pkg/api/util"
)

// List of registered API versions.
// The list is in the order of most preferred to the least.
var RegisteredVersions []string

func init() {
	validGroupVersions := map[string]bool{
		"v1":                       true,
		"extensions/v1beta1":       true,
		"componentconfig/v1alpha1": true,
		"metrics/v1alpha1":         true,
	}

	// The default list of supported api versions, in order of most preferred to the least.
	defaultSupportedVersions := "v1,extensions/v1beta1,componentconfig/v1alpha1"
	// Env var KUBE_API_VERSIONS is a comma separated list of API versions that should be registered in the scheme.
	// The versions should be in the order of most preferred to the least.
	supportedVersions := os.Getenv("KUBE_API_VERSIONS")
	if supportedVersions == "" {
		supportedVersions = defaultSupportedVersions
	}
	versions := strings.Split(supportedVersions, ",")
	for _, version := range versions {
		// Verify that the version is valid.
		valid, ok := validGroupVersions[version]
		if !ok || !valid {
			// Not a valid API version.
			glog.Fatalf("invalid api version: %s in KUBE_API_VERSIONS: %s. List of valid API versions: %v",
				version, os.Getenv("KUBE_API_VERSIONS"), validGroupVersions)
		}
		RegisteredVersions = append(RegisteredVersions, version)
	}
}

// Returns true if the given api version is one of the registered api versions.
func IsRegisteredAPIVersion(version string) bool {
	for _, apiVersion := range RegisteredVersions {
		if apiVersion == version {
			return true
		}
	}
	return false
}

// GroupVersionsForGroup returns the registered versions of a group in the form
// of "group/version".
func GroupVersionsForGroup(group string) []string {
	ret := []string{}
	for _, v := range RegisteredVersions {
		if apiutil.GetGroup(v) == group {
			ret = append(ret, v)
		}
	}
	return ret
}
<|endoftext|>


<commit_before>package control

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"encoding/base64"

	"github.com/rancher/k3s/pkg/daemons/config"
	"github.com/sirupsen/logrus"
	"go.etcd.io/etcd/clientv3"
)

const (
	etcdDialTimeout    = 5 * time.Second
	k3sRuntimeEtcdPath = "/k3s/runtime"
	bootstrapTypeNone  = "none"
	bootstrapTypeRead  = "read"
	bootstrapTypeWrite = "write"
	bootstrapTypeFull  = "full"
)

type serverBootstrap struct {
	ServerCAData           string `json:"serverCAData,omitempty"`
	ServerCAKeyData        string `json:"serverCAKeyData,omitempty"`
	ClientCAData           string `json:"clientCAData,omitempty"`
	ClientCAKeyData        string `json:"clientCAKeyData,omitempty"`
	ServiceKeyData         string `json:"serviceKeyData,omitempty"`
	PasswdFileData         string `json:"passwdFileData,omitempty"`
	RequestHeaderCAData    string `json:"requestHeaderCAData,omitempty"`
	RequestHeaderCAKeyData string `json:"requestHeaderCAKeyData,omitempty"`
}

var validBootstrapTypes = map[string]bool{
	bootstrapTypeRead:  true,
	bootstrapTypeWrite: true,
	bootstrapTypeFull:  true,
}

func fetchBootstrapData(cfg *config.Control) error {
	if valid, err := checkBootstrapArgs(cfg, map[string]bool{
		bootstrapTypeFull: true,
		bootstrapTypeRead: true,
	}); !valid {
		if err != nil {
			logrus.Warnf("Not fetching bootstrap data: %v", err)
		}
		return nil
	}

	tlsConfig, err := genBootstrapTLSConfig(cfg)
	if err != nil {
		return err
	}

	endpoints := strings.Split(cfg.StorageEndpoint, ",")
	cli, err := clientv3.New(clientv3.Config{
		Endpoints:   endpoints,
		DialTimeout: etcdDialTimeout,
		TLS:         tlsConfig,
	})
	if err != nil {
		return err
	}
	defer cli.Close()

	gr, err := cli.Get(context.TODO(), k3sRuntimeEtcdPath)
	if err != nil {
		return err
	}
	if len(gr.Kvs) == 0 {
		return nil
	}

	runtimeJSON, err := base64.URLEncoding.DecodeString(string(gr.Kvs[0].Value))
	if err != nil {
		return err
	}
	serverRuntime := &serverBootstrap{}
	if err := json.Unmarshal(runtimeJSON, serverRuntime); err != nil {
		return err
	}
	return writeRuntimeBootstrapData(cfg.Runtime, serverRuntime)
}

func storeBootstrapData(cfg *config.Control) error {
	if valid, err := checkBootstrapArgs(cfg, map[string]bool{
		bootstrapTypeFull:  true,
		bootstrapTypeWrite: true,
	}); !valid {
		if err != nil {
			logrus.Warnf("Not storing boostrap data: %v", err)
		}
		return nil
	}

	tlsConfig, err := genBootstrapTLSConfig(cfg)
	if err != nil {
		return err
	}

	endpoints := strings.Split(cfg.StorageEndpoint, ",")
	cli, err := clientv3.New(clientv3.Config{
		Endpoints:   endpoints,
		DialTimeout: etcdDialTimeout,
		TLS:         tlsConfig,
	})
	if err != nil {
		return err
	}
	defer cli.Close()

	gr, err := cli.Get(context.TODO(), k3sRuntimeEtcdPath)
	if err != nil {
		return err
	}
	if len(gr.Kvs) > 0 && string(gr.Kvs[0].Value) != "" {
		return nil
	}
	certData, err := readRuntimeBootstrapData(cfg.Runtime)
	if err != nil {
		return err
	}

	runtimeBase64 := base64.StdEncoding.EncodeToString(certData)
	_, err = cli.Put(context.TODO(), k3sRuntimeEtcdPath, runtimeBase64)
	if err != nil {
		return err
	}

	return nil
}

func checkBootstrapArgs(cfg *config.Control, accepted map[string]bool) (bool, error) {
	if cfg.BootstrapType == "" || cfg.BootstrapType == bootstrapTypeNone {
		return false, nil
	}
	if !validBootstrapTypes[cfg.BootstrapType] {
		return false, fmt.Errorf("unsupported bootstrap type [%s]", cfg.BootstrapType)
	}
	if cfg.StorageBackend != "etcd3" {
		return false, errors.New("bootstrap only supported with etcd3 as storage backend")
	}
	if !accepted[cfg.BootstrapType] {
		return false, nil
	}
	return true, nil
}

func genBootstrapTLSConfig(cfg *config.Control) (*tls.Config, error) {
	tlsConfig := &tls.Config{}
	if cfg.StorageCertFile != "" && cfg.StorageKeyFile != "" {
		certPem, err := ioutil.ReadFile(cfg.StorageCertFile)
		if err != nil {
			return nil, err
		}
		keyPem, err := ioutil.ReadFile(cfg.StorageKeyFile)
		if err != nil {
			return nil, err
		}
		tlsCert, err := tls.X509KeyPair(certPem, keyPem)
		if err != nil {
			return nil, err
		}
		tlsConfig.Certificates = []tls.Certificate{tlsCert}
	}
	if cfg.StorageCAFile != "" {
		caData, err := ioutil.ReadFile(cfg.StorageCAFile)
		if err != nil {
			return nil, err
		}
		certPool := x509.NewCertPool()
		certPool.AppendCertsFromPEM(caData)
		tlsConfig.RootCAs = certPool
	}
	return tlsConfig, nil
}

func readRuntimeBootstrapData(runtime *config.ControlRuntime) ([]byte, error) {
	serverBootstrapFiles := map[string]string{
		runtime.ServerCA:           "",
		runtime.ServerCAKey:        "",
		runtime.ClientCA:           "",
		runtime.ClientCAKey:        "",
		runtime.ServiceKey:         "",
		runtime.PasswdFile:         "",
		runtime.RequestHeaderCA:    "",
		runtime.RequestHeaderCAKey: "",
	}
	for k := range serverBootstrapFiles {
		data, err := ioutil.ReadFile(k)
		if err != nil {
			return nil, err
		}
		serverBootstrapFiles[k] = string(data)
	}
	serverBootstrapFileData := &serverBootstrap{
		ServerCAData:           serverBootstrapFiles[runtime.ServerCA],
		ServerCAKeyData:        serverBootstrapFiles[runtime.ServerCAKey],
		ClientCAData:           serverBootstrapFiles[runtime.ClientCA],
		ClientCAKeyData:        serverBootstrapFiles[runtime.ClientCAKey],
		ServiceKeyData:         serverBootstrapFiles[runtime.ServiceKey],
		PasswdFileData:         serverBootstrapFiles[runtime.PasswdFile],
		RequestHeaderCAData:    serverBootstrapFiles[runtime.RequestHeaderCA],
		RequestHeaderCAKeyData: serverBootstrapFiles[runtime.RequestHeaderCAKey],
	}
	return json.Marshal(serverBootstrapFileData)
}

func writeRuntimeBootstrapData(runtime *config.ControlRuntime, runtimeData *serverBootstrap) error {
	runtimePathValue := map[string]string{
		runtime.ServerCA:           runtimeData.ServerCAData,
		runtime.ServerCAKey:        runtimeData.ServerCAKeyData,
		runtime.ClientCA:           runtimeData.ClientCAData,
		runtime.ClientCAKey:        runtimeData.ClientCAKeyData,
		runtime.ServiceKey:         runtimeData.ServiceKeyData,
		runtime.PasswdFile:         runtimeData.PasswdFileData,
		runtime.RequestHeaderCA:    runtimeData.RequestHeaderCAData,
		runtime.RequestHeaderCAKey: runtimeData.RequestHeaderCAKeyData,
	}
	for k, v := range runtimePathValue {
		if _, err := os.Stat(k); os.IsNotExist(err) {
			if err := ioutil.WriteFile(k, []byte(v), 600); err != nil {
				return err
			}
		}
	}
	return nil
}
<commit_msg>Enforce explicit read or write for bootstrap<commit_after>package control

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"time"

	"encoding/base64"

	"github.com/rancher/k3s/pkg/daemons/config"
	"github.com/sirupsen/logrus"
	"go.etcd.io/etcd/clientv3"
)

const (
	etcdDialTimeout    = 5 * time.Second
	k3sRuntimeEtcdPath = "/k3s/runtime"
	bootstrapTypeNone  = "none"
	bootstrapTypeRead  = "read"
	bootstrapTypeWrite = "write"
	bootstrapTypeFull  = "full"
)

type serverBootstrap struct {
	ServerCAData           string `json:"serverCAData,omitempty"`
	ServerCAKeyData        string `json:"serverCAKeyData,omitempty"`
	ClientCAData           string `json:"clientCAData,omitempty"`
	ClientCAKeyData        string `json:"clientCAKeyData,omitempty"`
	ServiceKeyData         string `json:"serviceKeyData,omitempty"`
	PasswdFileData         string `json:"passwdFileData,omitempty"`
	RequestHeaderCAData    string `json:"requestHeaderCAData,omitempty"`
	RequestHeaderCAKeyData string `json:"requestHeaderCAKeyData,omitempty"`
}

var validBootstrapTypes = map[string]bool{
	bootstrapTypeRead:  true,
	bootstrapTypeWrite: true,
	bootstrapTypeFull:  true,
}

func fetchBootstrapData(cfg *config.Control) error {
	if valid, err := checkBootstrapArgs(cfg, map[string]bool{
		bootstrapTypeFull: true,
		bootstrapTypeRead: true,
	}); !valid {
		if err != nil {
			logrus.Warnf("Not fetching bootstrap data: %v", err)
		}
		return nil
	}

	tlsConfig, err := genBootstrapTLSConfig(cfg)
	if err != nil {
		return err
	}

	endpoints := strings.Split(cfg.StorageEndpoint, ",")
	cli, err := clientv3.New(clientv3.Config{
		Endpoints:   endpoints,
		DialTimeout: etcdDialTimeout,
		TLS:         tlsConfig,
	})
	if err != nil {
		return err
	}
	defer cli.Close()

	gr, err := cli.Get(context.TODO(), k3sRuntimeEtcdPath)
	if err != nil {
		return err
	}
	if len(gr.Kvs) == 0 {
		if cfg.BootstrapType != bootstrapTypeRead {
			return nil
		}
		return errors.New("Unable to read bootstrap data from server")
	}

	runtimeJSON, err := base64.URLEncoding.DecodeString(string(gr.Kvs[0].Value))
	if err != nil {
		return err
	}
	serverRuntime := &serverBootstrap{}
	if err := json.Unmarshal(runtimeJSON, serverRuntime); err != nil {
		return err
	}
	return writeRuntimeBootstrapData(cfg.Runtime, serverRuntime)
}

func storeBootstrapData(cfg *config.Control) error {
	if valid, err := checkBootstrapArgs(cfg, map[string]bool{
		bootstrapTypeFull:  true,
		bootstrapTypeWrite: true,
	}); !valid {
		if err != nil {
			logrus.Warnf("Not storing boostrap data: %v", err)
		}
		return nil
	}

	tlsConfig, err := genBootstrapTLSConfig(cfg)
	if err != nil {
		return err
	}

	endpoints := strings.Split(cfg.StorageEndpoint, ",")
	cli, err := clientv3.New(clientv3.Config{
		Endpoints:   endpoints,
		DialTimeout: etcdDialTimeout,
		TLS:         tlsConfig,
	})
	if err != nil {
		return err
	}
	defer cli.Close()

	if cfg.BootstrapType != bootstrapTypeWrite {
		gr, err := cli.Get(context.TODO(), k3sRuntimeEtcdPath)
		if err != nil {
			return err
		}
		if len(gr.Kvs) > 0 && string(gr.Kvs[0].Value) != "" {
			return nil
		}
	}

	certData, err := readRuntimeBootstrapData(cfg.Runtime)
	if err != nil {
		return err
	}

	runtimeBase64 := base64.StdEncoding.EncodeToString(certData)
	_, err = cli.Put(context.TODO(), k3sRuntimeEtcdPath, runtimeBase64)
	if err != nil {
		return err
	}

	return nil
}

func checkBootstrapArgs(cfg *config.Control, accepted map[string]bool) (bool, error) {
	if cfg.BootstrapType == "" || cfg.BootstrapType == bootstrapTypeNone {
		return false, nil
	}
	if !validBootstrapTypes[cfg.BootstrapType] {
		return false, fmt.Errorf("unsupported bootstrap type [%s]", cfg.BootstrapType)
	}
	if cfg.StorageBackend != "etcd3" {
		return false, errors.New("bootstrap only supported with etcd3 as storage backend")
	}
	if !accepted[cfg.BootstrapType] {
		return false, nil
	}
	return true, nil
}

func genBootstrapTLSConfig(cfg *config.Control) (*tls.Config, error) {
	tlsConfig := &tls.Config{}
	if cfg.StorageCertFile != "" && cfg.StorageKeyFile != "" {
		certPem, err := ioutil.ReadFile(cfg.StorageCertFile)
		if err != nil {
			return nil, err
		}
		keyPem, err := ioutil.ReadFile(cfg.StorageKeyFile)
		if err != nil {
			return nil, err
		}
		tlsCert, err := tls.X509KeyPair(certPem, keyPem)
		if err != nil {
			return nil, err
		}
		tlsConfig.Certificates = []tls.Certificate{tlsCert}
	}
	if cfg.StorageCAFile != "" {
		caData, err := ioutil.ReadFile(cfg.StorageCAFile)
		if err != nil {
			return nil, err
		}
		certPool := x509.NewCertPool()
		certPool.AppendCertsFromPEM(caData)
		tlsConfig.RootCAs = certPool
	}
	return tlsConfig, nil
}

func readRuntimeBootstrapData(runtime *config.ControlRuntime) ([]byte, error) {
	serverBootstrapFiles := map[string]string{
		runtime.ServerCA:           "",
		runtime.ServerCAKey:        "",
		runtime.ClientCA:           "",
		runtime.ClientCAKey:        "",
		runtime.ServiceKey:         "",
		runtime.PasswdFile:         "",
		runtime.RequestHeaderCA:    "",
		runtime.RequestHeaderCAKey: "",
	}
	for k := range serverBootstrapFiles {
		data, err := ioutil.ReadFile(k)
		if err != nil {
			return nil, err
		}
		serverBootstrapFiles[k] = string(data)
	}
	serverBootstrapFileData := &serverBootstrap{
		ServerCAData:           serverBootstrapFiles[runtime.ServerCA],
		ServerCAKeyData:        serverBootstrapFiles[runtime.ServerCAKey],
		ClientCAData:           serverBootstrapFiles[runtime.ClientCA],
		ClientCAKeyData:        serverBootstrapFiles[runtime.ClientCAKey],
		ServiceKeyData:         serverBootstrapFiles[runtime.ServiceKey],
		PasswdFileData:         serverBootstrapFiles[runtime.PasswdFile],
		RequestHeaderCAData:    serverBootstrapFiles[runtime.RequestHeaderCA],
		RequestHeaderCAKeyData: serverBootstrapFiles[runtime.RequestHeaderCAKey],
	}
	return json.Marshal(serverBootstrapFileData)
}

func writeRuntimeBootstrapData(runtime *config.ControlRuntime, runtimeData *serverBootstrap) error {
	runtimePathValue := map[string]string{
		runtime.ServerCA:           runtimeData.ServerCAData,
		runtime.ServerCAKey:        runtimeData.ServerCAKeyData,
		runtime.ClientCA:           runtimeData.ClientCAData,
		runtime.ClientCAKey:        runtimeData.ClientCAKeyData,
		runtime.ServiceKey:         runtimeData.ServiceKeyData,
		runtime.PasswdFile:         runtimeData.PasswdFileData,
		runtime.RequestHeaderCA:    runtimeData.RequestHeaderCAData,
		runtime.RequestHeaderCAKey: runtimeData.RequestHeaderCAKeyData,
	}
	for k, v := range runtimePathValue {
		if _, err := os.Stat(k); os.IsNotExist(err) {
			if err := ioutil.WriteFile(k, []byte(v), 600); err != nil {
				return err
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package watch

import (
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"testing"

	"github.com/GoogleCloudPlatform/skaffold/pkg/skaffold/config"
	testutil "github.com/GoogleCloudPlatform/skaffold/test"
	"github.com/rjeczalik/notify"
	"github.com/sirupsen/logrus"

	"github.com/spf13/afero"
)

var mockFS = map[string]string{
	"/tmp/skaffold/workspace_1/Dockerfile":             workspace1Dockerfile,
	"/tmp/skaffold/workspace_1/Dockerfile.MISSINGFILE": workspace1DockerfileMissingFile,
	"/tmp/skaffold/workspace_1/Dockerfile.SymlinkDep":  workspace1DockerfileSymlink,
	"/tmp/skaffold/workspace_1/file_a":                 "a",
	"/tmp/skaffold/workspace_1/file_b":                 "b",
	"/tmp/skaffold/workspace_1/file.symlink":           "/tmp/skaffold/workspace_1/file_a",

	"/tmp/skaffold/workspace_2/Dockerfile":              workspace2Dockerfile,
	"/tmp/skaffold/workspace_2/Dockerfile.build":        workspace2DockerfileBuild,
	"/tmp/skaffold/workspace_2/Dockerfile.ignored_path": dockerfileIgnoredFile,
	"/tmp/skaffold/workspace_2/file_a":                  "a",
	"/tmp/skaffold/workspace_2/vendor/vendor_file":      "vendor",
	"/tmp/skaffold/workspace_2/dir_a/file_a":            "b",
}

const (
	workspace1Dockerfile = `
FROM test
COPY file_a dst_a
COPY file_b dst_b
CMD ls
`

	workspace1DockerfileSymlink = `
FROM test
COPY file.symlink dst_a
COPY file_b dst_b
CMD ls`

	workspace1DockerfileMissingFile = `
FROM test
COPY file_MISSING dst_a
COPY file_b dst_b
CMD ls
`

	workspace2DockerfileBuild = `
FROM test
COPY * /
CMD ls`

	workspace2Dockerfile = `
FROM test
COPY file_a /
ADD dir_a/file_a /
CMD ls`

	dockerfileIgnoredFile = `
FROM test
COPY file_a /
ADD vendor/vendor_file /
CMD ls`
)

var artifactA = &config.Artifact{
	Workspace:      "/tmp/skaffold/workspace_1",
	DockerfilePath: "Dockerfile",
}

var artifactDefaultDockerfilePath = &config.Artifact{
	Workspace: "/tmp/skaffold/workspace_1",
}

var artifactMissingFile = &config.Artifact{
	Workspace:      "/tmp/skaffold/workspace_1",
	DockerfilePath: "Dockerfile.MISSINGFILE",
}

var artifactMissingDockerfile = &config.Artifact{
	Workspace:      "/tmp/skaffold/workspace_1",
	DockerfilePath: "NOT_FOUND",
}

var artifactDockerfileWithIgnoredDep = &config.Artifact{
	Workspace:      "/tmp/skaffold/workspace_2",
	DockerfilePath: "Dockerfile.ignored_path",
}

var artifactSymlinkDep = &config.Artifact{
	Workspace:      "/tmp/skaffold/workspace_1",
	DockerfilePath: "Dockerfile.SymlinkDep",
}

func initFS() {
	for p, contents := range mockFS {
		dir := filepath.Dir(p)

		if err := fs.MkdirAll(dir, 0750); err != nil {
			logrus.Fatalf("making mock fs dir %s", err)
		}
		if strings.HasSuffix(p, "symlink") {
			if err := os.Symlink(contents, p); err != nil {
				logrus.Fatalf("creating symlink file: %s", err)
			}
			continue
		}
		if err := afero.WriteFile(fs, p, []byte(contents), 0640); err != nil {
			logrus.Fatalf("writing mock fs file: %s", err)
		}
	}
}

func write(t *testing.T, path, contents string) {
	if err := afero.WriteFile(fs, path, []byte(contents), 0640); err != nil {
		t.Errorf("writing mock fs file: %s", err)
	}
}

func TestWatch(t *testing.T) {
	var tests = []struct {
		description string
		artifacts   []*config.Artifact

		writes     []string
		expected   *WatchEvent
		sendCancel bool
		shouldErr  bool
	}{
		{
			description: "write file and ignored file",
			artifacts: []*config.Artifact{
				artifactA,
				artifactDockerfileWithIgnoredDep,
			},
			writes: []string{
				"/tmp/skaffold/workspace_2/vendor/vendor_file",
				"/tmp/skaffold/workspace_1/file_a",
			},
			expected: &WatchEvent{
				EventType:       "notify.Write",
				ChangedArtifact: artifactA,
			},
		},
		{
			description: "missing dockerfile",
			artifacts:   []*config.Artifact{artifactMissingDockerfile},
			shouldErr:   true,
		},
		{
			description: "send cancel",
			artifacts:   []*config.Artifact{artifactA},
			sendCancel:  true,
			expected: &WatchEvent{
				EventType: "WatchStop",
			},
		},
	}

	for _, test := range tests {
		t.Run(test.description, func(t *testing.T) {
			watcher := &FSWatcher{}
			eventCh := make(chan *WatchEvent, 1)
			readyCh := make(chan *WatchEvent, 1)
			errCh := make(chan error, 1)
			cancel := make(chan struct{}, 1)
			go func() {
				evt, err := watcher.Watch(test.artifacts, readyCh, cancel)
				if err != nil {
					errCh <- err
					return
				}

				eventCh <- evt
			}()

			select {
			case err := <-errCh:
				testutil.CheckError(t, test.shouldErr, err)
				return
			case readyEvt := <-readyCh:
				if readyEvt.EventType != WatchReady {
					t.Errorf("Got unknown watch event %s, expected %s", readyEvt.EventType, WatchReady)
				}
			}

			if test.sendCancel {
				cancel <- struct{}{}
			}

			for _, p := range test.writes {
				write(t, p, "")
			}
			// Now check to see if the watch registered a change event
			select {
			case e := <-eventCh:
				if !reflect.DeepEqual(e, test.expected) {
					t.Errorf("Expected %+v, Actual %+v", test.expected, e)
				}
			case err := <-errCh:
				testutil.CheckError(t, test.shouldErr, err)
				return
			}

		})
	}
}

func TestAddDepsForArtifact(t *testing.T) {
	var tests = []struct {
		description string
		artifact    *config.Artifact
		expected    map[string]*config.Artifact

		shouldErr bool
	}{
		{
			description: "add deps",
			artifact:    artifactA,
			expected: map[string]*config.Artifact{
				"/tmp/skaffold/workspace_1/file_a": artifactA,
				"/tmp/skaffold/workspace_1/file_b": artifactA,
			},
		},
		{
			description: "missing dockerfile",
			artifact:    artifactMissingDockerfile,
			expected:    map[string]*config.Artifact{},
			shouldErr:   true,
		},
		{
			description: "missing file",
			artifact:    artifactMissingFile,
			expected:    map[string]*config.Artifact{},
			shouldErr:   true,
		},
		{
			description: "symlink file",
			artifact:    artifactSymlinkDep,
			expected: map[string]*config.Artifact{
				"/tmp/skaffold/workspace_1/file_b": artifactSymlinkDep,
			},
		},
		{
			description: "default dockerfile path",
			artifact:    artifactDefaultDockerfilePath,
			expected: map[string]*config.Artifact{
				"/tmp/skaffold/workspace_1/file_a": artifactDefaultDockerfilePath,
				"/tmp/skaffold/workspace_1/file_b": artifactDefaultDockerfilePath,
			},
		},
	}

	for _, test := range tests {
		t.Run(test.description, func(t *testing.T) {
			m := map[string]*config.Artifact{}
			err := addDepsForArtifact(test.artifact, m)
			testutil.CheckErrorAndDeepEqual(t, test.shouldErr, err, test.expected, m)
		})
	}
}

func TestAddWatchForDeps(t *testing.T) {
	var tests = []struct {
		description     string
		depsToArtifacts map[string]*config.Artifact
		c               chan notify.EventInfo
		shouldErr       bool
	}{
		{
			description: "error bad path",
			depsToArtifacts: map[string]*config.Artifact{
				"bad.path": artifactA,
			},
			c:         make(chan notify.EventInfo, 1),
			shouldErr: true,
		},
	}

	for _, test := range tests {
		t.Run(test.description, func(t *testing.T) {
			err := addWatchForDeps(test.depsToArtifacts, test.c)
			testutil.CheckError(t, test.shouldErr, err)
		})
	}
}

func TestMain(m *testing.M) {
	cleanup := func() {
		if err := fs.RemoveAll("/tmp/skaffold"); err != nil {
			logrus.Fatalf("Removing testing temp dir: %s", err)
		}
	}

	cleanup()
	initFS()
	exit := m.Run()
	cleanup()
	os.Exit(exit)
}
<commit_msg>Refactor watch tests<commit_after>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package watch

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"strings"
	"testing"

	"github.com/GoogleCloudPlatform/skaffold/pkg/skaffold/config"
	testutil "github.com/GoogleCloudPlatform/skaffold/test"
	"github.com/sirupsen/logrus"

	"github.com/spf13/afero"
)

var tmpDir string

var mockFS = map[string][]string{
	"Dockerfile":              {"COPY 1 /", "ADD dir/2 /"},
	"Dockerfile.MISSINGFILE":  {"COPY file_MISSING /"},
	"Dockerfile.symlinkdep":   {"COPY 5 /", "COPY 5 /"},
	"Dockerfile.star":         {"COPY * /"},
	"Dockerfile.ignored_file": {"COPY vendor/3 /", "COPY 1 /"},
	// regular files
	"1":         nil,
	"dir/2":     nil,
	"vendor/3":  nil,
	"4.symlink": {"1"},
	"5":         nil,
}

func initFS() {
	for p, contentSlice := range mockFS {
		fullPath := filepath.Join(tmpDir, p)
		contents := strings.Join(contentSlice, "\n")
		dir := filepath.Dir(fullPath)
		if err := fs.MkdirAll(dir, 0750); err != nil {
			logrus.Fatalf("making mock fs dir %s", err)
		}
		if strings.HasSuffix(fullPath, "symlink") {
			if err := os.Symlink(filepath.Join(tmpDir, contents), fullPath); err != nil {
				logrus.Fatalf("creating symlink file: %s", err)
			}
			continue
		}
		if err := afero.WriteFile(fs, fullPath, []byte(contents), 0640); err != nil {
			logrus.Fatalf("writing mock fs file: %s", err)
		}
	}
}

func write(t *testing.T, path, contents string) {
	if err := afero.WriteFile(fs, filepath.Join(tmpDir, path), []byte(contents), 0640); err != nil {
		t.Errorf("writing mock fs file: %s", err)
	}
}

func TestWatch(t *testing.T) {
	var tests = []struct {
		description    string
		artifacts      []*config.Artifact
		dockerfiles    []string
		writes         []string
		expected       *WatchEvent
		expectedChange string
		sendCancel     bool
		shouldErr      bool
	}{
		{
			description: "write file and ignored file",
			dockerfiles: []string{"Dockerfile.ignored_file", "Dockerfile"},
			writes: []string{
				"vendor/3",
				"dir/2",
			},
			expectedChange: "Dockerfile",
		},
		{
			description: "missing dockerfile",
			dockerfiles: []string{"Dockerfile.MISSINGFILE"},
			shouldErr:   true,
		},
		// {
		// 	description: "send cancel",
		// 	dockerfiles: []string{"Dockerfile"},
		// 	sendCancel:  true,
		// 	expected: &WatchEvent{EventType: "WatchStop",},
		// },
	}

	for _, test := range tests {
		t.Run(test.description, func(t *testing.T) {
			artifacts := []*config.Artifact{}
			for _, d := range test.dockerfiles {
				artifacts = append(artifacts, &config.Artifact{
					DockerfilePath: d,
					Workspace:      tmpDir,
				})
			}
			watcher := &FSWatcher{}
			eventCh := make(chan *WatchEvent, 1)
			readyCh := make(chan *WatchEvent, 1)
			errCh := make(chan error, 1)
			cancel := make(chan struct{}, 1)
			go func() {
				evt, err := watcher.Watch(artifacts, readyCh, cancel)
				if err != nil {
					errCh <- err
					return
				}
				eventCh <- evt
			}()

			select {
			case err := <-errCh:
				testutil.CheckError(t, test.shouldErr, err)
				return
			case readyEvt := <-readyCh:
				if readyEvt.EventType != WatchReady {
					t.Errorf("Got unknown watch event %s, expected %s", readyEvt.EventType, WatchReady)
				}
			}

			if test.sendCancel {
				cancel <- struct{}{}
			}

			for _, p := range test.writes {
				write(t, p, "")
			}

			// Now check to see if the watch registered a change event
			select {
			case e := <-eventCh:
				if !reflect.DeepEqual(e.ChangedArtifact.DockerfilePath, test.expectedChange) {
					t.Errorf("Expected %+v, Actual %+v", test.expectedChange, e.ChangedArtifact.DockerfilePath)
				}
				return
			case err := <-errCh:
				testutil.CheckError(t, test.shouldErr, err)
				return
			}

		})
	}
}

func TestAddDepsForArtifact(t *testing.T) {
	var tests = []struct {
		description string
		dockerfile  string
		expected    []string

		shouldErr bool
	}{
		{
			description: "add deps",
			dockerfile:  "Dockerfile",
			expected:    []string{"1", "dir/2"},
		},
		{
			description: "missing dockerfile",
			dockerfile:  "not a real file",
			shouldErr:   true,
		},
		{
			description: "missing file",
			dockerfile:  "Dockerfile.MISSINGFILE",
			shouldErr:   true,
		},
		{
			description: "symlink deps ignored",
			dockerfile:  "Dockerfile.symlinkdep",
			expected:    []string{"5"},
		},
	}

	for _, test := range tests {
		t.Run(test.description, func(t *testing.T) {
			m := map[string]*config.Artifact{}
			a := &config.Artifact{
				Workspace:      tmpDir,
				DockerfilePath: test.dockerfile,
			}
			expectedMap := map[string]*config.Artifact{}
			for _, d := range test.expected {
				expectedMap[filepath.Join(tmpDir, d)] = a
			}
			err := addDepsForArtifact(a, m)
			testutil.CheckErrorAndDeepEqual(t, test.shouldErr, err, expectedMap, m)
		})
	}
}

func TestMain(m *testing.M) {
	var err error
	tmpDir, err = ioutil.TempDir("", "skaffold")
	if err != nil {
		logrus.Fatalf("Getting temp dir: %s", err)
	}
	// On macOS the /tmp is symlinked to /private/tmp
	// the dockerfile parser won't accept symlinks, so we evaluate
	// the symlink for our tmp dir
	tmpDir, err = filepath.EvalSymlinks(tmpDir)
	if err != nil {
		logrus.Fatalf("Evaluating possible temp dir symlink: %s", err)
	}
	cleanup := func() {
		if err := fs.RemoveAll(tmpDir); err != nil {
			logrus.Fatalf("Removing testing temp dir: %s", err)
		}
	}

	cleanup()
	initFS()
	exit := m.Run()
	cleanup()
	os.Exit(exit)
}
<|endoftext|>


<commit_before>package memory

import (
	asst "github.com/stretchr/testify/assert"
	"github.com/xephonhq/xephon-k/pkg/common"
	"testing"
)

func TestIntersect(t *testing.T) {
	assert := asst.New(t)
	l1 := []common.SeriesID{"n1", "n2", "n3"}
	l2 := []common.SeriesID{"n1", "n2", "n3", "n4"}
	l3 := []common.SeriesID{"n1", "n2"}
	l4 := []common.SeriesID{"n2"}

	assert.Equal([]common.SeriesID{"n2"}, Intersect(l1, l2, l3, l4))

	// longer list reaches end before short list does
	// NOTE: should see `break outer in 1 th list` if the log is enabled
	assert.Equal([]common.SeriesID{"1"}, Intersect([]common.SeriesID{"1", "8", "11"}, []common.SeriesID{"1", "4", "5", "6"}))
}

func TestUnion(t *testing.T) {
	assert := asst.New(t)

	// simple union without duplication
	l1 := []common.SeriesID{"n1", "n2"}
	l2 := []common.SeriesID{"n3"}
	assert.Equal([]common.SeriesID{"n1", "n2", "n3"}, Union(l1, l2))

	// duplication
	l3 := []common.SeriesID{"n1", "n2"}
	l4 := []common.SeriesID{"n1", "n2", "n3"}
	// FIXED: result is n1, n2, n2, by adding
	//if lastVal == smallestVal {
	//	continue
	//}
	assert.Equal([]common.SeriesID{"n1", "n2", "n3"}, Union(l3, l4))

	//log.Logger.EnableSourceLine()
	// three way
	l5 := []common.SeriesID{"n9"}
	//fmt.Println("index_test.go:44") // works for IDEA
	//fmt.Println("source=index_test.go:44") // does not works for IDEA
	//fmt.Println("source= index_test.go:44") // works for IDEA
	// FIXED: {"n1", "n2", "n9", "n3"} would show up randomly, sometimes the test just pass
	// Got!! when we deal with dup, we need to use its next value to compare if there is any, the random is caused by picking the first value
	assert.Equal([]common.SeriesID{"n1", "n2", "n3", "n9"}, Union(l3, l4, l5))
	//log.Logger.DisableSourceLine()

	// just want to write one more test
	l6 := []common.SeriesID{"n1", "n9"}
	assert.Equal([]common.SeriesID{"n1", "n2", "n3", "n9"}, Union(l3, l4, l5, l6))
}

func TestIndex_Filter(t *testing.T) {
	assert := asst.New(t)
	idx := NewIndex(5)
	idx.Add(common.SeriesID("n1"), "app", "nginx")
	idx.Add(common.SeriesID("n1"), nameTagKey, "response_time")
	idx.Add(common.SeriesID("n2"), "app", "apache")
	idx.Add(common.SeriesID("n2"), nameTagKey, "response_time")
	filterResponse := common.Filter{Type: "tag_match", Key: nameTagKey, Value: "response_time"}
	assert.Equal([]common.SeriesID{"n1", "n2"}, idx.Filter(&filterResponse))
	filterNginxResponse := common.Filter{Type: "and",
		LeftOperand:  &common.Filter{Type: "tag_match", Key: nameTagKey, Value: "response_time"},
		RightOperand: &common.Filter{Type: "tag_match", Key: "app", Value: "nginx"}}
	assert.Equal([]common.SeriesID{"n1"}, idx.Filter(&filterNginxResponse))
}

func TestIndex_Get(t *testing.T) {
	assert := asst.New(t)
	idx := NewIndex(1)
	idx.Add(common.SeriesID("n1"), "app", "nginx")
	assert.Equal([]common.SeriesID{"n1"}, idx.Get("app", "nginx"))
	assert.Equal([]common.SeriesID{}, idx.Get("foo", "bar"))
	assert.Equal(0, len(idx.Get("foo", "bar")))
}

func TestIndex_Add(t *testing.T) {
	assert := asst.New(t)
	idx := NewIndex(1)
	idx.Add(common.SeriesID("n1"), "app", "nginx")
	idx.Add(common.SeriesID("n2"), "app", "nginx")
	idx.Add(common.SeriesID("a1"), "app", "apache")
	assert.Equal(true, idx.tagKeyIndex["app"]["nginx"])
	assert.Equal(true, idx.tagKeyIndex["app"]["apache"])
	assert.Equal(false, idx.tagKeyIndex["app"]["IIS"])
	// FIXME: currently we don't add separator between tagKey and tagValue
	assert.Equal([]common.SeriesID{"n1", "n2"}, idx.invertedIndexes["appnginx"].Postings)
}

func TestInvertedIndex_Add(t *testing.T) {
	assert := asst.New(t)
	iidx := newInvertedIndex("foo")
	iidx.Add(common.SeriesID("d"))
	iidx.Add(common.SeriesID("e"))
	iidx.Add(common.SeriesID("a"))
	iidx.Add(common.SeriesID("a"))
	iidx.Add(common.SeriesID("b"))
	assert.Equal(iidx.Postings, []common.SeriesID{"a", "b", "d", "e"})
}

func TestMin(t *testing.T) {
	assert := asst.New(t)
	assert.Equal(1, min(1, 2))
	assert.Equal(1, min(2, 1))
}
<commit_msg>[common] Replace str with uint64 in index test #36<commit_after>package memory

import (
	asst "github.com/stretchr/testify/assert"
	"github.com/xephonhq/xephon-k/pkg/common"
	"testing"
)

func TestIntersect(t *testing.T) {
	assert := asst.New(t)
	l1 := []common.SeriesID{1, 2, 3}
	l2 := []common.SeriesID{1, 2, 3, 4}
	l3 := []common.SeriesID{1, 2}
	l4 := []common.SeriesID{2}

	assert.Equal([]common.SeriesID{2}, Intersect(l1, l2, l3, l4))

	// longer list reaches end before short list does
	// NOTE: should see `break outer in 1 th list` if the log is enabled
	assert.Equal([]common.SeriesID{1}, Intersect([]common.SeriesID{1, 8, 11}, []common.SeriesID{1, 4, 5, 6}))
}

func TestUnion(t *testing.T) {
	assert := asst.New(t)

	// simple union without duplication
	l1 := []common.SeriesID{1, 2}
	l2 := []common.SeriesID{3}
	assert.Equal([]common.SeriesID{1, 2, 3}, Union(l1, l2))

	// duplication
	l3 := []common.SeriesID{1, 2}
	l4 := []common.SeriesID{1, 2, 3}
	// FIXED: result is n1, n2, n2, by adding
	//if lastVal == smallestVal {
	//	continue
	//}
	assert.Equal([]common.SeriesID{1, 2, 3}, Union(l3, l4))

	//log.Logger.EnableSourceLine()
	// three way
	l5 := []common.SeriesID{9}
	//fmt.Println("index_test.go:44") // works for IDEA
	//fmt.Println("source=index_test.go:44") // does not works for IDEA
	//fmt.Println("source= index_test.go:44") // works for IDEA
	// FIXED: {1, 2, 9, 3} would show up randomly, sometimes the test just pass
	// Got!! when we deal with dup, we need to use its next value to compare if there is any, the random is caused by picking the first value
	assert.Equal([]common.SeriesID{1, 2, 3, 9}, Union(l3, l4, l5))
	//log.Logger.DisableSourceLine()

	// just want to write one more test
	l6 := []common.SeriesID{1, 9}
	assert.Equal([]common.SeriesID{1, 2, 3, 9}, Union(l3, l4, l5, l6))
}

func TestIndex_Filter(t *testing.T) {
	assert := asst.New(t)
	idx := NewIndex(5)
	idx.Add(common.SeriesID(1), "app", "nginx")
	idx.Add(common.SeriesID(1), nameTagKey, "response_time")
	idx.Add(common.SeriesID(2), "app", "apache")
	idx.Add(common.SeriesID(2), nameTagKey, "response_time")
	filterResponse := common.Filter{Type: "tag_match", Key: nameTagKey, Value: "response_time"}
	assert.Equal([]common.SeriesID{1, 2}, idx.Filter(&filterResponse))
	filterNginxResponse := common.Filter{Type: "and",
		LeftOperand:  &common.Filter{Type: "tag_match", Key: nameTagKey, Value: "response_time"},
		RightOperand: &common.Filter{Type: "tag_match", Key: "app", Value: "nginx"}}
	assert.Equal([]common.SeriesID{1}, idx.Filter(&filterNginxResponse))
}

func TestIndex_Get(t *testing.T) {
	assert := asst.New(t)
	idx := NewIndex(1)
	idx.Add(common.SeriesID(1), "app", "nginx")
	assert.Equal([]common.SeriesID{1}, idx.Get("app", "nginx"))
	assert.Equal([]common.SeriesID{}, idx.Get("foo", "bar"))
	assert.Equal(0, len(idx.Get("foo", "bar")))
}

func TestIndex_Add(t *testing.T) {
	assert := asst.New(t)
	idx := NewIndex(1)
	idx.Add(common.SeriesID(1), "app", "nginx")
	// NOTE: they just share one tag pair, so they can have different series ID
	idx.Add(common.SeriesID(2), "app", "nginx")
	idx.Add(common.SeriesID(3), "app", "apache")
	assert.Equal(true, idx.tagKeyIndex["app"]["nginx"])
	assert.Equal(true, idx.tagKeyIndex["app"]["apache"])
	assert.Equal(false, idx.tagKeyIndex["app"]["IIS"])
	// FIXME: currently we don't add separator between tagKey and tagValue
	assert.Equal([]common.SeriesID{1, 2}, idx.invertedIndexes["appnginx"].Postings)
}

func TestInvertedIndex_Add(t *testing.T) {
	assert := asst.New(t)
	iidx := newInvertedIndex("foo")
	iidx.Add(common.SeriesID(5))
	iidx.Add(common.SeriesID(1))
	iidx.Add(common.SeriesID(2))
	iidx.Add(common.SeriesID(2))
	iidx.Add(common.SeriesID(4))
	assert.Equal(iidx.Postings, []common.SeriesID{1, 2, 4, 5})
}

func TestMin(t *testing.T) {
	assert := asst.New(t)
	assert.Equal(1, min(1, 2))
	assert.Equal(1, min(2, 1))
}
<|endoftext|>


<commit_before>package executioncontext

// Flag type
type Flag int

// Set flag
func (f Flag) Set(flag Flag) Flag { return f | flag }

// Clear flag
func (f Flag) Clear(flag Flag) Flag { return f &^ flag }

// Toggle flag
func (f Flag) Toggle(flag Flag) Flag { return f ^ flag }

// Has flag
func (f Flag) Has(flag Flag) bool { return f&flag != 0 }
<commit_msg>Use uint8 type<commit_after>package executioncontext

// Flag type
type Flag uint8

// Set flag
func (f Flag) Set(flag Flag) Flag { return f | flag }

// Clear flag
func (f Flag) Clear(flag Flag) Flag { return f &^ flag }

// Toggle flag
func (f Flag) Toggle(flag Flag) Flag { return f ^ flag }

// Has flag
func (f Flag) Has(flag Flag) bool { return f&flag != 0 }
<|endoftext|>


<commit_before>// Copyright 2020 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package relay

import (
	"context"
	"io"
	"time"

	observerpb "github.com/cilium/cilium/api/v1/observer"
	"github.com/cilium/cilium/pkg/hubble/relay/queue"
	"github.com/sirupsen/logrus"

	"golang.org/x/sync/errgroup"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// ensure that Server implements the observer.ObserverServer interface.
var _ observerpb.ObserverServer = (*Server)(nil)

func retrieveFlowsFromPeer(
	ctx context.Context,
	conn *grpc.ClientConn,
	req *observerpb.GetFlowsRequest,
	flows chan<- *observerpb.GetFlowsResponse,
) error {
	client := observerpb.NewObserverClient(conn)
	c, err := client.GetFlows(ctx, req)
	if err != nil {
		return err
	}
	for {
		flow, err := c.Recv()
		switch err {
		case io.EOF, context.Canceled:
			return nil
		case nil:
			select {
			case flows <- flow:
			case <-ctx.Done():
				return nil
			}
		default:
			if status.Code(err) != codes.Canceled {
				return err
			}
			return nil
		}
	}
}

func sortFlows(
	ctx context.Context,
	flows <-chan *observerpb.GetFlowsResponse,
	qlen int,
	bufferDrainTimeout time.Duration,
) <-chan *observerpb.GetFlowsResponse {
	pq := queue.NewPriorityQueue(qlen)
	sortedFlows := make(chan *observerpb.GetFlowsResponse, qlen)

	go func() {
		defer close(sortedFlows)

	flowsLoop:
		for {
			select {
			case flow, ok := <-flows:
				if !ok {
					break flowsLoop
				}
				if pq.Len() == qlen {
					f := pq.Pop()
					select {
					case sortedFlows <- f:
					case <-ctx.Done():
						return
					}
				}
				pq.Push(flow)
			case <-time.After(bufferDrainTimeout): // make sure to drain the queue when no new flow responses are received
				if f := pq.Pop(); f != nil {
					select {
					case sortedFlows <- f:
					case <-ctx.Done():
						return
					}
				}
			case <-ctx.Done():
				return
			}
		}
		// drain the queue
		for f := pq.Pop(); f != nil; f = pq.Pop() {
			select {
			case sortedFlows <- f:
			case <-ctx.Done():
				return
			}
		}

	}()

	return sortedFlows
}

// GetFlows implements observer.ObserverServer.GetFlows by proxying requests to
// the hubble instance the proxy is connected to.
func (s *Server) GetFlows(req *observerpb.GetFlowsRequest, stream observerpb.Observer_GetFlowsServer) error {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func() {
		select {
		case <-s.stop:
			cancel()
		case <-ctx.Done():
		}
	}()

	peers := s.peerList()
	qlen := s.opts.BufferMaxLen // we don't want to buffer too many flows
	if nqlen := req.GetNumber() * uint64(len(peers)); nqlen > 0 && nqlen < uint64(qlen) {
		// don't make the queue bigger than necessary as it would be a problem
		// with the priority queue (we pop out when the queue is full)
		qlen = int(nqlen)
	}

	g, gctx := errgroup.WithContext(ctx)
	flows := make(chan *observerpb.GetFlowsResponse, qlen)
	for _, p := range peers {
		p := p
		if p.conn == nil || p.connErr != nil {
			s.log.WithField("address", p.Address.String()).Infof(
				"No connection to peer %s, skipping", p.Name,
			)
			go s.connectPeer(p.Name, p.Address.String())
			continue
		}
		g.Go(func() error {
			// retrieveFlowsFromPeer returns blocks until the peer finishes
			// the request by closing the connection, an error occurs,
			// or gctx expires.
			err := retrieveFlowsFromPeer(gctx, p.conn, req, flows)
			if err != nil {
				s.log.WithFields(logrus.Fields{
					"error": err,
					"peer":  p,
				}).Warning("Failed to retrieve flows from peer")
			}
			return nil
		})
	}
	go func() {
		g.Wait()
		close(flows)
	}()

	sortedFlows := sortFlows(ctx, flows, qlen, s.opts.BufferDrainTimeout)

sortedFlowsLoop:
	for {
		select {
		case flow, ok := <-sortedFlows:
			if !ok {
				break sortedFlowsLoop
			}
			if err := stream.Send(flow); err != nil {
				return err
			}
		case <-ctx.Done():
			break sortedFlowsLoop
		}
	}
	return g.Wait()
}

// ServerStatus implements observer.ObserverServer.ServerStatus by aggregating
// the ServerStatus answer of all hubble peers.
func (s *Server) ServerStatus(ctx context.Context, req *observerpb.ServerStatusRequest) (*observerpb.ServerStatusResponse, error) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	g, ctx := errgroup.WithContext(ctx)
	go func() {
		select {
		case <-s.stop:
			cancel()
		case <-ctx.Done():
		}
	}()

	peers := s.peerList()
	statuses := make(chan *observerpb.ServerStatusResponse, len(peers))
	for _, p := range peers {
		p := p
		if p.conn == nil || p.connErr != nil {
			s.log.WithField("address", p.Address.String()).Infof(
				"No connection to peer %s, skipping", p.Name,
			)
			go s.connectPeer(p.Name, p.Address.String())
			continue
		}
		g.Go(func() error {
			client := observerpb.NewObserverClient(p.conn)
			status, err := client.ServerStatus(ctx, req)
			if err != nil {
				s.log.WithFields(logrus.Fields{
					"error": err,
					"peer":  p,
				}).Warning("Failed to retrieve server status")
				return nil
			}
			select {
			case statuses <- status:
			case <-ctx.Done():
			}
			return nil
		})
	}
	go func() {
		g.Wait()
		close(statuses)
	}()
	resp := &observerpb.ServerStatusResponse{}
	for status := range statuses {
		if status == nil {
			continue
		}
		resp.MaxFlows += status.MaxFlows
		resp.NumFlows += status.NumFlows
		resp.SeenFlows += status.SeenFlows
		// use the oldest uptime as a reference for the uptime as cumulating
		// values would make little sense
		if resp.UptimeNs == 0 || resp.UptimeNs > status.UptimeNs {
			resp.UptimeNs = status.UptimeNs
		}
	}
	return resp, g.Wait()
}
<commit_msg>hubble-relay: Send node status events in GetFlows<commit_after>// Copyright 2020 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package relay

import (
	"context"
	"io"
	"time"

	observerpb "github.com/cilium/cilium/api/v1/observer"
	relaypb "github.com/cilium/cilium/api/v1/relay"
	"github.com/cilium/cilium/pkg/hubble/relay/queue"
	nodeTypes "github.com/cilium/cilium/pkg/node/types"

	"github.com/golang/protobuf/ptypes"
	"github.com/sirupsen/logrus"
	"golang.org/x/sync/errgroup"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// ensure that Server implements the observer.ObserverServer interface.
var _ observerpb.ObserverServer = (*Server)(nil)

func retrieveFlowsFromPeer(
	ctx context.Context,
	conn *grpc.ClientConn,
	req *observerpb.GetFlowsRequest,
	flows chan<- *observerpb.GetFlowsResponse,
) error {
	client := observerpb.NewObserverClient(conn)
	c, err := client.GetFlows(ctx, req)
	if err != nil {
		return err
	}
	for {
		flow, err := c.Recv()
		switch err {
		case io.EOF, context.Canceled:
			return nil
		case nil:
			select {
			case flows <- flow:
			case <-ctx.Done():
				return nil
			}
		default:
			if status.Code(err) != codes.Canceled {
				return err
			}
			return nil
		}
	}
}

func sortFlows(
	ctx context.Context,
	flows <-chan *observerpb.GetFlowsResponse,
	qlen int,
	bufferDrainTimeout time.Duration,
) <-chan *observerpb.GetFlowsResponse {
	pq := queue.NewPriorityQueue(qlen)
	sortedFlows := make(chan *observerpb.GetFlowsResponse, qlen)

	go func() {
		defer close(sortedFlows)

	flowsLoop:
		for {
			select {
			case flow, ok := <-flows:
				if !ok {
					break flowsLoop
				}
				if pq.Len() == qlen {
					f := pq.Pop()
					select {
					case sortedFlows <- f:
					case <-ctx.Done():
						return
					}
				}
				pq.Push(flow)
			case <-time.After(bufferDrainTimeout): // make sure to drain the queue when no new flow responses are received
				if f := pq.Pop(); f != nil {
					select {
					case sortedFlows <- f:
					case <-ctx.Done():
						return
					}
				}
			case <-ctx.Done():
				return
			}
		}
		// drain the queue
		for f := pq.Pop(); f != nil; f = pq.Pop() {
			select {
			case sortedFlows <- f:
			case <-ctx.Done():
				return
			}
		}

	}()

	return sortedFlows
}

func nodeStatusError(err error, nodeNames ...string) *observerpb.GetFlowsResponse {
	msg := err.Error()
	if s, ok := status.FromError(err); ok && s.Code() == codes.Unknown {
		msg = s.Message()
	}

	return &observerpb.GetFlowsResponse{
		NodeName: nodeTypes.GetName(),
		Time:     ptypes.TimestampNow(),
		ResponseTypes: &observerpb.GetFlowsResponse_NodeStatus{
			NodeStatus: &relaypb.NodeStatusEvent{
				StateChange: relaypb.NodeState_NODE_ERROR,
				NodeNames:   nodeNames,
				Message:     msg,
			},
		},
	}
}

func nodeStatusEvent(state relaypb.NodeState, nodeNames ...string) *observerpb.GetFlowsResponse {
	return &observerpb.GetFlowsResponse{
		NodeName: nodeTypes.GetName(),
		Time:     ptypes.TimestampNow(),
		ResponseTypes: &observerpb.GetFlowsResponse_NodeStatus{
			NodeStatus: &relaypb.NodeStatusEvent{
				StateChange: state,
				NodeNames:   nodeNames,
			},
		},
	}
}

// GetFlows implements observer.ObserverServer.GetFlows by proxying requests to
// the hubble instance the proxy is connected to.
func (s *Server) GetFlows(req *observerpb.GetFlowsRequest, stream observerpb.Observer_GetFlowsServer) error {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	go func() {
		select {
		case <-s.stop:
			cancel()
		case <-ctx.Done():
		}
	}()

	peers := s.peerList()
	qlen := s.opts.BufferMaxLen // we don't want to buffer too many flows
	if nqlen := req.GetNumber() * uint64(len(peers)); nqlen > 0 && nqlen < uint64(qlen) {
		// don't make the queue bigger than necessary as it would be a problem
		// with the priority queue (we pop out when the queue is full)
		qlen = int(nqlen)
	}

	g, gctx := errgroup.WithContext(ctx)
	flows := make(chan *observerpb.GetFlowsResponse, qlen)
	var connectedNodes, unavailableNodes []string

	for _, p := range peers {
		p := p
		if p.conn == nil || p.connErr != nil {
			s.log.WithField("address", p.Address.String()).Infof(
				"No connection to peer %s, skipping", p.Name,
			)
			unavailableNodes = append(unavailableNodes, p.Name)
			go s.connectPeer(p.Name, p.Address.String())
			continue
		}
		connectedNodes = append(connectedNodes, p.Name)
		g.Go(func() error {
			// retrieveFlowsFromPeer returns blocks until the peer finishes
			// the request by closing the connection, an error occurs,
			// or gctx expires.
			err := retrieveFlowsFromPeer(gctx, p.conn, req, flows)
			if err != nil {
				s.log.WithFields(logrus.Fields{
					"error": err,
					"peer":  p,
				}).Warning("Failed to retrieve flows from peer")
				select {
				case flows <- nodeStatusError(err, p.Name):
				case <-gctx.Done():
				}
			}
			return nil
		})
	}
	go func() {
		g.Wait()
		close(flows)
	}()

	sortedFlows := sortFlows(ctx, flows, qlen, s.opts.BufferDrainTimeout)

	// inform the client about the nodes from which we expect to receive flows first
	if len(connectedNodes) > 0 {
		status := nodeStatusEvent(relaypb.NodeState_NODE_CONNECTED, connectedNodes...)
		if err := stream.Send(status); err != nil {
			return err
		}
	}
	if len(unavailableNodes) > 0 {
		status := nodeStatusEvent(relaypb.NodeState_NODE_UNAVAILABLE, unavailableNodes...)
		if err := stream.Send(status); err != nil {
			return err
		}
	}

sortedFlowsLoop:
	for {
		select {
		case flow, ok := <-sortedFlows:
			if !ok {
				break sortedFlowsLoop
			}
			if err := stream.Send(flow); err != nil {
				return err
			}
		case <-ctx.Done():
			break sortedFlowsLoop
		}
	}
	return g.Wait()
}

// ServerStatus implements observer.ObserverServer.ServerStatus by aggregating
// the ServerStatus answer of all hubble peers.
func (s *Server) ServerStatus(ctx context.Context, req *observerpb.ServerStatusRequest) (*observerpb.ServerStatusResponse, error) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()
	g, ctx := errgroup.WithContext(ctx)
	go func() {
		select {
		case <-s.stop:
			cancel()
		case <-ctx.Done():
		}
	}()

	peers := s.peerList()
	statuses := make(chan *observerpb.ServerStatusResponse, len(peers))
	for _, p := range peers {
		p := p
		if p.conn == nil || p.connErr != nil {
			s.log.WithField("address", p.Address.String()).Infof(
				"No connection to peer %s, skipping", p.Name,
			)
			go s.connectPeer(p.Name, p.Address.String())
			continue
		}
		g.Go(func() error {
			client := observerpb.NewObserverClient(p.conn)
			status, err := client.ServerStatus(ctx, req)
			if err != nil {
				s.log.WithFields(logrus.Fields{
					"error": err,
					"peer":  p,
				}).Warning("Failed to retrieve server status")
				return nil
			}
			select {
			case statuses <- status:
			case <-ctx.Done():
			}
			return nil
		})
	}
	go func() {
		g.Wait()
		close(statuses)
	}()
	resp := &observerpb.ServerStatusResponse{}
	for status := range statuses {
		if status == nil {
			continue
		}
		resp.MaxFlows += status.MaxFlows
		resp.NumFlows += status.NumFlows
		resp.SeenFlows += status.SeenFlows
		// use the oldest uptime as a reference for the uptime as cumulating
		// values would make little sense
		if resp.UptimeNs == 0 || resp.UptimeNs > status.UptimeNs {
			resp.UptimeNs = status.UptimeNs
		}
	}
	return resp, g.Wait()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"path"
	"strings"
	"time"
)

const (
	project  = "http://metadata.google.internal/computeMetadata/v1/project/"
	instance = "http://metadata.google.internal/computeMetadata/v1/instance/"
)

// ProviderGCP is the type implementing the Provider interface for GCP
type ProviderGCP struct {
}

// NewGCP returns a new ProviderGCP
func NewGCP() *ProviderGCP {
	return &ProviderGCP{}
}

func (p *ProviderGCP) String() string {
	return "GCP"
}

// Probe checks if we are running on GCP
func (p *ProviderGCP) Probe() bool {
	// Getting the hostname should always work...
	_, err := gcpGet(instance + "hostname")
	return (err == nil)
}

// Extract gets both the GCP specific and generic userdata
func (p *ProviderGCP) Extract() ([]byte, error) {
	// Get host name. This must not fail
	hostname, err := gcpGet(instance + "hostname")
	if err != nil {
		return nil, err
	}
	err = ioutil.WriteFile(path.Join(ConfigPath, Hostname), hostname, 0644)
	if err != nil {
		return nil, fmt.Errorf("GCP: Failed to write hostname: %s", err)
	}

	// SSH keys:
	// TODO also retrieve the instance keys and respect block
	//      project keys see:
	//      https://cloud.google.com/compute/docs/instances/ssh-keys
	// The keys have usernames attached, but as a simplification
	// we are going to add them all to one root file
	// TODO split them into individual user files and make the ssh
	//      container construct those users
	sshKeys, err := gcpGet(project + "attributes/sshKeys")
	if err == nil {
		rootKeys := ""
		for _, line := range strings.Split(string(sshKeys), "\n") {
			parts := strings.SplitN(line, ":", 2)
			// ignoring username for now
			if len(parts) == 2 {
				rootKeys = rootKeys + parts[1] + "\n"
			}
		}
		err = ioutil.WriteFile(path.Join(ConfigPath, SSH, "authorized_keys"), []byte(rootKeys), 0600)
		if err != nil {
			log.Printf("GCP: Failed to write ssh keys: %s", err)
		}
	}

	// Generic userdata
	userData, err := gcpGet(instance + "attributes/userdata")
	if err != nil {
		log.Printf("GCP: Failed to get user-data: %s", err)
		// This is not an error
		return nil, nil
	}
	return userData, nil
}

// gcpGet requests and extracts the requested URL
func gcpGet(url string) ([]byte, error) {
	var client = &http.Client{
		Timeout: time.Second * 2,
	}

	req, err := http.NewRequest("", url, nil)
	if err != nil {
		return nil, fmt.Errorf("GCP: http.NewRequest failed: %s", err)
	}
	req.Header.Set("Metadata-Flavor", "Google")

	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("GCP: Could not contact metadata service: %s", err)
	}
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("GCP: Status not ok: %d", resp.StatusCode)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("GCP: Failed to read http response: %s", err)
	}
	return body, nil
}
<commit_msg>metadata: Make sure the ssh directory exists before writing to it<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"strings"
	"time"
)

const (
	project  = "http://metadata.google.internal/computeMetadata/v1/project/"
	instance = "http://metadata.google.internal/computeMetadata/v1/instance/"
)

// ProviderGCP is the type implementing the Provider interface for GCP
type ProviderGCP struct {
}

// NewGCP returns a new ProviderGCP
func NewGCP() *ProviderGCP {
	return &ProviderGCP{}
}

func (p *ProviderGCP) String() string {
	return "GCP"
}

// Probe checks if we are running on GCP
func (p *ProviderGCP) Probe() bool {
	// Getting the hostname should always work...
	_, err := gcpGet(instance + "hostname")
	return (err == nil)
}

// Extract gets both the GCP specific and generic userdata
func (p *ProviderGCP) Extract() ([]byte, error) {
	// Get host name. This must not fail
	hostname, err := gcpGet(instance + "hostname")
	if err != nil {
		return nil, err
	}
	err = ioutil.WriteFile(path.Join(ConfigPath, Hostname), hostname, 0644)
	if err != nil {
		return nil, fmt.Errorf("GCP: Failed to write hostname: %s", err)
	}

	// SSH keys:
	// TODO also retrieve the instance keys and respect block
	//      project keys see:
	//      https://cloud.google.com/compute/docs/instances/ssh-keys
	// The keys have usernames attached, but as a simplification
	// we are going to add them all to one root file
	// TODO split them into individual user files and make the ssh
	//      container construct those users
	sshKeys, err := gcpGet(project + "attributes/sshKeys")
	if err == nil {
		if err := os.Mkdir(path.Join(ConfigPath, SSH), 0755); err != nil {
			log.Printf("Failed to create %s: %s", SSH, err)
			goto ErrorSSH
		}
		rootKeys := ""
		for _, line := range strings.Split(string(sshKeys), "\n") {
			parts := strings.SplitN(line, ":", 2)
			// ignoring username for now
			if len(parts) == 2 {
				rootKeys = rootKeys + parts[1] + "\n"
			}
		}
		err = ioutil.WriteFile(path.Join(ConfigPath, SSH, "authorized_keys"), []byte(rootKeys), 0600)
		if err != nil {
			log.Printf("GCP: Failed to write ssh keys: %s", err)
		}
	}

ErrorSSH:
	// Generic userdata
	userData, err := gcpGet(instance + "attributes/userdata")
	if err != nil {
		log.Printf("GCP: Failed to get user-data: %s", err)
		// This is not an error
		return nil, nil
	}
	return userData, nil
}

// gcpGet requests and extracts the requested URL
func gcpGet(url string) ([]byte, error) {
	var client = &http.Client{
		Timeout: time.Second * 2,
	}

	req, err := http.NewRequest("", url, nil)
	if err != nil {
		return nil, fmt.Errorf("GCP: http.NewRequest failed: %s", err)
	}
	req.Header.Set("Metadata-Flavor", "Google")

	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("GCP: Could not contact metadata service: %s", err)
	}
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("GCP: Status not ok: %d", resp.StatusCode)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("GCP: Failed to read http response: %s", err)
	}
	return body, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"strings"
	"time"
)

const (
	project  = "http://metadata.google.internal/computeMetadata/v1/project/"
	instance = "http://metadata.google.internal/computeMetadata/v1/instance/"
)

// ProviderGCP is the type implementing the Provider interface for GCP
type ProviderGCP struct {
}

// NewGCP returns a new ProviderGCP
func NewGCP() *ProviderGCP {
	return &ProviderGCP{}
}

func (p *ProviderGCP) String() string {
	return "GCP"
}

// Probe checks if we are running on GCP
func (p *ProviderGCP) Probe() bool {
	// Getting the hostname should always work...
	_, err := gcpGet(instance + "hostname")
	return (err == nil)
}

// Extract gets both the GCP specific and generic userdata
func (p *ProviderGCP) Extract() ([]byte, error) {
	// Get host name. This must not fail
	hostname, err := gcpGet(instance + "hostname")
	if err != nil {
		return nil, err
	}
	err = ioutil.WriteFile(path.Join(ConfigPath, Hostname), hostname, 0644)
	if err != nil {
		return nil, fmt.Errorf("GCP: Failed to write hostname: %s", err)
	}

	// SSH keys:
	// TODO also retrieve the instance keys and respect block
	//      project keys see:
	//      https://cloud.google.com/compute/docs/instances/ssh-keys
	// The keys have usernames attached, but as a simplification
	// we are going to add them all to one root file
	// TODO split them into individual user files and make the ssh
	//      container construct those users
	sshKeys, err := gcpGet(project + "attributes/sshKeys")
	if err == nil {
		if err := os.Mkdir(path.Join(ConfigPath, SSH), 0755); err != nil {
			log.Printf("Failed to create %s: %s", SSH, err)
			goto ErrorSSH
		}
		rootKeys := ""
		for _, line := range strings.Split(string(sshKeys), "\n") {
			parts := strings.SplitN(line, ":", 2)
			// ignoring username for now
			if len(parts) == 2 {
				rootKeys = rootKeys + parts[1] + "\n"
			}
		}
		err = ioutil.WriteFile(path.Join(ConfigPath, SSH, "authorized_keys"), []byte(rootKeys), 0600)
		if err != nil {
			log.Printf("GCP: Failed to write ssh keys: %s", err)
		}
	}

ErrorSSH:
	// Generic userdata
	userData, err := gcpGet(instance + "attributes/userdata")
	if err != nil {
		log.Printf("GCP: Failed to get user-data: %s", err)
		// This is not an error
		return nil, nil
	}
	return userData, nil
}

// gcpGet requests and extracts the requested URL
func gcpGet(url string) ([]byte, error) {
	var client = &http.Client{
		Timeout: time.Second * 2,
	}

	req, err := http.NewRequest("", url, nil)
	if err != nil {
		return nil, fmt.Errorf("GCP: http.NewRequest failed: %s", err)
	}
	req.Header.Set("Metadata-Flavor", "Google")

	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("GCP: Could not contact metadata service: %s", err)
	}
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("GCP: Status not ok: %d", resp.StatusCode)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("GCP: Failed to read http response: %s", err)
	}
	return body, nil
}
<commit_msg>metadata: Simplify the GCP SSH code<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"strings"
	"time"
)

const (
	project  = "http://metadata.google.internal/computeMetadata/v1/project/"
	instance = "http://metadata.google.internal/computeMetadata/v1/instance/"
)

// ProviderGCP is the type implementing the Provider interface for GCP
type ProviderGCP struct {
}

// NewGCP returns a new ProviderGCP
func NewGCP() *ProviderGCP {
	return &ProviderGCP{}
}

func (p *ProviderGCP) String() string {
	return "GCP"
}

// Probe checks if we are running on GCP
func (p *ProviderGCP) Probe() bool {
	// Getting the hostname should always work...
	_, err := gcpGet(instance + "hostname")
	return (err == nil)
}

// Extract gets both the GCP specific and generic userdata
func (p *ProviderGCP) Extract() ([]byte, error) {
	// Get host name. This must not fail
	hostname, err := gcpGet(instance + "hostname")
	if err != nil {
		return nil, err
	}
	err = ioutil.WriteFile(path.Join(ConfigPath, Hostname), hostname, 0644)
	if err != nil {
		return nil, fmt.Errorf("GCP: Failed to write hostname: %s", err)
	}

	if err := p.handleSSH(); err != nil {
		log.Printf("GCP: Failed to get ssh data: %s", err)
	}

	// Generic userdata
	userData, err := gcpGet(instance + "attributes/userdata")
	if err != nil {
		log.Printf("GCP: Failed to get user-data: %s", err)
		// This is not an error
		return nil, nil
	}
	return userData, nil
}

// gcpGet requests and extracts the requested URL
func gcpGet(url string) ([]byte, error) {
	var client = &http.Client{
		Timeout: time.Second * 2,
	}

	req, err := http.NewRequest("", url, nil)
	if err != nil {
		return nil, fmt.Errorf("GCP: http.NewRequest failed: %s", err)
	}
	req.Header.Set("Metadata-Flavor", "Google")

	resp, err := client.Do(req)
	if err != nil {
		return nil, fmt.Errorf("GCP: Could not contact metadata service: %s", err)
	}
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("GCP: Status not ok: %d", resp.StatusCode)
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("GCP: Failed to read http response: %s", err)
	}
	return body, nil
}

// SSH keys:
// TODO also retrieve the instance keys and respect block
//      project keys see:
//      https://cloud.google.com/compute/docs/instances/ssh-keys
// The keys have usernames attached, but as a simplification
// we are going to add them all to one root file
// TODO split them into individual user files and make the ssh
//      container construct those users
func (p *ProviderGCP) handleSSH() error {
	sshKeys, err := gcpGet(project + "attributes/sshKeys")
	if err != nil {
		return fmt.Errorf("Failed to get sshKeys: %s", err)
	}

	if err := os.Mkdir(path.Join(ConfigPath, SSH), 0755); err != nil {
		return fmt.Errorf("Failed to create %s: %s", SSH, err)
	}

	rootKeys := ""
	for _, line := range strings.Split(string(sshKeys), "\n") {
		parts := strings.SplitN(line, ":", 2)
		// ignoring username for now
		if len(parts) == 2 {
			rootKeys = rootKeys + parts[1] + "\n"
		}
	}
	err = ioutil.WriteFile(path.Join(ConfigPath, SSH, "authorized_keys"), []byte(rootKeys), 0600)
	if err != nil {
		return fmt.Errorf("Failed to write ssh keys: %s", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package operator

import (
	"context"
	"time"

	"github.com/coreos-inc/vault-operator/pkg/spec"
	"github.com/coreos-inc/vault-operator/pkg/util/k8sutil"

	"github.com/Sirupsen/logrus"
	vaultapi "github.com/hashicorp/vault/api"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
)

// monitorAndUpdateStaus monitors the vault service and replicas statuses, and
// updates the status resrouce in the vault CR item.
func (vs *Vaults) monitorAndUpdateStaus(ctx context.Context, name, namespace string) {
	s := spec.VaultStatus{}

	for {
		select {
		case err := <-ctx.Done():
			logrus.Infof("stopped monitoring vault: %s (%v)", name, err)
		case <-time.After(10 * time.Second):
		}

		vs.updateLocalVaultCRStatus(ctx, name, namespace, &s)

		err := vs.updateVaultCRStatus(ctx, name, namespace, s)
		if err != nil {
			logrus.Errorf("failed updating the status for the vault service: %s (%v)", name, err)
		}
	}
}

// updateLocalVaultCRStatus updates local vault CR status by querying each vault pod's API.
func (vs *Vaults) updateLocalVaultCRStatus(ctx context.Context, name, namespace string, s *spec.VaultStatus) {
	sel := k8sutil.PodsLabelsForVault(name)
	// TODO: handle upgrades when pods from two replicaset can co-exist :(
	opt := metav1.ListOptions{LabelSelector: labels.SelectorFromSet(sel).String()}
	pods, err := vs.kubecli.CoreV1().Pods(namespace).List(opt)
	if err != nil {
		logrus.Errorf("failed to update vault replica status: failed listing pods for the vault service (%s.%s): %v", name, namespace, err)
		return
	}

	// If it can't talk to any vault pod, we are not changing the state.
	sealNodes := s.SealedNodes
	inited := s.Initialized

	for _, p := range pods.Items {
		cfg := vaultapi.DefaultConfig()
		// TODO: change to https.
		// TODO: use FQDN?
		podURL := "http://" + p.Status.PodIP + ":8200"
		cfg.Address = podURL
		vapi, err := vaultapi.NewClient(cfg)
		if err != nil {
			logrus.Errorf("failed to update vault replica status: failed creating client for the vault pod (%s/%s): %v", namespace, p.GetName(), err)
			continue
		}

		hr, err := vapi.Sys().Health()
		if err != nil {
			logrus.Errorf("failed to update vault replica status: failed requesting health info for the vault pod (%s/%s): %v", namespace, p.GetName(), err)
			continue
		}
		// is active node?
		// TODO: add to vaultutil?
		if hr.Initialized && !hr.Sealed && !hr.Standby {
			s.ActiveNode = podURL
		}
		if hr.Sealed {
			sealNodes = append(sealNodes, podURL)
		}
		if hr.Initialized {
			inited = true
		}
	}

	s.SealedNodes = sealNodes
	s.Initialized = inited
}

// updateVaultCRStatus updates the status field of the Vault CR.
func (vs *Vaults) updateVaultCRStatus(ctx context.Context, name, namespace string, status spec.VaultStatus) error {
	vault, err := vs.vaultsCRCli.Get(ctx, namespace, name)
	if err != nil {
		return err
	}
	vault.Status = status
	_, err = vs.vaultsCRCli.Update(ctx, vault)
	return err
}
<commit_msg>pkg/operator: do not persevere sealed nodes<commit_after>package operator

import (
	"context"
	"time"

	"github.com/coreos-inc/vault-operator/pkg/spec"
	"github.com/coreos-inc/vault-operator/pkg/util/k8sutil"

	"github.com/Sirupsen/logrus"
	vaultapi "github.com/hashicorp/vault/api"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
)

// monitorAndUpdateStaus monitors the vault service and replicas statuses, and
// updates the status resrouce in the vault CR item.
func (vs *Vaults) monitorAndUpdateStaus(ctx context.Context, name, namespace string) {
	s := spec.VaultStatus{}

	for {
		select {
		case err := <-ctx.Done():
			logrus.Infof("stopped monitoring vault: %s (%v)", name, err)
		case <-time.After(10 * time.Second):
		}

		vs.updateLocalVaultCRStatus(ctx, name, namespace, &s)

		err := vs.updateVaultCRStatus(ctx, name, namespace, s)
		if err != nil {
			logrus.Errorf("failed updating the status for the vault service: %s (%v)", name, err)
		}
	}
}

// updateLocalVaultCRStatus updates local vault CR status by querying each vault pod's API.
func (vs *Vaults) updateLocalVaultCRStatus(ctx context.Context, name, namespace string, s *spec.VaultStatus) {
	sel := k8sutil.PodsLabelsForVault(name)
	// TODO: handle upgrades when pods from two replicaset can co-exist :(
	opt := metav1.ListOptions{LabelSelector: labels.SelectorFromSet(sel).String()}
	pods, err := vs.kubecli.CoreV1().Pods(namespace).List(opt)
	if err != nil {
		logrus.Errorf("failed to update vault replica status: failed listing pods for the vault service (%s.%s): %v", name, namespace, err)
		return
	}

	var sealNodes []string
	// If it can't talk to any vault pod, we are not changing the state.
	inited := s.Initialized

	for _, p := range pods.Items {
		cfg := vaultapi.DefaultConfig()
		// TODO: change to https.
		// TODO: use FQDN?
		podURL := "http://" + p.Status.PodIP + ":8200"
		cfg.Address = podURL
		vapi, err := vaultapi.NewClient(cfg)
		if err != nil {
			logrus.Errorf("failed to update vault replica status: failed creating client for the vault pod (%s/%s): %v", namespace, p.GetName(), err)
			continue
		}

		hr, err := vapi.Sys().Health()
		if err != nil {
			logrus.Errorf("failed to update vault replica status: failed requesting health info for the vault pod (%s/%s): %v", namespace, p.GetName(), err)
			continue
		}
		// is active node?
		// TODO: add to vaultutil?
		if hr.Initialized && !hr.Sealed && !hr.Standby {
			s.ActiveNode = podURL
		}
		if hr.Sealed {
			sealNodes = append(sealNodes, podURL)
		}
		if hr.Initialized {
			inited = true
		}
	}

	s.SealedNodes = sealNodes
	s.Initialized = inited
}

// updateVaultCRStatus updates the status field of the Vault CR.
func (vs *Vaults) updateVaultCRStatus(ctx context.Context, name, namespace string, status spec.VaultStatus) error {
	vault, err := vs.vaultsCRCli.Get(ctx, namespace, name)
	if err != nil {
		return err
	}
	vault.Status = status
	_, err = vs.vaultsCRCli.Update(ctx, vault)
	return err
}
<|endoftext|>


<commit_before>// Copyright 2021 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Package rbeconfigsgen contains utilities to generate C++ & Java Toolchain configs for Bazel to be
// used to run RBE builds
package rbeconfigsgen

import (
	"fmt"
	"log"
	"path"
	"strings"

	"github.com/bazelbuild/bazelisk/core"
	"github.com/bazelbuild/bazelisk/repositories"
)

// Options are the options to tweak Bazel C++/Java Toolchain config generation.
type Options struct {
	// BazelVersion is the version of Bazel to generate configs for. If unset, the latest Bazel
	// version is automatically populated into this field when Validate() is called.
	BazelVersion string
	// BazelPath is the path within the container where Bazel is preinstalled. If unspecified,
	// Bazelisk will be downloaded and installed.
	BazelPath string
	// ToolchainContainer is the docker image of the toolchain container to generate configs for.
	ToolchainContainer string
	// ExecOS is the OS of the toolchain container image or the OS in which the build actions will
	// execute.
	ExecOS string
	// TargetOS is the OS to be used as the target platform in the generated platform rule. This
	// is the OS that artifacts built by Bazel will be executed on.
	TargetOS string
	// OutputTarball is the path at with a tarball will be generated containing the C++/Java
	// configs.
	OutputTarball string
	// OutputSourceRoot is the path where the root of the source repository where generated configs
	// should be copied to. This directory is expected to have a Bazel WORKSPACE file.
	OutputSourceRoot string
	// OutputConfigPath is the path relative to OutputSourceRoot where the generated configs will
	// be copied to.
	OutputConfigPath string
	// OutputManifest is a path where a text file containing details about the generated configs.
	// The manifest aims to be easily parseable by shell utilities like grep/sed.
	OutputManifest string
	// PlatformParams specify platform specific constraints used to generate a BUILD file with the
	// toolchain & platform targets in the generated configs. This is set to default values and not
	// directly configurable.
	PlatformParams *PlatformToolchainsTemplateParams

	// C++ Config generation options.
	// GenCPPConfigs determines whether C++ configs are generated.
	GenCPPConfigs bool
	// CPPConfigTargets are the Bazel targets that will be used to make Bazel auto-generated C++
	// configs.
	// This field can be auto-populated with a default value. See DefaultExecOptions below.
	CPPConfigTargets []string
	// CPPConfigRepo is the name of the Bazel external repo (i.e., the repo name without the '@')
	// whose Bazel build output directory will be used to extract generated C++ config files.
	// This field can be auto-populated with a default value. See DefaultExecOptions below.
	CPPConfigRepo string
	// CppBazelCmd is the Bazel command that'll be executed (build|query) on CppConfigTargets to
	// generate CppConfigTargets. So if CppConfigTargets is @foo//:blah & CppBazelCmd is build,
	// this tool will run bazel build @foo//:blah.
	// This field can be auto-populated with a default value. See DefaultExecOptions below.
	CppBazelCmd string
	// CppGenEnv are the environment variables that'll be set when running the Bazel command to
	// generate C++ configs inside the toolchain container. Only one of CppGenEnv or CppGenEnvJSON
	// can be specified.
	CppGenEnv map[string]string
	// CppGenEnvJSON is a JSON file with environment variables that'll be set when running the Bazel
	// command to generate C++ configs inside the toolchain container. Only one of CppGenEnv or
	// CppGenEnvJSON can be specified.
	CppGenEnvJSON string
	// CPPToolchainTarget is the toolchain to be used by the cpp configs.
	CPPToolchainTargetName string

	// Java config generation options.
	// GenJavaConfigs determines whether Java configs are generated.
	GenJavaConfigs bool
	// JavaUseLocalRuntime forces the generated java toolchain to use the local_java_runtime
	// rule instead of java_runtime. Otherwise, the Bazel version will be used to infer which rule
	// to use. Older Bazel versions use java_runtime.
	JavaUseLocalRuntime bool
	// TempWorkDir is a temporary directory that will be used by this tool to store intermediate
	// files. If unspecified, a temporary directory will be requested from the OS.
	TempWorkDir string
	// Cleanup determines whether the running container & intermediate files will be deleted once
	// config generation is done. Setting it to false is useful for debugging intermediate state.
	Cleanup bool
}

// DefaultOptions are some option values that are populated as default values for certain fields
// of "Options". See "Options" for explanation on what the fields mean.
type DefaultOptions struct {
	PlatformParams         PlatformToolchainsTemplateParams
	CPPConfigTargets       []string
	CPPConfigRepo          string
	CppBazelCmd            string
	CppGenEnv              map[string]string
	CPPToolchainTargetName string
}

const (
	// OSLinux represents Linux when selecting platforms.
	OSLinux = "linux"
	// OSWindows represents Windows when selecting platforms.
	OSWindows = "windows"
)

var (
	validOS = []string{
		OSLinux,
		OSWindows,
	}

	// DefaultExecOptions is a map from the ExecOS to default values for certain fields in Options
	// that vary based on the execution environment.
	DefaultExecOptions = map[string]DefaultOptions{
		OSLinux: {
			PlatformParams: PlatformToolchainsTemplateParams{
				ExecConstraints: []string{
					"@bazel_tools//platforms:linux",
					"@bazel_tools//platforms:x86_64",
					"@bazel_tools//tools/cpp:clang",
				},
				TargetConstraints: []string{
					"@bazel_tools//platforms:linux",
					"@bazel_tools//platforms:x86_64",
				},
				OSFamily: "Linux",
			},
			CPPConfigTargets: []string{"@local_config_cc//..."},
			CPPConfigRepo:    "local_config_cc",
			CppBazelCmd:      "build",
			CppGenEnv: map[string]string{
				"ABI_LIBC_VERSION":    "glibc_2.19",
				"ABI_VERSION":         "clang",
				"BAZEL_COMPILER":      "clang",
				"BAZEL_HOST_SYSTEM":   "i686-unknown-linux-gnu",
				"BAZEL_TARGET_CPU":    "k8",
				"BAZEL_TARGET_LIBC":   "glibc_2.19",
				"BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu",
				"CC":                  "clang",
				"CC_TOOLCHAIN_NAME":   "linux_gnu_x86",
			},
			CPPToolchainTargetName: "cc-compiler-k8",
		},
		OSWindows: {
			PlatformParams: PlatformToolchainsTemplateParams{
				ExecConstraints: []string{
					"@bazel_tools//platforms:windows",
					"@bazel_tools//platforms:x86_64",
				},
				TargetConstraints: []string{
					"@bazel_tools//platforms:windows",
					"@bazel_tools//platforms:x86_64",
				},
				OSFamily: "Windows",
			},
			CPPConfigTargets:       []string{"@local_config_cc//..."},
			CPPConfigRepo:          "local_config_cc",
			CppBazelCmd:            "query",
			CPPToolchainTargetName: "cc-compiler-x64_windows",
		},
	}
)

func strListContains(l []string, s string) bool {
	for _, i := range l {
		if i == s {
			return true
		}
	}
	return false
}

// ApplyDefaults applies platform specific default values to the given options for the given
// OS.
func (o *Options) ApplyDefaults(os string) error {
	dopts, ok := DefaultExecOptions[os]
	if !ok {
		return fmt.Errorf("got unknown OS %q, want one of %s", os, strings.Join(validOS, ", "))
	}
	o.PlatformParams = new(PlatformToolchainsTemplateParams)
	*o.PlatformParams = dopts.PlatformParams
	o.CPPConfigTargets = dopts.CPPConfigTargets
	o.CPPConfigRepo = dopts.CPPConfigRepo
	o.CppBazelCmd = dopts.CppBazelCmd
	// Only apply C++ env defaults if the options didn't already specify defaults and no JSON file
	// to read environment variables from was specified.
	if len(o.CppGenEnv) == 0 && len(o.CppGenEnvJSON) == 0 {
		o.CppGenEnv = dopts.CppGenEnv
	}
	if o.CPPToolchainTargetName == "" {
		o.CPPToolchainTargetName = dopts.CPPToolchainTargetName
	}
	return nil
}

// latestBazelVersion uses Bazelisk to determine the latest available Bazel version.
func latestBazelVersion() (string, error) {
	r := core.CreateRepositories(&repositories.GCSRepo{}, nil, nil, nil, false)
	v, _, err := r.ResolveVersion("", "", "latest")
	if err != nil {
		return "", fmt.Errorf("unable to determine the latest available Bazel release using Bazelisk: %w", err)
	}
	return v, nil
}

// Validate verifies that mandatory arguments were provided and argument values don't conflict in
// certain cases.
func (o *Options) Validate() error {
	if o.BazelVersion == "" {
		v, err := latestBazelVersion()
		if err != nil {
			return fmt.Errorf("BazelVersion wasn't specified and was unable to determine the latest available Bazel version: %w", err)
		}
		o.BazelVersion = v
	}
	if o.ToolchainContainer == "" {
		return fmt.Errorf("ToolchainContainer was not specified")
	}
	if o.ExecOS == "" {
		return fmt.Errorf("ExecOS was not specified")
	}
	if !strListContains(validOS, o.ExecOS) {
		return fmt.Errorf("invalid exec_os, got %q, want one of %s", o.ExecOS, strings.Join(validOS, ", "))
	}
	if o.TargetOS == "" {
		return fmt.Errorf("TargetOS was not specified")
	}
	if !strListContains(validOS, o.TargetOS) {
		return fmt.Errorf("invalid TargetOS, got %q, want one of %s", o.TargetOS, strings.Join(validOS, ", "))
	}
	if o.OutputTarball == "" && o.OutputSourceRoot == "" {
		return fmt.Errorf("atleast one of OutputTarball or OutputSourceRoot must be specified or this tool won't generate any output")
	}
	if o.OutputSourceRoot == "" && o.OutputConfigPath != "" {
		return fmt.Errorf("OutputSourceRoot is required because OutputConfigPath was specified")
	}
	if path.IsAbs(o.OutputConfigPath) {
		return fmt.Errorf("OutputConfigPath should be a relative path")
	}
	if o.PlatformParams == nil {
		return fmt.Errorf("PlatformParams was not initialized")
	}
	if !o.GenCPPConfigs && !o.GenJavaConfigs {
		return fmt.Errorf("both GenCPPConfigs & GenJavaConfigs were set to false which means there's no configs to generate")
	}
	if o.GenCPPConfigs && len(o.CPPConfigTargets) == 0 {
		return fmt.Errorf("GenCPPConfigs was true but CppConfigTargets was not specified")
	}
	if o.GenCPPConfigs && len(o.CppBazelCmd) == 0 {
		return fmt.Errorf("GenCPPConfigs was true but CppBazelCmd was not specified")
	}
	if len(o.CppGenEnv) != 0 && len(o.CppGenEnvJSON) != 0 {
		return fmt.Errorf("only one of CppGenEnv=%v or CppGenEnvJSON=%q must be specified", o.CppGenEnv, o.CppGenEnvJSON)
	}
	log.Printf("rbeconfigsgen.Options:")
	log.Printf("BazelVersion=%q", o.BazelVersion)
	log.Printf("ToolchainContainer=%q", o.ToolchainContainer)
	log.Printf("ExecOS=%q", o.ExecOS)
	log.Printf("TargetOS=%q", o.TargetOS)
	log.Printf("OutputTarball=%q", o.OutputTarball)
	log.Printf("OutputSourceRoot=%q", o.OutputSourceRoot)
	log.Printf("OutputConfigPath=%q", o.OutputConfigPath)
	log.Printf("OutputManifest=%q", o.OutputManifest)
	log.Printf("PlatformParams=%v", *o.PlatformParams)
	log.Printf("GenCPPConfigs=%v", o.GenCPPConfigs)
	log.Printf("CPPConfigTargets=%v", o.CPPConfigTargets)
	log.Printf("CPPConfigRepo=%q", o.CPPConfigRepo)
	log.Printf("CppBazelCmd=%q", o.CppBazelCmd)
	log.Printf("CppGenEnv=%v", o.CppGenEnv)
	log.Printf("CppGenEnvJSON=%q", o.CppGenEnvJSON)
	log.Printf("GenJavaConfigs=%v", o.GenJavaConfigs)
	log.Printf("JavaUseLocalRuntime=%v", o.JavaUseLocalRuntime)
	log.Printf("TempWorkDir=%q", o.TempWorkDir)
	log.Printf("Cleanup=%v", o.Cleanup)
	return nil
}
<commit_msg>Support new style platforms. See bazel issue 8622 (#989)<commit_after>// Copyright 2021 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//	http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Package rbeconfigsgen contains utilities to generate C++ & Java Toolchain configs for Bazel to be
// used to run RBE builds
package rbeconfigsgen

import (
	"fmt"
	"log"
	"path"
	"strings"

	"github.com/bazelbuild/bazelisk/core"
	"github.com/bazelbuild/bazelisk/repositories"
)

// Options are the options to tweak Bazel C++/Java Toolchain config generation.
type Options struct {
	// BazelVersion is the version of Bazel to generate configs for. If unset, the latest Bazel
	// version is automatically populated into this field when Validate() is called.
	BazelVersion string
	// BazelPath is the path within the container where Bazel is preinstalled. If unspecified,
	// Bazelisk will be downloaded and installed.
	BazelPath string
	// ToolchainContainer is the docker image of the toolchain container to generate configs for.
	ToolchainContainer string
	// ExecOS is the OS of the toolchain container image or the OS in which the build actions will
	// execute.
	ExecOS string
	// TargetOS is the OS to be used as the target platform in the generated platform rule. This
	// is the OS that artifacts built by Bazel will be executed on.
	TargetOS string
	// OutputTarball is the path at with a tarball will be generated containing the C++/Java
	// configs.
	OutputTarball string
	// OutputSourceRoot is the path where the root of the source repository where generated configs
	// should be copied to. This directory is expected to have a Bazel WORKSPACE file.
	OutputSourceRoot string
	// OutputConfigPath is the path relative to OutputSourceRoot where the generated configs will
	// be copied to.
	OutputConfigPath string
	// OutputManifest is a path where a text file containing details about the generated configs.
	// The manifest aims to be easily parseable by shell utilities like grep/sed.
	OutputManifest string
	// PlatformParams specify platform specific constraints used to generate a BUILD file with the
	// toolchain & platform targets in the generated configs. This is set to default values and not
	// directly configurable.
	PlatformParams *PlatformToolchainsTemplateParams

	// C++ Config generation options.
	// GenCPPConfigs determines whether C++ configs are generated.
	GenCPPConfigs bool
	// CPPConfigTargets are the Bazel targets that will be used to make Bazel auto-generated C++
	// configs.
	// This field can be auto-populated with a default value. See DefaultExecOptions below.
	CPPConfigTargets []string
	// CPPConfigRepo is the name of the Bazel external repo (i.e., the repo name without the '@')
	// whose Bazel build output directory will be used to extract generated C++ config files.
	// This field can be auto-populated with a default value. See DefaultExecOptions below.
	CPPConfigRepo string
	// CppBazelCmd is the Bazel command that'll be executed (build|query) on CppConfigTargets to
	// generate CppConfigTargets. So if CppConfigTargets is @foo//:blah & CppBazelCmd is build,
	// this tool will run bazel build @foo//:blah.
	// This field can be auto-populated with a default value. See DefaultExecOptions below.
	CppBazelCmd string
	// CppGenEnv are the environment variables that'll be set when running the Bazel command to
	// generate C++ configs inside the toolchain container. Only one of CppGenEnv or CppGenEnvJSON
	// can be specified.
	CppGenEnv map[string]string
	// CppGenEnvJSON is a JSON file with environment variables that'll be set when running the Bazel
	// command to generate C++ configs inside the toolchain container. Only one of CppGenEnv or
	// CppGenEnvJSON can be specified.
	CppGenEnvJSON string
	// CPPToolchainTarget is the toolchain to be used by the cpp configs.
	CPPToolchainTargetName string

	// Java config generation options.
	// GenJavaConfigs determines whether Java configs are generated.
	GenJavaConfigs bool
	// JavaUseLocalRuntime forces the generated java toolchain to use the local_java_runtime
	// rule instead of java_runtime. Otherwise, the Bazel version will be used to infer which rule
	// to use. Older Bazel versions use java_runtime.
	JavaUseLocalRuntime bool
	// TempWorkDir is a temporary directory that will be used by this tool to store intermediate
	// files. If unspecified, a temporary directory will be requested from the OS.
	TempWorkDir string
	// Cleanup determines whether the running container & intermediate files will be deleted once
	// config generation is done. Setting it to false is useful for debugging intermediate state.
	Cleanup bool
}

// DefaultOptions are some option values that are populated as default values for certain fields
// of "Options". See "Options" for explanation on what the fields mean.
type DefaultOptions struct {
	PlatformParams         PlatformToolchainsTemplateParams
	CPPConfigTargets       []string
	CPPConfigRepo          string
	CppBazelCmd            string
	CppGenEnv              map[string]string
	CPPToolchainTargetName string
}

const (
	// OSLinux represents Linux when selecting platforms.
	OSLinux = "linux"
	// OSWindows represents Windows when selecting platforms.
	OSWindows = "windows"
)

var (
	validOS = []string{
		OSLinux,
		OSWindows,
	}

	// DefaultExecOptions is a map from the ExecOS to default values for certain fields in Options
	// that vary based on the execution environment.
	DefaultExecOptions = map[string]DefaultOptions{
		OSLinux: {
			PlatformParams: PlatformToolchainsTemplateParams{
				ExecConstraints: []string{
					"@platforms//os:linux",
					"@platforms//cpu:x86_64",
					"@bazel_tools//tools/cpp:clang",
				},
				TargetConstraints: []string{
					"@platforms//os:linux",
					"@platforms//cpu:x86_64",
				},
				OSFamily: "Linux",
			},
			CPPConfigTargets: []string{"@local_config_cc//..."},
			CPPConfigRepo:    "local_config_cc",
			CppBazelCmd:      "build",
			CppGenEnv: map[string]string{
				"ABI_LIBC_VERSION":    "glibc_2.19",
				"ABI_VERSION":         "clang",
				"BAZEL_COMPILER":      "clang",
				"BAZEL_HOST_SYSTEM":   "i686-unknown-linux-gnu",
				"BAZEL_TARGET_CPU":    "k8",
				"BAZEL_TARGET_LIBC":   "glibc_2.19",
				"BAZEL_TARGET_SYSTEM": "x86_64-unknown-linux-gnu",
				"CC":                  "clang",
				"CC_TOOLCHAIN_NAME":   "linux_gnu_x86",
			},
			CPPToolchainTargetName: "cc-compiler-k8",
		},
		OSWindows: {
			PlatformParams: PlatformToolchainsTemplateParams{
				ExecConstraints: []string{
					"@platforms//os:windows",
					"@platforms//cpu:x86_64",
				},
				TargetConstraints: []string{
					"@platforms//os:windows",
					"@platforms//cpu:x86_64",
				},
				OSFamily: "Windows",
			},
			CPPConfigTargets:       []string{"@local_config_cc//..."},
			CPPConfigRepo:          "local_config_cc",
			CppBazelCmd:            "query",
			CPPToolchainTargetName: "cc-compiler-x64_windows",
		},
	}
)

func strListContains(l []string, s string) bool {
	for _, i := range l {
		if i == s {
			return true
		}
	}
	return false
}

// ApplyDefaults applies platform specific default values to the given options for the given
// OS.
func (o *Options) ApplyDefaults(os string) error {
	dopts, ok := DefaultExecOptions[os]
	if !ok {
		return fmt.Errorf("got unknown OS %q, want one of %s", os, strings.Join(validOS, ", "))
	}
	o.PlatformParams = new(PlatformToolchainsTemplateParams)
	*o.PlatformParams = dopts.PlatformParams
	o.CPPConfigTargets = dopts.CPPConfigTargets
	o.CPPConfigRepo = dopts.CPPConfigRepo
	o.CppBazelCmd = dopts.CppBazelCmd
	// Only apply C++ env defaults if the options didn't already specify defaults and no JSON file
	// to read environment variables from was specified.
	if len(o.CppGenEnv) == 0 && len(o.CppGenEnvJSON) == 0 {
		o.CppGenEnv = dopts.CppGenEnv
	}
	if o.CPPToolchainTargetName == "" {
		o.CPPToolchainTargetName = dopts.CPPToolchainTargetName
	}
	return nil
}

// latestBazelVersion uses Bazelisk to determine the latest available Bazel version.
func latestBazelVersion() (string, error) {
	r := core.CreateRepositories(&repositories.GCSRepo{}, nil, nil, nil, false)
	v, _, err := r.ResolveVersion("", "", "latest")
	if err != nil {
		return "", fmt.Errorf("unable to determine the latest available Bazel release using Bazelisk: %w", err)
	}
	return v, nil
}

// Validate verifies that mandatory arguments were provided and argument values don't conflict in
// certain cases.
func (o *Options) Validate() error {
	if o.BazelVersion == "" {
		v, err := latestBazelVersion()
		if err != nil {
			return fmt.Errorf("BazelVersion wasn't specified and was unable to determine the latest available Bazel version: %w", err)
		}
		o.BazelVersion = v
	}
	if o.ToolchainContainer == "" {
		return fmt.Errorf("ToolchainContainer was not specified")
	}
	if o.ExecOS == "" {
		return fmt.Errorf("ExecOS was not specified")
	}
	if !strListContains(validOS, o.ExecOS) {
		return fmt.Errorf("invalid exec_os, got %q, want one of %s", o.ExecOS, strings.Join(validOS, ", "))
	}
	if o.TargetOS == "" {
		return fmt.Errorf("TargetOS was not specified")
	}
	if !strListContains(validOS, o.TargetOS) {
		return fmt.Errorf("invalid TargetOS, got %q, want one of %s", o.TargetOS, strings.Join(validOS, ", "))
	}
	if o.OutputTarball == "" && o.OutputSourceRoot == "" {
		return fmt.Errorf("atleast one of OutputTarball or OutputSourceRoot must be specified or this tool won't generate any output")
	}
	if o.OutputSourceRoot == "" && o.OutputConfigPath != "" {
		return fmt.Errorf("OutputSourceRoot is required because OutputConfigPath was specified")
	}
	if path.IsAbs(o.OutputConfigPath) {
		return fmt.Errorf("OutputConfigPath should be a relative path")
	}
	if o.PlatformParams == nil {
		return fmt.Errorf("PlatformParams was not initialized")
	}
	if !o.GenCPPConfigs && !o.GenJavaConfigs {
		return fmt.Errorf("both GenCPPConfigs & GenJavaConfigs were set to false which means there's no configs to generate")
	}
	if o.GenCPPConfigs && len(o.CPPConfigTargets) == 0 {
		return fmt.Errorf("GenCPPConfigs was true but CppConfigTargets was not specified")
	}
	if o.GenCPPConfigs && len(o.CppBazelCmd) == 0 {
		return fmt.Errorf("GenCPPConfigs was true but CppBazelCmd was not specified")
	}
	if len(o.CppGenEnv) != 0 && len(o.CppGenEnvJSON) != 0 {
		return fmt.Errorf("only one of CppGenEnv=%v or CppGenEnvJSON=%q must be specified", o.CppGenEnv, o.CppGenEnvJSON)
	}
	log.Printf("rbeconfigsgen.Options:")
	log.Printf("BazelVersion=%q", o.BazelVersion)
	log.Printf("ToolchainContainer=%q", o.ToolchainContainer)
	log.Printf("ExecOS=%q", o.ExecOS)
	log.Printf("TargetOS=%q", o.TargetOS)
	log.Printf("OutputTarball=%q", o.OutputTarball)
	log.Printf("OutputSourceRoot=%q", o.OutputSourceRoot)
	log.Printf("OutputConfigPath=%q", o.OutputConfigPath)
	log.Printf("OutputManifest=%q", o.OutputManifest)
	log.Printf("PlatformParams=%v", *o.PlatformParams)
	log.Printf("GenCPPConfigs=%v", o.GenCPPConfigs)
	log.Printf("CPPConfigTargets=%v", o.CPPConfigTargets)
	log.Printf("CPPConfigRepo=%q", o.CPPConfigRepo)
	log.Printf("CppBazelCmd=%q", o.CppBazelCmd)
	log.Printf("CppGenEnv=%v", o.CppGenEnv)
	log.Printf("CppGenEnvJSON=%q", o.CppGenEnvJSON)
	log.Printf("GenJavaConfigs=%v", o.GenJavaConfigs)
	log.Printf("JavaUseLocalRuntime=%v", o.JavaUseLocalRuntime)
	log.Printf("TempWorkDir=%q", o.TempWorkDir)
	log.Printf("Cleanup=%v", o.Cleanup)
	return nil
}
<|endoftext|>