<commit_before>package agent

import (
	"github.com/hashicorp/consul/consul/structs"
	"github.com/hashicorp/consul/testutil"
	"os"
	"reflect"
	"testing"
	"time"
)

func TestAgentAntiEntropy_Services(t *testing.T) {
	conf := nextConfig()
	dir, agent := makeAgent(t, conf)
	defer os.RemoveAll(dir)
	defer agent.Shutdown()

	testutil.WaitForLeader(t, agent.RPC, "dc1")

	// Register info
	args := &structs.RegisterRequest{
		Datacenter: "dc1",
		Node:       agent.config.NodeName,
		Address:    "127.0.0.1",
	}

	// Exists both, same (noop)
	var out struct{}
	srv1 := &structs.NodeService{
		ID:      "mysql",
		Service: "mysql",
		Tags:    []string{"master"},
		Port:    5000,
	}
	agent.state.AddService(srv1)
	args.Service = srv1
	if err := agent.RPC("Catalog.Register", args, &out); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Exists both, different (update)
	srv2 := &structs.NodeService{
		ID:      "redis",
		Service: "redis",
		Tags:    nil,
		Port:    8000,
	}
	agent.state.AddService(srv2)

	srv2_mod := new(structs.NodeService)
	*srv2_mod = *srv2
	srv2_mod.Port = 9000
	args.Service = srv2_mod
	if err := agent.RPC("Catalog.Register", args, &out); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Exists local (create)
	srv3 := &structs.NodeService{
		ID:      "web",
		Service: "web",
		Tags:    nil,
		Port:    80,
	}
	agent.state.AddService(srv3)

	// Exists remote (delete)
	srv4 := &structs.NodeService{
		ID:      "lb",
		Service: "lb",
		Tags:    nil,
		Port:    443,
	}
	args.Service = srv4
	if err := agent.RPC("Catalog.Register", args, &out); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Trigger anti-entropy run and wait
	agent.StartSync()
	time.Sleep(100 * time.Millisecond)

	// Verify that we are in sync
	req := structs.NodeSpecificRequest{
		Datacenter: "dc1",
		Node:       agent.config.NodeName,
	}
	var services structs.IndexedNodeServices
	if err := agent.RPC("Catalog.NodeServices", &req, &services); err != nil {
		t.Fatalf("err: %v", err)
	}

	// We should have 4 services (consul included)
	if len(services.NodeServices.Services) != 4 {
		t.Fatalf("bad: %v", services.NodeServices.Services)
	}

	// All the services should match
	for id, serv := range services.NodeServices.Services {
		switch id {
		case "mysql":
			if !reflect.DeepEqual(serv, srv1) {
				t.Fatalf("bad: %v %v", serv, srv1)
			}
		case "redis":
			if !reflect.DeepEqual(serv, srv2) {
				t.Fatalf("bad: %v %v", serv, srv2)
			}
		case "web":
			if !reflect.DeepEqual(serv, srv3) {
				t.Fatalf("bad: %v %v", serv, srv3)
			}
		case "consul":
			// ignore
		default:
			t.Fatalf("unexpected service: %v", id)
		}
	}

	// Check the local state
	if len(agent.state.services) != 3 {
		t.Fatalf("bad: %v", agent.state.services)
	}
	if len(agent.state.serviceStatus) != 3 {
		t.Fatalf("bad: %v", agent.state.serviceStatus)
	}
	for name, status := range agent.state.serviceStatus {
		if !status.inSync {
			t.Fatalf("should be in sync: %v %v", name, status)
		}
	}
}

func TestAgentAntiEntropy_Checks(t *testing.T) {
	conf := nextConfig()
	dir, agent := makeAgent(t, conf)
	defer os.RemoveAll(dir)
	defer agent.Shutdown()

	testutil.WaitForLeader(t, agent.RPC, "dc1")

	// Register info
	args := &structs.RegisterRequest{
		Datacenter: "dc1",
		Node:       agent.config.NodeName,
		Address:    "127.0.0.1",
	}

	// Exists both, same (noop)
	var out struct{}
	chk1 := &structs.HealthCheck{
		Node:    agent.config.NodeName,
		CheckID: "mysql",
		Name:    "mysql",
		Status:  structs.HealthPassing,
	}
	agent.state.AddCheck(chk1)
	args.Check = chk1
	if err := agent.RPC("Catalog.Register", args, &out); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Exists both, different (update)
	chk2 := &structs.HealthCheck{
		Node:    agent.config.NodeName,
		CheckID: "redis",
		Name:    "redis",
		Status:  structs.HealthPassing,
	}
	agent.state.AddCheck(chk2)

	chk2_mod := new(structs.HealthCheck)
	*chk2_mod = *chk2
	chk2_mod.Status = structs.HealthUnknown
	args.Check = chk2_mod
	if err := agent.RPC("Catalog.Register", args, &out); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Exists local (create)
	chk3 := &structs.HealthCheck{
		Node:    agent.config.NodeName,
		CheckID: "web",
		Name:    "web",
		Status:  structs.HealthPassing,
	}
	agent.state.AddCheck(chk3)

	// Exists remote (delete)
	chk4 := &structs.HealthCheck{
		Node:    agent.config.NodeName,
		CheckID: "lb",
		Name:    "lb",
		Status:  structs.HealthPassing,
	}
	args.Check = chk4
	if err := agent.RPC("Catalog.Register", args, &out); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Trigger anti-entropy run and wait
	agent.StartSync()
	time.Sleep(100 * time.Millisecond)

	// Verify that we are in sync
	req := structs.NodeSpecificRequest{
		Datacenter: "dc1",
		Node:       agent.config.NodeName,
	}
	var checks structs.IndexedHealthChecks
	if err := agent.RPC("Health.NodeChecks", &req, &checks); err != nil {
		t.Fatalf("err: %v", err)
	}

	// We should have 4 services (serf included)
	if len(checks.HealthChecks) != 4 {
		t.Fatalf("bad: %v", checks)
	}

	// All the checks should match
	for _, chk := range checks.HealthChecks {
		switch chk.CheckID {
		case "mysql":
			if !reflect.DeepEqual(chk, chk1) {
				t.Fatalf("bad: %v %v", chk, chk1)
			}
		case "redis":
			if !reflect.DeepEqual(chk, chk2) {
				t.Fatalf("bad: %v %v", chk, chk2)
			}
		case "web":
			if !reflect.DeepEqual(chk, chk3) {
				t.Fatalf("bad: %v %v", chk, chk3)
			}
		case "serfHealth":
			// ignore
		default:
			t.Fatalf("unexpected check: %v", chk)
		}
	}

	// Check the local state
	if len(agent.state.checks) != 3 {
		t.Fatalf("bad: %v", agent.state.checks)
	}
	if len(agent.state.checkStatus) != 3 {
		t.Fatalf("bad: %v", agent.state.checkStatus)
	}
	for name, status := range agent.state.checkStatus {
		if !status.inSync {
			t.Fatalf("should be in sync: %v %v", name, status)
		}
	}
}
<commit_msg>Increase wait time after `agent.StartSync()`<commit_after>package agent

import (
	"github.com/hashicorp/consul/consul/structs"
	"github.com/hashicorp/consul/testutil"
	"os"
	"reflect"
	"testing"
	"time"
)

func TestAgentAntiEntropy_Services(t *testing.T) {
	conf := nextConfig()
	dir, agent := makeAgent(t, conf)
	defer os.RemoveAll(dir)
	defer agent.Shutdown()

	testutil.WaitForLeader(t, agent.RPC, "dc1")

	// Register info
	args := &structs.RegisterRequest{
		Datacenter: "dc1",
		Node:       agent.config.NodeName,
		Address:    "127.0.0.1",
	}

	// Exists both, same (noop)
	var out struct{}
	srv1 := &structs.NodeService{
		ID:      "mysql",
		Service: "mysql",
		Tags:    []string{"master"},
		Port:    5000,
	}
	agent.state.AddService(srv1)
	args.Service = srv1
	if err := agent.RPC("Catalog.Register", args, &out); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Exists both, different (update)
	srv2 := &structs.NodeService{
		ID:      "redis",
		Service: "redis",
		Tags:    nil,
		Port:    8000,
	}
	agent.state.AddService(srv2)

	srv2_mod := new(structs.NodeService)
	*srv2_mod = *srv2
	srv2_mod.Port = 9000
	args.Service = srv2_mod
	if err := agent.RPC("Catalog.Register", args, &out); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Exists local (create)
	srv3 := &structs.NodeService{
		ID:      "web",
		Service: "web",
		Tags:    nil,
		Port:    80,
	}
	agent.state.AddService(srv3)

	// Exists remote (delete)
	srv4 := &structs.NodeService{
		ID:      "lb",
		Service: "lb",
		Tags:    nil,
		Port:    443,
	}
	args.Service = srv4
	if err := agent.RPC("Catalog.Register", args, &out); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Trigger anti-entropy run and wait
	agent.StartSync()
	time.Sleep(200 * time.Millisecond)

	// Verify that we are in sync
	req := structs.NodeSpecificRequest{
		Datacenter: "dc1",
		Node:       agent.config.NodeName,
	}
	var services structs.IndexedNodeServices
	if err := agent.RPC("Catalog.NodeServices", &req, &services); err != nil {
		t.Fatalf("err: %v", err)
	}

	// We should have 4 services (consul included)
	if len(services.NodeServices.Services) != 4 {
		t.Fatalf("bad: %v", services.NodeServices.Services)
	}

	// All the services should match
	for id, serv := range services.NodeServices.Services {
		switch id {
		case "mysql":
			if !reflect.DeepEqual(serv, srv1) {
				t.Fatalf("bad: %v %v", serv, srv1)
			}
		case "redis":
			if !reflect.DeepEqual(serv, srv2) {
				t.Fatalf("bad: %v %v", serv, srv2)
			}
		case "web":
			if !reflect.DeepEqual(serv, srv3) {
				t.Fatalf("bad: %v %v", serv, srv3)
			}
		case "consul":
			// ignore
		default:
			t.Fatalf("unexpected service: %v", id)
		}
	}

	// Check the local state
	if len(agent.state.services) != 3 {
		t.Fatalf("bad: %v", agent.state.services)
	}
	if len(agent.state.serviceStatus) != 3 {
		t.Fatalf("bad: %v", agent.state.serviceStatus)
	}
	for name, status := range agent.state.serviceStatus {
		if !status.inSync {
			t.Fatalf("should be in sync: %v %v", name, status)
		}
	}
}

func TestAgentAntiEntropy_Checks(t *testing.T) {
	conf := nextConfig()
	dir, agent := makeAgent(t, conf)
	defer os.RemoveAll(dir)
	defer agent.Shutdown()

	testutil.WaitForLeader(t, agent.RPC, "dc1")

	// Register info
	args := &structs.RegisterRequest{
		Datacenter: "dc1",
		Node:       agent.config.NodeName,
		Address:    "127.0.0.1",
	}

	// Exists both, same (noop)
	var out struct{}
	chk1 := &structs.HealthCheck{
		Node:    agent.config.NodeName,
		CheckID: "mysql",
		Name:    "mysql",
		Status:  structs.HealthPassing,
	}
	agent.state.AddCheck(chk1)
	args.Check = chk1
	if err := agent.RPC("Catalog.Register", args, &out); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Exists both, different (update)
	chk2 := &structs.HealthCheck{
		Node:    agent.config.NodeName,
		CheckID: "redis",
		Name:    "redis",
		Status:  structs.HealthPassing,
	}
	agent.state.AddCheck(chk2)

	chk2_mod := new(structs.HealthCheck)
	*chk2_mod = *chk2
	chk2_mod.Status = structs.HealthUnknown
	args.Check = chk2_mod
	if err := agent.RPC("Catalog.Register", args, &out); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Exists local (create)
	chk3 := &structs.HealthCheck{
		Node:    agent.config.NodeName,
		CheckID: "web",
		Name:    "web",
		Status:  structs.HealthPassing,
	}
	agent.state.AddCheck(chk3)

	// Exists remote (delete)
	chk4 := &structs.HealthCheck{
		Node:    agent.config.NodeName,
		CheckID: "lb",
		Name:    "lb",
		Status:  structs.HealthPassing,
	}
	args.Check = chk4
	if err := agent.RPC("Catalog.Register", args, &out); err != nil {
		t.Fatalf("err: %v", err)
	}

	// Trigger anti-entropy run and wait
	agent.StartSync()
	time.Sleep(200 * time.Millisecond)

	// Verify that we are in sync
	req := structs.NodeSpecificRequest{
		Datacenter: "dc1",
		Node:       agent.config.NodeName,
	}
	var checks structs.IndexedHealthChecks
	if err := agent.RPC("Health.NodeChecks", &req, &checks); err != nil {
		t.Fatalf("err: %v", err)
	}

	// We should have 4 services (serf included)
	if len(checks.HealthChecks) != 4 {
		t.Fatalf("bad: %v", checks)
	}

	// All the checks should match
	for _, chk := range checks.HealthChecks {
		switch chk.CheckID {
		case "mysql":
			if !reflect.DeepEqual(chk, chk1) {
				t.Fatalf("bad: %v %v", chk, chk1)
			}
		case "redis":
			if !reflect.DeepEqual(chk, chk2) {
				t.Fatalf("bad: %v %v", chk, chk2)
			}
		case "web":
			if !reflect.DeepEqual(chk, chk3) {
				t.Fatalf("bad: %v %v", chk, chk3)
			}
		case "serfHealth":
			// ignore
		default:
			t.Fatalf("unexpected check: %v", chk)
		}
	}

	// Check the local state
	if len(agent.state.checks) != 3 {
		t.Fatalf("bad: %v", agent.state.checks)
	}
	if len(agent.state.checkStatus) != 3 {
		t.Fatalf("bad: %v", agent.state.checkStatus)
	}
	for name, status := range agent.state.checkStatus {
		if !status.inSync {
			t.Fatalf("should be in sync: %v %v", name, status)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package install

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/containernetworking/cni/libcni"
	"github.com/pkg/errors"

	"istio.io/istio/cni/pkg/install-cni/pkg/config"
	"istio.io/istio/cni/pkg/install-cni/pkg/util"
	"istio.io/istio/pkg/file"
	"istio.io/pkg/log"
)

type pluginConfig struct {
	mountedCNINetDir string
	cniConfName      string
	chainedCNIPlugin bool
}

type cniConfigTemplate struct {
	cniNetworkConfigFile string
	cniNetworkConfig     string
}

type cniConfigVars struct {
	cniNetDir          string
	kubeconfigFilename string
	logLevel           string
	k8sServiceHost     string
	k8sServicePort     string
	k8sNodeName        string
}

func getPluginConfig(cfg *config.Config) pluginConfig {
	return pluginConfig{
		mountedCNINetDir: cfg.MountedCNINetDir,
		cniConfName:      cfg.CNIConfName,
		chainedCNIPlugin: cfg.ChainedCNIPlugin,
	}
}

func getCNIConfigTemplate(cfg *config.Config) cniConfigTemplate {
	return cniConfigTemplate{
		cniNetworkConfigFile: cfg.CNINetworkConfigFile,
		cniNetworkConfig:     cfg.CNINetworkConfig,
	}
}

func getCNIConfigVars(cfg *config.Config) cniConfigVars {
	return cniConfigVars{
		cniNetDir:          cfg.CNINetDir,
		kubeconfigFilename: cfg.KubeconfigFilename,
		logLevel:           cfg.LogLevel,
		k8sServiceHost:     cfg.K8sServiceHost,
		k8sServicePort:     cfg.K8sServicePort,
		k8sNodeName:        cfg.K8sNodeName,
	}
}

func createCNIConfigFile(ctx context.Context, cfg *config.Config, saToken string) (string, error) {
	cniConfig, err := readCNIConfigTemplate(getCNIConfigTemplate(cfg))
	if err != nil {
		return "", err
	}

	cniConfig = replaceCNIConfigVars(cniConfig, getCNIConfigVars(cfg), saToken)

	return writeCNIConfig(ctx, cniConfig, getPluginConfig(cfg))
}

func readCNIConfigTemplate(template cniConfigTemplate) ([]byte, error) {
	if file.Exists(template.cniNetworkConfigFile) {
		cniConfig, err := ioutil.ReadFile(template.cniNetworkConfigFile)
		if err != nil {
			return nil, err
		}
		log.Infof("Using CNI config template from %s", template.cniNetworkConfigFile)
		return cniConfig, nil
	}

	if len(template.cniNetworkConfig) > 0 {
		log.Infof("Using CNI config template from CNI_NETWORK_CONFIG environment variable.")
		return []byte(template.cniNetworkConfig), nil
	}

	return nil, errors.New("need CNI_NETWORK_CONFIG or CNI_NETWORK_CONFIG_FILE to be set")
}

func replaceCNIConfigVars(cniConfig []byte, vars cniConfigVars, saToken string) []byte {
	cniConfigStr := string(cniConfig)

	cniConfigStr = strings.ReplaceAll(cniConfigStr, "__LOG_LEVEL__", vars.logLevel)
	cniConfigStr = strings.ReplaceAll(cniConfigStr, "__KUBECONFIG_FILENAME__", vars.kubeconfigFilename)
	cniConfigStr = strings.ReplaceAll(cniConfigStr, "__KUBECONFIG_FILEPATH__", filepath.Join(vars.cniNetDir, vars.kubeconfigFilename))
	cniConfigStr = strings.ReplaceAll(cniConfigStr, "__KUBERNETES_SERVICE_HOST__", vars.k8sServiceHost)
	cniConfigStr = strings.ReplaceAll(cniConfigStr, "__KUBERNETES_SERVICE_PORT__", vars.k8sServicePort)
	cniConfigStr = strings.ReplaceAll(cniConfigStr, "__KUBERNETES_NODE_NAME__", vars.k8sNodeName)

	// Log the config file before inserting service account token.
	// This way auth token is not visible in the logs.
	log.Infof("CNI config: %s", cniConfigStr)

	cniConfigStr = strings.ReplaceAll(cniConfigStr, "__SERVICEACCOUNT_TOKEN__", saToken)

	return []byte(cniConfigStr)
}

func writeCNIConfig(ctx context.Context, cniConfig []byte, cfg pluginConfig) (string, error) {
	cniConfigFilepath, err := getCNIConfigFilepath(ctx, cfg)
	if err != nil {
		return "", err
	}

	if cfg.chainedCNIPlugin {
		if !file.Exists(cniConfigFilepath) {
			return "", fmt.Errorf("CNI config file %s removed during configuration", cniConfigFilepath)
		}
		// This section overwrites an existing plugins list entry for istio-cni
		existingCNIConfig, err := ioutil.ReadFile(cniConfigFilepath)
		if err != nil {
			return "", err
		}
		cniConfig, err = insertCNIConfig(cniConfig, existingCNIConfig)
		if err != nil {
			return "", err
		}
	}

	if err = file.AtomicWrite(cniConfigFilepath, cniConfig, os.FileMode(0644)); err != nil {
		return "", err
	}

	if cfg.chainedCNIPlugin && strings.HasSuffix(cniConfigFilepath, ".conf") {
		// If the old CNI config filename ends with .conf, rename it to .conflist, because it has to be changed to a list
		log.Infof("Renaming %s extension to .conflist", cniConfigFilepath)
		err = os.Rename(cniConfigFilepath, cniConfigFilepath+"list")
		if err != nil {
			return "", err
		}
		cniConfigFilepath += "list"
	}

	log.Infof("Created CNI config %s", cniConfigFilepath)
	return cniConfigFilepath, nil
}

// If configured as chained CNI plugin, waits indefinitely for a main CNI config file to exist before returning
// Or until cancelled by parent context
func getCNIConfigFilepath(ctx context.Context, cfg pluginConfig) (string, error) {
	filename := cfg.cniConfName

	if !cfg.chainedCNIPlugin {
		if len(filename) == 0 {
			filename = "YYY-istio-cni.conf"
		}
		return filepath.Join(cfg.mountedCNINetDir, filename), nil
	}

	watcher, fileModified, errChan, err := util.CreateFileWatcher(cfg.mountedCNINetDir)
	if err != nil {
		return "", err
	}
	defer func() {
		_ = watcher.Close()
	}()

	for len(filename) == 0 {
		filename, err = getDefaultCNINetwork(cfg.mountedCNINetDir)
		if err == nil {
			break
		}
		if err = util.WaitForFileMod(ctx, fileModified, errChan); err != nil {
			return "", err
		}
	}

	cniConfigFilepath := filepath.Join(cfg.mountedCNINetDir, filename)

	for !file.Exists(cniConfigFilepath) {
		if strings.HasSuffix(cniConfigFilepath, ".conf") && file.Exists(cniConfigFilepath+"list") {
			log.Infof("%s doesn't exist, but %[1]slist does; Using it as the CNI config file instead.", cniConfigFilepath)
			cniConfigFilepath += "list"
		} else if strings.HasSuffix(cniConfigFilepath, ".conflist") && file.Exists(cniConfigFilepath[:len(cniConfigFilepath)-4]) {
			log.Infof("%s doesn't exist, but %s does; Using it as the CNI config file instead.", cniConfigFilepath, cniConfigFilepath[:len(cniConfigFilepath)-4])
			cniConfigFilepath = cniConfigFilepath[:len(cniConfigFilepath)-4]
		} else {
			log.Infof("CNI config file %s does not exist. Waiting for file to be written...", cniConfigFilepath)
			if err = util.WaitForFileMod(ctx, fileModified, errChan); err != nil {
				return "", err
			}
		}
	}

	log.Infof("CNI config file %s exists. Proceeding.", cniConfigFilepath)

	return cniConfigFilepath, err
}

// Follows the same semantics as kubelet
// https://github.com/kubernetes/kubernetes/blob/954996e231074dc7429f7be1256a579bedd8344c/pkg/kubelet/dockershim/network/cni/cni.go#L144-L184
func getDefaultCNINetwork(confDir string) (string, error) {
	files, err := libcni.ConfFiles(confDir, []string{".conf", ".conflist"})
	switch {
	case err != nil:
		return "", err
	case len(files) == 0:
		return "", fmt.Errorf("no networks found in %s", confDir)
	}

	sort.Strings(files)
	for _, confFile := range files {
		var confList *libcni.NetworkConfigList
		if strings.HasSuffix(confFile, ".conflist") {
			confList, err = libcni.ConfListFromFile(confFile)
			if err != nil {
				log.Warnf("Error loading CNI config list file %s: %v", confFile, err)
				continue
			}
		} else {
			conf, err := libcni.ConfFromFile(confFile)
			if err != nil {
				log.Warnf("Error loading CNI config file %s: %v", confFile, err)
				continue
			}
			// Ensure the config has a "type" so we know what plugin to run.
			// Also catches the case where somebody put a conflist into a conf file.
			if conf.Network.Type == "" {
				log.Warnf("Error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", confFile)
				continue
			}

			confList, err = libcni.ConfListFromConf(conf)
			if err != nil {
				log.Warnf("Error converting CNI config file %s to list: %v", confFile, err)
				continue
			}
		}
		if len(confList.Plugins) == 0 {
			log.Warnf("CNI config list %s has no networks, skipping", confList.Name)
			continue
		}

		return filepath.Base(confFile), nil
	}

	return "", fmt.Errorf("no valid networks found in %s", confDir)
}

// newCNIConfig = istio-cni config, that should be inserted into existingCNIConfig
func insertCNIConfig(newCNIConfig, existingCNIConfig []byte) ([]byte, error) {
	var istioMap map[string]interface{}
	err := json.Unmarshal(newCNIConfig, &istioMap)
	if err != nil {
		return nil, fmt.Errorf("error loading Istio CNI config (JSON error): %v", err)
	}

	var existingMap map[string]interface{}
	err = json.Unmarshal(existingCNIConfig, &existingMap)
	if err != nil {
		return nil, fmt.Errorf("error loading existing CNI config (JSON error): %v", err)
	}

	delete(istioMap, "cniVersion")

	var newMap map[string]interface{}

	if _, ok := existingMap["type"]; ok {
		// Assume it is a regular network conf file
		delete(existingMap, "cniVersion")

		plugins := make([]map[string]interface{}, 2)
		plugins[0] = existingMap
		plugins[1] = istioMap

		newMap = map[string]interface{}{
			"name":       "k8s-pod-network",
			"cniVersion": "0.3.1",
			"plugins":    plugins,
		}
	} else {
		// Assume it is a network list file
		newMap = existingMap
		plugins, err := util.GetPlugins(newMap)
		if err != nil {
			return nil, fmt.Errorf("existing CNI config: %v", err)
		}

		for i, rawPlugin := range plugins {
			plugin, err := util.GetPlugin(rawPlugin)
			if err != nil {
				return nil, fmt.Errorf("existing CNI plugin: %v", err)
			}
			if plugin["type"] == "istio-cni" {
				plugins = append(plugins[:i], plugins[i+1:]...)
				break
			}
		}

		newMap["plugins"] = append(plugins, istioMap)
	}

	return util.MarshalCNIConfig(newMap)
}
<commit_msg>Add log before busy waiting for CNI network config. (#32889)<commit_after>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package install

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/containernetworking/cni/libcni"
	"github.com/pkg/errors"

	"istio.io/istio/cni/pkg/install-cni/pkg/config"
	"istio.io/istio/cni/pkg/install-cni/pkg/util"
	"istio.io/istio/pkg/file"
	"istio.io/pkg/log"
)

type pluginConfig struct {
	mountedCNINetDir string
	cniConfName      string
	chainedCNIPlugin bool
}

type cniConfigTemplate struct {
	cniNetworkConfigFile string
	cniNetworkConfig     string
}

type cniConfigVars struct {
	cniNetDir          string
	kubeconfigFilename string
	logLevel           string
	k8sServiceHost     string
	k8sServicePort     string
	k8sNodeName        string
}

func getPluginConfig(cfg *config.Config) pluginConfig {
	return pluginConfig{
		mountedCNINetDir: cfg.MountedCNINetDir,
		cniConfName:      cfg.CNIConfName,
		chainedCNIPlugin: cfg.ChainedCNIPlugin,
	}
}

func getCNIConfigTemplate(cfg *config.Config) cniConfigTemplate {
	return cniConfigTemplate{
		cniNetworkConfigFile: cfg.CNINetworkConfigFile,
		cniNetworkConfig:     cfg.CNINetworkConfig,
	}
}

func getCNIConfigVars(cfg *config.Config) cniConfigVars {
	return cniConfigVars{
		cniNetDir:          cfg.CNINetDir,
		kubeconfigFilename: cfg.KubeconfigFilename,
		logLevel:           cfg.LogLevel,
		k8sServiceHost:     cfg.K8sServiceHost,
		k8sServicePort:     cfg.K8sServicePort,
		k8sNodeName:        cfg.K8sNodeName,
	}
}

func createCNIConfigFile(ctx context.Context, cfg *config.Config, saToken string) (string, error) {
	cniConfig, err := readCNIConfigTemplate(getCNIConfigTemplate(cfg))
	if err != nil {
		return "", err
	}

	cniConfig = replaceCNIConfigVars(cniConfig, getCNIConfigVars(cfg), saToken)

	return writeCNIConfig(ctx, cniConfig, getPluginConfig(cfg))
}

func readCNIConfigTemplate(template cniConfigTemplate) ([]byte, error) {
	if file.Exists(template.cniNetworkConfigFile) {
		cniConfig, err := ioutil.ReadFile(template.cniNetworkConfigFile)
		if err != nil {
			return nil, err
		}
		log.Infof("Using CNI config template from %s", template.cniNetworkConfigFile)
		return cniConfig, nil
	}

	if len(template.cniNetworkConfig) > 0 {
		log.Infof("Using CNI config template from CNI_NETWORK_CONFIG environment variable.")
		return []byte(template.cniNetworkConfig), nil
	}

	return nil, errors.New("need CNI_NETWORK_CONFIG or CNI_NETWORK_CONFIG_FILE to be set")
}

func replaceCNIConfigVars(cniConfig []byte, vars cniConfigVars, saToken string) []byte {
	cniConfigStr := string(cniConfig)

	cniConfigStr = strings.ReplaceAll(cniConfigStr, "__LOG_LEVEL__", vars.logLevel)
	cniConfigStr = strings.ReplaceAll(cniConfigStr, "__KUBECONFIG_FILENAME__", vars.kubeconfigFilename)
	cniConfigStr = strings.ReplaceAll(cniConfigStr, "__KUBECONFIG_FILEPATH__", filepath.Join(vars.cniNetDir, vars.kubeconfigFilename))
	cniConfigStr = strings.ReplaceAll(cniConfigStr, "__KUBERNETES_SERVICE_HOST__", vars.k8sServiceHost)
	cniConfigStr = strings.ReplaceAll(cniConfigStr, "__KUBERNETES_SERVICE_PORT__", vars.k8sServicePort)
	cniConfigStr = strings.ReplaceAll(cniConfigStr, "__KUBERNETES_NODE_NAME__", vars.k8sNodeName)

	// Log the config file before inserting service account token.
	// This way auth token is not visible in the logs.
	log.Infof("CNI config: %s", cniConfigStr)

	cniConfigStr = strings.ReplaceAll(cniConfigStr, "__SERVICEACCOUNT_TOKEN__", saToken)

	return []byte(cniConfigStr)
}

func writeCNIConfig(ctx context.Context, cniConfig []byte, cfg pluginConfig) (string, error) {
	cniConfigFilepath, err := getCNIConfigFilepath(ctx, cfg)
	if err != nil {
		return "", err
	}

	if cfg.chainedCNIPlugin {
		if !file.Exists(cniConfigFilepath) {
			return "", fmt.Errorf("CNI config file %s removed during configuration", cniConfigFilepath)
		}
		// This section overwrites an existing plugins list entry for istio-cni
		existingCNIConfig, err := ioutil.ReadFile(cniConfigFilepath)
		if err != nil {
			return "", err
		}
		cniConfig, err = insertCNIConfig(cniConfig, existingCNIConfig)
		if err != nil {
			return "", err
		}
	}

	if err = file.AtomicWrite(cniConfigFilepath, cniConfig, os.FileMode(0644)); err != nil {
		return "", err
	}

	if cfg.chainedCNIPlugin && strings.HasSuffix(cniConfigFilepath, ".conf") {
		// If the old CNI config filename ends with .conf, rename it to .conflist, because it has to be changed to a list
		log.Infof("Renaming %s extension to .conflist", cniConfigFilepath)
		err = os.Rename(cniConfigFilepath, cniConfigFilepath+"list")
		if err != nil {
			return "", err
		}
		cniConfigFilepath += "list"
	}

	log.Infof("Created CNI config %s", cniConfigFilepath)
	return cniConfigFilepath, nil
}

// If configured as chained CNI plugin, waits indefinitely for a main CNI config file to exist before returning
// Or until cancelled by parent context
func getCNIConfigFilepath(ctx context.Context, cfg pluginConfig) (string, error) {
	filename := cfg.cniConfName

	if !cfg.chainedCNIPlugin {
		if len(filename) == 0 {
			filename = "YYY-istio-cni.conf"
		}
		return filepath.Join(cfg.mountedCNINetDir, filename), nil
	}

	watcher, fileModified, errChan, err := util.CreateFileWatcher(cfg.mountedCNINetDir)
	if err != nil {
		return "", err
	}
	defer func() {
		_ = watcher.Close()
	}()

	for len(filename) == 0 {
		filename, err = getDefaultCNINetwork(cfg.mountedCNINetDir)
		if err == nil {
			break
		}
		log.Warnf("Istio CNI is configured as chained plugin, but cannot find existing CNI network config: %v", err)
		log.Infof("Waiting for CNI network config file to be written in %v...", cfg.mountedCNINetDir)
		if err = util.WaitForFileMod(ctx, fileModified, errChan); err != nil {
			return "", err
		}
	}

	cniConfigFilepath := filepath.Join(cfg.mountedCNINetDir, filename)

	for !file.Exists(cniConfigFilepath) {
		if strings.HasSuffix(cniConfigFilepath, ".conf") && file.Exists(cniConfigFilepath+"list") {
			log.Infof("%s doesn't exist, but %[1]slist does; Using it as the CNI config file instead.", cniConfigFilepath)
			cniConfigFilepath += "list"
		} else if strings.HasSuffix(cniConfigFilepath, ".conflist") && file.Exists(cniConfigFilepath[:len(cniConfigFilepath)-4]) {
			log.Infof("%s doesn't exist, but %s does; Using it as the CNI config file instead.", cniConfigFilepath, cniConfigFilepath[:len(cniConfigFilepath)-4])
			cniConfigFilepath = cniConfigFilepath[:len(cniConfigFilepath)-4]
		} else {
			log.Infof("CNI config file %s does not exist. Waiting for file to be written...", cniConfigFilepath)
			if err = util.WaitForFileMod(ctx, fileModified, errChan); err != nil {
				return "", err
			}
		}
	}

	log.Infof("CNI config file %s exists. Proceeding.", cniConfigFilepath)

	return cniConfigFilepath, err
}

// Follows the same semantics as kubelet
// https://github.com/kubernetes/kubernetes/blob/954996e231074dc7429f7be1256a579bedd8344c/pkg/kubelet/dockershim/network/cni/cni.go#L144-L184
func getDefaultCNINetwork(confDir string) (string, error) {
	files, err := libcni.ConfFiles(confDir, []string{".conf", ".conflist"})
	switch {
	case err != nil:
		return "", err
	case len(files) == 0:
		return "", fmt.Errorf("no networks found in %s", confDir)
	}

	sort.Strings(files)
	for _, confFile := range files {
		var confList *libcni.NetworkConfigList
		if strings.HasSuffix(confFile, ".conflist") {
			confList, err = libcni.ConfListFromFile(confFile)
			if err != nil {
				log.Warnf("Error loading CNI config list file %s: %v", confFile, err)
				continue
			}
		} else {
			conf, err := libcni.ConfFromFile(confFile)
			if err != nil {
				log.Warnf("Error loading CNI config file %s: %v", confFile, err)
				continue
			}
			// Ensure the config has a "type" so we know what plugin to run.
			// Also catches the case where somebody put a conflist into a conf file.
			if conf.Network.Type == "" {
				log.Warnf("Error loading CNI config file %s: no 'type'; perhaps this is a .conflist?", confFile)
				continue
			}

			confList, err = libcni.ConfListFromConf(conf)
			if err != nil {
				log.Warnf("Error converting CNI config file %s to list: %v", confFile, err)
				continue
			}
		}
		if len(confList.Plugins) == 0 {
			log.Warnf("CNI config list %s has no networks, skipping", confList.Name)
			continue
		}

		return filepath.Base(confFile), nil
	}

	return "", fmt.Errorf("no valid networks found in %s", confDir)
}

// newCNIConfig = istio-cni config, that should be inserted into existingCNIConfig
func insertCNIConfig(newCNIConfig, existingCNIConfig []byte) ([]byte, error) {
	var istioMap map[string]interface{}
	err := json.Unmarshal(newCNIConfig, &istioMap)
	if err != nil {
		return nil, fmt.Errorf("error loading Istio CNI config (JSON error): %v", err)
	}

	var existingMap map[string]interface{}
	err = json.Unmarshal(existingCNIConfig, &existingMap)
	if err != nil {
		return nil, fmt.Errorf("error loading existing CNI config (JSON error): %v", err)
	}

	delete(istioMap, "cniVersion")

	var newMap map[string]interface{}

	if _, ok := existingMap["type"]; ok {
		// Assume it is a regular network conf file
		delete(existingMap, "cniVersion")

		plugins := make([]map[string]interface{}, 2)
		plugins[0] = existingMap
		plugins[1] = istioMap

		newMap = map[string]interface{}{
			"name":       "k8s-pod-network",
			"cniVersion": "0.3.1",
			"plugins":    plugins,
		}
	} else {
		// Assume it is a network list file
		newMap = existingMap
		plugins, err := util.GetPlugins(newMap)
		if err != nil {
			return nil, fmt.Errorf("existing CNI config: %v", err)
		}

		for i, rawPlugin := range plugins {
			plugin, err := util.GetPlugin(rawPlugin)
			if err != nil {
				return nil, fmt.Errorf("existing CNI plugin: %v", err)
			}
			if plugin["type"] == "istio-cni" {
				plugins = append(plugins[:i], plugins[i+1:]...)
				break
			}
		}

		newMap["plugins"] = append(plugins, istioMap)
	}

	return util.MarshalCNIConfig(newMap)
}
<|endoftext|>


<commit_before>package alloc

import (
	"sync"
)

type BufferPool struct {
	chain     chan []byte
	allocator *sync.Pool
}

func NewBufferPool(bufferSize, poolSize int) *BufferPool {
	pool := &BufferPool{
		chain: make(chan []byte, poolSize),
		allocator: &sync.Pool{
			New: func() interface{} { return make([]byte, bufferSize) },
		},
	}
	for i := 0; i < poolSize/2; i++ {
		pool.chain <- make([]byte, bufferSize)
	}
	return pool
}

func (p *BufferPool) Allocate() *Buffer {
	var b []byte
	select {
	case b = <-p.chain:
	default:
		b = p.allocator.Get().([]byte)
	}
	return &Buffer{
		head:   b,
		pool:   p,
		Value:  b[defaultOffset:],
		offset: defaultOffset,
	}
}

func (p *BufferPool) Free(buffer *Buffer) {
	rawBuffer := buffer.head
	if rawBuffer == nil {
		return
	}
	select {
	case p.chain <- rawBuffer:
	default:
		p.allocator.Put(rawBuffer)
	}
}

const (
	SmallBufferSize = 1024 - defaultOffset
	BufferSize      = 8*1024 - defaultOffset
	LargeBufferSize = 64*1024 - defaultOffset
)

var smallPool = NewBufferPool(1024, 64)
var mediumPool = NewBufferPool(8*1024, 128)
var largePool = NewBufferPool(64*1024, 64)
<commit_msg>enlarge small buffer to fix common mtu size<commit_after>package alloc

import (
	"sync"
)

type BufferPool struct {
	chain     chan []byte
	allocator *sync.Pool
}

func NewBufferPool(bufferSize, poolSize int) *BufferPool {
	pool := &BufferPool{
		chain: make(chan []byte, poolSize),
		allocator: &sync.Pool{
			New: func() interface{} { return make([]byte, bufferSize) },
		},
	}
	for i := 0; i < poolSize/2; i++ {
		pool.chain <- make([]byte, bufferSize)
	}
	return pool
}

func (p *BufferPool) Allocate() *Buffer {
	var b []byte
	select {
	case b = <-p.chain:
	default:
		b = p.allocator.Get().([]byte)
	}
	return &Buffer{
		head:   b,
		pool:   p,
		Value:  b[defaultOffset:],
		offset: defaultOffset,
	}
}

func (p *BufferPool) Free(buffer *Buffer) {
	rawBuffer := buffer.head
	if rawBuffer == nil {
		return
	}
	select {
	case p.chain <- rawBuffer:
	default:
		p.allocator.Put(rawBuffer)
	}
}

const (
	SmallBufferSize = 1600 - defaultOffset
	BufferSize      = 8*1024 - defaultOffset
	LargeBufferSize = 64*1024 - defaultOffset
)

var smallPool = NewBufferPool(1600, 128)
var mediumPool = NewBufferPool(8*1024, 128)
var largePool = NewBufferPool(64*1024, 64)
<|endoftext|>


<commit_before>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package dataflow contains the Dataflow runner for submitting pipelines
// to Google Cloud Dataflow.
package dataflow

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"path"
	"strings"
	"sync/atomic"
	"time"

	"cloud.google.com/go/storage"
	"github.com/apache/beam/sdks/go/pkg/beam"
	"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/graphx"
	"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/pipelinex"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/hooks"
	"github.com/apache/beam/sdks/go/pkg/beam/internal/errors"
	"github.com/apache/beam/sdks/go/pkg/beam/log"
	"github.com/apache/beam/sdks/go/pkg/beam/options/gcpopts"
	"github.com/apache/beam/sdks/go/pkg/beam/options/jobopts"
	"github.com/apache/beam/sdks/go/pkg/beam/runners/dataflow/dataflowlib"
	"github.com/apache/beam/sdks/go/pkg/beam/util/gcsx"
	"github.com/apache/beam/sdks/go/pkg/beam/x/hooks/perf"
	"github.com/golang/protobuf/proto"
)

// TODO(herohde) 5/16/2017: the Dataflow flags should match the other SDKs.

var (
	endpoint             = flag.String("dataflow_endpoint", "", "Dataflow endpoint (optional).")
	stagingLocation      = flag.String("staging_location", "", "GCS staging location (required).")
	image                = flag.String("worker_harness_container_image", "", "Worker harness container image (required).")
	labels               = flag.String("labels", "", "JSON-formatted map[string]string of job labels (optional).")
	serviceAccountEmail  = flag.String("service_account_email", "", "Service account email (optional).")
	numWorkers           = flag.Int64("num_workers", 0, "Number of workers (optional).")
	maxNumWorkers        = flag.Int64("max_num_workers", 0, "Maximum number of workers during scaling (optional).")
	diskSizeGb           = flag.Int64("disk_size_gb", 0, "Size of root disk for VMs, in GB (optional).")
	autoscalingAlgorithm = flag.String("autoscaling_algorithm", "", "Autoscaling mode to use (optional).")
	zone                 = flag.String("zone", "", "GCP zone (optional)")
	network              = flag.String("network", "", "GCP network (optional)")
	subnetwork           = flag.String("subnetwork", "", "GCP subnetwork (optional)")
	noUsePublicIPs       = flag.Bool("no_use_public_ips", false, "Workers must not use public IP addresses (optional)")
	tempLocation         = flag.String("temp_location", "", "Temp location (optional)")
	machineType          = flag.String("worker_machine_type", "", "GCE machine type (optional)")
	minCPUPlatform       = flag.String("min_cpu_platform", "", "GCE minimum cpu platform (optional)")
	workerJar            = flag.String("dataflow_worker_jar", "", "Dataflow worker jar (optional)")
	workerRegion         = flag.String("worker_region", "", "Dataflow worker region (optional)")
	workerZone           = flag.String("worker_zone", "", "Dataflow worker zone (optional)")

	executeAsync   = flag.Bool("execute_async", false, "Asynchronous execution. Submit the job and return immediately.")
	dryRun         = flag.Bool("dry_run", false, "Dry run. Just print the job, but don't submit it.")
	teardownPolicy = flag.String("teardown_policy", "", "Job teardown policy (internal only).")

	// SDK options
	cpuProfiling     = flag.String("cpu_profiling", "", "Job records CPU profiles to this GCS location (optional)")
	sessionRecording = flag.String("session_recording", "", "Job records session transcripts")
)

func init() {
	// Note that we also _ import harness/init to setup the remote execution hook.
	beam.RegisterRunner("dataflow", Execute)
	beam.RegisterRunner("DataflowRunner", Execute)

	perf.RegisterProfCaptureHook("gcs_profile_writer", gcsRecorderHook)
}

var unique int32

// Execute runs the given pipeline on Google Cloud Dataflow. It uses the
// default application credentials to submit the job.
func Execute(ctx context.Context, p *beam.Pipeline) (beam.PipelineResult, error) {
	// (1) Gather job options

	project := *gcpopts.Project
	if project == "" {
		return nil, errors.New("no Google Cloud project specified. Use --project=<project>")
	}
	region := gcpopts.GetRegion(ctx)
	if region == "" {
		return nil, errors.New("No Google Cloud region specified. Use --region=<region>. See https://cloud.google.com/dataflow/docs/concepts/regional-endpoints")
	}
	if *stagingLocation == "" {
		return nil, errors.New("no GCS staging location specified. Use --staging_location=gs://<bucket>/<path>")
	}
	var jobLabels map[string]string
	if *labels != "" {
		if err := json.Unmarshal([]byte(*labels), &jobLabels); err != nil {
			return nil, errors.Wrapf(err, "error reading --label flag as JSON")
		}
	}

	if *cpuProfiling != "" {
		perf.EnableProfCaptureHook("gcs_profile_writer", *cpuProfiling)
	}

	if *sessionRecording != "" {
		// TODO(wcn): BEAM-4017
		// It's a bit inconvenient for GCS because the whole object is written in
		// one pass, whereas the session logs are constantly appended. We wouldn't
		// want to hold all the logs in memory to flush at the end of the pipeline
		// as we'd blow out memory on the worker. The implementation of the
		// CaptureHook should create an internal buffer and write chunks out to GCS
		// once they get to an appropriate size (50M or so?)
	}
	if *autoscalingAlgorithm != "" {
		if *autoscalingAlgorithm != "NONE" && *autoscalingAlgorithm != "THROUGHPUT_BASED" {
			return nil, errors.New("invalid autoscaling algorithm. Use --autoscaling_algorithm=(NONE|THROUGHPUT_BASED)")
		}
	}

	hooks.SerializeHooksToOptions()

	experiments := jobopts.GetExperiments()
	// Always use runner v2, unless set already.
	var v2set, portaSubmission bool
	for _, e := range experiments {
		if strings.Contains(e, "use_runner_v2") || strings.Contains(e, "use_unified_worker") {
			v2set = true
		}
		if strings.Contains(e, "use_portable_job_submission") {
			portaSubmission = true
		}
	}
	// Enable by default unified worker, and portable job submission.
	if !v2set {
		experiments = append(experiments, "use_unified_worker")
	}
	if !portaSubmission {
		experiments = append(experiments, "use_portable_job_submission")
	}
	// TODO(BEAM-11779) remove shuffle_mode=appliance with runner v2 once issue is resolved.
	experiments = append(experiments, "shuffle_mode=appliance")

	if *minCPUPlatform != "" {
		experiments = append(experiments, fmt.Sprintf("min_cpu_platform=%v", *minCPUPlatform))
	}

	opts := &dataflowlib.JobOptions{
		Name:                jobopts.GetJobName(),
		Experiments:         experiments,
		Options:             beam.PipelineOptions.Export(),
		Project:             project,
		Region:              region,
		Zone:                *zone,
		Network:             *network,
		Subnetwork:          *subnetwork,
		NoUsePublicIPs:      *noUsePublicIPs,
		NumWorkers:          *numWorkers,
		MaxNumWorkers:       *maxNumWorkers,
		DiskSizeGb:          *diskSizeGb,
		Algorithm:           *autoscalingAlgorithm,
		MachineType:         *machineType,
		Labels:              jobLabels,
		ServiceAccountEmail: *serviceAccountEmail,
		TempLocation:        *tempLocation,
		Worker:              *jobopts.WorkerBinary,
		WorkerJar:           *workerJar,
		WorkerRegion:        *workerRegion,
		WorkerZone:          *workerZone,
		TeardownPolicy:      *teardownPolicy,
		ContainerImage:      getContainerImage(ctx),
	}
	if opts.TempLocation == "" {
		opts.TempLocation = gcsx.Join(*stagingLocation, "tmp")
	}

	// (1) Build and submit
	// NOTE(herohde) 10/8/2018: the last segment of the names must be "worker" and "dataflow-worker.jar".
	id := fmt.Sprintf("go-%v-%v", atomic.AddInt32(&unique, 1), time.Now().UnixNano())

	modelURL := gcsx.Join(*stagingLocation, id, "model")
	workerURL := gcsx.Join(*stagingLocation, id, "worker")
	jarURL := gcsx.Join(*stagingLocation, id, "dataflow-worker.jar")
	xlangURL := gcsx.Join(*stagingLocation, id, "xlang")

	edges, _, err := p.Build()
	if err != nil {
		return nil, err
	}
	artifactURLs, err := dataflowlib.ResolveXLangArtifacts(ctx, edges, opts.Project, xlangURL)
	if err != nil {
		return nil, errors.WithContext(err, "resolving cross-language artifacts")
	}
	opts.ArtifactURLs = artifactURLs
	environment, err := graphx.CreateEnvironment(ctx, jobopts.GetEnvironmentUrn(ctx), getContainerImage)
	if err != nil {
		return nil, errors.WithContext(err, "creating environment for model pipeline")
	}
	model, err := graphx.Marshal(edges, &graphx.Options{Environment: environment})
	if err != nil {
		return nil, errors.WithContext(err, "generating model pipeline")
	}
	err = pipelinex.ApplySdkImageOverrides(model, jobopts.GetSdkImageOverrides())
	if err != nil {
		return nil, errors.WithContext(err, "applying container image overrides")
	}

	if *dryRun {
		log.Info(ctx, "Dry-run: not submitting job!")

		log.Info(ctx, proto.MarshalTextString(model))
		job, err := dataflowlib.Translate(ctx, model, opts, workerURL, jarURL, modelURL)
		if err != nil {
			return nil, err
		}
		dataflowlib.PrintJob(ctx, job)
		return nil, nil
	}

	return dataflowlib.Execute(ctx, model, opts, workerURL, jarURL, modelURL, *endpoint, *executeAsync)
}
func gcsRecorderHook(opts []string) perf.CaptureHook {
	bucket, prefix, err := gcsx.ParseObject(opts[0])
	if err != nil {
		panic(fmt.Sprintf("Invalid hook configuration for gcsRecorderHook: %s", opts))
	}

	return func(ctx context.Context, spec string, r io.Reader) error {
		client, err := gcsx.NewClient(ctx, storage.ScopeReadWrite)
		if err != nil {
			return errors.WithContext(err, "establishing GCS client")
		}
		return gcsx.WriteObject(ctx, client, bucket, path.Join(prefix, spec), r)
	}
}

func getContainerImage(ctx context.Context) string {
	urn := jobopts.GetEnvironmentUrn(ctx)
	if urn == "" || urn == "beam:env:docker:v1" {
		if *image != "" {
			return *image
		}
		return jobopts.GetEnvironmentConfig(ctx)
	}
	panic(fmt.Sprintf("Unsupported environment %v", urn))
}
<commit_msg>[BEAM-11779] Remove appliance restriction.<commit_after>// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements.  See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License.  You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package dataflow contains the Dataflow runner for submitting pipelines
// to Google Cloud Dataflow.
package dataflow

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"path"
	"strings"
	"sync/atomic"
	"time"

	"cloud.google.com/go/storage"
	"github.com/apache/beam/sdks/go/pkg/beam"
	"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/graphx"
	"github.com/apache/beam/sdks/go/pkg/beam/core/runtime/pipelinex"
	"github.com/apache/beam/sdks/go/pkg/beam/core/util/hooks"
	"github.com/apache/beam/sdks/go/pkg/beam/internal/errors"
	"github.com/apache/beam/sdks/go/pkg/beam/log"
	"github.com/apache/beam/sdks/go/pkg/beam/options/gcpopts"
	"github.com/apache/beam/sdks/go/pkg/beam/options/jobopts"
	"github.com/apache/beam/sdks/go/pkg/beam/runners/dataflow/dataflowlib"
	"github.com/apache/beam/sdks/go/pkg/beam/util/gcsx"
	"github.com/apache/beam/sdks/go/pkg/beam/x/hooks/perf"
	"github.com/golang/protobuf/proto"
)

// TODO(herohde) 5/16/2017: the Dataflow flags should match the other SDKs.

var (
	endpoint             = flag.String("dataflow_endpoint", "", "Dataflow endpoint (optional).")
	stagingLocation      = flag.String("staging_location", "", "GCS staging location (required).")
	image                = flag.String("worker_harness_container_image", "", "Worker harness container image (required).")
	labels               = flag.String("labels", "", "JSON-formatted map[string]string of job labels (optional).")
	serviceAccountEmail  = flag.String("service_account_email", "", "Service account email (optional).")
	numWorkers           = flag.Int64("num_workers", 0, "Number of workers (optional).")
	maxNumWorkers        = flag.Int64("max_num_workers", 0, "Maximum number of workers during scaling (optional).")
	diskSizeGb           = flag.Int64("disk_size_gb", 0, "Size of root disk for VMs, in GB (optional).")
	autoscalingAlgorithm = flag.String("autoscaling_algorithm", "", "Autoscaling mode to use (optional).")
	zone                 = flag.String("zone", "", "GCP zone (optional)")
	network              = flag.String("network", "", "GCP network (optional)")
	subnetwork           = flag.String("subnetwork", "", "GCP subnetwork (optional)")
	noUsePublicIPs       = flag.Bool("no_use_public_ips", false, "Workers must not use public IP addresses (optional)")
	tempLocation         = flag.String("temp_location", "", "Temp location (optional)")
	machineType          = flag.String("worker_machine_type", "", "GCE machine type (optional)")
	minCPUPlatform       = flag.String("min_cpu_platform", "", "GCE minimum cpu platform (optional)")
	workerJar            = flag.String("dataflow_worker_jar", "", "Dataflow worker jar (optional)")
	workerRegion         = flag.String("worker_region", "", "Dataflow worker region (optional)")
	workerZone           = flag.String("worker_zone", "", "Dataflow worker zone (optional)")

	executeAsync   = flag.Bool("execute_async", false, "Asynchronous execution. Submit the job and return immediately.")
	dryRun         = flag.Bool("dry_run", false, "Dry run. Just print the job, but don't submit it.")
	teardownPolicy = flag.String("teardown_policy", "", "Job teardown policy (internal only).")

	// SDK options
	cpuProfiling     = flag.String("cpu_profiling", "", "Job records CPU profiles to this GCS location (optional)")
	sessionRecording = flag.String("session_recording", "", "Job records session transcripts")
)

func init() {
	// Note that we also _ import harness/init to setup the remote execution hook.
	beam.RegisterRunner("dataflow", Execute)
	beam.RegisterRunner("DataflowRunner", Execute)

	perf.RegisterProfCaptureHook("gcs_profile_writer", gcsRecorderHook)
}

var unique int32

// Execute runs the given pipeline on Google Cloud Dataflow. It uses the
// default application credentials to submit the job.
func Execute(ctx context.Context, p *beam.Pipeline) (beam.PipelineResult, error) {
	// (1) Gather job options

	project := *gcpopts.Project
	if project == "" {
		return nil, errors.New("no Google Cloud project specified. Use --project=<project>")
	}
	region := gcpopts.GetRegion(ctx)
	if region == "" {
		return nil, errors.New("No Google Cloud region specified. Use --region=<region>. See https://cloud.google.com/dataflow/docs/concepts/regional-endpoints")
	}
	if *stagingLocation == "" {
		return nil, errors.New("no GCS staging location specified. Use --staging_location=gs://<bucket>/<path>")
	}
	var jobLabels map[string]string
	if *labels != "" {
		if err := json.Unmarshal([]byte(*labels), &jobLabels); err != nil {
			return nil, errors.Wrapf(err, "error reading --label flag as JSON")
		}
	}

	if *cpuProfiling != "" {
		perf.EnableProfCaptureHook("gcs_profile_writer", *cpuProfiling)
	}

	if *sessionRecording != "" {
		// TODO(wcn): BEAM-4017
		// It's a bit inconvenient for GCS because the whole object is written in
		// one pass, whereas the session logs are constantly appended. We wouldn't
		// want to hold all the logs in memory to flush at the end of the pipeline
		// as we'd blow out memory on the worker. The implementation of the
		// CaptureHook should create an internal buffer and write chunks out to GCS
		// once they get to an appropriate size (50M or so?)
	}
	if *autoscalingAlgorithm != "" {
		if *autoscalingAlgorithm != "NONE" && *autoscalingAlgorithm != "THROUGHPUT_BASED" {
			return nil, errors.New("invalid autoscaling algorithm. Use --autoscaling_algorithm=(NONE|THROUGHPUT_BASED)")
		}
	}

	hooks.SerializeHooksToOptions()

	experiments := jobopts.GetExperiments()
	// Always use runner v2, unless set already.
	var v2set, portaSubmission bool
	for _, e := range experiments {
		if strings.Contains(e, "use_runner_v2") || strings.Contains(e, "use_unified_worker") {
			v2set = true
		}
		if strings.Contains(e, "use_portable_job_submission") {
			portaSubmission = true
		}
	}
	// Enable by default unified worker, and portable job submission.
	if !v2set {
		experiments = append(experiments, "use_unified_worker")
	}
	if !portaSubmission {
		experiments = append(experiments, "use_portable_job_submission")
	}

	if *minCPUPlatform != "" {
		experiments = append(experiments, fmt.Sprintf("min_cpu_platform=%v", *minCPUPlatform))
	}

	opts := &dataflowlib.JobOptions{
		Name:                jobopts.GetJobName(),
		Experiments:         experiments,
		Options:             beam.PipelineOptions.Export(),
		Project:             project,
		Region:              region,
		Zone:                *zone,
		Network:             *network,
		Subnetwork:          *subnetwork,
		NoUsePublicIPs:      *noUsePublicIPs,
		NumWorkers:          *numWorkers,
		MaxNumWorkers:       *maxNumWorkers,
		DiskSizeGb:          *diskSizeGb,
		Algorithm:           *autoscalingAlgorithm,
		MachineType:         *machineType,
		Labels:              jobLabels,
		ServiceAccountEmail: *serviceAccountEmail,
		TempLocation:        *tempLocation,
		Worker:              *jobopts.WorkerBinary,
		WorkerJar:           *workerJar,
		WorkerRegion:        *workerRegion,
		WorkerZone:          *workerZone,
		TeardownPolicy:      *teardownPolicy,
		ContainerImage:      getContainerImage(ctx),
	}
	if opts.TempLocation == "" {
		opts.TempLocation = gcsx.Join(*stagingLocation, "tmp")
	}

	// (1) Build and submit
	// NOTE(herohde) 10/8/2018: the last segment of the names must be "worker" and "dataflow-worker.jar".
	id := fmt.Sprintf("go-%v-%v", atomic.AddInt32(&unique, 1), time.Now().UnixNano())

	modelURL := gcsx.Join(*stagingLocation, id, "model")
	workerURL := gcsx.Join(*stagingLocation, id, "worker")
	jarURL := gcsx.Join(*stagingLocation, id, "dataflow-worker.jar")
	xlangURL := gcsx.Join(*stagingLocation, id, "xlang")

	edges, _, err := p.Build()
	if err != nil {
		return nil, err
	}
	artifactURLs, err := dataflowlib.ResolveXLangArtifacts(ctx, edges, opts.Project, xlangURL)
	if err != nil {
		return nil, errors.WithContext(err, "resolving cross-language artifacts")
	}
	opts.ArtifactURLs = artifactURLs
	environment, err := graphx.CreateEnvironment(ctx, jobopts.GetEnvironmentUrn(ctx), getContainerImage)
	if err != nil {
		return nil, errors.WithContext(err, "creating environment for model pipeline")
	}
	model, err := graphx.Marshal(edges, &graphx.Options{Environment: environment})
	if err != nil {
		return nil, errors.WithContext(err, "generating model pipeline")
	}
	err = pipelinex.ApplySdkImageOverrides(model, jobopts.GetSdkImageOverrides())
	if err != nil {
		return nil, errors.WithContext(err, "applying container image overrides")
	}

	if *dryRun {
		log.Info(ctx, "Dry-run: not submitting job!")

		log.Info(ctx, proto.MarshalTextString(model))
		job, err := dataflowlib.Translate(ctx, model, opts, workerURL, jarURL, modelURL)
		if err != nil {
			return nil, err
		}
		dataflowlib.PrintJob(ctx, job)
		return nil, nil
	}

	return dataflowlib.Execute(ctx, model, opts, workerURL, jarURL, modelURL, *endpoint, *executeAsync)
}
func gcsRecorderHook(opts []string) perf.CaptureHook {
	bucket, prefix, err := gcsx.ParseObject(opts[0])
	if err != nil {
		panic(fmt.Sprintf("Invalid hook configuration for gcsRecorderHook: %s", opts))
	}

	return func(ctx context.Context, spec string, r io.Reader) error {
		client, err := gcsx.NewClient(ctx, storage.ScopeReadWrite)
		if err != nil {
			return errors.WithContext(err, "establishing GCS client")
		}
		return gcsx.WriteObject(ctx, client, bucket, path.Join(prefix, spec), r)
	}
}

func getContainerImage(ctx context.Context) string {
	urn := jobopts.GetEnvironmentUrn(ctx)
	if urn == "" || urn == "beam:env:docker:v1" {
		if *image != "" {
			return *image
		}
		return jobopts.GetEnvironmentConfig(ctx)
	}
	panic(fmt.Sprintf("Unsupported environment %v", urn))
}
<|endoftext|>


<commit_before>package commandevaluators

import (
	"errors"
	"log"

	"github.com/byuoitav/av-api/base"
	"github.com/byuoitav/av-api/dbo"
	"github.com/byuoitav/configuration-database-microservice/accessors"
)

//ChangeVideoInputVideoswitcher the struct that implements the CommandEvaluation struct
type ChangeVideoInputVideoswitcher struct {
}

//Evaluate f
func (c *ChangeVideoInputVideoswitcher) Evaluate(room base.PublicRoom) ([]base.ActionStructure, error) {
	actionList := []base.ActionStructure{}

	if len(room.CurrentVideoInput) != 0 {
		devices, err := dbo.GetDevicesByBuildingAndRoomAndRole(room.Building, room.Room, "VideoOut")
		if err != nil {
			return []base.ActionStructure{}, err
		}

		for _, device := range devices {
			GetSwitcherAndCreateAction(room, device)
		}
	}

	// if there is at least one display
	if len(room.Displays) != 0 {

		// interate through all displays in the room, create an ActionStructure if it has an input
		for _, display := range room.Displays {

			// if the display has an input, create the action
			if len(display.Input) != 0 {
				// GetSwitcherAndCreateAction(room, display.Device) //?
			}
		}

	}
	return actionList, nil
}

//GetSwitcherAndCreateAction gets the videoswitcher in a room, matches the destination port to the new port
// and creates an action
func GetSwitcherAndCreateAction(room base.PublicRoom, device accessors.Device) (base.ActionStructure, error) {
	switcher, err := dbo.GetDevicesByBuildingAndRoomAndRole(room.Building, room.Room, "VideoSwitcher")
	if err != nil {
		return base.ActionStructure{}, err
	}
	if len(switcher) != 1 {
		return base.ActionStructure{}, errors.New("too many switchers/none available")
	}

	for _, port := range switcher[0].Ports {
		if port.Destination == device.Name && port.Source == room.CurrentVideoInput {
			m := make(map[string]string)
			m["port"] = port.Name

			tempAction := base.ActionStructure{
				Action:              "ChangeInput",
				GeneratingEvaluator: "ChangeVideoInputVideoswitcher",
				Device:              switcher[0],
				Parameters:          m,
				DeviceSpecific:      false,
				Overridden:          false,
			}

			return tempAction, nil
		}
	}

	return base.ActionStructure{}, errors.New("no switcher found with the matching port")
}

//Validate f
func (c *ChangeVideoInputVideoswitcher) Validate(action base.ActionStructure) error {
	log.Printf("Validating action for command %v", action.Action)

	// check if ChangeInput is a valid name of a command (ok is a bool)
	ok, _ := CheckCommands(action.Device.Commands, "ChangeInput")

	// returns and error if the ChangeInput command doesn't exist or if the command isn't ChangeInput
	if !ok || action.Action != "ChangeInput" {
		log.Printf("ERROR. %s is an invalid command for %s", action.Action, action.Device.Name)
		return errors.New(action.Action + "is not an invalid command for " + action.Device.Name)
	}

	log.Print("done.")
	return nil
}

//GetIncompatibleCommands f
func (c *ChangeVideoInputVideoswitcher) GetIncompatibleCommands() []string {
	return nil
}
<commit_msg>Delete Change-Video-Input-Videoswitcher.go<commit_after><|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"runtime"
	"testing"
)

/*

	Unit tests. Skipped the tests that would have required mocking the Docker API
	for now, may make them actually create package files.

*/

func Test_majorMinorVersionOnly(t *testing.T) {
	assert.Equal(t, majorMinorVersionOnly("2.1.34"), "2.1")
	assert.Equal(t, majorMinorVersionOnly("1.9.3-p635"), "1.9")
}

func Test_rubyDownloadUrl(t *testing.T) {
	assert.Equal(t, rubyDownloadUrl("2.1.34"), "http://cache.ruby-lang.org/pub/ruby/2.1/ruby-2.1.34.tar.gz")
	assert.Equal(t, rubyDownloadUrl("2.0.34-p451"), "http://cache.ruby-lang.org/pub/ruby/2.0/ruby-2.0.34-p451.tar.gz")
}

// Could do with pushing this out to go-bindata or similar
func Test_dockerFileFromTemplate(t *testing.T) {
	dockerfile_putput := fmt.Sprintf(`FROM ubuntu:12.04
RUN apt-get update
RUN apt-get install -y ruby1.9.3 build-essential \
    libc6-dev libffi-dev libgdbm-dev libncurses5-dev \
    libreadline-dev libssl-dev libyaml-dev zlib1g-dev
RUN gem install fpm --bindir=/usr/bin --no-rdoc --no-ri
RUN apt-get install -y curl
RUN curl http://cache.ruby-lang.org/pub/ruby/2.1/ruby-2.1.34.tar.gz|tar oxzC /tmp
WORKDIR /tmp/ruby-2.1.34
RUN CFLAGS="-march=native -O3" ./configure \
  --prefix=/opt/ruby2.1.34 \
  --enable-shared \
  --disable-install-doc \
  --enable-load-relative
RUN make -j%d install DESTDIR=/tmp/fpm

WORKDIR /
RUN fpm \
    -s dir \
    -t deb \
    -n ruby-2.1.34 \
    -a amd64 \
    -v 2.1.34 \
    --iteration 37s~precise \
    -d libc6-dev \
    -d libffi-dev \
    -d libgdbm-dev \
    -d libncurses5-dev \
    -d libreadline-dev \
    -d libssl-dev \
    -d libyaml-dev \
    -d zlib1g-dev \
    -C /tmp/fpm \
    -p /ruby-2.1.34_37s~precise_amd64.deb \
    opt
`, runtime.NumCPU())

	assert.Equal(t, dockerFileFromTemplate("ubuntu:12.04", "2.1.34", "amd64", "37s~precise").String(), dockerfile_putput)
}

// Could do with pushing this out to go-bindata or similar
func Test_dockerFileFromTemplate_lucid(t *testing.T) {
	dockerfile_putput := fmt.Sprintf(`FROM ubuntu:10.04
RUN echo "deb http://security.ubuntu.com/ubuntu lucid-security main" >> /etc/apt/sources.list
RUN apt-get update
RUN apt-get install -y ruby1.9.1-full build-essential \
    libc6-dev libffi-dev libgdbm-dev libncurses5-dev \
    libreadline-dev libssl-dev libyaml-dev zlib1g-dev \
    libopenssl-ruby1.9.1 ruby1.9.1-dev curl
RUN curl http://production.cf.rubygems.org/rubygems/rubygems-2.4.2.tgz |tar oxzC /tmp
RUN cd /tmp/rubygems-2.4.2 && ruby1.9.1 setup.rb
RUN gem1.9.1 install fpm --bindir=/usr/bin --no-rdoc --no-ri
RUN curl http://cache.ruby-lang.org/pub/ruby/2.1/ruby-2.1.34.tar.gz|tar oxzC /tmp
WORKDIR /tmp/ruby-2.1.34
RUN CFLAGS="-march=native -O3" ./configure \
  --prefix=/opt/ruby2.1.34 \
  --enable-shared \
  --disable-install-doc \
  --enable-load-relative
RUN make -j%d install DESTDIR=/tmp/fpm

WORKDIR /
RUN fpm \
    -s dir \
    -t deb \
    -n ruby-2.1.34 \
    -a amd64 \
    -v 2.1.34 \
    --iteration 37s~lucid \
    -d libc6-dev \
    -d libffi-dev \
    -d libgdbm-dev \
    -d libncurses5-dev \
    -d libreadline-dev \
    -d libssl-dev \
    -d libyaml-dev \
    -d zlib1g-dev \
    -C /tmp/fpm \
    -p /ruby-2.1.34_37s~lucid_amd64.deb \
    opt
`, runtime.NumCPU())

	assert.Equal(t, dockerFileFromTemplate("ubuntu:10.04", "2.1.34", "amd64", "37s~lucid").String(), dockerfile_putput)
}

func Test_rubyPackageFileName(t *testing.T) {
	assert.Equal(t, "ruby-2.1.0_37s~raring_amd64.deb", rubyPackageFileName("2.1.0", "37s~raring", "amd64"))
}

func Test_createTarFileFromDockerfile(t *testing.T) {
	tar_in_buffer := createTarFileFromDockerfile(bytes.NewBufferString("foo"))

	var tar_out *tar.Reader = tar.NewReader(tar_in_buffer)

	// Get the header so we can check the name of the only entry we care about
	tar_header, err := tar_out.Next()
	if err != nil {
		panic(err)
	}

	// Get the data to make sure our foo made it through
	var out_bytes []byte
	out_bytes, err = ioutil.ReadAll(tar_out)
	if err != nil {
		panic(err)
	}

	assert.Equal(t, tar_header.Name, "Dockerfile")
	assert.Equal(t, string(out_bytes), "foo")
}

func Test_copyPackageFromContainerToLocalFs(t *testing.T) {
	t.Skip()
}

func Test_buildRuby(t *testing.T) {
	t.Skip()
}
<commit_msg>Fix tests for segmented fpm install<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"github.com/stretchr/testify/assert"
	"io/ioutil"
	"runtime"
	"testing"
)

/*

	Unit tests. Skipped the tests that would have required mocking the Docker API
	for now, may make them actually create package files.

*/

func Test_majorMinorVersionOnly(t *testing.T) {
	assert.Equal(t, majorMinorVersionOnly("2.1.34"), "2.1")
	assert.Equal(t, majorMinorVersionOnly("1.9.3-p635"), "1.9")
}

func Test_rubyDownloadUrl(t *testing.T) {
	assert.Equal(t, rubyDownloadUrl("2.1.34"), "http://cache.ruby-lang.org/pub/ruby/2.1/ruby-2.1.34.tar.gz")
	assert.Equal(t, rubyDownloadUrl("2.0.34-p451"), "http://cache.ruby-lang.org/pub/ruby/2.0/ruby-2.0.34-p451.tar.gz")
}

// Could do with pushing this out to go-bindata or similar
func Test_dockerFileFromTemplate(t *testing.T) {
	dockerfile_putput := fmt.Sprintf(`FROM ubuntu:12.04
RUN apt-get update
RUN apt-get install -y ruby1.9.3 build-essential \
    libc6-dev libffi-dev libgdbm-dev libncurses5-dev \
    libreadline-dev libssl-dev libyaml-dev zlib1g-dev
RUN ["/usr/bin/gem", "install", "fpm", "--bindir=/usr/bin", "--no-rdoc", "--no-ri"]
RUN apt-get install -y curl
RUN curl http://cache.ruby-lang.org/pub/ruby/2.1/ruby-2.1.34.tar.gz|tar oxzC /tmp
WORKDIR /tmp/ruby-2.1.34
RUN CFLAGS="-march=native -O3" ./configure \
  --prefix=/opt/ruby2.1.34 \
  --enable-shared \
  --disable-install-doc \
  --enable-load-relative
RUN make -j%d install DESTDIR=/tmp/fpm

WORKDIR /
RUN fpm \
    -s dir \
    -t deb \
    -n ruby-2.1.34 \
    -a amd64 \
    -v 2.1.34 \
    --iteration 37s~precise \
    -d libc6-dev \
    -d libffi-dev \
    -d libgdbm-dev \
    -d libncurses5-dev \
    -d libreadline-dev \
    -d libssl-dev \
    -d libyaml-dev \
    -d zlib1g-dev \
    -C /tmp/fpm \
    -p /ruby-2.1.34_37s~precise_amd64.deb \
    opt
`, runtime.NumCPU())

	assert.Equal(t, dockerFileFromTemplate("ubuntu:12.04", "2.1.34", "amd64", "37s~precise").String(), dockerfile_putput)
}

// Could do with pushing this out to go-bindata or similar
func Test_dockerFileFromTemplate_lucid(t *testing.T) {
	dockerfile_putput := fmt.Sprintf(`FROM ubuntu:10.04
RUN echo "deb http://security.ubuntu.com/ubuntu lucid-security main" >> /etc/apt/sources.list
RUN apt-get update
RUN apt-get install -y ruby1.9.1-full build-essential \
    libc6-dev libffi-dev libgdbm-dev libncurses5-dev \
    libreadline-dev libssl-dev libyaml-dev zlib1g-dev \
    libopenssl-ruby1.9.1 ruby1.9.1-dev curl
RUN curl http://production.cf.rubygems.org/rubygems/rubygems-2.4.2.tgz |tar oxzC /tmp
RUN cd /tmp/rubygems-2.4.2 && ruby1.9.1 setup.rb
RUN gem1.9.1 install fpm --bindir=/usr/bin --no-rdoc --no-ri
RUN curl http://cache.ruby-lang.org/pub/ruby/2.1/ruby-2.1.34.tar.gz|tar oxzC /tmp
WORKDIR /tmp/ruby-2.1.34
RUN CFLAGS="-march=native -O3" ./configure \
  --prefix=/opt/ruby2.1.34 \
  --enable-shared \
  --disable-install-doc \
  --enable-load-relative
RUN make -j%d install DESTDIR=/tmp/fpm

WORKDIR /
RUN fpm \
    -s dir \
    -t deb \
    -n ruby-2.1.34 \
    -a amd64 \
    -v 2.1.34 \
    --iteration 37s~lucid \
    -d libc6-dev \
    -d libffi-dev \
    -d libgdbm-dev \
    -d libncurses5-dev \
    -d libreadline-dev \
    -d libssl-dev \
    -d libyaml-dev \
    -d zlib1g-dev \
    -C /tmp/fpm \
    -p /ruby-2.1.34_37s~lucid_amd64.deb \
    opt
`, runtime.NumCPU())

	assert.Equal(t, dockerFileFromTemplate("ubuntu:10.04", "2.1.34", "amd64", "37s~lucid").String(), dockerfile_putput)
}

func Test_rubyPackageFileName(t *testing.T) {
	assert.Equal(t, "ruby-2.1.0_37s~raring_amd64.deb", rubyPackageFileName("2.1.0", "37s~raring", "amd64"))
}

func Test_createTarFileFromDockerfile(t *testing.T) {
	tar_in_buffer := createTarFileFromDockerfile(bytes.NewBufferString("foo"))

	var tar_out *tar.Reader = tar.NewReader(tar_in_buffer)

	// Get the header so we can check the name of the only entry we care about
	tar_header, err := tar_out.Next()
	if err != nil {
		panic(err)
	}

	// Get the data to make sure our foo made it through
	var out_bytes []byte
	out_bytes, err = ioutil.ReadAll(tar_out)
	if err != nil {
		panic(err)
	}

	assert.Equal(t, tar_header.Name, "Dockerfile")
	assert.Equal(t, string(out_bytes), "foo")
}

func Test_copyPackageFromContainerToLocalFs(t *testing.T) {
	t.Skip()
}

func Test_buildRuby(t *testing.T) {
	t.Skip()
}
<|endoftext|>


<commit_before>package whale

import (
	"fmt"
	"strconv"
)

import (
	"gokogiri/xml"
)

// Converting JSON to XML nodes

func json_to_node(jsonVal interface{}, jsonDoc *xml.XmlDocument) (jsonNode xml.Node)  {
  switch jsonVal.(type) {
  case nil:
  	// generates <null />
    jsonNode = jsonDoc.CreateElementNode("null")
  case bool:
  	// generates <true /> or <false />
    if jsonVal == false {
    	jsonNode = jsonDoc.CreateElementNode("false")
    } else {
    	jsonNode = jsonDoc.CreateElementNode("true")
    }
  case float64:
  	// generates <number>[digits]</number>
    jsonNode = jsonDoc.CreateElementNode("number")
    jsonNode.SetContent(fmt.Sprintf("%v", jsonVal.(float64)))
  case string:
  	// generates <string>[chars]</string>
    jsonNode = jsonDoc.CreateElementNode("string")
    jsonNode.SetContent(jsonVal.(string))
  case []interface{}:
  	// generates <array>[json nodes]...</array>
    jsonNode = jsonDoc.CreateElementNode("array")
    for _, elem := range jsonVal.([]interface{}) {
      elemNode := json_to_node(elem, jsonDoc)
      jsonNode.AddChild(elemNode)
    }
  case map[string]interface{}:
  	// generates <object><member name="key1">[json node]</member>...</object>
    jsonNode = jsonDoc.CreateElementNode("object")
    for name, value := range jsonVal.(map[string]interface{}) {
      memberNode := jsonDoc.CreateElementNode("member")
      memberNode.SetAttr("name", name)
      valueNode := json_to_node(value, jsonDoc)
      memberNode.AddChild(valueNode)
      jsonNode.AddChild(memberNode)
    }
  }
  return
}

// Converting XML nodes back to JSON (goes with the preceding json_to_node function)

func node_to_json(node xml.Node) interface{} {
	if node == nil {
		return nil
	}
  switch node.Name() {
  case "null":
    return nil
  case "false":
    return false
  case "true":
    return true
  case "number":
    f, err := strconv.ParseFloat(node.Content(), 64)
    if err != nil {
      return nil
    }
    return f
  case "string":
    return node.Content()
  case "array":
    length := node.CountChildren()
    array := make([]interface{}, length)
    for elem, i := node.FirstChild(), 0; elem != nil; elem, i = elem.NextSibling(), i+1 {
      array[i] = node_to_json(elem)
    }
    return array
  case "object":
    object := make(map[string]interface{})
    for member := node.FirstChild(); member != nil; member = member.NextSibling() {
      if member.Name() != "member" || member.Attribute("name") == nil {
        // TODO: log a debugging message here
        continue // just skip nodes that aren't name-value pairs
      }
      object[member.Attr("name")] = node_to_json(member.FirstChild())
    }
    return object
  }
  // TODO: log a debugging message if we get to this point
  return nil
}

// Converting XML to JSON (version 1; will deprecate)

func NodeToJson(node xml.Node) (retVal interface{}) {
	defer func() {
		if err := recover(); err != nil {
			retVal = WrapJsonError(err)
		}
	}()

	if node.Name() == "array" {
		retVal = NodeToJsonArray(node, make([]interface{}, 0))
	} else if node.Name() == "hash" {
		retVal = NodeToJsonHash(node, make(map[string]interface{}))
	}
	return
}

func NodeToJsonHash(node xml.Node, json map[string]interface{}) map[string]interface{} {
	if node.Name() != "hash" {
		panic("Incorrect JSON format, trying to serialize a " + node.Name() + " into a json hash.")
	}
	for n := node.FirstChild(); n != nil; n = n.NextSibling() {
		if n.Name() == "elem" && len(n.Attr("key")) > 0 {
			child := n.FirstChild()
			if child != nil && child.Name() == "array" {
				json[n.Attr("key")] = NodeToJsonArray(child, make([]interface{}, 0))
			} else if child != nil && child.Name() == "hash" {
				json[n.Attr("key")] = NodeToJsonHash(child, make(map[string]interface{}))
			} else {
				json[n.Attr("key")] = n.InnerHtml()
			}
		}
	}
	return json
}

func NodeToJsonArray(node xml.Node, json []interface{}) []interface{} {
	if node.Name() != "array" {
		panic("Incorrect JSON format, trying to serialize a " + node.Name() + " into a json array.")
	}
	for n := node.FirstChild(); n != nil; n = n.NextSibling() {
		if n.Name() == "elem" {
			child := n.FirstChild()
			if child != nil && child.Name() == "array" {
				json = append(json, NodeToJsonArray(child, make([]interface{}, 0)))
			} else if child != nil && child.Name() == "hash" {
				json = append(json, NodeToJsonHash(child, make(map[string]interface{})))
			} else {
				json = append(json, n.InnerHtml())
			}
		}
	}
	return json
}

func WrapJsonError(err interface{}) map[string]string {
	jsonErr := make(map[string]string)
	switch v := err.(type) {
	case error:
		jsonErr["tritium_error"] = v.Error()
	case string:
		jsonErr["tritium_error"] = v
	default:
		jsonErr["tritium_error"] = fmt.Sprintf("unknown type: %s", v)
	}
	return jsonErr
}
<commit_msg>go fmt<commit_after>package whale

import (
	"fmt"
	"strconv"
)

import (
	"gokogiri/xml"
)

// Converting JSON to XML nodes

func json_to_node(jsonVal interface{}, jsonDoc *xml.XmlDocument) (jsonNode xml.Node) {
	switch jsonVal.(type) {
	case nil:
		// generates <null />
		jsonNode = jsonDoc.CreateElementNode("null")
	case bool:
		// generates <true /> or <false />
		if jsonVal == false {
			jsonNode = jsonDoc.CreateElementNode("false")
		} else {
			jsonNode = jsonDoc.CreateElementNode("true")
		}
	case float64:
		// generates <number>[digits]</number>
		jsonNode = jsonDoc.CreateElementNode("number")
		jsonNode.SetContent(fmt.Sprintf("%v", jsonVal.(float64)))
	case string:
		// generates <string>[chars]</string>
		jsonNode = jsonDoc.CreateElementNode("string")
		jsonNode.SetContent(jsonVal.(string))
	case []interface{}:
		// generates <array>[json nodes]...</array>
		jsonNode = jsonDoc.CreateElementNode("array")
		for _, elem := range jsonVal.([]interface{}) {
			elemNode := json_to_node(elem, jsonDoc)
			jsonNode.AddChild(elemNode)
		}
	case map[string]interface{}:
		// generates <object><member name="key1">[json node]</member>...</object>
		jsonNode = jsonDoc.CreateElementNode("object")
		for name, value := range jsonVal.(map[string]interface{}) {
			memberNode := jsonDoc.CreateElementNode("member")
			memberNode.SetAttr("name", name)
			valueNode := json_to_node(value, jsonDoc)
			memberNode.AddChild(valueNode)
			jsonNode.AddChild(memberNode)
		}
	}
	return
}

// Converting XML nodes back to JSON (goes with the preceding json_to_node function)

func node_to_json(node xml.Node) interface{} {
	if node == nil {
		return nil
	}
	switch node.Name() {
	case "null":
		return nil
	case "false":
		return false
	case "true":
		return true
	case "number":
		f, err := strconv.ParseFloat(node.Content(), 64)
		if err != nil {
			return nil
		}
		return f
	case "string":
		return node.Content()
	case "array":
		length := node.CountChildren()
		array := make([]interface{}, length)
		for elem, i := node.FirstChild(), 0; elem != nil; elem, i = elem.NextSibling(), i+1 {
			array[i] = node_to_json(elem)
		}
		return array
	case "object":
		object := make(map[string]interface{})
		for member := node.FirstChild(); member != nil; member = member.NextSibling() {
			if member.Name() != "member" || member.Attribute("name") == nil {
				// TODO: log a debugging message here
				continue // just skip nodes that aren't name-value pairs
			}
			object[member.Attr("name")] = node_to_json(member.FirstChild())
		}
		return object
	}
	// TODO: log a debugging message if we get to this point
	return nil
}

// Converting XML to JSON (version 1; will deprecate)

func NodeToJson(node xml.Node) (retVal interface{}) {
	defer func() {
		if err := recover(); err != nil {
			retVal = WrapJsonError(err)
		}
	}()

	if node.Name() == "array" {
		retVal = NodeToJsonArray(node, make([]interface{}, 0))
	} else if node.Name() == "hash" {
		retVal = NodeToJsonHash(node, make(map[string]interface{}))
	}
	return
}

func NodeToJsonHash(node xml.Node, json map[string]interface{}) map[string]interface{} {
	if node.Name() != "hash" {
		panic("Incorrect JSON format, trying to serialize a " + node.Name() + " into a json hash.")
	}
	for n := node.FirstChild(); n != nil; n = n.NextSibling() {
		if n.Name() == "elem" && len(n.Attr("key")) > 0 {
			child := n.FirstChild()
			if child != nil && child.Name() == "array" {
				json[n.Attr("key")] = NodeToJsonArray(child, make([]interface{}, 0))
			} else if child != nil && child.Name() == "hash" {
				json[n.Attr("key")] = NodeToJsonHash(child, make(map[string]interface{}))
			} else {
				json[n.Attr("key")] = n.InnerHtml()
			}
		}
	}
	return json
}

func NodeToJsonArray(node xml.Node, json []interface{}) []interface{} {
	if node.Name() != "array" {
		panic("Incorrect JSON format, trying to serialize a " + node.Name() + " into a json array.")
	}
	for n := node.FirstChild(); n != nil; n = n.NextSibling() {
		if n.Name() == "elem" {
			child := n.FirstChild()
			if child != nil && child.Name() == "array" {
				json = append(json, NodeToJsonArray(child, make([]interface{}, 0)))
			} else if child != nil && child.Name() == "hash" {
				json = append(json, NodeToJsonHash(child, make(map[string]interface{})))
			} else {
				json = append(json, n.InnerHtml())
			}
		}
	}
	return json
}

func WrapJsonError(err interface{}) map[string]string {
	jsonErr := make(map[string]string)
	switch v := err.(type) {
	case error:
		jsonErr["tritium_error"] = v.Error()
	case string:
		jsonErr["tritium_error"] = v
	default:
		jsonErr["tritium_error"] = fmt.Sprintf("unknown type: %s", v)
	}
	return jsonErr
}
<|endoftext|>