<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"testing"

	"github.com/stretchr/testify/require"

	"github.com/GoogleCloudPlatform/terraform-validator/converters/google"
)

// TestCLI tests the "convert" and "validate" subcommand against a generated .tfplan file.
func TestCLI(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping integration test in short mode.")
		return
	}
	// Define the reusable constraints to be use for the test cases.
	type constraint struct {
		name            string
		wantViolation   bool
		wantOutputRegex string
	}
	// Currently, we only test one rule. Moving forward, resource specific rules
	// should be added to increase the coverage.
	alwaysViolate := constraint{name: "always_violate", wantViolation: true, wantOutputRegex: "Constraint GCPAlwaysViolatesConstraintV1.always_violates_all on resource"}

	// Test cases for each type of resource is defined here.
	cases := []struct {
		name                 string
		constraints          []constraint
		compareConvertOutput compareConvertOutputFunc
	}{
		{name: "bucket"},
		{name: "bucket_iam"},
		{name: "disk"},
		{name: "firewall"},
		{name: "instance"},
		{name: "sql"},
		{name: "example_bigquery_dataset"},
		{name: "example_compute_disk"},
		{name: "example_compute_firewall"},
		{name: "example_compute_instance"},
		{name: "example_container_cluster"},
		{name: "example_organization_iam_binding"},
		{name: "example_organization_iam_member"},
		{name: "example_organization_iam_policy"},
		{name: "example_pubsub_topic"},
		{name: "example_project"},
		{name: "example_project_in_org"},
		{name: "example_project_in_folder"},
		{name: "example_project_iam"},
		{name: "example_project_iam_binding", compareConvertOutput: compareMergedIamBindingOutput},
		{name: "example_project_iam_member", compareConvertOutput: compareMergedIamMemberOutput},
		{name: "example_project_iam_policy"},
		{name: "example_project_service"},
		{name: "example_sql_database_instance"},
		{name: "example_storage_bucket"},
		{name: "full_compute_firewall"},
		{name: "full_compute_instance"},
		{name: "full_container_cluster"},
		{name: "full_container_node_pool"},
		{name: "full_sql_database_instance"},
		{name: "full_storage_bucket"},
	}

	// Map of cases to skip to reasons for the skip
	skipCases := map[string]string{
		"TestCLI/v=0.12/tf=example_compute_instance/offline=true/cmd=convert":                            "compute_instance doesn't work in offline mode - github.com/hashicorp/terraform-provider-google/issues/8489",
		"TestCLI/v=0.12/tf=example_compute_instance/offline=true/cmd=validate/constraint=always_violate": "compute_instance doesn't work in offline mode - github.com/hashicorp/terraform-provider-google/issues/8489",
	}
	for i := range cases {
		// Allocate a variable to make sure test can run in parallel.
		c := cases[i]
		// Add default constraints if not set.
		if len(c.constraints) == 0 {
			c.constraints = []constraint{alwaysViolate}
		}

		// Add default convert comparison func if not set
		if c.compareConvertOutput == nil {
			c.compareConvertOutput = compareUnmergedConvertOutput
		}

		// Test both offline and online mode.
		for _, offline := range []bool{true, false} {
			offline := offline
			t.Run(fmt.Sprintf("v=0.12/tf=%s/offline=%t", c.name, offline), func(t *testing.T) {
				t.Parallel()
				// Create a temporary directory for running terraform.
				dir, err := ioutil.TempDir(tmpDir, "terraform")
				if err != nil {
					log.Fatal(err)
				}
				defer os.RemoveAll(dir)

				// Generate the <name>.tf and <name>_assets.json files into the temporary directory.
				generateTestFiles(t, "../testdata/templates", dir, c.name+".tf")
				generateTestFiles(t, "../testdata/templates", dir, c.name+".json")

				terraform(t, dir, c.name)

				t.Run("cmd=convert", func(t *testing.T) {
					if reason, exists := skipCases[t.Name()]; exists {
						t.Skip(reason)
					}
					testConvertCommand(t, dir, c.name, offline, c.compareConvertOutput)
				})

				for _, ct := range c.constraints {
					t.Run(fmt.Sprintf("cmd=validate/constraint=%s", ct.name), func(t *testing.T) {
						if reason, exists := skipCases[t.Name()]; exists {
							t.Skip(reason)
						}
						testValidateCommand(t, ct.wantViolation, ct.wantOutputRegex, dir, c.name, offline, ct.name)
					})
				}
			})
		}
	}
}

type compareConvertOutputFunc func(t *testing.T, expected []google.Asset, actual []google.Asset, offline bool)

func compareUnmergedConvertOutput(t *testing.T, expected []google.Asset, actual []google.Asset, offline bool) {
	expectedJSON := normalizeAssets(t, expected, offline)
	actualJSON := normalizeAssets(t, actual, offline)
	require.JSONEq(t, string(expectedJSON), string(actualJSON))
}

// For merged IAM members, only consider whether the expected members are present.
func compareMergedIamMemberOutput(t *testing.T, expected []google.Asset, actual []google.Asset, offline bool) {
	var normalizedActual []google.Asset
	for i := range expected {
		expectedAsset := expected[i]
		actualAsset := actual[i]

		normalizedActualAsset := google.Asset{
			Name:     actualAsset.Name,
			Type:     actualAsset.Type,
			Ancestry: actualAsset.Ancestry,
		}

		expectedBindings := map[string]map[string]struct{}{}
		for _, binding := range expectedAsset.IAMPolicy.Bindings {
			expectedBindings[binding.Role] = map[string]struct{}{}
			for _, member := range binding.Members {
				expectedBindings[binding.Role][member] = struct{}{}
			}
		}

		iamPolicy := google.IAMPolicy{}
		for _, binding := range actualAsset.IAMPolicy.Bindings {
			if expectedMembers, exists := expectedBindings[binding.Role]; exists {
				iamBinding := google.IAMBinding{
					Role: binding.Role,
				}
				for _, member := range binding.Members {
					if _, exists := expectedMembers[member]; exists {
						iamBinding.Members = append(iamBinding.Members, member)
					}
				}
				iamPolicy.Bindings = append(iamPolicy.Bindings, iamBinding)
			}
		}
		normalizedActualAsset.IAMPolicy = &iamPolicy
		normalizedActual = append(normalizedActual, normalizedActualAsset)
	}

	expectedJSON := normalizeAssets(t, expected, offline)
	actualJSON := normalizeAssets(t, normalizedActual, offline)
	require.JSONEq(t, string(expectedJSON), string(actualJSON))
}

// For merged IAM bindings, only consider whether the expected bindings are as expected.
func compareMergedIamBindingOutput(t *testing.T, expected []google.Asset, actual []google.Asset, offline bool) {
	var normalizedActual []google.Asset
	for i := range expected {
		expectedAsset := expected[i]
		actualAsset := actual[i]

		normalizedActualAsset := google.Asset{
			Name:     actualAsset.Name,
			Type:     actualAsset.Type,
			Ancestry: actualAsset.Ancestry,
		}

		expectedBindings := map[string]struct{}{}
		for _, binding := range expectedAsset.IAMPolicy.Bindings {
			expectedBindings[binding.Role] = struct{}{}
		}

		iamPolicy := google.IAMPolicy{}
		for _, binding := range actualAsset.IAMPolicy.Bindings {
			if _, exists := expectedBindings[binding.Role]; exists {
				iamPolicy.Bindings = append(iamPolicy.Bindings, binding)
			}
		}
		normalizedActualAsset.IAMPolicy = &iamPolicy
		normalizedActual = append(normalizedActual, normalizedActualAsset)
	}

	expectedJSON := normalizeAssets(t, expected, offline)
	actualJSON := normalizeAssets(t, normalizedActual, offline)
	require.JSONEq(t, string(expectedJSON), string(actualJSON))
}

func testConvertCommand(t *testing.T, dir, name string, offline bool, compare compareConvertOutputFunc) {
	var payload []byte

	// Load expected assets
	testfile := filepath.Join(dir, name+".json")
	payload, err := ioutil.ReadFile(testfile)
	if err != nil {
		t.Fatalf("Error reading %v: %v", testfile, err)
	}
	var expected []google.Asset
	if err := json.Unmarshal(payload, &expected); err != nil {
		t.Fatalf("unmarshaling: %v", err)
	}

	// Get converted assets
	payload = tfvConvert(t, dir, name+".tfplan.json", offline)
	var actual []google.Asset
	err = json.Unmarshal(payload, &actual)
	if err != nil {
		t.Fatalf("unmarshaling: %v", err)
	}

	compare(t, expected, actual, offline)
}

func testValidateCommand(t *testing.T, wantViolation bool, want, dir, name string, offline bool, constraintName string) {
	cwd, err := os.Getwd()
	if err != nil {
		t.Fatalf("cannot get current directory: %v", err)
	}
	policyPath := filepath.Join(cwd, samplePolicyPath, constraintName)
	var got []byte
	got = tfvValidate(t, wantViolation, dir, name+".tfplan.json", policyPath, offline)
	wantRe := regexp.MustCompile(want)
	if want != "" && !wantRe.Match(got) {
		t.Fatalf("binary did not return expect output, \ngot=%s \nwant (regex)=%s", string(got), want)
	}
}

func terraform(t *testing.T, dir, name string) {
	terraformInit(t, "terraform", dir)
	terraformPlan(t, "terraform", dir, name+".tfplan")
	payload := terraformShow(t, "terraform", dir, name+".tfplan")
	saveFile(t, dir, name+".tfplan.json", payload)
}

func terraformInit(t *testing.T, executable, dir string) {
	terraformExec(t, executable, dir, "init", "-input=false")
}

func terraformPlan(t *testing.T, executable, dir, tfplan string) {
	terraformExec(t, executable, dir, "plan", "-input=false", "--out", tfplan)
}

func terraformShow(t *testing.T, executable, dir, tfplan string) []byte {
	return terraformExec(t, executable, dir, "show", "--json", tfplan)
}

func terraformExec(t *testing.T, executable, dir string, args ...string) []byte {
	cmd := exec.Command(executable, args...)
	cmd.Env = []string{"HOME=" + filepath.Join(dir, "fakehome")}
	cmd.Dir = dir
	wantError := false
	payload, _ := run(t, cmd, wantError)
	return payload
}

func saveFile(t *testing.T, dir, filename string, payload []byte) {
	fullpath := filepath.Join(dir, filename)
	f, err := os.Create(fullpath)
	if err != nil {
		t.Fatalf("error while creating file %s, error %v", fullpath, err)
	}
	_, err = f.Write(payload)
	if err != nil {
		t.Fatalf("error while writing to file %s, error %v", fullpath, err)
	}
}

func tfvConvert(t *testing.T, dir, tfplan string, offline bool) []byte {
	executable := tfvBinary
	wantError := false
	args := []string{"convert", "--project", data.Provider["project"]}
	if offline {
		args = append(args, "--offline", "--ancestry", data.Ancestry)
	}
	args = append(args, tfplan)
	cmd := exec.Command(executable, args...)
	// Remove environment variables inherited from the test runtime.
	cmd.Env = []string{}
	// Add credentials back.
	if data.Provider["credentials"] != "" {
		cmd.Env = append(cmd.Env, "GOOGLE_APPLICATION_CREDENTIALS="+data.Provider["credentials"])
	}
	cmd.Dir = dir
	payload, _ := run(t, cmd, wantError)
	return payload
}

func tfvValidate(t *testing.T, wantError bool, dir, tfplan, policyPath string, offline bool) []byte {
	executable := tfvBinary
	args := []string{"validate", "--project", data.Provider["project"], "--policy-path", policyPath}
	if offline {
		args = append(args, "--offline", "--ancestry", data.Ancestry)
	}
	args = append(args, tfplan)
	cmd := exec.Command(executable, args...)
	cmd.Env = []string{"GOOGLE_APPLICATION_CREDENTIALS=" + data.Provider["credentials"]}
	cmd.Dir = dir
	payload, _ := run(t, cmd, wantError)
	return payload
}

// run a command and call t.Fatal on non-zero exit.
func run(t *testing.T, cmd *exec.Cmd, wantError bool) ([]byte, []byte) {
	var stderr, stdout bytes.Buffer
	cmd.Stderr, cmd.Stdout = &stderr, &stdout
	err := cmd.Run()
	if gotError := (err != nil); gotError != wantError {
		t.Fatalf("running %s: \nerror=%v \nstderr=%s \nstdout=%s", cmdToString(cmd), err, stderr.String(), stdout.String())
	}
	// Print env, stdout and stderr if verbose flag is used.
	if len(cmd.Env) != 0 {
		t.Logf("=== Environment Variable of %s ===", cmdToString(cmd))
		t.Log(strings.Join(cmd.Env, "\n"))
	}
	if stdout.String() != "" {
		t.Logf("=== STDOUT of %s ===", cmdToString(cmd))
		t.Log(stdout.String())
	}
	if stderr.String() != "" {
		t.Logf("=== STDERR of %s ===", cmdToString(cmd))
		t.Log(stderr.String())
	}
	return stdout.Bytes(), stderr.Bytes()
}

// cmdToString clones the logic of https://golang.org/pkg/os/exec/#Cmd.String.
func cmdToString(c *exec.Cmd) string {
	// report the exact executable path (plus args)
	b := new(strings.Builder)
	b.WriteString(c.Path)
	for _, a := range c.Args[1:] {
		b.WriteByte(' ')
		b.WriteString(a)
	}
	return b.String()
}
<commit_msg>Skipped example_project_iam online tests because merge can't be handled generically<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"testing"

	"github.com/stretchr/testify/require"

	"github.com/GoogleCloudPlatform/terraform-validator/converters/google"
)

// TestCLI tests the "convert" and "validate" subcommand against a generated .tfplan file.
func TestCLI(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping integration test in short mode.")
		return
	}
	// Define the reusable constraints to be use for the test cases.
	type constraint struct {
		name            string
		wantViolation   bool
		wantOutputRegex string
	}
	// Currently, we only test one rule. Moving forward, resource specific rules
	// should be added to increase the coverage.
	alwaysViolate := constraint{name: "always_violate", wantViolation: true, wantOutputRegex: "Constraint GCPAlwaysViolatesConstraintV1.always_violates_all on resource"}

	// Test cases for each type of resource is defined here.
	cases := []struct {
		name                 string
		constraints          []constraint
		compareConvertOutput compareConvertOutputFunc
	}{
		{name: "bucket"},
		{name: "bucket_iam"},
		{name: "disk"},
		{name: "firewall"},
		{name: "instance"},
		{name: "sql"},
		{name: "example_bigquery_dataset"},
		{name: "example_compute_disk"},
		{name: "example_compute_firewall"},
		{name: "example_compute_instance"},
		{name: "example_container_cluster"},
		{name: "example_organization_iam_binding"},
		{name: "example_organization_iam_member"},
		{name: "example_organization_iam_policy"},
		{name: "example_pubsub_topic"},
		{name: "example_project"},
		{name: "example_project_in_org"},
		{name: "example_project_in_folder"},
		{name: "example_project_iam"},
		{name: "example_project_iam_binding", compareConvertOutput: compareMergedIamBindingOutput},
		{name: "example_project_iam_member", compareConvertOutput: compareMergedIamMemberOutput},
		{name: "example_project_iam_policy"},
		{name: "example_project_service"},
		{name: "example_sql_database_instance"},
		{name: "example_storage_bucket"},
		{name: "full_compute_firewall"},
		{name: "full_compute_instance"},
		{name: "full_container_cluster"},
		{name: "full_container_node_pool"},
		{name: "full_sql_database_instance"},
		{name: "full_storage_bucket"},
	}

	// Map of cases to skip to reasons for the skip
	skipCases := map[string]string{
		"TestCLI/v=0.12/tf=example_compute_instance/offline=true/cmd=convert":                            "compute_instance doesn't work in offline mode - github.com/hashicorp/terraform-provider-google/issues/8489",
		"TestCLI/v=0.12/tf=example_compute_instance/offline=true/cmd=validate/constraint=always_violate": "compute_instance doesn't work in offline mode - github.com/hashicorp/terraform-provider-google/issues/8489",
		"TestCLI/v=0.12/tf=example_project_iam/offline=false/cmd=convert":                                "example_project_iam is too complex to untangle merges with online data generically",
	}
	for i := range cases {
		// Allocate a variable to make sure test can run in parallel.
		c := cases[i]
		// Add default constraints if not set.
		if len(c.constraints) == 0 {
			c.constraints = []constraint{alwaysViolate}
		}

		// Add default convert comparison func if not set
		if c.compareConvertOutput == nil {
			c.compareConvertOutput = compareUnmergedConvertOutput
		}

		// Test both offline and online mode.
		for _, offline := range []bool{true, false} {
			offline := offline
			t.Run(fmt.Sprintf("v=0.12/tf=%s/offline=%t", c.name, offline), func(t *testing.T) {
				t.Parallel()
				// Create a temporary directory for running terraform.
				dir, err := ioutil.TempDir(tmpDir, "terraform")
				if err != nil {
					log.Fatal(err)
				}
				defer os.RemoveAll(dir)

				// Generate the <name>.tf and <name>_assets.json files into the temporary directory.
				generateTestFiles(t, "../testdata/templates", dir, c.name+".tf")
				generateTestFiles(t, "../testdata/templates", dir, c.name+".json")

				terraform(t, dir, c.name)

				t.Run("cmd=convert", func(t *testing.T) {
					if reason, exists := skipCases[t.Name()]; exists {
						t.Skip(reason)
					}
					testConvertCommand(t, dir, c.name, offline, c.compareConvertOutput)
				})

				for _, ct := range c.constraints {
					t.Run(fmt.Sprintf("cmd=validate/constraint=%s", ct.name), func(t *testing.T) {
						if reason, exists := skipCases[t.Name()]; exists {
							t.Skip(reason)
						}
						testValidateCommand(t, ct.wantViolation, ct.wantOutputRegex, dir, c.name, offline, ct.name)
					})
				}
			})
		}
	}
}

type compareConvertOutputFunc func(t *testing.T, expected []google.Asset, actual []google.Asset, offline bool)

func compareUnmergedConvertOutput(t *testing.T, expected []google.Asset, actual []google.Asset, offline bool) {
	expectedJSON := normalizeAssets(t, expected, offline)
	actualJSON := normalizeAssets(t, actual, offline)
	require.JSONEq(t, string(expectedJSON), string(actualJSON))
}

// For merged IAM members, only consider whether the expected members are present.
func compareMergedIamMemberOutput(t *testing.T, expected []google.Asset, actual []google.Asset, offline bool) {
	var normalizedActual []google.Asset
	for i := range expected {
		expectedAsset := expected[i]
		actualAsset := actual[i]

		normalizedActualAsset := google.Asset{
			Name:     actualAsset.Name,
			Type:     actualAsset.Type,
			Ancestry: actualAsset.Ancestry,
		}

		expectedBindings := map[string]map[string]struct{}{}
		for _, binding := range expectedAsset.IAMPolicy.Bindings {
			expectedBindings[binding.Role] = map[string]struct{}{}
			for _, member := range binding.Members {
				expectedBindings[binding.Role][member] = struct{}{}
			}
		}

		iamPolicy := google.IAMPolicy{}
		for _, binding := range actualAsset.IAMPolicy.Bindings {
			if expectedMembers, exists := expectedBindings[binding.Role]; exists {
				iamBinding := google.IAMBinding{
					Role: binding.Role,
				}
				for _, member := range binding.Members {
					if _, exists := expectedMembers[member]; exists {
						iamBinding.Members = append(iamBinding.Members, member)
					}
				}
				iamPolicy.Bindings = append(iamPolicy.Bindings, iamBinding)
			}
		}
		normalizedActualAsset.IAMPolicy = &iamPolicy
		normalizedActual = append(normalizedActual, normalizedActualAsset)
	}

	expectedJSON := normalizeAssets(t, expected, offline)
	actualJSON := normalizeAssets(t, normalizedActual, offline)
	require.JSONEq(t, string(expectedJSON), string(actualJSON))
}

// For merged IAM bindings, only consider whether the expected bindings are as expected.
func compareMergedIamBindingOutput(t *testing.T, expected []google.Asset, actual []google.Asset, offline bool) {
	var normalizedActual []google.Asset
	for i := range expected {
		expectedAsset := expected[i]
		actualAsset := actual[i]

		normalizedActualAsset := google.Asset{
			Name:     actualAsset.Name,
			Type:     actualAsset.Type,
			Ancestry: actualAsset.Ancestry,
		}

		expectedBindings := map[string]struct{}{}
		for _, binding := range expectedAsset.IAMPolicy.Bindings {
			expectedBindings[binding.Role] = struct{}{}
		}

		iamPolicy := google.IAMPolicy{}
		for _, binding := range actualAsset.IAMPolicy.Bindings {
			if _, exists := expectedBindings[binding.Role]; exists {
				iamPolicy.Bindings = append(iamPolicy.Bindings, binding)
			}
		}
		normalizedActualAsset.IAMPolicy = &iamPolicy
		normalizedActual = append(normalizedActual, normalizedActualAsset)
	}

	expectedJSON := normalizeAssets(t, expected, offline)
	actualJSON := normalizeAssets(t, normalizedActual, offline)
	require.JSONEq(t, string(expectedJSON), string(actualJSON))
}

func testConvertCommand(t *testing.T, dir, name string, offline bool, compare compareConvertOutputFunc) {
	var payload []byte

	// Load expected assets
	testfile := filepath.Join(dir, name+".json")
	payload, err := ioutil.ReadFile(testfile)
	if err != nil {
		t.Fatalf("Error reading %v: %v", testfile, err)
	}
	var expected []google.Asset
	if err := json.Unmarshal(payload, &expected); err != nil {
		t.Fatalf("unmarshaling: %v", err)
	}

	// Get converted assets
	payload = tfvConvert(t, dir, name+".tfplan.json", offline)
	var actual []google.Asset
	err = json.Unmarshal(payload, &actual)
	if err != nil {
		t.Fatalf("unmarshaling: %v", err)
	}

	compare(t, expected, actual, offline)
}

func testValidateCommand(t *testing.T, wantViolation bool, want, dir, name string, offline bool, constraintName string) {
	cwd, err := os.Getwd()
	if err != nil {
		t.Fatalf("cannot get current directory: %v", err)
	}
	policyPath := filepath.Join(cwd, samplePolicyPath, constraintName)
	var got []byte
	got = tfvValidate(t, wantViolation, dir, name+".tfplan.json", policyPath, offline)
	wantRe := regexp.MustCompile(want)
	if want != "" && !wantRe.Match(got) {
		t.Fatalf("binary did not return expect output, \ngot=%s \nwant (regex)=%s", string(got), want)
	}
}

func terraform(t *testing.T, dir, name string) {
	terraformInit(t, "terraform", dir)
	terraformPlan(t, "terraform", dir, name+".tfplan")
	payload := terraformShow(t, "terraform", dir, name+".tfplan")
	saveFile(t, dir, name+".tfplan.json", payload)
}

func terraformInit(t *testing.T, executable, dir string) {
	terraformExec(t, executable, dir, "init", "-input=false")
}

func terraformPlan(t *testing.T, executable, dir, tfplan string) {
	terraformExec(t, executable, dir, "plan", "-input=false", "--out", tfplan)
}

func terraformShow(t *testing.T, executable, dir, tfplan string) []byte {
	return terraformExec(t, executable, dir, "show", "--json", tfplan)
}

func terraformExec(t *testing.T, executable, dir string, args ...string) []byte {
	cmd := exec.Command(executable, args...)
	cmd.Env = []string{"HOME=" + filepath.Join(dir, "fakehome")}
	cmd.Dir = dir
	wantError := false
	payload, _ := run(t, cmd, wantError)
	return payload
}

func saveFile(t *testing.T, dir, filename string, payload []byte) {
	fullpath := filepath.Join(dir, filename)
	f, err := os.Create(fullpath)
	if err != nil {
		t.Fatalf("error while creating file %s, error %v", fullpath, err)
	}
	_, err = f.Write(payload)
	if err != nil {
		t.Fatalf("error while writing to file %s, error %v", fullpath, err)
	}
}

func tfvConvert(t *testing.T, dir, tfplan string, offline bool) []byte {
	executable := tfvBinary
	wantError := false
	args := []string{"convert", "--project", data.Provider["project"]}
	if offline {
		args = append(args, "--offline", "--ancestry", data.Ancestry)
	}
	args = append(args, tfplan)
	cmd := exec.Command(executable, args...)
	// Remove environment variables inherited from the test runtime.
	cmd.Env = []string{}
	// Add credentials back.
	if data.Provider["credentials"] != "" {
		cmd.Env = append(cmd.Env, "GOOGLE_APPLICATION_CREDENTIALS="+data.Provider["credentials"])
	}
	cmd.Dir = dir
	payload, _ := run(t, cmd, wantError)
	return payload
}

func tfvValidate(t *testing.T, wantError bool, dir, tfplan, policyPath string, offline bool) []byte {
	executable := tfvBinary
	args := []string{"validate", "--project", data.Provider["project"], "--policy-path", policyPath}
	if offline {
		args = append(args, "--offline", "--ancestry", data.Ancestry)
	}
	args = append(args, tfplan)
	cmd := exec.Command(executable, args...)
	cmd.Env = []string{"GOOGLE_APPLICATION_CREDENTIALS=" + data.Provider["credentials"]}
	cmd.Dir = dir
	payload, _ := run(t, cmd, wantError)
	return payload
}

// run a command and call t.Fatal on non-zero exit.
func run(t *testing.T, cmd *exec.Cmd, wantError bool) ([]byte, []byte) {
	var stderr, stdout bytes.Buffer
	cmd.Stderr, cmd.Stdout = &stderr, &stdout
	err := cmd.Run()
	if gotError := (err != nil); gotError != wantError {
		t.Fatalf("running %s: \nerror=%v \nstderr=%s \nstdout=%s", cmdToString(cmd), err, stderr.String(), stdout.String())
	}
	// Print env, stdout and stderr if verbose flag is used.
	if len(cmd.Env) != 0 {
		t.Logf("=== Environment Variable of %s ===", cmdToString(cmd))
		t.Log(strings.Join(cmd.Env, "\n"))
	}
	if stdout.String() != "" {
		t.Logf("=== STDOUT of %s ===", cmdToString(cmd))
		t.Log(stdout.String())
	}
	if stderr.String() != "" {
		t.Logf("=== STDERR of %s ===", cmdToString(cmd))
		t.Log(stderr.String())
	}
	return stdout.Bytes(), stderr.Bytes()
}

// cmdToString clones the logic of https://golang.org/pkg/os/exec/#Cmd.String.
func cmdToString(c *exec.Cmd) string {
	// report the exact executable path (plus args)
	b := new(strings.Builder)
	b.WriteString(c.Path)
	for _, a := range c.Args[1:] {
		b.WriteByte(' ')
		b.WriteString(a)
	}
	return b.String()
}
<|endoftext|>


<commit_before>package bot

import (
	"encoding/json"
	"io/ioutil"
	_ "log" // go-imports pajaSWA
	"net/http"
	"strings"

	"github.com/pajlada/pajbot2/common"
)

type bttvApi struct {
	Emotes []map[string]interface{} `json:"emotes"`
}

// LoadBttvEmotes should load emotes from redis, but this should do for now
func (bot *Bot) LoadBttvEmotes() {
	req, err := http.Get("https://api.betterttv.net/emotes")
	if err != nil {
		log.Fatal(err)
	}
	blob, _ := ioutil.ReadAll(req.Body)
	var data bttvApi
	err = json.Unmarshal(blob, &data)
	if err != nil {
		log.Fatal(err)
	}
	if data.Emotes == nil {
		log.Fatal("no data")
	}
	for _, e := range data.Emotes {
		name := e["regex"].(string)
		spl := strings.Split(e["url"].(string), "/emote/")[1]
		id := spl[:len(spl)-3] // remove /1x
		sizeX := e["width"].(float64)
		sizeY := e["height"].(float64)
		isGif := e["imageType"].(string) == "gif"
		emote := common.Emote{
			Name:  name,
			ID:    id,
			Type:  "bttv",
			SizeX: int(sizeX),
			SizeY: int(sizeY),
			IsGif: isGif,
			Count: 1,
		}
		bot.Channel.BttvEmotes[name] = emote
		log.Debug(emote)
	}
}

// regex would probably be better but im a regex noob ¯\_(ツ)_/¯
func (bot *Bot) parseBttvEmotes(msg *common.Msg) {
	m := strings.Split(msg.Text, " ")
	for _, word := range m {
		for _, emote := range bot.Channel.BttvEmotes {
			if word == emote.Name {
				msg.Emotes = append(msg.Emotes, emote)
				log.Debug(emote)
			}
		}
	}
}
<commit_msg>removed useless loop<commit_after>package bot

import (
	"encoding/json"
	"io/ioutil"
	_ "log" // go-imports pajaSWA
	"net/http"
	"strings"

	"github.com/pajlada/pajbot2/common"
)

type bttvApi struct {
	Emotes []map[string]interface{} `json:"emotes"`
}

// LoadBttvEmotes should load emotes from redis, but this should do for now
func (bot *Bot) LoadBttvEmotes() {
	req, err := http.Get("https://api.betterttv.net/emotes")
	if err != nil {
		log.Fatal(err)
	}
	blob, _ := ioutil.ReadAll(req.Body)
	var data bttvApi
	err = json.Unmarshal(blob, &data)
	if err != nil {
		log.Fatal(err)
	}
	if data.Emotes == nil {
		log.Fatal("no data")
	}
	for _, e := range data.Emotes {
		name := e["regex"].(string)
		spl := strings.Split(e["url"].(string), "/emote/")[1]
		id := spl[:len(spl)-3] // remove /1x
		sizeX := e["width"].(float64)
		sizeY := e["height"].(float64)
		isGif := e["imageType"].(string) == "gif"
		emote := common.Emote{
			Name:  name,
			ID:    id,
			Type:  "bttv",
			SizeX: int(sizeX),
			SizeY: int(sizeY),
			IsGif: isGif,
			Count: 1,
		}
		bot.Channel.BttvEmotes[name] = emote
		log.Debug(emote)
	}
}

// regex would probably be better but im a regex noob ¯\_(ツ)_/¯
func (bot *Bot) parseBttvEmotes(msg *common.Msg) {
	m := strings.Split(msg.Text, " ")
	for _, word := range m {
		if emote, ok := bot.Channel.BttvEmotes[word]; ok {
			msg.Emotes = append(msg.Emotes, emote)
			log.Debug(emote)
		}
	}
}
<|endoftext|>


<commit_before>package plugin

import (
	"fmt"
	"log"
	"os"
)

// plugin is launched by xdrive.   The following code provides some simple debugging
// aid that logs some message.  Real impl should consider a better logging mechanism
var g_dbgLog *os.File

func StartDbgLog() {
	//
	// Change/comment this line, for debugging.
	//
	if true {
		var err error
		fn := fmt.Sprintf("/tmp/xdrive_fsplugin.%d.out", os.Getpid())
		g_dbgLog, err = os.OpenFile(fn, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			log.Fatalf("Cannot open log file for debugging outputs.")
		}
		log.SetOutput(g_dbgLog)
		log.Printf("Switch log output to %s\n", fn)
	}
}

func StopDbgLog() {
	if g_dbgLog != nil {
		log.Printf("Stop log.\n")
		g_dbgLog.Close()
	}
	g_dbgLog = nil
}

func DbgLog(msg string, args ...interface{}) {
	if g_dbgLog != nil {
		log.Printf(msg, args...)
	}
}

func DbgLogIfErr(err error, msg string, args ...interface{}) {
	if err != nil && g_dbgLog != nil {
		log.Printf(msg+": error "+err.Error()+"\n", args...)
	}
}

func FatalIfErr(err error, msg string, args ...interface{}) {
	// fatal: is FATAL, regardless g_dbgLog, we die.
	if err != nil {
		log.Fatalf(msg+": error "+err.Error()+"\n", args...)
	}
}

func FatalIf(shouldDie bool, msg string, args ...interface{}) {
	if shouldDie {
		log.Fatalf(msg, args...)
	}
}
<commit_msg>add StartDbgLogWithPrefix<commit_after>package plugin

import (
	"fmt"
	"log"
	"os"
	"time"
//	"syscall"
)

// plugin is launched by xdrive.   The following code provides some simple debugging
// aid that logs some message.  Real impl should consider a better logging mechanism
var g_dbgLog *os.File

func StartDbgLogWithPrefix(prefix string) {
	var err error
	fn := fmt.Sprintf("%s-%s.log", prefix, time.Now().Local().Format("2006-01-02"))
//	fn := fmt.Sprintf("%s-%d.log", prefix, os.Getpid())
	g_dbgLog, err = os.OpenFile(fn, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		log.Fatalf("Cannot open log file for debugging outputs.")
	}
/*
	err = syscall.Dup2(int(g_dbgLog.Fd()), int(os.Stderr.Fd()))
	if err != nil {
		log.Fatalf("Failed to redirect stderr to file: %v", err)
	}
*/
	log.SetOutput(g_dbgLog)
	log.Printf("Switch log output to %s\n", fn)
	log.Printf("PID=%d\n", os.Getpid())
}

func StartDbgLog() {
	//
	// Change/comment this line, for debugging.
	//
	if true {
		var err error
		fn := fmt.Sprintf("/tmp/xdrive_fsplugin.%d.out", os.Getpid())
		g_dbgLog, err = os.OpenFile(fn, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
		if err != nil {
			log.Fatalf("Cannot open log file for debugging outputs.")
		}
		log.SetOutput(g_dbgLog)
		log.Printf("Switch log output to %s\n", fn)
	}
}

func StopDbgLog() {
	if g_dbgLog != nil {
		log.Printf("Stop log.\n")
		g_dbgLog.Close()
	}
	g_dbgLog = nil
}

func DbgLog(msg string, args ...interface{}) {
	if g_dbgLog != nil {
		log.Printf(msg, args...)
	}
}

func DbgLogIfErr(err error, msg string, args ...interface{}) {
	if err != nil && g_dbgLog != nil {
		log.Printf(msg+": error "+err.Error()+"\n", args...)
	}
}

func FatalIfErr(err error, msg string, args ...interface{}) {
	// fatal: is FATAL, regardless g_dbgLog, we die.
	if err != nil {
		log.Fatalf(msg+": error "+err.Error()+"\n", args...)
	}
}

func FatalIf(shouldDie bool, msg string, args ...interface{}) {
	if shouldDie {
		log.Fatalf(msg, args...)
	}
}

<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package factory

import (
	"fmt"
	"regexp"
	"strings"
	"sync"

	"k8s.io/kubernetes/pkg/util/sets"
	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/predicates"
	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/priorities"
	schedulerapi "k8s.io/kubernetes/plugin/pkg/scheduler/api"

	"github.com/golang/glog"
)

// PluginFactoryArgs are passed to all plugin factory functions.
type PluginFactoryArgs struct {
	PodLister                      algorithm.PodLister
	ServiceLister                  algorithm.ServiceLister
	ControllerLister               algorithm.ControllerLister
	ReplicaSetLister               algorithm.ReplicaSetLister
	NodeLister                     algorithm.NodeLister
	NodeInfo                       predicates.NodeInfo
	PVInfo                         predicates.PersistentVolumeInfo
	PVCInfo                        predicates.PersistentVolumeClaimInfo
	HardPodAffinitySymmetricWeight int
	FailureDomains                 []string
}

// A FitPredicateFactory produces a FitPredicate from the given args.
type FitPredicateFactory func(PluginFactoryArgs) algorithm.FitPredicate

// A PriorityFunctionFactory produces a PriorityConfig from the given args.
type PriorityFunctionFactory func(PluginFactoryArgs) algorithm.PriorityFunction

// A PriorityConfigFactory produces a PriorityConfig from the given function and weight
type PriorityConfigFactory struct {
	Function PriorityFunctionFactory
	Weight   int
}

var (
	schedulerFactoryMutex sync.Mutex

	// maps that hold registered algorithm types
	fitPredicateMap      = make(map[string]FitPredicateFactory)
	priorityFunctionMap  = make(map[string]PriorityConfigFactory)
	algorithmProviderMap = make(map[string]AlgorithmProviderConfig)
)

const (
	DefaultProvider = "DefaultProvider"
)

type AlgorithmProviderConfig struct {
	FitPredicateKeys     sets.String
	PriorityFunctionKeys sets.String
}

// RegisterFitPredicate registers a fit predicate with the algorithm
// registry. Returns the name with which the predicate was registered.
func RegisterFitPredicate(name string, predicate algorithm.FitPredicate) string {
	return RegisterFitPredicateFactory(name, func(PluginFactoryArgs) algorithm.FitPredicate { return predicate })
}

// RegisterFitPredicateFactory registers a fit predicate factory with the
// algorithm registry. Returns the name with which the predicate was registered.
func RegisterFitPredicateFactory(name string, predicateFactory FitPredicateFactory) string {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	validateAlgorithmNameOrDie(name)
	fitPredicateMap[name] = predicateFactory
	return name
}

// Registers a custom fit predicate with the algorithm registry.
// Returns the name, with which the predicate was registered.
func RegisterCustomFitPredicate(policy schedulerapi.PredicatePolicy) string {
	var predicateFactory FitPredicateFactory
	var ok bool

	validatePredicateOrDie(policy)

	// generate the predicate function, if a custom type is requested
	if policy.Argument != nil {
		if policy.Argument.ServiceAffinity != nil {
			predicateFactory = func(args PluginFactoryArgs) algorithm.FitPredicate {
				return predicates.NewServiceAffinityPredicate(
					args.PodLister,
					args.ServiceLister,
					args.NodeInfo,
					policy.Argument.ServiceAffinity.Labels,
				)
			}
		} else if policy.Argument.LabelsPresence != nil {
			predicateFactory = func(args PluginFactoryArgs) algorithm.FitPredicate {
				return predicates.NewNodeLabelPredicate(
					policy.Argument.LabelsPresence.Labels,
					policy.Argument.LabelsPresence.Presence,
				)
			}
		}
	} else if predicateFactory, ok = fitPredicateMap[policy.Name]; ok {
		// checking to see if a pre-defined predicate is requested
		glog.V(2).Infof("Predicate type %s already registered, reusing.", policy.Name)
		return policy.Name
	}

	if predicateFactory == nil {
		glog.Fatalf("Invalid configuration: Predicate type not found for %s", policy.Name)
	}

	return RegisterFitPredicateFactory(policy.Name, predicateFactory)
}

// This check is useful for testing providers.
func IsFitPredicateRegistered(name string) bool {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	_, ok := fitPredicateMap[name]
	return ok
}

// Registers a priority function with the algorithm registry. Returns the name,
// with which the function was registered.
func RegisterPriorityFunction(name string, function algorithm.PriorityFunction, weight int) string {
	return RegisterPriorityConfigFactory(name, PriorityConfigFactory{
		Function: func(PluginFactoryArgs) algorithm.PriorityFunction {
			return function
		},
		Weight: weight,
	})
}

func RegisterPriorityConfigFactory(name string, pcf PriorityConfigFactory) string {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	validateAlgorithmNameOrDie(name)
	priorityFunctionMap[name] = pcf
	return name
}

// Registers a custom priority function with the algorithm registry.
// Returns the name, with which the priority function was registered.
func RegisterCustomPriorityFunction(policy schedulerapi.PriorityPolicy) string {
	var pcf *PriorityConfigFactory

	validatePriorityOrDie(policy)

	// generate the priority function, if a custom priority is requested
	if policy.Argument != nil {
		if policy.Argument.ServiceAntiAffinity != nil {
			pcf = &PriorityConfigFactory{
				Function: func(args PluginFactoryArgs) algorithm.PriorityFunction {
					return priorities.NewServiceAntiAffinityPriority(
						args.PodLister,
						args.ServiceLister,
						policy.Argument.ServiceAntiAffinity.Label,
					)
				},
				Weight: policy.Weight,
			}
		} else if policy.Argument.LabelPreference != nil {
			pcf = &PriorityConfigFactory{
				Function: func(args PluginFactoryArgs) algorithm.PriorityFunction {
					return priorities.NewNodeLabelPriority(
						policy.Argument.LabelPreference.Label,
						policy.Argument.LabelPreference.Presence,
					)
				},
				Weight: policy.Weight,
			}
		}
	} else if existing_pcf, ok := priorityFunctionMap[policy.Name]; ok {
		glog.V(2).Infof("Priority type %s already registered, reusing.", policy.Name)
		// set/update the weight based on the policy
		pcf = &PriorityConfigFactory{
			Function: existing_pcf.Function,
			Weight:   policy.Weight,
		}
	}

	if pcf == nil {
		glog.Fatalf("Invalid configuration: Priority type not found for %s", policy.Name)
	}

	return RegisterPriorityConfigFactory(policy.Name, *pcf)
}

// This check is useful for testing providers.
func IsPriorityFunctionRegistered(name string) bool {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	_, ok := priorityFunctionMap[name]
	return ok
}

// Registers a new algorithm provider with the algorithm registry. This should
// be called from the init function in a provider plugin.
func RegisterAlgorithmProvider(name string, predicateKeys, priorityKeys sets.String) string {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	validateAlgorithmNameOrDie(name)
	algorithmProviderMap[name] = AlgorithmProviderConfig{
		FitPredicateKeys:     predicateKeys,
		PriorityFunctionKeys: priorityKeys,
	}
	return name
}

// This function should not be used to modify providers. It is publicly visible for testing.
func GetAlgorithmProvider(name string) (*AlgorithmProviderConfig, error) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	var provider AlgorithmProviderConfig
	provider, ok := algorithmProviderMap[name]
	if !ok {
		return nil, fmt.Errorf("plugin %q has not been registered", name)
	}

	return &provider, nil
}

func getFitPredicateFunctions(names sets.String, args PluginFactoryArgs) (map[string]algorithm.FitPredicate, error) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	predicates := map[string]algorithm.FitPredicate{}
	for _, name := range names.List() {
		factory, ok := fitPredicateMap[name]
		if !ok {
			return nil, fmt.Errorf("Invalid predicate name %q specified - no corresponding function found", name)
		}
		predicates[name] = factory(args)
	}
	return predicates, nil
}

func getPriorityFunctionConfigs(names sets.String, args PluginFactoryArgs) ([]algorithm.PriorityConfig, error) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	configs := []algorithm.PriorityConfig{}
	for _, name := range names.List() {
		factory, ok := priorityFunctionMap[name]
		if !ok {
			return nil, fmt.Errorf("Invalid priority name %s specified - no corresponding function found", name)
		}
		configs = append(configs, algorithm.PriorityConfig{
			Function: factory.Function(args),
			Weight:   factory.Weight,
		})
	}
	return configs, nil
}

var validName = regexp.MustCompile("^[a-zA-Z0-9]([-a-zA-Z0-9]*[a-zA-Z0-9])$")

func validateAlgorithmNameOrDie(name string) {
	if !validName.MatchString(name) {
		glog.Fatalf("Algorithm name %v does not match the name validation regexp \"%v\".", name, validName)
	}
}

func validatePredicateOrDie(predicate schedulerapi.PredicatePolicy) {
	if predicate.Argument != nil {
		numArgs := 0
		if predicate.Argument.ServiceAffinity != nil {
			numArgs++
		}
		if predicate.Argument.LabelsPresence != nil {
			numArgs++
		}
		if numArgs != 1 {
			glog.Fatalf("Exactly 1 predicate argument is required, numArgs: %v, Predicate: %s", numArgs, predicate.Name)
		}
	}
}

func validatePriorityOrDie(priority schedulerapi.PriorityPolicy) {
	if priority.Argument != nil {
		numArgs := 0
		if priority.Argument.ServiceAntiAffinity != nil {
			numArgs++
		}
		if priority.Argument.LabelPreference != nil {
			numArgs++
		}
		if numArgs != 1 {
			glog.Fatalf("Exactly 1 priority argument is required, numArgs: %v, Priority: %s", numArgs, priority.Name)
		}
	}
}

// ListAlgorithmProviders is called when listing all available algorithm providers in `kube-scheduler --help`
func ListAlgorithmProviders() string {
	var availableAlgorithmProviders []string
	for name := range algorithmProviderMap {
		availableAlgorithmProviders = append(availableAlgorithmProviders, name)
	}
	return strings.Join(availableAlgorithmProviders, " | ")
}
<commit_msg>Doc page for scheduler is not stable (for man pages)<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package factory

import (
	"fmt"
	"regexp"
	"sort"
	"strings"
	"sync"

	"k8s.io/kubernetes/pkg/util/sets"
	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm"
	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/predicates"
	"k8s.io/kubernetes/plugin/pkg/scheduler/algorithm/priorities"
	schedulerapi "k8s.io/kubernetes/plugin/pkg/scheduler/api"

	"github.com/golang/glog"
)

// PluginFactoryArgs are passed to all plugin factory functions.
type PluginFactoryArgs struct {
	PodLister                      algorithm.PodLister
	ServiceLister                  algorithm.ServiceLister
	ControllerLister               algorithm.ControllerLister
	ReplicaSetLister               algorithm.ReplicaSetLister
	NodeLister                     algorithm.NodeLister
	NodeInfo                       predicates.NodeInfo
	PVInfo                         predicates.PersistentVolumeInfo
	PVCInfo                        predicates.PersistentVolumeClaimInfo
	HardPodAffinitySymmetricWeight int
	FailureDomains                 []string
}

// A FitPredicateFactory produces a FitPredicate from the given args.
type FitPredicateFactory func(PluginFactoryArgs) algorithm.FitPredicate

// A PriorityFunctionFactory produces a PriorityConfig from the given args.
type PriorityFunctionFactory func(PluginFactoryArgs) algorithm.PriorityFunction

// A PriorityConfigFactory produces a PriorityConfig from the given function and weight
type PriorityConfigFactory struct {
	Function PriorityFunctionFactory
	Weight   int
}

var (
	schedulerFactoryMutex sync.Mutex

	// maps that hold registered algorithm types
	fitPredicateMap      = make(map[string]FitPredicateFactory)
	priorityFunctionMap  = make(map[string]PriorityConfigFactory)
	algorithmProviderMap = make(map[string]AlgorithmProviderConfig)
)

const (
	DefaultProvider = "DefaultProvider"
)

type AlgorithmProviderConfig struct {
	FitPredicateKeys     sets.String
	PriorityFunctionKeys sets.String
}

// RegisterFitPredicate registers a fit predicate with the algorithm
// registry. Returns the name with which the predicate was registered.
func RegisterFitPredicate(name string, predicate algorithm.FitPredicate) string {
	return RegisterFitPredicateFactory(name, func(PluginFactoryArgs) algorithm.FitPredicate { return predicate })
}

// RegisterFitPredicateFactory registers a fit predicate factory with the
// algorithm registry. Returns the name with which the predicate was registered.
func RegisterFitPredicateFactory(name string, predicateFactory FitPredicateFactory) string {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	validateAlgorithmNameOrDie(name)
	fitPredicateMap[name] = predicateFactory
	return name
}

// Registers a custom fit predicate with the algorithm registry.
// Returns the name, with which the predicate was registered.
func RegisterCustomFitPredicate(policy schedulerapi.PredicatePolicy) string {
	var predicateFactory FitPredicateFactory
	var ok bool

	validatePredicateOrDie(policy)

	// generate the predicate function, if a custom type is requested
	if policy.Argument != nil {
		if policy.Argument.ServiceAffinity != nil {
			predicateFactory = func(args PluginFactoryArgs) algorithm.FitPredicate {
				return predicates.NewServiceAffinityPredicate(
					args.PodLister,
					args.ServiceLister,
					args.NodeInfo,
					policy.Argument.ServiceAffinity.Labels,
				)
			}
		} else if policy.Argument.LabelsPresence != nil {
			predicateFactory = func(args PluginFactoryArgs) algorithm.FitPredicate {
				return predicates.NewNodeLabelPredicate(
					policy.Argument.LabelsPresence.Labels,
					policy.Argument.LabelsPresence.Presence,
				)
			}
		}
	} else if predicateFactory, ok = fitPredicateMap[policy.Name]; ok {
		// checking to see if a pre-defined predicate is requested
		glog.V(2).Infof("Predicate type %s already registered, reusing.", policy.Name)
		return policy.Name
	}

	if predicateFactory == nil {
		glog.Fatalf("Invalid configuration: Predicate type not found for %s", policy.Name)
	}

	return RegisterFitPredicateFactory(policy.Name, predicateFactory)
}

// This check is useful for testing providers.
func IsFitPredicateRegistered(name string) bool {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	_, ok := fitPredicateMap[name]
	return ok
}

// Registers a priority function with the algorithm registry. Returns the name,
// with which the function was registered.
func RegisterPriorityFunction(name string, function algorithm.PriorityFunction, weight int) string {
	return RegisterPriorityConfigFactory(name, PriorityConfigFactory{
		Function: func(PluginFactoryArgs) algorithm.PriorityFunction {
			return function
		},
		Weight: weight,
	})
}

func RegisterPriorityConfigFactory(name string, pcf PriorityConfigFactory) string {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	validateAlgorithmNameOrDie(name)
	priorityFunctionMap[name] = pcf
	return name
}

// Registers a custom priority function with the algorithm registry.
// Returns the name, with which the priority function was registered.
func RegisterCustomPriorityFunction(policy schedulerapi.PriorityPolicy) string {
	var pcf *PriorityConfigFactory

	validatePriorityOrDie(policy)

	// generate the priority function, if a custom priority is requested
	if policy.Argument != nil {
		if policy.Argument.ServiceAntiAffinity != nil {
			pcf = &PriorityConfigFactory{
				Function: func(args PluginFactoryArgs) algorithm.PriorityFunction {
					return priorities.NewServiceAntiAffinityPriority(
						args.PodLister,
						args.ServiceLister,
						policy.Argument.ServiceAntiAffinity.Label,
					)
				},
				Weight: policy.Weight,
			}
		} else if policy.Argument.LabelPreference != nil {
			pcf = &PriorityConfigFactory{
				Function: func(args PluginFactoryArgs) algorithm.PriorityFunction {
					return priorities.NewNodeLabelPriority(
						policy.Argument.LabelPreference.Label,
						policy.Argument.LabelPreference.Presence,
					)
				},
				Weight: policy.Weight,
			}
		}
	} else if existing_pcf, ok := priorityFunctionMap[policy.Name]; ok {
		glog.V(2).Infof("Priority type %s already registered, reusing.", policy.Name)
		// set/update the weight based on the policy
		pcf = &PriorityConfigFactory{
			Function: existing_pcf.Function,
			Weight:   policy.Weight,
		}
	}

	if pcf == nil {
		glog.Fatalf("Invalid configuration: Priority type not found for %s", policy.Name)
	}

	return RegisterPriorityConfigFactory(policy.Name, *pcf)
}

// This check is useful for testing providers.
func IsPriorityFunctionRegistered(name string) bool {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	_, ok := priorityFunctionMap[name]
	return ok
}

// Registers a new algorithm provider with the algorithm registry. This should
// be called from the init function in a provider plugin.
func RegisterAlgorithmProvider(name string, predicateKeys, priorityKeys sets.String) string {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()
	validateAlgorithmNameOrDie(name)
	algorithmProviderMap[name] = AlgorithmProviderConfig{
		FitPredicateKeys:     predicateKeys,
		PriorityFunctionKeys: priorityKeys,
	}
	return name
}

// This function should not be used to modify providers. It is publicly visible for testing.
func GetAlgorithmProvider(name string) (*AlgorithmProviderConfig, error) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	var provider AlgorithmProviderConfig
	provider, ok := algorithmProviderMap[name]
	if !ok {
		return nil, fmt.Errorf("plugin %q has not been registered", name)
	}

	return &provider, nil
}

func getFitPredicateFunctions(names sets.String, args PluginFactoryArgs) (map[string]algorithm.FitPredicate, error) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	predicates := map[string]algorithm.FitPredicate{}
	for _, name := range names.List() {
		factory, ok := fitPredicateMap[name]
		if !ok {
			return nil, fmt.Errorf("Invalid predicate name %q specified - no corresponding function found", name)
		}
		predicates[name] = factory(args)
	}
	return predicates, nil
}

func getPriorityFunctionConfigs(names sets.String, args PluginFactoryArgs) ([]algorithm.PriorityConfig, error) {
	schedulerFactoryMutex.Lock()
	defer schedulerFactoryMutex.Unlock()

	configs := []algorithm.PriorityConfig{}
	for _, name := range names.List() {
		factory, ok := priorityFunctionMap[name]
		if !ok {
			return nil, fmt.Errorf("Invalid priority name %s specified - no corresponding function found", name)
		}
		configs = append(configs, algorithm.PriorityConfig{
			Function: factory.Function(args),
			Weight:   factory.Weight,
		})
	}
	return configs, nil
}

var validName = regexp.MustCompile("^[a-zA-Z0-9]([-a-zA-Z0-9]*[a-zA-Z0-9])$")

func validateAlgorithmNameOrDie(name string) {
	if !validName.MatchString(name) {
		glog.Fatalf("Algorithm name %v does not match the name validation regexp \"%v\".", name, validName)
	}
}

func validatePredicateOrDie(predicate schedulerapi.PredicatePolicy) {
	if predicate.Argument != nil {
		numArgs := 0
		if predicate.Argument.ServiceAffinity != nil {
			numArgs++
		}
		if predicate.Argument.LabelsPresence != nil {
			numArgs++
		}
		if numArgs != 1 {
			glog.Fatalf("Exactly 1 predicate argument is required, numArgs: %v, Predicate: %s", numArgs, predicate.Name)
		}
	}
}

func validatePriorityOrDie(priority schedulerapi.PriorityPolicy) {
	if priority.Argument != nil {
		numArgs := 0
		if priority.Argument.ServiceAntiAffinity != nil {
			numArgs++
		}
		if priority.Argument.LabelPreference != nil {
			numArgs++
		}
		if numArgs != 1 {
			glog.Fatalf("Exactly 1 priority argument is required, numArgs: %v, Priority: %s", numArgs, priority.Name)
		}
	}
}

// ListAlgorithmProviders is called when listing all available algorithm providers in `kube-scheduler --help`
func ListAlgorithmProviders() string {
	var availableAlgorithmProviders []string
	for name := range algorithmProviderMap {
		availableAlgorithmProviders = append(availableAlgorithmProviders, name)
	}
	sort.Strings(availableAlgorithmProviders)
	return strings.Join(availableAlgorithmProviders, " | ")
}
<|endoftext|>


<commit_before>package diff

import (
	"bytes"
	"fmt"
	"io"
	"strings"

	"gopkg.in/src-d/go-git.v4/plumbing"
)

const (
	diffInit = "diff --git a/%s b/%s\n"

	chunkStart  = "@@ -"
	chunkMiddle = " +"
	chunkEnd    = " @@%s\n"
	chunkCount  = "%d,%d"

	noFilePath = "/dev/null"
	aDir       = "a/"
	bDir       = "b/"

	fPath  = "--- %s\n"
	tPath  = "+++ %s\n"
	binary = "Binary files %s and %s differ\n"

	addLine    = "+%s\n"
	deleteLine = "-%s\n"
	equalLine  = " %s\n"

	oldMode         = "old mode %o\n"
	newMode         = "new mode %o\n"
	deletedFileMode = "deleted file mode %o\n"
	newFileMode     = "new file mode %o\n"

	renameFrom     = "from"
	renameTo       = "to"
	renameFileMode = "rename %s %s\n"

	indexAndMode = "index %s..%s %o\n"
	indexNoMode  = "index %s..%s\n"

	DefaultContextLines = 3
)

// UnifiedEncoder encodes an unified diff into the provided Writer.
// There are some unsupported features:
//     - Similarity index for renames
//     - Sort hash representation
type UnifiedEncoder struct {
	io.Writer

	// ctxLines is the count of unchanged lines that will appear
	// surrounding a change.
	ctxLines int

	buf bytes.Buffer
}

func NewUnifiedEncoder(w io.Writer, ctxLines int) *UnifiedEncoder {
	return &UnifiedEncoder{ctxLines: ctxLines, Writer: w}
}

func (e *UnifiedEncoder) Encode(patch Patch) error {
	e.printMessage(patch.Message())

	if err := e.encodeFilePatch(patch.FilePatches()); err != nil {
		return err
	}

	_, err := e.buf.WriteTo(e)

	return err
}

func (e *UnifiedEncoder) encodeFilePatch(filePatches []FilePatch) error {
	for _, p := range filePatches {
		f, t := p.Files()
		if err := e.header(f, t, p.IsBinary()); err != nil {
			return err
		}

		g := newHunksGenerator(p.Chunks(), e.ctxLines)
		for _, c := range g.Generate() {
			c.WriteTo(&e.buf)
		}
	}

	return nil
}

func (e *UnifiedEncoder) printMessage(message string) {
	isEmpty := message == ""
	hasSuffix := strings.HasSuffix(message, "\n")
	if !isEmpty && !hasSuffix {
		message = message + "\n"
	}

	e.buf.WriteString(message)
}

func (e *UnifiedEncoder) header(from, to File, isBinary bool) error {
	switch {
	case from == nil && to == nil:
		return nil
	case from != nil && to != nil:
		hashEquals := from.Hash() == to.Hash()

		fmt.Fprintf(&e.buf, diffInit, from.Path(), to.Path())

		if from.Mode() != to.Mode() {
			fmt.Fprintf(&e.buf, oldMode+newMode, from.Mode(), to.Mode())
		}

		if from.Path() != to.Path() {
			fmt.Fprintf(&e.buf,
				renameFileMode+renameFileMode,
				renameFrom, from.Path(), renameTo, to.Path())
		}

		if from.Mode() != to.Mode() && !hashEquals {
			fmt.Fprintf(&e.buf, indexNoMode, from.Hash(), to.Hash())
		} else if !hashEquals {
			fmt.Fprintf(&e.buf, indexAndMode, from.Hash(), to.Hash(), from.Mode())
		}

		if !hashEquals {
			e.pathLines(isBinary, aDir+from.Path(), bDir+to.Path())
		}
	case from == nil:
		fmt.Fprintf(&e.buf, diffInit, to.Path(), to.Path())
		fmt.Fprintf(&e.buf, newFileMode, to.Mode())
		fmt.Fprintf(&e.buf, indexNoMode, plumbing.ZeroHash, to.Hash())
		e.pathLines(isBinary, noFilePath, bDir+to.Path())
	case to == nil:
		fmt.Fprintf(&e.buf, diffInit, from.Path(), from.Path())
		fmt.Fprintf(&e.buf, deletedFileMode, from.Mode())
		fmt.Fprintf(&e.buf, indexNoMode, from.Hash(), plumbing.ZeroHash)
		e.pathLines(isBinary, aDir+from.Path(), noFilePath)
	}

	return nil
}

func (e *UnifiedEncoder) pathLines(isBinary bool, fromPath, toPath string) {
	format := fPath + tPath
	if isBinary {
		format = binary
	}

	fmt.Fprintf(&e.buf, format, fromPath, toPath)
}

type hunksGenerator struct {
	fromLine, toLine            int
	ctxLines                    int
	chunks                      []Chunk
	current                     *hunk
	hunks                       []*hunk
	beforeContext, afterContext []string
}

func newHunksGenerator(chunks []Chunk, ctxLines int) *hunksGenerator {
	return &hunksGenerator{
		chunks:   chunks,
		ctxLines: ctxLines,
	}
}

func (c *hunksGenerator) Generate() []*hunk {
	for i, chunk := range c.chunks {
		ls := splitLines(chunk.Content())
		lsLen := len(ls)

		switch chunk.Type() {
		case Equal:
			c.fromLine += lsLen
			c.toLine += lsLen
			c.processEqualsLines(ls, i)
		case Delete:
			if lsLen != 0 {
				c.fromLine++
			}

			c.processHunk(i, chunk.Type())
			c.fromLine += lsLen - 1
			c.current.AddOp(chunk.Type(), ls...)
		case Add:
			if lsLen != 0 {
				c.toLine++
			}
			c.processHunk(i, chunk.Type())
			c.toLine += lsLen - 1
			c.current.AddOp(chunk.Type(), ls...)
		}

		if i == len(c.chunks)-1 && c.current != nil {
			c.hunks = append(c.hunks, c.current)
		}
	}

	return c.hunks
}

func (c *hunksGenerator) processHunk(i int, op Operation) {
	if c.current != nil {
		return
	}

	var ctxPrefix string
	linesBefore := len(c.beforeContext)
	if linesBefore > c.ctxLines {
		ctxPrefix = " " + c.beforeContext[linesBefore-c.ctxLines-1]
		c.beforeContext = c.beforeContext[linesBefore-c.ctxLines:]
		linesBefore = c.ctxLines
	}

	c.current = &hunk{ctxPrefix: ctxPrefix}
	c.current.AddOp(Equal, c.beforeContext...)

	switch op {
	case Delete:
		c.current.fromLine, c.current.toLine =
			c.addLineNumbers(c.fromLine, c.toLine, linesBefore, i, Add)
	case Add:
		c.current.toLine, c.current.fromLine =
			c.addLineNumbers(c.toLine, c.fromLine, linesBefore, i, Delete)
	}

	c.beforeContext = nil
}

// addLineNumbers obtains the line numbers in a new chunk
func (c *hunksGenerator) addLineNumbers(la, lb int, linesBefore int, i int, op Operation) (cla, clb int) {
	cla = la - linesBefore
	// we need to search for a reference for the next diff
	switch {
	case linesBefore != 0 && c.ctxLines != 0:
		clb = lb - c.ctxLines + 1
	case c.ctxLines == 0:
		clb = lb - c.ctxLines
	case i != len(c.chunks)-1:
		next := c.chunks[i+1]
		if next.Type() == op || next.Type() == Equal {
			// this diff will be into this chunk
			clb = lb + 1
		}
	}

	return
}

func (c *hunksGenerator) processEqualsLines(ls []string, i int) {
	if c.current == nil {
		c.beforeContext = append(c.beforeContext, ls...)
		return
	}

	c.afterContext = append(c.afterContext, ls...)
	if len(c.afterContext) <= c.ctxLines*2 && i != len(c.chunks)-1 {
		c.current.AddOp(Equal, c.afterContext...)
		c.afterContext = nil
	} else {
		c.current.AddOp(Equal, c.afterContext[:c.ctxLines]...)
		c.hunks = append(c.hunks, c.current)

		c.current = nil
		c.beforeContext = c.afterContext[c.ctxLines:]
		c.afterContext = nil
	}
}

func splitLines(s string) []string {
	out := strings.Split(s, "\n")
	if out[len(out)-1] == "" {
		out = out[:len(out)-1]
	}

	return out
}

type hunk struct {
	fromLine int
	toLine   int

	fromCount int
	toCount   int

	ctxPrefix string
	ops       []*op
}

func (c *hunk) WriteTo(buf *bytes.Buffer) {
	buf.WriteString(chunkStart)

	if c.fromCount == 1 {
		fmt.Fprintf(buf, "%d", c.fromLine)
	} else {
		fmt.Fprintf(buf, chunkCount, c.fromLine, c.fromCount)
	}

	buf.WriteString(chunkMiddle)

	if c.toCount == 1 {
		fmt.Fprintf(buf, "%d", c.toLine)
	} else {
		fmt.Fprintf(buf, chunkCount, c.toLine, c.toCount)
	}

	fmt.Fprintf(buf, chunkEnd, c.ctxPrefix)

	for _, d := range c.ops {
		buf.WriteString(d.String())
	}
}

func (c *hunk) AddOp(t Operation, s ...string) {
	ls := len(s)
	switch t {
	case Add:
		c.toCount += ls
	case Delete:
		c.fromCount += ls
	case Equal:
		c.toCount += ls
		c.fromCount += ls
	}

	for _, l := range s {
		c.ops = append(c.ops, &op{l, t})
	}
}

type op struct {
	text string
	t    Operation
}

func (o *op) String() string {
	var prefix string
	switch o.t {
	case Add:
		prefix = addLine
	case Delete:
		prefix = deleteLine
	case Equal:
		prefix = equalLine
	}

	return fmt.Sprintf(prefix, o.text)
}
<commit_msg>fix crash when generating a unified diff with a small ending equal-chunk<commit_after>package diff

import (
	"bytes"
	"fmt"
	"io"
	"strings"

	"gopkg.in/src-d/go-git.v4/plumbing"
)

const (
	diffInit = "diff --git a/%s b/%s\n"

	chunkStart  = "@@ -"
	chunkMiddle = " +"
	chunkEnd    = " @@%s\n"
	chunkCount  = "%d,%d"

	noFilePath = "/dev/null"
	aDir       = "a/"
	bDir       = "b/"

	fPath  = "--- %s\n"
	tPath  = "+++ %s\n"
	binary = "Binary files %s and %s differ\n"

	addLine    = "+%s\n"
	deleteLine = "-%s\n"
	equalLine  = " %s\n"

	oldMode         = "old mode %o\n"
	newMode         = "new mode %o\n"
	deletedFileMode = "deleted file mode %o\n"
	newFileMode     = "new file mode %o\n"

	renameFrom     = "from"
	renameTo       = "to"
	renameFileMode = "rename %s %s\n"

	indexAndMode = "index %s..%s %o\n"
	indexNoMode  = "index %s..%s\n"

	DefaultContextLines = 3
)

// UnifiedEncoder encodes an unified diff into the provided Writer.
// There are some unsupported features:
//     - Similarity index for renames
//     - Sort hash representation
type UnifiedEncoder struct {
	io.Writer

	// ctxLines is the count of unchanged lines that will appear
	// surrounding a change.
	ctxLines int

	buf bytes.Buffer
}

func NewUnifiedEncoder(w io.Writer, ctxLines int) *UnifiedEncoder {
	return &UnifiedEncoder{ctxLines: ctxLines, Writer: w}
}

func (e *UnifiedEncoder) Encode(patch Patch) error {
	e.printMessage(patch.Message())

	if err := e.encodeFilePatch(patch.FilePatches()); err != nil {
		return err
	}

	_, err := e.buf.WriteTo(e)

	return err
}

func (e *UnifiedEncoder) encodeFilePatch(filePatches []FilePatch) error {
	for _, p := range filePatches {
		f, t := p.Files()
		if err := e.header(f, t, p.IsBinary()); err != nil {
			return err
		}

		g := newHunksGenerator(p.Chunks(), e.ctxLines)
		for _, c := range g.Generate() {
			c.WriteTo(&e.buf)
		}
	}

	return nil
}

func (e *UnifiedEncoder) printMessage(message string) {
	isEmpty := message == ""
	hasSuffix := strings.HasSuffix(message, "\n")
	if !isEmpty && !hasSuffix {
		message = message + "\n"
	}

	e.buf.WriteString(message)
}

func (e *UnifiedEncoder) header(from, to File, isBinary bool) error {
	switch {
	case from == nil && to == nil:
		return nil
	case from != nil && to != nil:
		hashEquals := from.Hash() == to.Hash()

		fmt.Fprintf(&e.buf, diffInit, from.Path(), to.Path())

		if from.Mode() != to.Mode() {
			fmt.Fprintf(&e.buf, oldMode+newMode, from.Mode(), to.Mode())
		}

		if from.Path() != to.Path() {
			fmt.Fprintf(&e.buf,
				renameFileMode+renameFileMode,
				renameFrom, from.Path(), renameTo, to.Path())
		}

		if from.Mode() != to.Mode() && !hashEquals {
			fmt.Fprintf(&e.buf, indexNoMode, from.Hash(), to.Hash())
		} else if !hashEquals {
			fmt.Fprintf(&e.buf, indexAndMode, from.Hash(), to.Hash(), from.Mode())
		}

		if !hashEquals {
			e.pathLines(isBinary, aDir+from.Path(), bDir+to.Path())
		}
	case from == nil:
		fmt.Fprintf(&e.buf, diffInit, to.Path(), to.Path())
		fmt.Fprintf(&e.buf, newFileMode, to.Mode())
		fmt.Fprintf(&e.buf, indexNoMode, plumbing.ZeroHash, to.Hash())
		e.pathLines(isBinary, noFilePath, bDir+to.Path())
	case to == nil:
		fmt.Fprintf(&e.buf, diffInit, from.Path(), from.Path())
		fmt.Fprintf(&e.buf, deletedFileMode, from.Mode())
		fmt.Fprintf(&e.buf, indexNoMode, from.Hash(), plumbing.ZeroHash)
		e.pathLines(isBinary, aDir+from.Path(), noFilePath)
	}

	return nil
}

func (e *UnifiedEncoder) pathLines(isBinary bool, fromPath, toPath string) {
	format := fPath + tPath
	if isBinary {
		format = binary
	}

	fmt.Fprintf(&e.buf, format, fromPath, toPath)
}

type hunksGenerator struct {
	fromLine, toLine            int
	ctxLines                    int
	chunks                      []Chunk
	current                     *hunk
	hunks                       []*hunk
	beforeContext, afterContext []string
}

func newHunksGenerator(chunks []Chunk, ctxLines int) *hunksGenerator {
	return &hunksGenerator{
		chunks:   chunks,
		ctxLines: ctxLines,
	}
}

func (c *hunksGenerator) Generate() []*hunk {
	for i, chunk := range c.chunks {
		ls := splitLines(chunk.Content())
		lsLen := len(ls)

		switch chunk.Type() {
		case Equal:
			c.fromLine += lsLen
			c.toLine += lsLen
			c.processEqualsLines(ls, i)
		case Delete:
			if lsLen != 0 {
				c.fromLine++
			}

			c.processHunk(i, chunk.Type())
			c.fromLine += lsLen - 1
			c.current.AddOp(chunk.Type(), ls...)
		case Add:
			if lsLen != 0 {
				c.toLine++
			}
			c.processHunk(i, chunk.Type())
			c.toLine += lsLen - 1
			c.current.AddOp(chunk.Type(), ls...)
		}

		if i == len(c.chunks)-1 && c.current != nil {
			c.hunks = append(c.hunks, c.current)
		}
	}

	return c.hunks
}

func (c *hunksGenerator) processHunk(i int, op Operation) {
	if c.current != nil {
		return
	}

	var ctxPrefix string
	linesBefore := len(c.beforeContext)
	if linesBefore > c.ctxLines {
		ctxPrefix = " " + c.beforeContext[linesBefore-c.ctxLines-1]
		c.beforeContext = c.beforeContext[linesBefore-c.ctxLines:]
		linesBefore = c.ctxLines
	}

	c.current = &hunk{ctxPrefix: ctxPrefix}
	c.current.AddOp(Equal, c.beforeContext...)

	switch op {
	case Delete:
		c.current.fromLine, c.current.toLine =
			c.addLineNumbers(c.fromLine, c.toLine, linesBefore, i, Add)
	case Add:
		c.current.toLine, c.current.fromLine =
			c.addLineNumbers(c.toLine, c.fromLine, linesBefore, i, Delete)
	}

	c.beforeContext = nil
}

// addLineNumbers obtains the line numbers in a new chunk
func (c *hunksGenerator) addLineNumbers(la, lb int, linesBefore int, i int, op Operation) (cla, clb int) {
	cla = la - linesBefore
	// we need to search for a reference for the next diff
	switch {
	case linesBefore != 0 && c.ctxLines != 0:
		clb = lb - c.ctxLines + 1
	case c.ctxLines == 0:
		clb = lb - c.ctxLines
	case i != len(c.chunks)-1:
		next := c.chunks[i+1]
		if next.Type() == op || next.Type() == Equal {
			// this diff will be into this chunk
			clb = lb + 1
		}
	}

	return
}

func (c *hunksGenerator) processEqualsLines(ls []string, i int) {
	if c.current == nil {
		c.beforeContext = append(c.beforeContext, ls...)
		return
	}

	c.afterContext = append(c.afterContext, ls...)
	if len(c.afterContext) <= c.ctxLines*2 && i != len(c.chunks)-1 {
		c.current.AddOp(Equal, c.afterContext...)
		c.afterContext = nil
	} else {
		ctxLines := c.ctxLines
		if ctxLines > len(c.afterContext) {
			ctxLines = len(c.afterContext)
		}
		c.current.AddOp(Equal, c.afterContext[:ctxLines]...)
		c.hunks = append(c.hunks, c.current)

		c.current = nil
		c.beforeContext = c.afterContext[ctxLines:]
		c.afterContext = nil
	}
}

func splitLines(s string) []string {
	out := strings.Split(s, "\n")
	if out[len(out)-1] == "" {
		out = out[:len(out)-1]
	}

	return out
}

type hunk struct {
	fromLine int
	toLine   int

	fromCount int
	toCount   int

	ctxPrefix string
	ops       []*op
}

func (c *hunk) WriteTo(buf *bytes.Buffer) {
	buf.WriteString(chunkStart)

	if c.fromCount == 1 {
		fmt.Fprintf(buf, "%d", c.fromLine)
	} else {
		fmt.Fprintf(buf, chunkCount, c.fromLine, c.fromCount)
	}

	buf.WriteString(chunkMiddle)

	if c.toCount == 1 {
		fmt.Fprintf(buf, "%d", c.toLine)
	} else {
		fmt.Fprintf(buf, chunkCount, c.toLine, c.toCount)
	}

	fmt.Fprintf(buf, chunkEnd, c.ctxPrefix)

	for _, d := range c.ops {
		buf.WriteString(d.String())
	}
}

func (c *hunk) AddOp(t Operation, s ...string) {
	ls := len(s)
	switch t {
	case Add:
		c.toCount += ls
	case Delete:
		c.fromCount += ls
	case Equal:
		c.toCount += ls
		c.fromCount += ls
	}

	for _, l := range s {
		c.ops = append(c.ops, &op{l, t})
	}
}

type op struct {
	text string
	t    Operation
}

func (o *op) String() string {
	var prefix string
	switch o.t {
	case Add:
		prefix = addLine
	case Delete:
		prefix = deleteLine
	case Equal:
		prefix = equalLine
	}

	return fmt.Sprintf(prefix, o.text)
}
<|endoftext|>


<commit_before>package forestdb

//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

//#include <libforestdb/forestdb.h>
import "C"

import (
	"unsafe"
)

// ForestDB iterator options
type IteratorOpt uint8

const (
	// Return both key and value through iterator
	ITR_NONE IteratorOpt = 0x00
	// Return key and its metadata only through iterator
	ITR_META_ONLY IteratorOpt = 0x01
	// Return only non-deleted items through iterator
	ITR_NO_DELETES IteratorOpt = 0x02
	// The lowest key specified will not be returned by the iterator
	FDB_ITR_SKIP_MIN_KEY IteratorOpt = 0x04
	//The highest key specified will not be returned by the iterator
	FDB_ITR_SKIP_MAX_KEY IteratorOpt = 0x08
)

// ForestDB seek options
type SeekOpt uint8

const (
	// If seek_key does not exist return the next sorted key higher than it
	FDB_ITR_SEEK_HIGHER SeekOpt = 0x00
	// If seek_key does not exist return the previous sorted key lower than it
	FDB_ITR_SEEK_LOWER SeekOpt = 0x01
)

// Iterator handle
type Iterator struct {
	iter *C.fdb_iterator
}

// Prev advances the iterator backwards
func (i *Iterator) Prev() error {
	errNo := C.fdb_iterator_prev(i.iter)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// Next advances the iterator forward
func (i *Iterator) Next() error {
	errNo := C.fdb_iterator_next(i.iter)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// Get gets the current item (key, metadata, doc body) from the iterator
func (i *Iterator) Get() (*Doc, error) {
	rv := Doc{}
	errNo := C.fdb_iterator_get_metaonly(i.iter, &rv.doc)
	if errNo != RESULT_SUCCESS {
		return nil, Error(errNo)
	}
	return &rv, nil
}

// GetMetaOnly gets the current item (key, metadata, offset to doc body) from the iterator
func (i *Iterator) GetMetaOnly() (*Doc, error) {
	rv := Doc{}
	errNo := C.fdb_iterator_get_metaonly(i.iter, &rv.doc)
	if errNo != RESULT_SUCCESS {
		return nil, Error(errNo)
	}
	return &rv, nil
}

// Seek fast forward / backward an iterator to
// return documents starting from
// the given seek_key. If the seek key does not
// exist, the iterator is positioned based on
// the specified dir (either before or after).
func (i *Iterator) Seek(seekKey []byte, dir SeekOpt) error {
	var sk unsafe.Pointer
	lensk := len(seekKey)
	if lensk != 0 {
		sk = unsafe.Pointer(&seekKey[0])
	}
	errNo := C.fdb_iterator_seek(i.iter, sk, C.size_t(lensk), C.fdb_iterator_seek_opt_t(dir))
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// SeekMin moves iterator to the smallest key
// of the iteration
func (i *Iterator) SeekMin() error {
	errNo := C.fdb_iterator_seek_to_min(i.iter)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// SeekMax moves iterator to the largest key
// of the iteration
func (i *Iterator) SeekMax() error {
	errNo := C.fdb_iterator_seek_to_max(i.iter)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// Close the iterator and free its associated resources
func (i *Iterator) Close() error {
	errNo := C.fdb_iterator_close(i.iter)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// IteratorInit creates an iterator to traverse a ForestDB snapshot by key range
func (k *KVStore) IteratorInit(startKey, endKey []byte, opt IteratorOpt) (*Iterator, error) {
	var sk, ek unsafe.Pointer

	lensk := len(startKey)
	lenek := len(endKey)

	if lensk != 0 {
		sk = unsafe.Pointer(&startKey[0])
	}

	if lenek != 0 {
		ek = unsafe.Pointer(&endKey[0])
	}

	rv := Iterator{}
	errNo := C.fdb_iterator_init(k.db, &rv.iter, sk, C.size_t(lensk), ek, C.size_t(lenek), C.fdb_iterator_opt_t(opt))
	if errNo != RESULT_SUCCESS {
		return nil, Error(errNo)
	}
	return &rv, nil
}

// IteratorSequenceInit create an iterator to traverse a ForestDB snapshot by sequence number range
func (k *KVStore) IteratorSequenceInit(startSeq, endSeq SeqNum, opt IteratorOpt) (*Iterator, error) {
	rv := Iterator{}
	errNo := C.fdb_iterator_sequence_init(k.db, &rv.iter, C.fdb_seqnum_t(startSeq), C.fdb_seqnum_t(endSeq), C.fdb_iterator_opt_t(opt))
	if errNo != RESULT_SUCCESS {
		return nil, Error(errNo)
	}
	return &rv, nil
}
<commit_msg>Remove ITR_META_ONLY iterator option.<commit_after>package forestdb

//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

//#include <libforestdb/forestdb.h>
import "C"

import (
	"unsafe"
)

// ForestDB iterator options
type IteratorOpt uint16

const (
	// Return both key and value through iterator
	ITR_NONE IteratorOpt = 0x00
	// Return only non-deleted items through iterator
	ITR_NO_DELETES IteratorOpt = 0x02
	// The lowest key specified will not be returned by the iterator
	FDB_ITR_SKIP_MIN_KEY IteratorOpt = 0x04
	//The highest key specified will not be returned by the iterator
	FDB_ITR_SKIP_MAX_KEY IteratorOpt = 0x08
)

// ForestDB seek options
type SeekOpt uint8

const (
	// If seek_key does not exist return the next sorted key higher than it
	FDB_ITR_SEEK_HIGHER SeekOpt = 0x00
	// If seek_key does not exist return the previous sorted key lower than it
	FDB_ITR_SEEK_LOWER SeekOpt = 0x01
)

// Iterator handle
type Iterator struct {
	iter *C.fdb_iterator
}

// Prev advances the iterator backwards
func (i *Iterator) Prev() error {
	errNo := C.fdb_iterator_prev(i.iter)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// Next advances the iterator forward
func (i *Iterator) Next() error {
	errNo := C.fdb_iterator_next(i.iter)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// Get gets the current item (key, metadata, doc body) from the iterator
func (i *Iterator) Get() (*Doc, error) {
	rv := Doc{}
	errNo := C.fdb_iterator_get_metaonly(i.iter, &rv.doc)
	if errNo != RESULT_SUCCESS {
		return nil, Error(errNo)
	}
	return &rv, nil
}

// GetMetaOnly gets the current item (key, metadata, offset to doc body) from the iterator
func (i *Iterator) GetMetaOnly() (*Doc, error) {
	rv := Doc{}
	errNo := C.fdb_iterator_get_metaonly(i.iter, &rv.doc)
	if errNo != RESULT_SUCCESS {
		return nil, Error(errNo)
	}
	return &rv, nil
}

// Seek fast forward / backward an iterator to
// return documents starting from
// the given seek_key. If the seek key does not
// exist, the iterator is positioned based on
// the specified dir (either before or after).
func (i *Iterator) Seek(seekKey []byte, dir SeekOpt) error {
	var sk unsafe.Pointer
	lensk := len(seekKey)
	if lensk != 0 {
		sk = unsafe.Pointer(&seekKey[0])
	}
	errNo := C.fdb_iterator_seek(i.iter, sk, C.size_t(lensk), C.fdb_iterator_seek_opt_t(dir))
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// SeekMin moves iterator to the smallest key
// of the iteration
func (i *Iterator) SeekMin() error {
	errNo := C.fdb_iterator_seek_to_min(i.iter)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// SeekMax moves iterator to the largest key
// of the iteration
func (i *Iterator) SeekMax() error {
	errNo := C.fdb_iterator_seek_to_max(i.iter)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// Close the iterator and free its associated resources
func (i *Iterator) Close() error {
	errNo := C.fdb_iterator_close(i.iter)
	if errNo != RESULT_SUCCESS {
		return Error(errNo)
	}
	return nil
}

// IteratorInit creates an iterator to traverse a ForestDB snapshot by key range
func (k *KVStore) IteratorInit(startKey, endKey []byte, opt IteratorOpt) (*Iterator, error) {
	var sk, ek unsafe.Pointer

	lensk := len(startKey)
	lenek := len(endKey)

	if lensk != 0 {
		sk = unsafe.Pointer(&startKey[0])
	}

	if lenek != 0 {
		ek = unsafe.Pointer(&endKey[0])
	}

	rv := Iterator{}
	errNo := C.fdb_iterator_init(k.db, &rv.iter, sk, C.size_t(lensk), ek, C.size_t(lenek), C.fdb_iterator_opt_t(opt))
	if errNo != RESULT_SUCCESS {
		return nil, Error(errNo)
	}
	return &rv, nil
}

// IteratorSequenceInit create an iterator to traverse a ForestDB snapshot by sequence number range
func (k *KVStore) IteratorSequenceInit(startSeq, endSeq SeqNum, opt IteratorOpt) (*Iterator, error) {
	rv := Iterator{}
	errNo := C.fdb_iterator_sequence_init(k.db, &rv.iter, C.fdb_seqnum_t(startSeq), C.fdb_seqnum_t(endSeq), C.fdb_iterator_opt_t(opt))
	if errNo != RESULT_SUCCESS {
		return nil, Error(errNo)
	}
	return &rv, nil
}
<|endoftext|>


<commit_before>// Package jsonutil provides JSON serialization of AWS requests and responses.
package jsonutil

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"reflect"
	"sort"
	"strconv"
	"time"

	"github.com/aws/aws-sdk-go/private/protocol"
)

var timeType = reflect.ValueOf(time.Time{}).Type()
var byteSliceType = reflect.ValueOf([]byte{}).Type()

// BuildJSON builds a JSON string for a given object v.
func BuildJSON(v interface{}) ([]byte, error) {
	var buf bytes.Buffer

	err := buildAny(reflect.ValueOf(v), &buf, "")
	return buf.Bytes(), err
}

func buildAny(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error {
	value = reflect.Indirect(value)
	if !value.IsValid() {
		return nil
	}

	vtype := value.Type()

	t := tag.Get("type")
	if t == "" {
		switch vtype.Kind() {
		case reflect.Struct:
			// also it can't be a time object
			if value.Type() != timeType {
				t = "structure"
			}
		case reflect.Slice:
			// also it can't be a byte slice
			if _, ok := value.Interface().([]byte); !ok {
				t = "list"
			}
		case reflect.Map:
			t = "map"
		}
	}

	switch t {
	case "structure":
		if field, ok := vtype.FieldByName("_"); ok {
			tag = field.Tag
		}
		return buildStruct(value, buf, tag)
	case "list":
		return buildList(value, buf, tag)
	case "map":
		return buildMap(value, buf, tag)
	default:
		return buildScalar(value, buf, tag)
	}
}

func buildStruct(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error {
	if !value.IsValid() {
		return nil
	}

	// unwrap payloads
	if payload := tag.Get("payload"); payload != "" {
		field, _ := value.Type().FieldByName(payload)
		tag = field.Tag
		value = elemOf(value.FieldByName(payload))

		if !value.IsValid() {
			return nil
		}
	}

	buf.WriteByte('{')

	t := value.Type()
	first := true
	for i := 0; i < t.NumField(); i++ {
		member := value.Field(i)
		field := t.Field(i)

		if field.PkgPath != "" {
			continue // ignore unexported fields
		}
		if field.Tag.Get("json") == "-" {
			continue
		}
		if field.Tag.Get("location") != "" {
			continue // ignore non-body elements
		}
		if field.Tag.Get("ignore") != "" {
			continue
		}

		if protocol.CanSetIdempotencyToken(member, field) {
			token := protocol.GetIdempotencyToken()
			member = reflect.ValueOf(&token)
		}

		if (member.Kind() == reflect.Ptr || member.Kind() == reflect.Slice || member.Kind() == reflect.Map) && member.IsNil() {
			continue // ignore unset fields
		}

		if first {
			first = false
		} else {
			buf.WriteByte(',')
		}

		// figure out what this field is called
		name := field.Name
		if locName := field.Tag.Get("locationName"); locName != "" {
			name = locName
		}

		writeString(name, buf)
		buf.WriteString(`:`)

		err := buildAny(member, buf, field.Tag)
		if err != nil {
			return err
		}

	}

	buf.WriteString("}")

	return nil
}

func buildList(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error {
	buf.WriteString("[")

	for i := 0; i < value.Len(); i++ {
		buildAny(value.Index(i), buf, "")

		if i < value.Len()-1 {
			buf.WriteString(",")
		}
	}

	buf.WriteString("]")

	return nil
}

type sortedValues []reflect.Value

func (sv sortedValues) Len() int           { return len(sv) }
func (sv sortedValues) Swap(i, j int)      { sv[i], sv[j] = sv[j], sv[i] }
func (sv sortedValues) Less(i, j int) bool { return sv[i].String() < sv[j].String() }

func buildMap(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error {
	buf.WriteString("{")

	sv := sortedValues(value.MapKeys())
	sort.Sort(sv)

	for i, k := range sv {
		if i > 0 {
			buf.WriteByte(',')
		}

		writeString(k.String(), buf)
		buf.WriteString(`:`)

		buildAny(value.MapIndex(k), buf, "")
	}

	buf.WriteString("}")

	return nil
}

func buildScalar(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error {
	switch value.Kind() {
	case reflect.String:
		writeString(value.String(), buf)
	case reflect.Bool:
		buf.WriteString(strconv.FormatBool(value.Bool()))
	case reflect.Int64:
		buf.WriteString(strconv.FormatInt(value.Int(), 10))
	case reflect.Float64:
		buf.WriteString(strconv.FormatFloat(value.Float(), 'f', -1, 64))
	default:
		switch value.Type() {
		case timeType:
			converted := value.Interface().(time.Time)
			buf.WriteString(strconv.FormatInt(converted.UTC().Unix(), 10))
		case byteSliceType:
			if !value.IsNil() {
				converted := value.Interface().([]byte)
				buf.WriteByte('"')
				if len(converted) < 1024 {
					// for small buffers, using Encode directly is much faster.
					dst := make([]byte, base64.StdEncoding.EncodedLen(len(converted)))
					base64.StdEncoding.Encode(dst, converted)
					buf.Write(dst)
				} else {
					// for large buffers, avoid unnecessary extra temporary
					// buffer space.
					enc := base64.NewEncoder(base64.StdEncoding, buf)
					enc.Write(converted)
					enc.Close()
				}
				buf.WriteByte('"')
			}
		default:
			return fmt.Errorf("unsupported JSON value %v (%s)", value.Interface(), value.Type())
		}
	}
	return nil
}

func writeString(s string, buf *bytes.Buffer) {
	buf.WriteByte('"')
	for _, r := range s {
		if r == '"' {
			buf.WriteString(`\"`)
		} else if r == '\\' {
			buf.WriteString(`\\`)
		} else if r == '\b' {
			buf.WriteString(`\b`)
		} else if r == '\f' {
			buf.WriteString(`\f`)
		} else if r == '\r' {
			buf.WriteString(`\r`)
		} else if r == '\t' {
			buf.WriteString(`\t`)
		} else if r == '\n' {
			buf.WriteString(`\n`)
		} else if r < 32 {
			fmt.Fprintf(buf, "\\u%0.4x", r)
		} else {
			buf.WriteRune(r)
		}
	}
	buf.WriteByte('"')
}

// Returns the reflection element of a value, if it is a pointer.
func elemOf(value reflect.Value) reflect.Value {
	for value.Kind() == reflect.Ptr {
		value = value.Elem()
	}
	return value
}
<commit_msg>Optimizing marshaling (#1038)<commit_after>// Package jsonutil provides JSON serialization of AWS requests and responses.
package jsonutil

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"reflect"
	"sort"
	"strconv"
	"time"

	"github.com/aws/aws-sdk-go/private/protocol"
)

var timeType = reflect.ValueOf(time.Time{}).Type()
var byteSliceType = reflect.ValueOf([]byte{}).Type()

// BuildJSON builds a JSON string for a given object v.
func BuildJSON(v interface{}) ([]byte, error) {
	var buf bytes.Buffer

	err := buildAny(reflect.ValueOf(v), &buf, "")
	return buf.Bytes(), err
}

func buildAny(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error {
	origVal := value
	value = reflect.Indirect(value)
	if !value.IsValid() {
		return nil
	}

	vtype := value.Type()

	t := tag.Get("type")
	if t == "" {
		switch vtype.Kind() {
		case reflect.Struct:
			// also it can't be a time object
			if value.Type() != timeType {
				t = "structure"
			}
		case reflect.Slice:
			// also it can't be a byte slice
			if _, ok := value.Interface().([]byte); !ok {
				t = "list"
			}
		case reflect.Map:
			t = "map"
		}
	}

	switch t {
	case "structure":
		if field, ok := vtype.FieldByName("_"); ok {
			tag = field.Tag
		}
		return buildStruct(value, buf, tag)
	case "list":
		return buildList(value, buf, tag)
	case "map":
		return buildMap(value, buf, tag)
	default:
		return buildScalar(origVal, buf, tag)
	}
}

func buildStruct(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error {
	if !value.IsValid() {
		return nil
	}

	// unwrap payloads
	if payload := tag.Get("payload"); payload != "" {
		field, _ := value.Type().FieldByName(payload)
		tag = field.Tag
		value = elemOf(value.FieldByName(payload))

		if !value.IsValid() {
			return nil
		}
	}

	buf.WriteByte('{')

	t := value.Type()
	first := true
	for i := 0; i < t.NumField(); i++ {
		member := value.Field(i)

		// This allocates the most memory.
		// Additionally, we cannot skip nil fields due to
		// idempotency auto filling.
		field := t.Field(i)

		if field.PkgPath != "" {
			continue // ignore unexported fields
		}
		if field.Tag.Get("json") == "-" {
			continue
		}
		if field.Tag.Get("location") != "" {
			continue // ignore non-body elements
		}
		if field.Tag.Get("ignore") != "" {
			continue
		}

		if protocol.CanSetIdempotencyToken(member, field) {
			token := protocol.GetIdempotencyToken()
			member = reflect.ValueOf(&token)
		}

		if (member.Kind() == reflect.Ptr || member.Kind() == reflect.Slice || member.Kind() == reflect.Map) && member.IsNil() {
			continue // ignore unset fields
		}

		if first {
			first = false
		} else {
			buf.WriteByte(',')
		}

		// figure out what this field is called
		name := field.Name
		if locName := field.Tag.Get("locationName"); locName != "" {
			name = locName
		}

		writeString(name, buf)
		buf.WriteString(`:`)

		err := buildAny(member, buf, field.Tag)
		if err != nil {
			return err
		}

	}

	buf.WriteString("}")

	return nil
}

func buildList(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error {
	buf.WriteString("[")

	for i := 0; i < value.Len(); i++ {
		buildAny(value.Index(i), buf, "")

		if i < value.Len()-1 {
			buf.WriteString(",")
		}
	}

	buf.WriteString("]")

	return nil
}

type sortedValues []reflect.Value

func (sv sortedValues) Len() int           { return len(sv) }
func (sv sortedValues) Swap(i, j int)      { sv[i], sv[j] = sv[j], sv[i] }
func (sv sortedValues) Less(i, j int) bool { return sv[i].String() < sv[j].String() }

func buildMap(value reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error {
	buf.WriteString("{")

	sv := sortedValues(value.MapKeys())
	sort.Sort(sv)

	for i, k := range sv {
		if i > 0 {
			buf.WriteByte(',')
		}

		writeString(k.String(), buf)
		buf.WriteString(`:`)

		buildAny(value.MapIndex(k), buf, "")
	}

	buf.WriteString("}")

	return nil
}

func buildScalar(v reflect.Value, buf *bytes.Buffer, tag reflect.StructTag) error {
	// prevents allocation on the heap.
	scratch := [64]byte{}
	switch value := reflect.Indirect(v); value.Kind() {
	case reflect.String:
		writeString(value.String(), buf)
	case reflect.Bool:
		if value.Bool() {
			buf.WriteString("true")
		} else {
			buf.WriteString("false")
		}
	case reflect.Int64:
		buf.Write(strconv.AppendInt(scratch[:0], value.Int(), 10))
	case reflect.Float64:
		buf.Write(strconv.AppendFloat(scratch[:0], value.Float(), 'f', -1, 64))
	default:
		switch value.Type() {
		case timeType:
			converted := v.Interface().(*time.Time)

			buf.Write(strconv.AppendInt(scratch[:0], converted.UTC().Unix(), 10))
		case byteSliceType:
			if !value.IsNil() {
				converted := value.Interface().([]byte)
				buf.WriteByte('"')
				if len(converted) < 1024 {
					// for small buffers, using Encode directly is much faster.
					dst := make([]byte, base64.StdEncoding.EncodedLen(len(converted)))
					base64.StdEncoding.Encode(dst, converted)
					buf.Write(dst)
				} else {
					// for large buffers, avoid unnecessary extra temporary
					// buffer space.
					enc := base64.NewEncoder(base64.StdEncoding, buf)
					enc.Write(converted)
					enc.Close()
				}
				buf.WriteByte('"')
			}
		default:
			return fmt.Errorf("unsupported JSON value %v (%s)", value.Interface(), value.Type())
		}
	}
	return nil
}

var hex = "0123456789abcdef"

func writeString(s string, buf *bytes.Buffer) {
	buf.WriteByte('"')
	for i := 0; i < len(s); i++ {
		if s[i] == '"' {
			buf.WriteString(`\"`)
		} else if s[i] == '\\' {
			buf.WriteString(`\\`)
		} else if s[i] == '\b' {
			buf.WriteString(`\b`)
		} else if s[i] == '\f' {
			buf.WriteString(`\f`)
		} else if s[i] == '\r' {
			buf.WriteString(`\r`)
		} else if s[i] == '\t' {
			buf.WriteString(`\t`)
		} else if s[i] == '\n' {
			buf.WriteString(`\n`)
		} else if s[i] < 32 {
			buf.WriteString("\\u00")
			buf.WriteByte(hex[s[i]>>4])
			buf.WriteByte(hex[s[i]&0xF])
		} else {
			buf.WriteByte(s[i])
		}
	}
	buf.WriteByte('"')
}

// Returns the reflection element of a value, if it is a pointer.
func elemOf(value reflect.Value) reflect.Value {
	for value.Kind() == reflect.Ptr {
		value = value.Elem()
	}
	return value
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package gce

import (
	"code.google.com/p/google-api-go-client/compute/v1"
	"github.com/juju/errors"

	"github.com/juju/juju/instance"
	"github.com/juju/juju/network"
)

type environInstance struct {
	id         instance.Id
	env        *environ
	zone       string
	rootDiskMB uint64

	// TODO(ericsnow) rename this to "raw"?
	gce *compute.Instance
}

var _ instance.Instance = (*environInstance)(nil)

func newInstance(raw *compute.Instance, env *environ) *environInstance {
	inst := environInstance{
		id:   instance.Id(raw.Name),
		env:  env,
		zone: zoneName(raw),
	}
	inst.update(raw)
	return &inst
}

func (inst *environInstance) Id() instance.Id {
	return inst.id
}

func (inst *environInstance) Status() string {
	return inst.gce.Status
}

func (inst *environInstance) update(raw *compute.Instance) {
	inst.gce = raw
}

func (inst *environInstance) updateDisk(raw *compute.Instance) {
	attached := rootDisk(raw)
	if diskSize, ok := inst.diskSize(attached); ok {
		inst.rootDiskMB = diskSize
	}
}

func (inst *environInstance) diskSize(attached *compute.AttachedDisk) (uint64, bool) {
	diskSizeGB, err := diskSizeGB(attached)
	if err != nil {
		logger.Warningf("error while getting root disk size: %v", err)
		disk, err := inst.env.gce.disk(attached.Source)
		if err != nil {
			logger.Warningf("error while getting root disk: %v", err)
			// Leave it what it was.
			return 0, false
		}
		diskSizeGB = disk.SizeGb
	}
	return uint64(diskSizeGB) * 1024, true
}

func (inst *environInstance) Refresh() error {
	env := inst.env.getSnapshot()

	raw, err := env.gce.instance(inst.zone, string(inst.id))
	if err != nil {
		return errors.Trace(err)
	}

	inst.update(raw)
	return nil
}

func (inst *environInstance) Addresses() ([]network.Address, error) {
	var addresses []network.Address

	for _, netif := range inst.gce.NetworkInterfaces {
		// Add public addresses.
		for _, accessConfig := range netif.AccessConfigs {
			if accessConfig.NatIP == "" {
				continue
			}
			address := network.Address{
				Value: accessConfig.NatIP,
				Type:  network.IPv4Address,
				Scope: network.ScopePublic,
			}
			addresses = append(addresses, address)

		}

		// Add private address.
		if netif.NetworkIP == "" {
			continue
		}
		address := network.Address{
			Value: netif.NetworkIP,
			Type:  network.IPv4Address,
			Scope: network.ScopeCloudLocal,
		}
		addresses = append(addresses, address)
	}

	return addresses, nil
}

func findInst(id instance.Id, instances []instance.Instance) instance.Instance {
	for _, inst := range instances {
		if id == inst.Id() {
			return inst
		}
	}
	return nil
}

// firewall stuff

// OpenPorts opens the given ports on the instance, which
// should have been started with the given machine id.
func (inst *environInstance) OpenPorts(machineId string, ports []network.PortRange) error {
	env := inst.env.getSnapshot()
	err := env.openPorts(machineId, ports)
	return errors.Trace(err)
}

// ClosePorts closes the given ports on the instance, which
// should have been started with the given machine id.
func (inst *environInstance) ClosePorts(machineId string, ports []network.PortRange) error {
	env := inst.env.getSnapshot()
	err := env.closePorts(machineId, ports)
	return errors.Trace(err)
}

// Ports returns the set of ports open on the instance, which
// should have been started with the given machine id.
// The ports are returned as sorted by SortPorts.
func (inst *environInstance) Ports(machineId string) ([]network.PortRange, error) {
	env := inst.env.getSnapshot()
	ports, err := env.ports(machineId)
	return ports, errors.Trace(err)
}
<commit_msg>provider/gce: Adjust the firewall name to the full machine name.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package gce

import (
	"code.google.com/p/google-api-go-client/compute/v1"
	"github.com/juju/errors"

	"github.com/juju/juju/instance"
	"github.com/juju/juju/network"
	"github.com/juju/juju/provider/common"
)

type environInstance struct {
	id         instance.Id
	env        *environ
	zone       string
	rootDiskMB uint64

	// TODO(ericsnow) rename this to "raw"?
	gce *compute.Instance
}

var _ instance.Instance = (*environInstance)(nil)

func newInstance(raw *compute.Instance, env *environ) *environInstance {
	inst := environInstance{
		id:   instance.Id(raw.Name),
		env:  env,
		zone: zoneName(raw),
	}
	inst.update(raw)
	return &inst
}

func (inst *environInstance) Id() instance.Id {
	return inst.id
}

func (inst *environInstance) Status() string {
	return inst.gce.Status
}

func (inst *environInstance) update(raw *compute.Instance) {
	inst.gce = raw
}

func (inst *environInstance) updateDisk(raw *compute.Instance) {
	attached := rootDisk(raw)
	if diskSize, ok := inst.diskSize(attached); ok {
		inst.rootDiskMB = diskSize
	}
}

func (inst *environInstance) diskSize(attached *compute.AttachedDisk) (uint64, bool) {
	diskSizeGB, err := diskSizeGB(attached)
	if err != nil {
		logger.Warningf("error while getting root disk size: %v", err)
		disk, err := inst.env.gce.disk(attached.Source)
		if err != nil {
			logger.Warningf("error while getting root disk: %v", err)
			// Leave it what it was.
			return 0, false
		}
		diskSizeGB = disk.SizeGb
	}
	return uint64(diskSizeGB) * 1024, true
}

func (inst *environInstance) Refresh() error {
	env := inst.env.getSnapshot()

	raw, err := env.gce.instance(inst.zone, string(inst.id))
	if err != nil {
		return errors.Trace(err)
	}

	inst.update(raw)
	return nil
}

func (inst *environInstance) Addresses() ([]network.Address, error) {
	var addresses []network.Address

	for _, netif := range inst.gce.NetworkInterfaces {
		// Add public addresses.
		for _, accessConfig := range netif.AccessConfigs {
			if accessConfig.NatIP == "" {
				continue
			}
			address := network.Address{
				Value: accessConfig.NatIP,
				Type:  network.IPv4Address,
				Scope: network.ScopePublic,
			}
			addresses = append(addresses, address)

		}

		// Add private address.
		if netif.NetworkIP == "" {
			continue
		}
		address := network.Address{
			Value: netif.NetworkIP,
			Type:  network.IPv4Address,
			Scope: network.ScopeCloudLocal,
		}
		addresses = append(addresses, address)
	}

	return addresses, nil
}

func findInst(id instance.Id, instances []instance.Instance) instance.Instance {
	for _, inst := range instances {
		if id == inst.Id() {
			return inst
		}
	}
	return nil
}

// firewall stuff

// OpenPorts opens the given ports on the instance, which
// should have been started with the given machine id.
func (inst *environInstance) OpenPorts(machineId string, ports []network.PortRange) error {
	// TODO(ericsnow) Make sure machineId matches inst.Id()?
	name := common.MachineFullName(inst.env, machineId)
	env := inst.env.getSnapshot()
	err := env.openPorts(name, ports)
	return errors.Trace(err)
}

// ClosePorts closes the given ports on the instance, which
// should have been started with the given machine id.
func (inst *environInstance) ClosePorts(machineId string, ports []network.PortRange) error {
	name := common.MachineFullName(inst.env, machineId)
	env := inst.env.getSnapshot()
	err := env.closePorts(name, ports)
	return errors.Trace(err)
}

// Ports returns the set of ports open on the instance, which
// should have been started with the given machine id.
// The ports are returned as sorted by SortPorts.
func (inst *environInstance) Ports(machineId string) ([]network.PortRange, error) {
	name := common.MachineFullName(inst.env, machineId)
	env := inst.env.getSnapshot()
	ports, err := env.ports(name)
	return ports, errors.Trace(err)
}
<|endoftext|>