<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"errors"
	"fmt"
	"reflect"
	"runtime"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	k8sRuntime "k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/diff"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apis/componentconfig"
	"k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
	"k8s.io/kubernetes/pkg/util/configz"
	"k8s.io/kubernetes/pkg/util/iptables"
	utilpointer "k8s.io/kubernetes/pkg/util/pointer"
)

type fakeNodeInterface struct {
	node api.Node
}

func (fake *fakeNodeInterface) Get(hostname string, options metav1.GetOptions) (*api.Node, error) {
	return &fake.node, nil
}

type fakeIPTablesVersioner struct {
	version string // what to return
	err     error  // what to return
}

func (fake *fakeIPTablesVersioner) GetVersion() (string, error) {
	return fake.version, fake.err
}

type fakeKernelCompatTester struct {
	ok bool
}

func (fake *fakeKernelCompatTester) IsCompatible() error {
	if !fake.ok {
		return fmt.Errorf("error")
	}
	return nil
}

func Test_getProxyMode(t *testing.T) {
	if runtime.GOOS != "linux" {
		t.Skip("skipping on non-Linux")
	}
	var cases = []struct {
		flag            string
		annotationKey   string
		annotationVal   string
		iptablesVersion string
		kernelCompat    bool
		iptablesError   error
		expected        string
	}{
		{ // flag says userspace
			flag:     "userspace",
			expected: proxyModeUserspace,
		},
		{ // flag says iptables, error detecting version
			flag:          "iptables",
			iptablesError: fmt.Errorf("oops!"),
			expected:      proxyModeUserspace,
		},
		{ // flag says iptables, version too low
			flag:            "iptables",
			iptablesVersion: "0.0.0",
			expected:        proxyModeUserspace,
		},
		{ // flag says iptables, version ok, kernel not compatible
			flag:            "iptables",
			iptablesVersion: iptables.MinCheckVersion,
			kernelCompat:    false,
			expected:        proxyModeUserspace,
		},
		{ // flag says iptables, version ok, kernel is compatible
			flag:            "iptables",
			iptablesVersion: iptables.MinCheckVersion,
			kernelCompat:    true,
			expected:        proxyModeIPTables,
		},
		{ // detect, error
			flag:          "",
			iptablesError: fmt.Errorf("oops!"),
			expected:      proxyModeUserspace,
		},
		{ // detect, version too low
			flag:            "",
			iptablesVersion: "0.0.0",
			expected:        proxyModeUserspace,
		},
		{ // detect, version ok, kernel not compatible
			flag:            "",
			iptablesVersion: iptables.MinCheckVersion,
			kernelCompat:    false,
			expected:        proxyModeUserspace,
		},
		{ // detect, version ok, kernel is compatible
			flag:            "",
			iptablesVersion: iptables.MinCheckVersion,
			kernelCompat:    true,
			expected:        proxyModeIPTables,
		},
	}
	for i, c := range cases {
		versioner := &fakeIPTablesVersioner{c.iptablesVersion, c.iptablesError}
		kcompater := &fakeKernelCompatTester{c.kernelCompat}
		r := getProxyMode(c.flag, versioner, kcompater)
		if r != c.expected {
			t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r)
		}
	}
}

// TestNewOptionsFailures tests failure modes for NewOptions()
func TestNewOptionsFailures(t *testing.T) {

	// Create a fake scheme builder that generates an error
	errString := fmt.Sprintf("Simulated error")
	genError := func(scheme *k8sRuntime.Scheme) error {
		return errors.New(errString)
	}
	fakeSchemeBuilder := k8sRuntime.NewSchemeBuilder(genError)

	simulatedErrorTest := func(target string) {
		var addToScheme *func(s *k8sRuntime.Scheme) error
		if target == "componentconfig" {
			addToScheme = &componentconfig.AddToScheme
		} else {
			addToScheme = &v1alpha1.AddToScheme
		}
		restoreValue := *addToScheme
		restore := func() {
			*addToScheme = restoreValue
		}
		defer restore()
		*addToScheme = fakeSchemeBuilder.AddToScheme
		_, err := NewOptions()
		assert.Error(t, err, fmt.Sprintf("Simulated error in component %s", target))
	}

	// Simulate errors in calls to AddToScheme()
	faultTargets := []string{"componentconfig", "v1alpha1"}
	for _, target := range faultTargets {
		simulatedErrorTest(target)
	}
}

// This test verifies that NewProxyServer does not crash when CleanupAndExit is true.
func TestProxyServerWithCleanupAndExit(t *testing.T) {
	// Each bind address below is a separate test case
	bindAddresses := []string{
		"0.0.0.0",
		"2001:db8::1",
	}
	for _, addr := range bindAddresses {
		options, err := NewOptions()
		if err != nil {
			t.Fatalf("Unexpected error with address %s: %v", addr, err)
		}

		options.config = &componentconfig.KubeProxyConfiguration{
			BindAddress: addr,
		}
		options.CleanupAndExit = true

		proxyserver, err := NewProxyServer(options.config, options.CleanupAndExit, options.scheme, options.master)

		assert.Nil(t, err, "unexpected error in NewProxyServer, addr: %s", addr)
		assert.NotNil(t, proxyserver, "nil proxy server obj, addr: %s", addr)
		assert.NotNil(t, proxyserver.IptInterface, "nil iptables intf, addr: %s", addr)
		assert.True(t, proxyserver.CleanupAndExit, "false CleanupAndExit, addr: %s", addr)

		// Clean up config for next test case
		configz.Delete("componentconfig")
	}
}

func TestGetConntrackMax(t *testing.T) {
	ncores := runtime.NumCPU()
	testCases := []struct {
		min        int32
		max        int32
		maxPerCore int32
		expected   int
		err        string
	}{
		{
			expected: 0,
		},
		{
			max:      12345,
			expected: 12345,
		},
		{
			max:        12345,
			maxPerCore: 67890,
			expected:   -1,
			err:        "mutually exclusive",
		},
		{
			maxPerCore: 67890, // use this if Max is 0
			min:        1,     // avoid 0 default
			expected:   67890 * ncores,
		},
		{
			maxPerCore: 1, // ensure that Min is considered
			min:        123456,
			expected:   123456,
		},
		{
			maxPerCore: 0, // leave system setting
			min:        123456,
			expected:   0,
		},
	}

	for i, tc := range testCases {
		cfg := componentconfig.KubeProxyConntrackConfiguration{
			Min:        tc.min,
			Max:        tc.max,
			MaxPerCore: tc.maxPerCore,
		}
		x, e := getConntrackMax(cfg)
		if e != nil {
			if tc.err == "" {
				t.Errorf("[%d] unexpected error: %v", i, e)
			} else if !strings.Contains(e.Error(), tc.err) {
				t.Errorf("[%d] expected an error containing %q: %v", i, tc.err, e)
			}
		} else if x != tc.expected {
			t.Errorf("[%d] expected %d, got %d", i, tc.expected, x)
		}
	}
}

// TestLoadConfig tests proper operation of loadConfig()
func TestLoadConfig(t *testing.T) {

	yamlTemplate := `apiVersion: componentconfig/v1alpha1
bindAddress: %s
clientConnection:
  acceptContentTypes: "abc"
  burst: 100
  contentType: content-type
  kubeconfig: "/path/to/kubeconfig"
  qps: 7
clusterCIDR: "%s"
configSyncPeriod: 15s
conntrack:
  max: 4
  maxPerCore: 2
  min: 1
  tcpCloseWaitTimeout: 10s
  tcpEstablishedTimeout: 20s
featureGates: "all"
healthzBindAddress: "%s"
hostnameOverride: "foo"
iptables:
  masqueradeAll: true
  masqueradeBit: 17
  minSyncPeriod: 10s
  syncPeriod: 60s
ipvs:
  minSyncPeriod: 10s
  syncPeriod: 60s
kind: KubeProxyConfiguration
metricsBindAddress: "%s"
mode: "iptables"
oomScoreAdj: 17
portRange: "2-7"
resourceContainer: /foo
udpTimeoutMilliseconds: 123ms
`

	testCases := []struct {
		name               string
		bindAddress        string
		clusterCIDR        string
		healthzBindAddress string
		metricsBindAddress string
	}{
		{
			name:               "IPv4 config",
			bindAddress:        "9.8.7.6",
			clusterCIDR:        "1.2.3.0/24",
			healthzBindAddress: "1.2.3.4:12345",
			metricsBindAddress: "2.3.4.5:23456",
		},
		{
			name:               "IPv6 config",
			bindAddress:        "2001:db8::1",
			clusterCIDR:        "fd00:1::0/64",
			healthzBindAddress: "[fd00:1::5]:12345",
			metricsBindAddress: "[fd00:2::5]:23456",
		},
	}

	for _, tc := range testCases {
		expected := &componentconfig.KubeProxyConfiguration{
			BindAddress: tc.bindAddress,
			ClientConnection: componentconfig.ClientConnectionConfiguration{
				AcceptContentTypes: "abc",
				Burst:              100,
				ContentType:        "content-type",
				KubeConfigFile:     "/path/to/kubeconfig",
				QPS:                7,
			},
			ClusterCIDR:      tc.clusterCIDR,
			ConfigSyncPeriod: metav1.Duration{Duration: 15 * time.Second},
			Conntrack: componentconfig.KubeProxyConntrackConfiguration{
				Max:                   4,
				MaxPerCore:            2,
				Min:                   1,
				TCPCloseWaitTimeout:   metav1.Duration{Duration: 10 * time.Second},
				TCPEstablishedTimeout: metav1.Duration{Duration: 20 * time.Second},
			},
			FeatureGates:       "all",
			HealthzBindAddress: tc.healthzBindAddress,
			HostnameOverride:   "foo",
			IPTables: componentconfig.KubeProxyIPTablesConfiguration{
				MasqueradeAll: true,
				MasqueradeBit: utilpointer.Int32Ptr(17),
				MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second},
				SyncPeriod:    metav1.Duration{Duration: 60 * time.Second},
			},
			IPVS: componentconfig.KubeProxyIPVSConfiguration{
				MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second},
				SyncPeriod:    metav1.Duration{Duration: 60 * time.Second},
			},
			MetricsBindAddress: tc.metricsBindAddress,
			Mode:               "iptables",
			// TODO: IPVS
			OOMScoreAdj:       utilpointer.Int32Ptr(17),
			PortRange:         "2-7",
			ResourceContainer: "/foo",
			UDPIdleTimeout:    metav1.Duration{Duration: 123 * time.Millisecond},
		}

		options, err := NewOptions()
		assert.NoError(t, err, "unexpected error for %s: %v", tc.name, err)

		yaml := fmt.Sprintf(
			yamlTemplate, tc.bindAddress, tc.clusterCIDR,
			tc.healthzBindAddress, tc.metricsBindAddress)
		config, err := options.loadConfig([]byte(yaml))
		assert.NoError(t, err, "unexpected error for %s: %v", tc.name, err)
		if !reflect.DeepEqual(expected, config) {
			t.Fatalf("unexpected config for %s test, diff = %s", tc.name, diff.ObjectDiff(config, expected))
		}
	}
}

// TestLoadConfigFailures tests failure modes for loadConfig()
func TestLoadConfigFailures(t *testing.T) {
	testCases := []struct {
		name   string
		config string
		expErr string
	}{
		{
			name:   "Decode error test",
			config: "Twas bryllyg, and ye slythy toves",
			expErr: "could not find expected ':'",
		},
		{
			name:   "Bad config type test",
			config: "kind: KubeSchedulerConfiguration",
			expErr: "unexpected config type",
		},
	}
	version := "apiVersion: componentconfig/v1alpha1"
	for _, tc := range testCases {
		options, _ := NewOptions()
		config := fmt.Sprintf("%s\n%s", version, tc.config)
		_, err := options.loadConfig([]byte(config))
		if assert.Error(t, err, tc.name) {
			assert.Contains(t, err.Error(), tc.expErr, tc.name)
		}
	}
}
<commit_msg>ut test load ipvs config<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"errors"
	"fmt"
	"reflect"
	"runtime"
	"strings"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	k8sRuntime "k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/util/diff"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apis/componentconfig"
	"k8s.io/kubernetes/pkg/apis/componentconfig/v1alpha1"
	"k8s.io/kubernetes/pkg/util/configz"
	"k8s.io/kubernetes/pkg/util/iptables"
	utilpointer "k8s.io/kubernetes/pkg/util/pointer"
)

type fakeNodeInterface struct {
	node api.Node
}

func (fake *fakeNodeInterface) Get(hostname string, options metav1.GetOptions) (*api.Node, error) {
	return &fake.node, nil
}

type fakeIPTablesVersioner struct {
	version string // what to return
	err     error  // what to return
}

func (fake *fakeIPTablesVersioner) GetVersion() (string, error) {
	return fake.version, fake.err
}

type fakeKernelCompatTester struct {
	ok bool
}

func (fake *fakeKernelCompatTester) IsCompatible() error {
	if !fake.ok {
		return fmt.Errorf("error")
	}
	return nil
}

func Test_getProxyMode(t *testing.T) {
	if runtime.GOOS != "linux" {
		t.Skip("skipping on non-Linux")
	}
	var cases = []struct {
		flag            string
		annotationKey   string
		annotationVal   string
		iptablesVersion string
		kernelCompat    bool
		iptablesError   error
		expected        string
	}{
		{ // flag says userspace
			flag:     "userspace",
			expected: proxyModeUserspace,
		},
		{ // flag says iptables, error detecting version
			flag:          "iptables",
			iptablesError: fmt.Errorf("oops!"),
			expected:      proxyModeUserspace,
		},
		{ // flag says iptables, version too low
			flag:            "iptables",
			iptablesVersion: "0.0.0",
			expected:        proxyModeUserspace,
		},
		{ // flag says iptables, version ok, kernel not compatible
			flag:            "iptables",
			iptablesVersion: iptables.MinCheckVersion,
			kernelCompat:    false,
			expected:        proxyModeUserspace,
		},
		{ // flag says iptables, version ok, kernel is compatible
			flag:            "iptables",
			iptablesVersion: iptables.MinCheckVersion,
			kernelCompat:    true,
			expected:        proxyModeIPTables,
		},
		{ // detect, error
			flag:          "",
			iptablesError: fmt.Errorf("oops!"),
			expected:      proxyModeUserspace,
		},
		{ // detect, version too low
			flag:            "",
			iptablesVersion: "0.0.0",
			expected:        proxyModeUserspace,
		},
		{ // detect, version ok, kernel not compatible
			flag:            "",
			iptablesVersion: iptables.MinCheckVersion,
			kernelCompat:    false,
			expected:        proxyModeUserspace,
		},
		{ // detect, version ok, kernel is compatible
			flag:            "",
			iptablesVersion: iptables.MinCheckVersion,
			kernelCompat:    true,
			expected:        proxyModeIPTables,
		},
	}
	for i, c := range cases {
		versioner := &fakeIPTablesVersioner{c.iptablesVersion, c.iptablesError}
		kcompater := &fakeKernelCompatTester{c.kernelCompat}
		r := getProxyMode(c.flag, versioner, kcompater)
		if r != c.expected {
			t.Errorf("Case[%d] Expected %q, got %q", i, c.expected, r)
		}
	}
}

// TestNewOptionsFailures tests failure modes for NewOptions()
func TestNewOptionsFailures(t *testing.T) {

	// Create a fake scheme builder that generates an error
	errString := fmt.Sprintf("Simulated error")
	genError := func(scheme *k8sRuntime.Scheme) error {
		return errors.New(errString)
	}
	fakeSchemeBuilder := k8sRuntime.NewSchemeBuilder(genError)

	simulatedErrorTest := func(target string) {
		var addToScheme *func(s *k8sRuntime.Scheme) error
		if target == "componentconfig" {
			addToScheme = &componentconfig.AddToScheme
		} else {
			addToScheme = &v1alpha1.AddToScheme
		}
		restoreValue := *addToScheme
		restore := func() {
			*addToScheme = restoreValue
		}
		defer restore()
		*addToScheme = fakeSchemeBuilder.AddToScheme
		_, err := NewOptions()
		assert.Error(t, err, fmt.Sprintf("Simulated error in component %s", target))
	}

	// Simulate errors in calls to AddToScheme()
	faultTargets := []string{"componentconfig", "v1alpha1"}
	for _, target := range faultTargets {
		simulatedErrorTest(target)
	}
}

// This test verifies that NewProxyServer does not crash when CleanupAndExit is true.
func TestProxyServerWithCleanupAndExit(t *testing.T) {
	// Each bind address below is a separate test case
	bindAddresses := []string{
		"0.0.0.0",
		"2001:db8::1",
	}
	for _, addr := range bindAddresses {
		options, err := NewOptions()
		if err != nil {
			t.Fatalf("Unexpected error with address %s: %v", addr, err)
		}

		options.config = &componentconfig.KubeProxyConfiguration{
			BindAddress: addr,
		}
		options.CleanupAndExit = true

		proxyserver, err := NewProxyServer(options.config, options.CleanupAndExit, options.scheme, options.master)

		assert.Nil(t, err, "unexpected error in NewProxyServer, addr: %s", addr)
		assert.NotNil(t, proxyserver, "nil proxy server obj, addr: %s", addr)
		assert.NotNil(t, proxyserver.IptInterface, "nil iptables intf, addr: %s", addr)
		assert.True(t, proxyserver.CleanupAndExit, "false CleanupAndExit, addr: %s", addr)

		// Clean up config for next test case
		configz.Delete("componentconfig")
	}
}

func TestGetConntrackMax(t *testing.T) {
	ncores := runtime.NumCPU()
	testCases := []struct {
		min        int32
		max        int32
		maxPerCore int32
		expected   int
		err        string
	}{
		{
			expected: 0,
		},
		{
			max:      12345,
			expected: 12345,
		},
		{
			max:        12345,
			maxPerCore: 67890,
			expected:   -1,
			err:        "mutually exclusive",
		},
		{
			maxPerCore: 67890, // use this if Max is 0
			min:        1,     // avoid 0 default
			expected:   67890 * ncores,
		},
		{
			maxPerCore: 1, // ensure that Min is considered
			min:        123456,
			expected:   123456,
		},
		{
			maxPerCore: 0, // leave system setting
			min:        123456,
			expected:   0,
		},
	}

	for i, tc := range testCases {
		cfg := componentconfig.KubeProxyConntrackConfiguration{
			Min:        tc.min,
			Max:        tc.max,
			MaxPerCore: tc.maxPerCore,
		}
		x, e := getConntrackMax(cfg)
		if e != nil {
			if tc.err == "" {
				t.Errorf("[%d] unexpected error: %v", i, e)
			} else if !strings.Contains(e.Error(), tc.err) {
				t.Errorf("[%d] expected an error containing %q: %v", i, tc.err, e)
			}
		} else if x != tc.expected {
			t.Errorf("[%d] expected %d, got %d", i, tc.expected, x)
		}
	}
}

// TestLoadConfig tests proper operation of loadConfig()
func TestLoadConfig(t *testing.T) {

	yamlTemplate := `apiVersion: componentconfig/v1alpha1
bindAddress: %s
clientConnection:
  acceptContentTypes: "abc"
  burst: 100
  contentType: content-type
  kubeconfig: "/path/to/kubeconfig"
  qps: 7
clusterCIDR: "%s"
configSyncPeriod: 15s
conntrack:
  max: 4
  maxPerCore: 2
  min: 1
  tcpCloseWaitTimeout: 10s
  tcpEstablishedTimeout: 20s
featureGates: "all"
healthzBindAddress: "%s"
hostnameOverride: "foo"
iptables:
  masqueradeAll: true
  masqueradeBit: 17
  minSyncPeriod: 10s
  syncPeriod: 60s
ipvs:
  minSyncPeriod: 10s
  syncPeriod: 60s
kind: KubeProxyConfiguration
metricsBindAddress: "%s"
mode: "%s"
oomScoreAdj: 17
portRange: "2-7"
resourceContainer: /foo
udpTimeoutMilliseconds: 123ms
`

	testCases := []struct {
		name               string
		mode               string
		bindAddress        string
		clusterCIDR        string
		healthzBindAddress string
		metricsBindAddress string
	}{
		{
			name:               "iptables mode, IPv4 config",
			mode:               "iptables",
			bindAddress:        "9.8.7.6",
			clusterCIDR:        "1.2.3.0/24",
			healthzBindAddress: "1.2.3.4:12345",
			metricsBindAddress: "2.3.4.5:23456",
		},
		{
			name:               "ipvs mode, IPv6 config",
			mode:               "ipvs",
			bindAddress:        "2001:db8::1",
			clusterCIDR:        "fd00:1::0/64",
			healthzBindAddress: "[fd00:1::5]:12345",
			metricsBindAddress: "[fd00:2::5]:23456",
		},
	}

	for _, tc := range testCases {
		expected := &componentconfig.KubeProxyConfiguration{
			BindAddress: tc.bindAddress,
			ClientConnection: componentconfig.ClientConnectionConfiguration{
				AcceptContentTypes: "abc",
				Burst:              100,
				ContentType:        "content-type",
				KubeConfigFile:     "/path/to/kubeconfig",
				QPS:                7,
			},
			ClusterCIDR:      tc.clusterCIDR,
			ConfigSyncPeriod: metav1.Duration{Duration: 15 * time.Second},
			Conntrack: componentconfig.KubeProxyConntrackConfiguration{
				Max:                   4,
				MaxPerCore:            2,
				Min:                   1,
				TCPCloseWaitTimeout:   metav1.Duration{Duration: 10 * time.Second},
				TCPEstablishedTimeout: metav1.Duration{Duration: 20 * time.Second},
			},
			FeatureGates:       "all",
			HealthzBindAddress: tc.healthzBindAddress,
			HostnameOverride:   "foo",
			IPTables: componentconfig.KubeProxyIPTablesConfiguration{
				MasqueradeAll: true,
				MasqueradeBit: utilpointer.Int32Ptr(17),
				MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second},
				SyncPeriod:    metav1.Duration{Duration: 60 * time.Second},
			},
			IPVS: componentconfig.KubeProxyIPVSConfiguration{
				MinSyncPeriod: metav1.Duration{Duration: 10 * time.Second},
				SyncPeriod:    metav1.Duration{Duration: 60 * time.Second},
			},
			MetricsBindAddress: tc.metricsBindAddress,
			Mode:               componentconfig.ProxyMode(tc.mode),
			OOMScoreAdj:        utilpointer.Int32Ptr(17),
			PortRange:          "2-7",
			ResourceContainer:  "/foo",
			UDPIdleTimeout:     metav1.Duration{Duration: 123 * time.Millisecond},
		}

		options, err := NewOptions()
		assert.NoError(t, err, "unexpected error for %s: %v", tc.name, err)

		yaml := fmt.Sprintf(
			yamlTemplate, tc.bindAddress, tc.clusterCIDR,
			tc.healthzBindAddress, tc.metricsBindAddress, tc.mode)
		config, err := options.loadConfig([]byte(yaml))
		assert.NoError(t, err, "unexpected error for %s: %v", tc.name, err)
		if !reflect.DeepEqual(expected, config) {
			t.Fatalf("unexpected config for %s test, diff = %s", tc.name, diff.ObjectDiff(config, expected))
		}
	}
}

// TestLoadConfigFailures tests failure modes for loadConfig()
func TestLoadConfigFailures(t *testing.T) {
	testCases := []struct {
		name   string
		config string
		expErr string
	}{
		{
			name:   "Decode error test",
			config: "Twas bryllyg, and ye slythy toves",
			expErr: "could not find expected ':'",
		},
		{
			name:   "Bad config type test",
			config: "kind: KubeSchedulerConfiguration",
			expErr: "unexpected config type",
		},
	}
	version := "apiVersion: componentconfig/v1alpha1"
	for _, tc := range testCases {
		options, _ := NewOptions()
		config := fmt.Sprintf("%s\n%s", version, tc.config)
		_, err := options.loadConfig([]byte(config))
		if assert.Error(t, err, tc.name) {
			assert.Contains(t, err.Error(), tc.expErr, tc.name)
		}
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"bytes"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"strings"
	"text/template"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/spf13/cobra"
	"github.com/tleyden/deepstyle/deepstylelib"
	"github.com/tleyden/go-couch"
)

const (
	DesignDocName = "unprocessed_jobs"
	ViewName      = "unprocessed_jobs"
)

// publish_cloudwatch_metricsCmd respresents the publish_cloudwatch_metrics command
var publish_cloudwatch_metricsCmd = &cobra.Command{
	Use:   "publish_cloudwatch_metrics",
	Short: "Publish queue metrics to CloudWatch in order to trigger auto-scale alarms",
	Long:  `Publish queue metrics to CloudWatch in order to trigger auto-scale alarms`,
	Run: func(cmd *cobra.Command, args []string) {

		if err := cmd.ParseFlags(args); err != nil {
			log.Printf("err: %v", err)
			return
		}

		awsKey := cmd.Flag("aws_key")

		awsKeyVal := awsKey.Value.String()

		log.Printf("awsKey: %v", awsKeyVal)
		if awsKeyVal == "" {
			log.Printf("ERROR: Missing: --aws_key.\n  %v", cmd.UsageString())
			return
		}

		urlFlag := cmd.Flag("admin_url")

		urlVal := urlFlag.Value.String()
		if urlVal == "" {
			log.Printf("ERROR: Missing: --url.\n  %v", cmd.UsageString())
			return
		}

		err := addCloudWatchMetrics(urlVal)
		if err != nil {
			log.Printf("ERROR: %v", err)
			return
		}

	},
}

func numJobsReadOrBeingProcessed(syncGwAdminUrl string) (metricValue float64, err error) {

	// try to query view
	//    curl localhost:4985/deepstyle/_design/unprocessed_jobs/_view/unprocessed_jobs
	// if we get a 404, then install the view and then requery

	// if it has a trailing slash, remove it
	rawUrl := strings.TrimSuffix(syncGwAdminUrl, "/")

	// url validation
	url, err := url.Parse(rawUrl)
	if err != nil {
		return 0.0, err
	}

	db, err := couch.Connect(url.String())
	if err != nil {
		return 0.0, fmt.Errorf("Error connecting to db: %v.  Err: %v", syncGwAdminUrl, err)
	}
	log.Printf("connected to db: %v", db)

	viewUrl := fmt.Sprintf("_design/%v/_view/%v", DesignDocName, ViewName)
	options := map[string]interface{}{}
	output := map[string]interface{}{}
	err = db.Query(viewUrl, options, &output)
	if err != nil {
		if strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "not_found") {
			// the view doesn't exist yet, attempt to install view
			if errInstallView := installView(rawUrl); errInstallView != nil {
				// failed to install view, give up
				return 0.0, errInstallView
			}
			// now retry
			errInner := db.Query(viewUrl, options, &output)
			if errInner != nil {
				// failed again, give up
				return 0.0, errInner
			}
		} else {
			return 0.0, err
		}
	}
	log.Printf("output: %+v", output)

	// TODO: count the number of rows / keys in the output and
	// convert to a float and return that value

	return 0.0, nil
}

type ViewParams struct {
	JobDocType string
	JobState1  string
	JobState2  string
	JobState3  string
}

func installView(syncGwAdminUrl string) error {

	viewJsonTemplate := `
{
    "views":{
        "unprocessed_jobs":{
            "map":"function (doc, meta) { if (doc.type != '{{.JobDocType}}') { return; } emit(meta.id, meta.id); }"
        }
    }
}
`

	viewParams := ViewParams{
		JobDocType: deepstylelib.Job,
		JobState1:  deepstylelib.StateNotReadyToProcess,
		JobState2:  deepstylelib.StateReadyToProcess,
		JobState3:  deepstylelib.StateBeingProcessed,
	}
	tmpl, err := template.New("UnprocessedJobsView").Parse(viewJsonTemplate)
	if err != nil {
		return err
	}

	var buffer bytes.Buffer // A Buffer needs no initialization.

	err = tmpl.Execute(&buffer, viewParams)
	if err != nil {
		return err
	}

	log.Printf("installView called")

	// curl -X PUT -H "Content-type: application/json" localhost:4985/todolite/_design/all_lists --data @testview
	viewUrl := fmt.Sprintf("%v/_design/%v", syncGwAdminUrl, DesignDocName)

	bufferBytes := buffer.Bytes()
	log.Printf("view: %v", string(bufferBytes))

	req, err := http.NewRequest("PUT", viewUrl, bytes.NewReader(bufferBytes))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	log.Printf("put view resp: %v", resp)

	return nil

}

func addCloudWatchMetrics(syncGwAdminUrl string) error {

	metricValue, err := numJobsReadOrBeingProcessed(syncGwAdminUrl)
	if err != nil {
		return err
	}

	// TODO: push something to CloudWatch
	// CLI example:
	//   aws cloudwatch put-metric-data
	//     --metric-name PageViewCount
	//      --namespace "MyService"
	//     --value 2
	//     --timestamp 2014-02-14T12:00:00.000Z
	cloudwatchSvc := cloudwatch.New(session.New(), &aws.Config{Region: aws.String("us-east-1")})

	log.Printf("cloudwatchSvc: %v", cloudwatchSvc)

	metricName := "NumJobsReadyOrBeingProcessed"
	timestamp := time.Now()

	metricDatum := &cloudwatch.MetricDatum{
		MetricName: &metricName,
		Value:      &metricValue,
		Timestamp:  &timestamp,
	}

	metricDatumSlice := []*cloudwatch.MetricDatum{metricDatum}
	namespace := "DeepStyleQueue"

	putMetricDataInput := &cloudwatch.PutMetricDataInput{
		MetricData: metricDatumSlice,
		Namespace:  &namespace,
	}

	out, err := cloudwatchSvc.PutMetricData(putMetricDataInput)
	if err != nil {
		log.Printf("ERROR adding metric data  %v", err)
		return err
	}
	log.Printf("Metric data output: %v", out)

	return nil

}

func init() {
	RootCmd.AddCommand(publish_cloudwatch_metricsCmd)

	// Here you will define your flags and configuration settings

	// Cobra supports Persistent Flags which will work for this command and all subcommands

	publish_cloudwatch_metricsCmd.PersistentFlags().String("aws_key", "", "AWS Key")

	publish_cloudwatch_metricsCmd.PersistentFlags().String("admin_url", "", "Sync Gateway Admin URL")

	// Cobra supports local flags which will only run when this command is called directly
	// publish_cloudwatch_metricsCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle" )

}
<commit_msg>returns unprocessed docs<commit_after>package cmd

import (
	"bytes"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"strings"
	"text/template"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatch"
	"github.com/spf13/cobra"
	"github.com/tleyden/deepstyle/deepstylelib"
	"github.com/tleyden/go-couch"
)

const (
	DesignDocName = "unprocessed_jobs"
	ViewName      = "unprocessed_jobs"
)

// publish_cloudwatch_metricsCmd respresents the publish_cloudwatch_metrics command
var publish_cloudwatch_metricsCmd = &cobra.Command{
	Use:   "publish_cloudwatch_metrics",
	Short: "Publish queue metrics to CloudWatch in order to trigger auto-scale alarms",
	Long:  `Publish queue metrics to CloudWatch in order to trigger auto-scale alarms`,
	Run: func(cmd *cobra.Command, args []string) {

		if err := cmd.ParseFlags(args); err != nil {
			log.Printf("err: %v", err)
			return
		}

		awsKey := cmd.Flag("aws_key")

		awsKeyVal := awsKey.Value.String()

		log.Printf("awsKey: %v", awsKeyVal)
		if awsKeyVal == "" {
			log.Printf("ERROR: Missing: --aws_key.\n  %v", cmd.UsageString())
			return
		}

		urlFlag := cmd.Flag("admin_url")

		urlVal := urlFlag.Value.String()
		if urlVal == "" {
			log.Printf("ERROR: Missing: --url.\n  %v", cmd.UsageString())
			return
		}

		err := addCloudWatchMetrics(urlVal)
		if err != nil {
			log.Printf("ERROR: %v", err)
			return
		}

	},
}

func numJobsReadOrBeingProcessed(syncGwAdminUrl string) (metricValue float64, err error) {

	// try to query view
	//    curl localhost:4985/deepstyle/_design/unprocessed_jobs/_view/unprocessed_jobs
	// if we get a 404, then install the view and then requery

	// if it has a trailing slash, remove it
	rawUrl := strings.TrimSuffix(syncGwAdminUrl, "/")

	// url validation
	url, err := url.Parse(rawUrl)
	if err != nil {
		return 0.0, err
	}

	db, err := couch.Connect(url.String())
	if err != nil {
		return 0.0, fmt.Errorf("Error connecting to db: %v.  Err: %v", syncGwAdminUrl, err)
	}
	log.Printf("connected to db: %v", db)

	viewUrl := fmt.Sprintf("_design/%v/_view/%v", DesignDocName, ViewName)
	options := map[string]interface{}{}
	output := map[string]interface{}{}
	err = db.Query(viewUrl, options, &output)
	if err != nil {
		if strings.Contains(err.Error(), "404") || strings.Contains(err.Error(), "not_found") {
			// the view doesn't exist yet, attempt to install view
			if errInstallView := installView(rawUrl); errInstallView != nil {
				// failed to install view, give up
				return 0.0, errInstallView
			}
			// now retry
			errInner := db.Query(viewUrl, options, &output)
			if errInner != nil {
				// failed again, give up
				return 0.0, errInner
			}
		} else {
			return 0.0, err
		}
	}
	log.Printf("output: %+v", output)

	// TODO: count the number of rows / keys in the output and
	// convert to a float and return that value

	return 0.0, nil
}

type ViewParams struct {
	JobDocType string
	JobState1  string
	JobState2  string
	JobState3  string
}

func installView(syncGwAdminUrl string) error {

	viewJsonTemplate := `
{
    "views":{
        "unprocessed_jobs":{
            "map":"function (doc, meta) { if (doc.type != '{{.JobDocType}}') { return; } if (doc.state == '{{.JobState1}}' || doc.state == '{{.JobState2}}' || doc.state == '{{.JobState3}}') { emit(meta.id, meta.id); }}"
        }
    }
}
`

	viewParams := ViewParams{
		JobDocType: deepstylelib.Job,
		JobState1:  deepstylelib.StateNotReadyToProcess,
		JobState2:  deepstylelib.StateReadyToProcess,
		JobState3:  deepstylelib.StateBeingProcessed,
	}
	tmpl, err := template.New("UnprocessedJobsView").Parse(viewJsonTemplate)
	if err != nil {
		return err
	}

	var buffer bytes.Buffer // A Buffer needs no initialization.

	err = tmpl.Execute(&buffer, viewParams)
	if err != nil {
		return err
	}

	log.Printf("installView called")

	// curl -X PUT -H "Content-type: application/json" localhost:4985/todolite/_design/all_lists --data @testview
	viewUrl := fmt.Sprintf("%v/_design/%v", syncGwAdminUrl, DesignDocName)

	bufferBytes := buffer.Bytes()
	log.Printf("view: %v", string(bufferBytes))

	req, err := http.NewRequest("PUT", viewUrl, bytes.NewReader(bufferBytes))
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}

	resp, err := client.Do(req)
	if err != nil {
		return err
	}

	log.Printf("put view resp: %v", resp)

	return nil

}

func addCloudWatchMetrics(syncGwAdminUrl string) error {

	metricValue, err := numJobsReadOrBeingProcessed(syncGwAdminUrl)
	if err != nil {
		return err
	}

	// TODO: push something to CloudWatch
	// CLI example:
	//   aws cloudwatch put-metric-data
	//     --metric-name PageViewCount
	//      --namespace "MyService"
	//     --value 2
	//     --timestamp 2014-02-14T12:00:00.000Z
	cloudwatchSvc := cloudwatch.New(session.New(), &aws.Config{Region: aws.String("us-east-1")})

	log.Printf("cloudwatchSvc: %v", cloudwatchSvc)

	metricName := "NumJobsReadyOrBeingProcessed"
	timestamp := time.Now()

	metricDatum := &cloudwatch.MetricDatum{
		MetricName: &metricName,
		Value:      &metricValue,
		Timestamp:  &timestamp,
	}

	metricDatumSlice := []*cloudwatch.MetricDatum{metricDatum}
	namespace := "DeepStyleQueue"

	putMetricDataInput := &cloudwatch.PutMetricDataInput{
		MetricData: metricDatumSlice,
		Namespace:  &namespace,
	}

	out, err := cloudwatchSvc.PutMetricData(putMetricDataInput)
	if err != nil {
		log.Printf("ERROR adding metric data  %v", err)
		return err
	}
	log.Printf("Metric data output: %v", out)

	return nil

}

func init() {
	RootCmd.AddCommand(publish_cloudwatch_metricsCmd)

	// Here you will define your flags and configuration settings

	// Cobra supports Persistent Flags which will work for this command and all subcommands

	publish_cloudwatch_metricsCmd.PersistentFlags().String("aws_key", "", "AWS Key")

	publish_cloudwatch_metricsCmd.PersistentFlags().String("admin_url", "", "Sync Gateway Admin URL")

	// Cobra supports local flags which will only run when this command is called directly
	// publish_cloudwatch_metricsCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle" )

}
<|endoftext|>


<commit_before>// Copyright 2021 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package structmask implements a functionality similar to
// google.protobuf.FieldMask, but which applies only to google.protobuf.Struct.
//
// A google.protobuf.FieldMask can refer only to valid protobuf fields and
// "google.golang.org/protobuf" asserts that when serializing the field mask.
// It makes this mechanism unusable for targeting "unusual" struct fields
// (for example ones containing '.'). Additionally, google.protobuf.FieldMask
// doesn't support wildcard matches (with '*', since it is not a valid proto
// field name).
package structmask

import (
	"fmt"

	"google.golang.org/protobuf/types/known/structpb"
)

// Filter knows how to use StructMask to filter google.protobuf.Struct.
//
// Construct it using NewFilter.
type Filter struct {
	root *node
}

// NewFilter returns a filter that filters structs according to the struct mask.
//
// Returns an error if the struct mask is malformed. If `mask` is empty, returns
// a filter that doesn't actually filter anything.
func NewFilter(mask []*StructMask) (*Filter, error) {
	root, err := parseMask(mask)
	if err != nil {
		return nil, err
	}
	return &Filter{root}, nil
}

// Apply returns a shallow copy of the struct, selecting only elements matching
// the mask.
//
// The result may reuse fields of the original struct (i.e. it copies pointers,
// whenever possible, not actual objects). In extreme case of mask `*` it will
// return `s` as is.
//
// If you need to modify the result, consider explicitly making a deep copy with
// proto.Clone first.
func (f *Filter) Apply(s *structpb.Struct) *structpb.Struct {
	if f.root == nil || f.root == leafNode {
		return s
	}
	filtered := f.root.filterStruct(s)
	if filtered == nil {
		return &structpb.Struct{}
	}
	// During merging we use `nil` as a stand in for "empty set after filtering",
	// in lists to make it distinct from NullValue representing the real `null`.
	// `nil` is not allowed in *structpb.Struct. Convert them all to real Nulls.
	fillNulls(filtered)
	return filtered.GetStructValue()
}

////////////////////////////////////////////////////////////////////////////////

// leafNode is a sentinel node meaning "grab the rest of the value unfiltered".
var leafNode = &node{}

// node contains a filter tree applying to some struct path element and its
// children.
//
// *node pointers have two special values:
//   nil - no filter is present (e.g. if `star == nil`, then do not recurse).
///  leafNode - a filter that grabs all remaining values unfiltered.
type node struct {
	star   *node            // a filter to apply to all dict fields or list indexes, if any
	fields map[string]*node // a filter for individual dict fields
}

// filter recursively applies the filter tree `n` to an input value.
//
// It returns a filtered value with possible "gaps" in lists (represented by
// nils in structpb.ListValue.Values slice). These gaps appear when the filter
// filters out the entire list element. They are needed because the parent
// node may still fill them in. It needs to know where gaps are to do so safely.
// Note that representing them with structpb.NullValue is dangerous, since
// structs can have genuine `null`s in them.
//
// For example, a filter `a.*.x` applied to a `{"a": [{"x": 1}, {"y": 2}]}`
// results in `{"a": [{"x": 1}, <gap>]}`. Similarly `*.*.y` applied to the same
// input results in `{"a": [<gap>, {"y": 2}]}`. When we join these filters, we
// get the result with all gaps filled in: `{"a": [{"x": 2}, {"y": 2}]}`. Note
// that since filter paths start with different tokens (`*` vs `a`) the filter
// nodes that actually produce gaps reside in different branches of the tree,
// separated by multiple layers. This necessitates the merging and gap filling
// to be recursive (see `merge`).
//
// Since a correctly constructed *structpb.Value isn't allowed to have `nil`s,
// all gaps left in the final result are converted to `null` at the very end of
// the filtering by `fillNulls`. This is documented in the StructMask proto doc
// in the section that talks about "exceptional conditions".
func (n *node) filter(val *structpb.Value) *structpb.Value {
	if n == leafNode {
		return val
	}

	// Since `n` is not a leafNode, it is actually `.<something>`, i.e. it needs
	// to filter inner guts of `val`. We can "dive" only into dicts and lists.
	// Trying to "explore" a scalar value results in "no match" result,
	// represented by `nil`. Note that it is distinct from NullValue.
	//
	// Also if `n` is the last `.*` of the mask, return `val` unchanged as is
	// without even diving into it or checking additional masks in `fields`. This
	// avoids useless memory allocation of structpb.Struct/structpb.ListValue
	// wrappers.
	switch v := val.Kind.(type) {
	case *structpb.Value_StructValue:
		if n.star == leafNode {
			return val
		}
		return n.filterStruct(v.StructValue)
	case *structpb.Value_ListValue:
		if n.star == leafNode {
			return val
		}
		return n.filterList(v.ListValue)
	default:
		return nil
	}
}

func (n *node) filterStruct(val *structpb.Struct) *structpb.Value {
	// Apply `*` mask first (if any).
	out := &structpb.Struct{}
	if n.star != nil {
		out.Fields = make(map[string]*structpb.Value, len(val.Fields))
		for k, v := range val.Fields {
			if filtered := n.star.filter(v); filtered != nil {
				out.Fields[k] = filtered
			}
		}
	} else {
		out.Fields = make(map[string]*structpb.Value, len(n.fields))
	}

	// Merge any additional values picked by field masks targeting individual
	// dict keys.
	for key, filter := range n.fields {
		if input, ok := val.Fields[key]; ok {
			if filtered := filter.filter(input); filtered != nil {
				out.Fields[key] = merge(out.Fields[key], filtered)
			}
		}
	}

	// If filtered out all keys, return an "empty set" value represented by nil.
	// Note that this drops genuinely empty dicts as well. Oh, well... This is
	// somewhat negated by the early return on leaf nodes in `filter`.
	if len(out.Fields) == 0 {
		return nil
	}

	return structpb.NewStructValue(out)
}

func (n *node) filterList(val *structpb.ListValue) *structpb.Value {
	// Only `*` is supported. Picking individual list indexes is not implemented
	// yet since it is not clear how to do merging step when there are different
	// masks that use `*` and concrete indexes at the same time. To do the correct
	// merging we need to "remember" original indexes of items in the filtered
	// list and there's no place for it in *structpb.ListValue.
	if n.star == nil {
		return nil
	}
	out := &structpb.ListValue{Values: make([]*structpb.Value, len(val.Values))}
	for i, v := range val.Values {
		// Note that this leaves a `nil` gap if the list element was completely
		// filtered out. It is important for `merge` to know where they are to do
		// merging correctly. These gaps are converted to nulls by `fillNulls` at
		// the very end when all merges are done.
		out.Values[i] = n.star.filter(v)
	}
	return structpb.NewListValue(out)
}

// merge merges `b` into `a`, returning a shallowly combined copy of both.
//
// Both `a` and `b` should be results of a filtering of the same value, thus
// they (if not nil) must have the same type and (if lists) have the same
// length. If they are scalars, they assumed to be equal already.
//
// `nil` represents "empty sets", i.e. if `a` is nil, `b` will be returned as
// is and vice-versa. If both are `nil`, returns `nil` as well.
func merge(a, b *structpb.Value) *structpb.Value {
	switch {
	case a == nil:
		return b
	case b == nil:
		return a
	case a == b:
		return a
	}
	switch a := a.Kind.(type) {
	case *structpb.Value_StructValue:
		// `b` *must* be a Struct here. Panic if not, it means there's a bug.
		return structpb.NewStructValue(mergeStruct(a.StructValue, b.Kind.(*structpb.Value_StructValue).StructValue))
	case *structpb.Value_ListValue:
		// `b` *must* be a List here. Panic if not, it means there's a bug.
		return structpb.NewListValue(mergeList(a.ListValue, b.Kind.(*structpb.Value_ListValue).ListValue))
	default:
		return b
	}
}

func mergeStruct(a, b *structpb.Struct) *structpb.Struct {
	l := len(a.Fields)
	if len(b.Fields) > l {
		l = len(b.Fields)
	}

	out := &structpb.Struct{
		Fields: make(map[string]*structpb.Value, l),
	}

	for key, aval := range a.Fields {
		out.Fields[key] = merge(aval, b.Fields[key])
	}

	for key, bval := range b.Fields {
		// Already dealt with A&B case above. Pick only B-A.
		if _, ok := a.Fields[key]; !ok {
			out.Fields[key] = bval
		}
	}

	return out
}

func mergeList(a, b *structpb.ListValue) *structpb.ListValue {
	if len(a.Values) != len(b.Values) {
		panic(fmt.Sprintf("unexpected list lengths %d != %d", len(a.Values), len(b.Values)))
	}
	out := &structpb.ListValue{
		Values: make([]*structpb.Value, len(a.Values)),
	}
	for idx := range a.Values {
		out.Values[idx] = merge(a.Values[idx], b.Values[idx])
	}
	return out
}

func fillNulls(v *structpb.Value) {
	switch v := v.Kind.(type) {
	case *structpb.Value_StructValue:
		for key, elem := range v.StructValue.Fields {
			if elem == nil {
				// We leave nils only in lists, not in structs.
				panic(fmt.Sprintf("unexpected nil for struct key %q", key))
			} else {
				fillNulls(elem)
			}
		}
	case *structpb.Value_ListValue:
		for idx, elem := range v.ListValue.Values {
			if elem == nil {
				v.ListValue.Values[idx] = structpb.NewNullValue()
			} else {
				fillNulls(elem)
			}
		}
	}
}
<commit_msg>[structmask] Applying a filter to `nil` is `nil`.<commit_after>// Copyright 2021 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package structmask implements a functionality similar to
// google.protobuf.FieldMask, but which applies only to google.protobuf.Struct.
//
// A google.protobuf.FieldMask can refer only to valid protobuf fields and
// "google.golang.org/protobuf" asserts that when serializing the field mask.
// It makes this mechanism unusable for targeting "unusual" struct fields
// (for example ones containing '.'). Additionally, google.protobuf.FieldMask
// doesn't support wildcard matches (with '*', since it is not a valid proto
// field name).
package structmask

import (
	"fmt"

	"google.golang.org/protobuf/types/known/structpb"
)

// Filter knows how to use StructMask to filter google.protobuf.Struct.
//
// Construct it using NewFilter.
type Filter struct {
	root *node
}

// NewFilter returns a filter that filters structs according to the struct mask.
//
// Returns an error if the struct mask is malformed. If `mask` is empty, returns
// a filter that doesn't actually filter anything.
func NewFilter(mask []*StructMask) (*Filter, error) {
	root, err := parseMask(mask)
	if err != nil {
		return nil, err
	}
	return &Filter{root}, nil
}

// Apply returns a shallow copy of the struct, selecting only elements matching
// the mask.
//
// The result may reuse fields of the original struct (i.e. it copies pointers,
// whenever possible, not actual objects). In extreme case of mask `*` it will
// return `s` as is.
//
// If you need to modify the result, consider explicitly making a deep copy with
// proto.Clone first.
//
// If given `nil`, returns `nil` as well.
func (f *Filter) Apply(s *structpb.Struct) *structpb.Struct {
	if s == nil || f.root == nil || f.root == leafNode {
		return s
	}
	filtered := f.root.filterStruct(s)
	if filtered == nil {
		return &structpb.Struct{}
	}
	// During merging we use `nil` as a stand in for "empty set after filtering",
	// in lists to make it distinct from NullValue representing the real `null`.
	// `nil` is not allowed in *structpb.Struct. Convert them all to real Nulls.
	fillNulls(filtered)
	return filtered.GetStructValue()
}

////////////////////////////////////////////////////////////////////////////////

// leafNode is a sentinel node meaning "grab the rest of the value unfiltered".
var leafNode = &node{}

// node contains a filter tree applying to some struct path element and its
// children.
//
// *node pointers have two special values:
//   nil - no filter is present (e.g. if `star == nil`, then do not recurse).
///  leafNode - a filter that grabs all remaining values unfiltered.
type node struct {
	star   *node            // a filter to apply to all dict fields or list indexes, if any
	fields map[string]*node // a filter for individual dict fields
}

// filter recursively applies the filter tree `n` to an input value.
//
// It returns a filtered value with possible "gaps" in lists (represented by
// nils in structpb.ListValue.Values slice). These gaps appear when the filter
// filters out the entire list element. They are needed because the parent
// node may still fill them in. It needs to know where gaps are to do so safely.
// Note that representing them with structpb.NullValue is dangerous, since
// structs can have genuine `null`s in them.
//
// For example, a filter `a.*.x` applied to a `{"a": [{"x": 1}, {"y": 2}]}`
// results in `{"a": [{"x": 1}, <gap>]}`. Similarly `*.*.y` applied to the same
// input results in `{"a": [<gap>, {"y": 2}]}`. When we join these filters, we
// get the result with all gaps filled in: `{"a": [{"x": 2}, {"y": 2}]}`. Note
// that since filter paths start with different tokens (`*` vs `a`) the filter
// nodes that actually produce gaps reside in different branches of the tree,
// separated by multiple layers. This necessitates the merging and gap filling
// to be recursive (see `merge`).
//
// Since a correctly constructed *structpb.Value isn't allowed to have `nil`s,
// all gaps left in the final result are converted to `null` at the very end of
// the filtering by `fillNulls`. This is documented in the StructMask proto doc
// in the section that talks about "exceptional conditions".
func (n *node) filter(val *structpb.Value) *structpb.Value {
	if n == leafNode {
		return val
	}

	// Since `n` is not a leafNode, it is actually `.<something>`, i.e. it needs
	// to filter inner guts of `val`. We can "dive" only into dicts and lists.
	// Trying to "explore" a scalar value results in "no match" result,
	// represented by `nil`. Note that it is distinct from NullValue.
	//
	// Also if `n` is the last `.*` of the mask, return `val` unchanged as is
	// without even diving into it or checking additional masks in `fields`. This
	// avoids useless memory allocation of structpb.Struct/structpb.ListValue
	// wrappers.
	switch v := val.Kind.(type) {
	case *structpb.Value_StructValue:
		if n.star == leafNode {
			return val
		}
		return n.filterStruct(v.StructValue)
	case *structpb.Value_ListValue:
		if n.star == leafNode {
			return val
		}
		return n.filterList(v.ListValue)
	default:
		return nil
	}
}

func (n *node) filterStruct(val *structpb.Struct) *structpb.Value {
	// Apply `*` mask first (if any).
	out := &structpb.Struct{}
	if n.star != nil {
		out.Fields = make(map[string]*structpb.Value, len(val.Fields))
		for k, v := range val.Fields {
			if filtered := n.star.filter(v); filtered != nil {
				out.Fields[k] = filtered
			}
		}
	} else {
		out.Fields = make(map[string]*structpb.Value, len(n.fields))
	}

	// Merge any additional values picked by field masks targeting individual
	// dict keys.
	for key, filter := range n.fields {
		if input, ok := val.Fields[key]; ok {
			if filtered := filter.filter(input); filtered != nil {
				out.Fields[key] = merge(out.Fields[key], filtered)
			}
		}
	}

	// If filtered out all keys, return an "empty set" value represented by nil.
	// Note that this drops genuinely empty dicts as well. Oh, well... This is
	// somewhat negated by the early return on leaf nodes in `filter`.
	if len(out.Fields) == 0 {
		return nil
	}

	return structpb.NewStructValue(out)
}

func (n *node) filterList(val *structpb.ListValue) *structpb.Value {
	// Only `*` is supported. Picking individual list indexes is not implemented
	// yet since it is not clear how to do merging step when there are different
	// masks that use `*` and concrete indexes at the same time. To do the correct
	// merging we need to "remember" original indexes of items in the filtered
	// list and there's no place for it in *structpb.ListValue.
	if n.star == nil {
		return nil
	}
	out := &structpb.ListValue{Values: make([]*structpb.Value, len(val.Values))}
	for i, v := range val.Values {
		// Note that this leaves a `nil` gap if the list element was completely
		// filtered out. It is important for `merge` to know where they are to do
		// merging correctly. These gaps are converted to nulls by `fillNulls` at
		// the very end when all merges are done.
		out.Values[i] = n.star.filter(v)
	}
	return structpb.NewListValue(out)
}

// merge merges `b` into `a`, returning a shallowly combined copy of both.
//
// Both `a` and `b` should be results of a filtering of the same value, thus
// they (if not nil) must have the same type and (if lists) have the same
// length. If they are scalars, they assumed to be equal already.
//
// `nil` represents "empty sets", i.e. if `a` is nil, `b` will be returned as
// is and vice-versa. If both are `nil`, returns `nil` as well.
func merge(a, b *structpb.Value) *structpb.Value {
	switch {
	case a == nil:
		return b
	case b == nil:
		return a
	case a == b:
		return a
	}
	switch a := a.Kind.(type) {
	case *structpb.Value_StructValue:
		// `b` *must* be a Struct here. Panic if not, it means there's a bug.
		return structpb.NewStructValue(mergeStruct(a.StructValue, b.Kind.(*structpb.Value_StructValue).StructValue))
	case *structpb.Value_ListValue:
		// `b` *must* be a List here. Panic if not, it means there's a bug.
		return structpb.NewListValue(mergeList(a.ListValue, b.Kind.(*structpb.Value_ListValue).ListValue))
	default:
		return b
	}
}

func mergeStruct(a, b *structpb.Struct) *structpb.Struct {
	l := len(a.Fields)
	if len(b.Fields) > l {
		l = len(b.Fields)
	}

	out := &structpb.Struct{
		Fields: make(map[string]*structpb.Value, l),
	}

	for key, aval := range a.Fields {
		out.Fields[key] = merge(aval, b.Fields[key])
	}

	for key, bval := range b.Fields {
		// Already dealt with A&B case above. Pick only B-A.
		if _, ok := a.Fields[key]; !ok {
			out.Fields[key] = bval
		}
	}

	return out
}

func mergeList(a, b *structpb.ListValue) *structpb.ListValue {
	if len(a.Values) != len(b.Values) {
		panic(fmt.Sprintf("unexpected list lengths %d != %d", len(a.Values), len(b.Values)))
	}
	out := &structpb.ListValue{
		Values: make([]*structpb.Value, len(a.Values)),
	}
	for idx := range a.Values {
		out.Values[idx] = merge(a.Values[idx], b.Values[idx])
	}
	return out
}

func fillNulls(v *structpb.Value) {
	switch v := v.Kind.(type) {
	case *structpb.Value_StructValue:
		for key, elem := range v.StructValue.Fields {
			if elem == nil {
				// We leave nils only in lists, not in structs.
				panic(fmt.Sprintf("unexpected nil for struct key %q", key))
			} else {
				fillNulls(elem)
			}
		}
	case *structpb.Value_ListValue:
		for idx, elem := range v.ListValue.Values {
			if elem == nil {
				v.ListValue.Values[idx] = structpb.NewNullValue()
			} else {
				fillNulls(elem)
			}
		}
	}
}
<|endoftext|>


<commit_before>package v7_test

import (
	"errors"

	"code.cloudfoundry.org/cli/actor/actionerror"
	"code.cloudfoundry.org/cli/actor/v7action"
	"code.cloudfoundry.org/cli/command/commandfakes"
	. "code.cloudfoundry.org/cli/command/v7"
	"code.cloudfoundry.org/cli/command/v7/v7fakes"
	"code.cloudfoundry.org/cli/resources"
	"code.cloudfoundry.org/cli/util/configv3"
	"code.cloudfoundry.org/cli/util/ui"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
)

var _ = Describe("routes Command", func() {
	var (
		cmd             RoutesCommand
		testUI          *ui.UI
		fakeConfig      *commandfakes.FakeConfig
		fakeSharedActor *commandfakes.FakeSharedActor
		fakeActor       *v7fakes.FakeActor
		executeErr      error
		args            []string
		binaryName      string
	)

	const tableHeaders = `space\s+host\s+domain\s+path\s+port\s+protocol\s+apps`

	BeforeEach(func() {
		testUI = ui.NewTestUI(nil, NewBuffer(), NewBuffer())
		fakeConfig = new(commandfakes.FakeConfig)
		fakeSharedActor = new(commandfakes.FakeSharedActor)
		fakeActor = new(v7fakes.FakeActor)
		args = nil

		binaryName = "faceman"
		fakeConfig.BinaryNameReturns(binaryName)

		cmd = RoutesCommand{
			BaseCommand: BaseCommand{
				UI:          testUI,
				Config:      fakeConfig,
				SharedActor: fakeSharedActor,
				Actor:       fakeActor,
			},
			Orglevel: false,
		}
	})

	JustBeforeEach(func() {
		executeErr = cmd.Execute(args)
	})

	When("the environment is not setup correctly", func() {
		When("checking target fails", func() {
			BeforeEach(func() {
				fakeSharedActor.CheckTargetReturns(actionerror.NotLoggedInError{BinaryName: binaryName})
			})

			It("returns an error", func() {
				Expect(executeErr).To(MatchError(actionerror.NotLoggedInError{BinaryName: binaryName}))

				Expect(fakeSharedActor.CheckTargetCallCount()).To(Equal(1))
				checkTargetedOrg, checkTargetedSpace := fakeSharedActor.CheckTargetArgsForCall(0)
				Expect(checkTargetedOrg).To(BeTrue())
				Expect(checkTargetedSpace).To(BeTrue())
			})
		})

		When("when there is no org targeted", func() {
			BeforeEach(func() {
				fakeSharedActor.CheckTargetReturns(actionerror.NoOrganizationTargetedError{BinaryName: binaryName})
			})

			It("returns an error", func() {
				Expect(executeErr).To(MatchError(actionerror.NoOrganizationTargetedError{BinaryName: binaryName}))
				checkTargetedOrg, checkTargetedSpace := fakeSharedActor.CheckTargetArgsForCall(0)
				Expect(checkTargetedOrg).To(BeTrue())
				Expect(checkTargetedSpace).To(BeTrue())
			})
		})
	})

	Context("When the environment is setup correctly", func() {
		BeforeEach(func() {
			fakeConfig.CurrentUserReturns(configv3.User{Name: "banana"}, nil)
			fakeConfig.TargetedOrganizationReturns(configv3.Organization{
				GUID: "some-org-guid",
				Name: "some-org",
			})

			fakeConfig.TargetedSpaceReturns(configv3.Space{
				GUID: "some-space-guid",
				Name: "some-space",
			})
		})

		When("getting routes succeeds", func() {
			var (
				routes []resources.Route
			)

			BeforeEach(func() {
				routes = []resources.Route{
					{GUID: "route-guid-3", Host: "host-1"},
					{GUID: "route-guid-1"},
					{GUID: "route-guid-2", Host: "host-3", Path: "/path/2"},
				}

				fakeActor.GetRoutesBySpaceReturns(
					routes,
					v7action.Warnings{"actor-warning-1"},
					nil,
				)
			})

			It("delegates to the actor for summaries", func() {
				Expect(fakeActor.GetRouteSummariesCallCount()).To(Equal(1))

				Expect(fakeActor.GetRouteSummariesArgsForCall(0)).To(Equal(routes))
			})

			When("getting route summaries succeeds", func() {
				var (
					routeSummaries []v7action.RouteSummary
				)

				BeforeEach(func() {
					routeSummaries = []v7action.RouteSummary{
						{
							DomainName: "domain1",
							SpaceName:  "space-1",
							Route:      resources.Route{GUID: "route-guid-1"},
						},
						{
							DomainName: "domain2",
							SpaceName:  "space-2",
							Route:      resources.Route{GUID: "route-guid-2", Host: "host-3", Path: "/path/2"},
						},
						{
							DomainName: "domain3",
							SpaceName:  "space-3",
							Route:      resources.Route{GUID: "route-guid-3", Host: "host-1"},
							AppNames:   []string{"app1", "app2"},
						},
						{
							DomainName: "tcp.domain",
							SpaceName:  "space-3",
							Route:      resources.Route{GUID: "route-guid-3", Port: 1024},
							AppNames:   []string{"app1", "app2"},
						},
					}

					fakeActor.GetRouteSummariesReturns(
						routeSummaries,
						v7action.Warnings{"actor-warning-2"},
						nil,
					)
				})

				It("prints routes in a table", func() {
					Expect(executeErr).NotTo(HaveOccurred())

					Expect(testUI.Err).To(Say("actor-warning-1"))
					Expect(testUI.Err).To(Say("actor-warning-2"))

					Expect(testUI.Out).To(Say(tableHeaders))
					Expect(testUI.Out).To(Say(`space-1\s+domain1\s+`))
					Expect(testUI.Out).To(Say(`space-2\s+host-3\s+domain2\s+\/path\/2`))
					Expect(testUI.Out).To(Say(`space-3\s+host-1\s+domain3\s+app1, app2`))
					Expect(testUI.Out).To(Say(`space-3\s+tcp\.domain\s+1024\s+app1, app2`))
				})
			})

			When("getting route summaries fails", func() {
				BeforeEach(func() {
					fakeActor.GetRouteSummariesReturns(
						nil,
						v7action.Warnings{"actor-warning-2", "actor-warning-3"},
						errors.New("summaries-error"),
					)
				})

				It("prints warnings and returns error", func() {
					Expect(executeErr).To(MatchError("summaries-error"))

					Expect(testUI.Err).To(Say("actor-warning-1"))
					Expect(testUI.Err).To(Say("actor-warning-2"))
					Expect(testUI.Err).To(Say("actor-warning-3"))
				})
			})
		})

		When("getting space routes fails", func() {
			var expectedErr error

			BeforeEach(func() {
				warnings := v7action.Warnings{"warning-1", "warning-2"}
				expectedErr = errors.New("some-error")
				fakeActor.GetRoutesBySpaceReturns(nil, warnings, expectedErr)
			})

			It("prints warnings and returns error", func() {
				Expect(executeErr).To(Equal(expectedErr))

				Expect(testUI.Err).To(Say("warning-1"))
				Expect(testUI.Err).To(Say("warning-2"))
				Expect(testUI.Out).ToNot(Say(tableHeaders))
			})
		})

		When("--org-level is passed and getting org routes fails", func() {
			var expectedErr error

			BeforeEach(func() {
				cmd.Orglevel = true
				warnings := v7action.Warnings{"warning-1", "warning-2"}
				expectedErr = errors.New("some-error")
				fakeActor.GetRoutesByOrgReturns(nil, warnings, expectedErr)
			})

			It("prints warnings and returns error", func() {
				Expect(executeErr).To(MatchError(expectedErr))

				Expect(testUI.Err).To(Say("warning-1"))
				Expect(testUI.Err).To(Say("warning-2"))
				Expect(testUI.Out).ToNot(Say(tableHeaders))
			})
		})

		When("--labels is passed in", func() {
			BeforeEach(func() {
				cmd.Labels = "some_label=fun"
			})
			It("passes the labels to the actor", func() {
				_, labels := fakeActor.GetRoutesBySpaceArgsForCall(0)
				Expect(labels).To(Equal("some_label=fun"))
			})
			When("--org-level is passed in", func() {
				BeforeEach(func() {
					cmd.Orglevel = true
				})
				It("passes the labels to the actor", func() {
					_, labels := fakeActor.GetRoutesByOrgArgsForCall(0)
					Expect(labels).To(Equal("some_label=fun"))
				})
			})
		})
	})
})
<commit_msg>Fix ordering of table headers for routes command unit test expectation<commit_after>package v7_test

import (
	"errors"

	"code.cloudfoundry.org/cli/actor/actionerror"
	"code.cloudfoundry.org/cli/actor/v7action"
	"code.cloudfoundry.org/cli/command/commandfakes"
	. "code.cloudfoundry.org/cli/command/v7"
	"code.cloudfoundry.org/cli/command/v7/v7fakes"
	"code.cloudfoundry.org/cli/resources"
	"code.cloudfoundry.org/cli/util/configv3"
	"code.cloudfoundry.org/cli/util/ui"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
)

var _ = Describe("routes Command", func() {
	var (
		cmd             RoutesCommand
		testUI          *ui.UI
		fakeConfig      *commandfakes.FakeConfig
		fakeSharedActor *commandfakes.FakeSharedActor
		fakeActor       *v7fakes.FakeActor
		executeErr      error
		args            []string
		binaryName      string
	)

	const tableHeaders = `space\s+host\s+domain\s+port\s+path\s+protocol\s+apps`

	BeforeEach(func() {
		testUI = ui.NewTestUI(nil, NewBuffer(), NewBuffer())
		fakeConfig = new(commandfakes.FakeConfig)
		fakeSharedActor = new(commandfakes.FakeSharedActor)
		fakeActor = new(v7fakes.FakeActor)
		args = nil

		binaryName = "faceman"
		fakeConfig.BinaryNameReturns(binaryName)

		cmd = RoutesCommand{
			BaseCommand: BaseCommand{
				UI:          testUI,
				Config:      fakeConfig,
				SharedActor: fakeSharedActor,
				Actor:       fakeActor,
			},
			Orglevel: false,
		}
	})

	JustBeforeEach(func() {
		executeErr = cmd.Execute(args)
	})

	When("the environment is not setup correctly", func() {
		When("checking target fails", func() {
			BeforeEach(func() {
				fakeSharedActor.CheckTargetReturns(actionerror.NotLoggedInError{BinaryName: binaryName})
			})

			It("returns an error", func() {
				Expect(executeErr).To(MatchError(actionerror.NotLoggedInError{BinaryName: binaryName}))

				Expect(fakeSharedActor.CheckTargetCallCount()).To(Equal(1))
				checkTargetedOrg, checkTargetedSpace := fakeSharedActor.CheckTargetArgsForCall(0)
				Expect(checkTargetedOrg).To(BeTrue())
				Expect(checkTargetedSpace).To(BeTrue())
			})
		})

		When("when there is no org targeted", func() {
			BeforeEach(func() {
				fakeSharedActor.CheckTargetReturns(actionerror.NoOrganizationTargetedError{BinaryName: binaryName})
			})

			It("returns an error", func() {
				Expect(executeErr).To(MatchError(actionerror.NoOrganizationTargetedError{BinaryName: binaryName}))
				checkTargetedOrg, checkTargetedSpace := fakeSharedActor.CheckTargetArgsForCall(0)
				Expect(checkTargetedOrg).To(BeTrue())
				Expect(checkTargetedSpace).To(BeTrue())
			})
		})
	})

	Context("When the environment is setup correctly", func() {
		BeforeEach(func() {
			fakeConfig.CurrentUserReturns(configv3.User{Name: "banana"}, nil)
			fakeConfig.TargetedOrganizationReturns(configv3.Organization{
				GUID: "some-org-guid",
				Name: "some-org",
			})

			fakeConfig.TargetedSpaceReturns(configv3.Space{
				GUID: "some-space-guid",
				Name: "some-space",
			})
		})

		When("getting routes succeeds", func() {
			var (
				routes []resources.Route
			)

			BeforeEach(func() {
				routes = []resources.Route{
					{GUID: "route-guid-3", Host: "host-1"},
					{GUID: "route-guid-1"},
					{GUID: "route-guid-2", Host: "host-3", Path: "/path/2"},
				}

				fakeActor.GetRoutesBySpaceReturns(
					routes,
					v7action.Warnings{"actor-warning-1"},
					nil,
				)
			})

			It("delegates to the actor for summaries", func() {
				Expect(fakeActor.GetRouteSummariesCallCount()).To(Equal(1))

				Expect(fakeActor.GetRouteSummariesArgsForCall(0)).To(Equal(routes))
			})

			When("getting route summaries succeeds", func() {
				var (
					routeSummaries []v7action.RouteSummary
				)

				BeforeEach(func() {
					routeSummaries = []v7action.RouteSummary{
						{
							DomainName: "domain1",
							SpaceName:  "space-1",
							Route:      resources.Route{GUID: "route-guid-1"},
						},
						{
							DomainName: "domain2",
							SpaceName:  "space-2",
							Route:      resources.Route{GUID: "route-guid-2", Host: "host-3", Path: "/path/2"},
						},
						{
							DomainName: "domain3",
							SpaceName:  "space-3",
							Route:      resources.Route{GUID: "route-guid-3", Host: "host-1"},
							AppNames:   []string{"app1", "app2"},
						},
						{
							DomainName: "tcp.domain",
							SpaceName:  "space-3",
							Route:      resources.Route{GUID: "route-guid-3", Port: 1024},
							AppNames:   []string{"app1", "app2"},
						},
					}

					fakeActor.GetRouteSummariesReturns(
						routeSummaries,
						v7action.Warnings{"actor-warning-2"},
						nil,
					)
				})

				It("prints routes in a table", func() {
					Expect(executeErr).NotTo(HaveOccurred())

					Expect(testUI.Err).To(Say("actor-warning-1"))
					Expect(testUI.Err).To(Say("actor-warning-2"))

					Expect(testUI.Out).To(Say(tableHeaders))
					Expect(testUI.Out).To(Say(`space-1\s+domain1\s+`))
					Expect(testUI.Out).To(Say(`space-2\s+host-3\s+domain2\s+\/path\/2`))
					Expect(testUI.Out).To(Say(`space-3\s+host-1\s+domain3\s+app1, app2`))
					Expect(testUI.Out).To(Say(`space-3\s+tcp\.domain\s+1024\s+app1, app2`))
				})
			})

			When("getting route summaries fails", func() {
				BeforeEach(func() {
					fakeActor.GetRouteSummariesReturns(
						nil,
						v7action.Warnings{"actor-warning-2", "actor-warning-3"},
						errors.New("summaries-error"),
					)
				})

				It("prints warnings and returns error", func() {
					Expect(executeErr).To(MatchError("summaries-error"))

					Expect(testUI.Err).To(Say("actor-warning-1"))
					Expect(testUI.Err).To(Say("actor-warning-2"))
					Expect(testUI.Err).To(Say("actor-warning-3"))
				})
			})
		})

		When("getting space routes fails", func() {
			var expectedErr error

			BeforeEach(func() {
				warnings := v7action.Warnings{"warning-1", "warning-2"}
				expectedErr = errors.New("some-error")
				fakeActor.GetRoutesBySpaceReturns(nil, warnings, expectedErr)
			})

			It("prints warnings and returns error", func() {
				Expect(executeErr).To(Equal(expectedErr))

				Expect(testUI.Err).To(Say("warning-1"))
				Expect(testUI.Err).To(Say("warning-2"))
				Expect(testUI.Out).ToNot(Say(tableHeaders))
			})
		})

		When("--org-level is passed and getting org routes fails", func() {
			var expectedErr error

			BeforeEach(func() {
				cmd.Orglevel = true
				warnings := v7action.Warnings{"warning-1", "warning-2"}
				expectedErr = errors.New("some-error")
				fakeActor.GetRoutesByOrgReturns(nil, warnings, expectedErr)
			})

			It("prints warnings and returns error", func() {
				Expect(executeErr).To(MatchError(expectedErr))

				Expect(testUI.Err).To(Say("warning-1"))
				Expect(testUI.Err).To(Say("warning-2"))
				Expect(testUI.Out).ToNot(Say(tableHeaders))
			})
		})

		When("--labels is passed in", func() {
			BeforeEach(func() {
				cmd.Labels = "some_label=fun"
			})
			It("passes the labels to the actor", func() {
				_, labels := fakeActor.GetRoutesBySpaceArgsForCall(0)
				Expect(labels).To(Equal("some_label=fun"))
			})
			When("--org-level is passed in", func() {
				BeforeEach(func() {
					cmd.Orglevel = true
				})
				It("passes the labels to the actor", func() {
					_, labels := fakeActor.GetRoutesByOrgArgsForCall(0)
					Expect(labels).To(Equal("some_label=fun"))
				})
			})
		})
	})
})
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"errors"
	"reflect"
	"testing"
)

func TestValidateAuthFlow(t *testing.T) {
	type FlagResult struct {
		Name  string
		Flow  string
		Error error
	}
	tests := []FlagResult{
		{Name: "validate gcr auth flow", Flow: gcrAuthFlow, Error: nil},
		{Name: "validate docker-cfg auth flow option", Flow: dockerConfigAuthFlow, Error: nil},
		{Name: "validate docker-cfg-url auth flow option", Flow: dockerConfigURLAuthFlow, Error: nil},
		{Name: "bad auth flow option", Flow: "bad-flow", Error: &AuthFlowFlagError{flagValue: "bad-flow"}},
		{Name: "empty auth flow option", Flow: "", Error: &AuthFlowFlagError{flagValue: ""}},
		{Name: "case-sensitive auth flow", Flow: "Gcrauthflow", Error: &AuthFlowFlagError{flagValue: "Gcrauthflow"}},
	}
	for _, tc := range tests {
		t.Run(tc.Name, func(t *testing.T) {
			err := validateFlags(&CredentialOptions{AuthFlow: tc.Flow})
			if tc.Error != nil {
				if err == nil {
					t.Fatalf("with flow %q did not get expected error %q", tc.Flow, err)
				}
				if !errors.Is(err, tc.Error) {
					t.Fatalf("with flow %q got unexpected error type %q (expected %q)", tc.Flow, reflect.TypeOf(err), reflect.TypeOf(tc.Error))
				}
				return
			}
			if err != nil {
				t.Fatalf("with flow %q unexpected error %q", tc.Flow, err)
			}
		})
	}
}

func TestProviderFromFlow(t *testing.T) {
	type ProviderResult struct {
		Name  string
		Flow  string
		Type  string
		Error error
	}
	tests := []ProviderResult{
		{Name: "gcr auth provider selection", Flow: gcrAuthFlow, Type: "ContainerRegistryProvider", Error: nil},
		{Name: "docker-cfg auth provider selection", Flow: dockerConfigAuthFlow, Type: "DockerConfigKeyProvider", Error: nil},
		{Name: "docker-cfg-url auth provider selection", Flow: dockerConfigURLAuthFlow, Type: "DockerConfigURLKeyProvider", Error: nil},
		{Name: "non-existent auth provider request", Flow: "bad-flow", Type: "", Error: &AuthFlowTypeError{requestedFlow: "bad-flow"}},
		{Name: "empty auth provider request", Flow: "", Type: "", Error: &AuthFlowTypeError{requestedFlow: ""}},
	}
	for _, tc := range tests {
		t.Run(tc.Name, func(t *testing.T) {
			provider, err := providerFromFlow(tc.Flow)
			if tc.Error != nil {
				if err == nil {
					t.Fatalf("with flow %q did not get expected error %q", tc.Flow, err)
				}
				if !errors.Is(err, tc.Error) {
					t.Fatalf("with flow %q got unexpected error type %q (expected %q)", tc.Flow, reflect.TypeOf(err), reflect.TypeOf(tc.Error))
				}
				return
			}
			if err != nil {
				t.Fatalf("with flow %q unexpected error %q", tc.Flow, err)
			}
			providerType := reflect.TypeOf(provider).String()
			if providerType != "*gcpcredential."+tc.Type {
				t.Errorf("with flow %q unexpected provider type %q", tc.Flow, providerType)
			}
		})
	}
}
<commit_msg>Remove redundant Error setting in getcredentials_test.go.<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"errors"
	"reflect"
	"testing"
)

func TestValidateAuthFlow(t *testing.T) {
	type FlagResult struct {
		Name  string
		Flow  string
		Error error
	}
	tests := []FlagResult{
		{Name: "validate gcr auth flow", Flow: gcrAuthFlow},
		{Name: "validate docker-cfg auth flow option", Flow: dockerConfigAuthFlow},
		{Name: "validate docker-cfg-url auth flow option", Flow: dockerConfigURLAuthFlow},
		{Name: "bad auth flow option", Flow: "bad-flow", Error: &AuthFlowFlagError{flagValue: "bad-flow"}},
		{Name: "empty auth flow option", Flow: "", Error: &AuthFlowFlagError{flagValue: ""}},
		{Name: "case-sensitive auth flow", Flow: "Gcrauthflow", Error: &AuthFlowFlagError{flagValue: "Gcrauthflow"}},
	}
	for _, tc := range tests {
		t.Run(tc.Name, func(t *testing.T) {
			err := validateFlags(&CredentialOptions{AuthFlow: tc.Flow})
			if tc.Error != nil {
				if err == nil {
					t.Fatalf("with flow %q did not get expected error %q", tc.Flow, err)
				}
				if !errors.Is(err, tc.Error) {
					t.Fatalf("with flow %q got unexpected error type %q (expected %q)", tc.Flow, reflect.TypeOf(err), reflect.TypeOf(tc.Error))
				}
				return
			}
			if err != nil {
				t.Fatalf("with flow %q unexpected error %q", tc.Flow, err)
			}
		})
	}
}

func TestProviderFromFlow(t *testing.T) {
	type ProviderResult struct {
		Name  string
		Flow  string
		Type  string
		Error error
	}
	tests := []ProviderResult{
		{Name: "gcr auth provider selection", Flow: gcrAuthFlow, Type: "ContainerRegistryProvider"},
		{Name: "docker-cfg auth provider selection", Flow: dockerConfigAuthFlow, Type: "DockerConfigKeyProvider"},
		{Name: "docker-cfg-url auth provider selection", Flow: dockerConfigURLAuthFlow, Type: "DockerConfigURLKeyProvider"},
		{Name: "non-existent auth provider request", Flow: "bad-flow", Type: "", Error: &AuthFlowTypeError{requestedFlow: "bad-flow"}},
		{Name: "empty auth provider request", Flow: "", Type: "", Error: &AuthFlowTypeError{requestedFlow: ""}},
	}
	for _, tc := range tests {
		t.Run(tc.Name, func(t *testing.T) {
			provider, err := providerFromFlow(tc.Flow)
			if tc.Error != nil {
				if err == nil {
					t.Fatalf("with flow %q did not get expected error %q", tc.Flow, err)
				}
				if !errors.Is(err, tc.Error) {
					t.Fatalf("with flow %q got unexpected error type %q (expected %q)", tc.Flow, reflect.TypeOf(err), reflect.TypeOf(tc.Error))
				}
				return
			}
			if err != nil {
				t.Fatalf("with flow %q unexpected error %q", tc.Flow, err)
			}
			providerType := reflect.TypeOf(provider).String()
			if providerType != "*gcpcredential."+tc.Type {
				t.Errorf("with flow %q unexpected provider type %q", tc.Flow, providerType)
			}
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"os"
	"strings"

	"github.com/docopt/docopt-go"
	clientv3 "github.com/projectcalico/libcalico-go/lib/clientv3"

	"github.com/projectcalico/calicoctl/v3/calicoctl/commands/clientmgr"
	"github.com/projectcalico/calicoctl/v3/calicoctl/commands/constants"
)

var VERSION string

func main() {
	doc := `Usage:
  calico_version_helper [options]

Options:
  -h --help                 Show this screen.
  -v --version=<version>    Version to set.
                            [default: ` + VERSION + `]
  -c --config=<config>      Path to the file containing connection
                            configuration in YAML or JSON format.
                            [default: ` + constants.DefaultConfigPath + `]
  --context=<context>       The name of the kubeconfig context to use.

Description:
  Set CalicoVersion in ClusterInformation.
`
	parsedArgs, err := docopt.ParseDoc(doc)
	if err != nil {
		fmt.Printf("Could not parse arguments: %s, err: %v\n", strings.Join(os.Args[1:], " "), err)
		os.Exit(1)
	}

	if context := parsedArgs["--context"]; context != nil {
		os.Setenv("K8S_CURRENT_CONTEXT", context.(string))
	}

	cf, _ := parsedArgs["--config"].(string)

	cfg, err := clientmgr.LoadClientConfig(cf)
	if err != nil {
		fmt.Printf("Could not load client config: %v\n", err)
		os.Exit(1)
	}

	// Get the backend client for updating cluster info and migrating IPAM.
	client, err := clientv3.New(*cfg)
	if err != nil {
		fmt.Printf("Could not create client: %v\n", err)
		os.Exit(1)
	}

	ctx := context.Background()

	calicoVersion, _ := parsedArgs["--version"].(string)

	if err := client.EnsureInitialized(ctx, calicoVersion, ""); err != nil {
		fmt.Printf("Could not set calico version to %s: %v\n", calicoVersion, err)
		os.Exit(1)
	}

	fmt.Printf("Calico version set to %s\n", calicoVersion)
}
<commit_msg>Add license to calico_version_helper.go (#2380)<commit_after>// Copyright (c) 2021 Tigera, Inc. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"fmt"
	"os"
	"strings"

	"github.com/docopt/docopt-go"
	clientv3 "github.com/projectcalico/libcalico-go/lib/clientv3"

	"github.com/projectcalico/calicoctl/v3/calicoctl/commands/clientmgr"
	"github.com/projectcalico/calicoctl/v3/calicoctl/commands/constants"
)

var VERSION string

func main() {
	doc := `Usage:
  calico_version_helper [options]

Options:
  -h --help                 Show this screen.
  -v --version=<version>    Version to set.
                            [default: ` + VERSION + `]
  -c --config=<config>      Path to the file containing connection
                            configuration in YAML or JSON format.
                            [default: ` + constants.DefaultConfigPath + `]
  --context=<context>       The name of the kubeconfig context to use.

Description:
  Set CalicoVersion in ClusterInformation.
`
	parsedArgs, err := docopt.ParseDoc(doc)
	if err != nil {
		fmt.Printf("Could not parse arguments: %s, err: %v\n", strings.Join(os.Args[1:], " "), err)
		os.Exit(1)
	}

	if context := parsedArgs["--context"]; context != nil {
		os.Setenv("K8S_CURRENT_CONTEXT", context.(string))
	}

	cf, _ := parsedArgs["--config"].(string)

	cfg, err := clientmgr.LoadClientConfig(cf)
	if err != nil {
		fmt.Printf("Could not load client config: %v\n", err)
		os.Exit(1)
	}

	// Get the backend client for updating cluster info and migrating IPAM.
	client, err := clientv3.New(*cfg)
	if err != nil {
		fmt.Printf("Could not create client: %v\n", err)
		os.Exit(1)
	}

	ctx := context.Background()

	calicoVersion, _ := parsedArgs["--version"].(string)

	if err := client.EnsureInitialized(ctx, calicoVersion, ""); err != nil {
		fmt.Printf("Could not set calico version to %s: %v\n", calicoVersion, err)
		os.Exit(1)
	}

	fmt.Printf("Calico version set to %s\n", calicoVersion)
}
<|endoftext|>


<commit_before>package mackerelplugin

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"math"
	"os"
	"strconv"
	"time"
)

type Metrics struct {
	Name    string  `json:"name"`
	Label   string  `json:"label"`
	Diff    bool    `json:"diff"`
	Type    string  `json:"type"`
	Stacked bool    `json:"stacked"`
	Scale   float64 `json:"scale"`
}

type Graphs struct {
	Label   string    `json:"label"`
	Unit    string    `json:"unit"`
	Metrics []Metrics `json:"metrics"`
}

type Plugin interface {
	FetchMetrics() (map[string]interface{}, error)
	GraphDefinition() map[string]Graphs
}

type MackerelPlugin struct {
	Plugin
	Tempfile string
}

func NewMackerelPlugin(plugin Plugin) MackerelPlugin {
	mp := MackerelPlugin{plugin, "/tmp/mackerel-plugin-default"}
	return mp
}

func (h *MackerelPlugin) printValue(w io.Writer, key string, value interface{}, now time.Time) {
	switch value.(type) {
	case uint32:
		fmt.Fprintf(w, "%s\t%d\t%d\n", key, value.(uint32), now.Unix())
	case uint64:
		fmt.Fprintf(w, "%s\t%d\t%d\n", key, value.(uint64), now.Unix())
	case float64:
		if math.IsNaN(value.(float64)) || math.IsInf(value.(float64), 0) {
			log.Printf("Invalid value: key = %s, value = %f\n", key, value)
		} else {
			fmt.Fprintf(w, "%s\t%f\t%d\n", key, value.(float64), now.Unix())
		}
	}
}

func (h *MackerelPlugin) fetchLastValues() (map[string]interface{}, time.Time, error) {
	lastTime := time.Now()

	f, err := os.Open(h.Tempfilename())
	if err != nil {
		if os.IsNotExist(err) {
			return nil, lastTime, nil
		}
		return nil, lastTime, err
	}
	defer f.Close()

	stat := make(map[string]interface{})
	decoder := json.NewDecoder(f)
	err = decoder.Decode(&stat)
	lastTime = time.Unix(int64(stat["_lastTime"].(float64)), 0)
	if err != nil {
		return stat, lastTime, err
	}
	return stat, lastTime, nil
}

func (h *MackerelPlugin) saveValues(values map[string]interface{}, now time.Time) error {
	f, err := os.Create(h.Tempfilename())
	if err != nil {
		return err
	}
	defer f.Close()

	values["_lastTime"] = float64(now.Unix())
	encoder := json.NewEncoder(f)
	err = encoder.Encode(values)
	if err != nil {
		return err
	}

	return nil
}

func (h *MackerelPlugin) calcDiff(value float64, now time.Time, lastValue float64, lastTime time.Time) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0, errors.New("Too long duration")
	}

	diff := (value - lastValue) * 60 / float64(diffTime)

	return diff, nil
}

func (h *MackerelPlugin) calcDiffUint32(value uint32, now time.Time, lastValue uint32, lastTime time.Time, lastDiff float64) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0, errors.New("Too long duration")
	}

	diff := float64((value-lastValue)*60) / float64(diffTime)

	if lastValue < value || diff < lastDiff*10 {
		return diff, nil
	}
	return 0.0, errors.New("Counter seems to be reseted.")

}

func (h *MackerelPlugin) calcDiffUint64(value uint64, now time.Time, lastValue uint64, lastTime time.Time, lastDiff float64) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0, errors.New("Too long duration")
	}

	diff := float64((value-lastValue)*60) / float64(diffTime)

	if lastValue < value || diff < lastDiff*10 {
		return diff, nil
	}
	return 0.0, errors.New("Counter seems to be reseted.")
}

func (h *MackerelPlugin) Tempfilename() string {
	return h.Tempfile
}

func (h *MackerelPlugin) OutputValues() {
	now := time.Now()
	stat, err := h.FetchMetrics()
	if err != nil {
		log.Fatalln("OutputValues: ", err)
	}

	lastStat, lastTime, err := h.fetchLastValues()
	if err != nil {
		log.Println("fetchLastValues (ignore):", err)
	}

	for key, graph := range h.GraphDefinition() {
		for _, metric := range graph.Metrics {
			var value interface{}
			value = stat[metric.Name]
			switch value.(type) {
			case string:
				switch metric.Type {
				case "uint32":
					value, _ = strconv.ParseUint(value.(string), 10, 32)
				case "uint64":
					value, _ = strconv.ParseUint(value.(string), 10, 64)
				default:
					value, _ = strconv.ParseFloat(value.(string), 64)
				}
			default:
				continue
			}

			if metric.Diff {
				_, ok := lastStat[metric.Name]
				if ok {
					lastDiff := lastStat[".last_diff."+metric.Name].(float64)
					switch metric.Type {
					case "uint32":
						value, err = h.calcDiffUint32(value.(uint32), now, lastStat[metric.Name].(uint32), lastTime, lastDiff)
						stat[".last_diff."+metric.Name] = value
					case "uint64":
						value, err = h.calcDiffUint64(value.(uint64), now, lastStat[metric.Name].(uint64), lastTime, lastDiff)
						stat[".last_diff."+metric.Name] = value
					default:
						value, err = h.calcDiff(value.(float64), now, lastStat[metric.Name].(float64), lastTime)
					}
					if err != nil {
						log.Println("OutputValues: ", err)
					}
				} else {
					log.Printf("%s is not exist at last fetch\n", metric.Name)
				}
			}

			if metric.Scale != 0 {
				switch metric.Type {
				case "uint32":
					value = value.(uint32) * uint32(metric.Scale)
				case "uint64":
					value = value.(uint64) * uint64(metric.Scale)
				default:
					value = value.(float64) * metric.Scale
				}
			}

			h.printValue(os.Stdout, key+"."+metric.Name, value, now)
		}
	}

	err = h.saveValues(stat, now)
	if err != nil {
		log.Fatalf("saveValues: ", err)
	}

}

type GraphDef struct {
	Graphs map[string]Graphs `json:"graphs"`
}

func (h *MackerelPlugin) OutputDefinitions() {
	fmt.Println("# mackerel-agent-plugin")
	var graphs GraphDef
	graphs.Graphs = h.GraphDefinition()

	b, err := json.Marshal(graphs)
	if err != nil {
		log.Fatalln("OutputDefinitions: ", err)
	}
	fmt.Println(string(b))
}
<commit_msg>fix to care type casting<commit_after>package mackerelplugin

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"math"
	"os"
	"strconv"
	"time"
)

type Metrics struct {
	Name    string  `json:"name"`
	Label   string  `json:"label"`
	Diff    bool    `json:"diff"`
	Type    string  `json:"type"`
	Stacked bool    `json:"stacked"`
	Scale   float64 `json:"scale"`
}

type Graphs struct {
	Label   string    `json:"label"`
	Unit    string    `json:"unit"`
	Metrics []Metrics `json:"metrics"`
}

type Plugin interface {
	FetchMetrics() (map[string]interface{}, error)
	GraphDefinition() map[string]Graphs
}

type MackerelPlugin struct {
	Plugin
	Tempfile string
}

func NewMackerelPlugin(plugin Plugin) MackerelPlugin {
	mp := MackerelPlugin{plugin, "/tmp/mackerel-plugin-default"}
	return mp
}

func (h *MackerelPlugin) printValue(w io.Writer, key string, value interface{}, now time.Time) {
	switch value.(type) {
	case uint32:
		fmt.Fprintf(w, "%s\t%d\t%d\n", key, value.(uint32), now.Unix())
	case uint64:
		fmt.Fprintf(w, "%s\t%d\t%d\n", key, value.(uint64), now.Unix())
	case float64:
		if math.IsNaN(value.(float64)) || math.IsInf(value.(float64), 0) {
			log.Printf("Invalid value: key = %s, value = %f\n", key, value)
		} else {
			fmt.Fprintf(w, "%s\t%f\t%d\n", key, value.(float64), now.Unix())
		}
	}
}

func (h *MackerelPlugin) fetchLastValues() (map[string]interface{}, time.Time, error) {
	lastTime := time.Now()

	f, err := os.Open(h.Tempfilename())
	if err != nil {
		if os.IsNotExist(err) {
			return nil, lastTime, nil
		}
		return nil, lastTime, err
	}
	defer f.Close()

	stat := make(map[string]interface{})
	decoder := json.NewDecoder(f)
	err = decoder.Decode(&stat)
	switch stat["_lastTime"].(type) {
	case float64:
		lastTime = time.Unix(int64(stat["_lastTime"].(float64)), 0)
	case int64:
		lastTime = time.Unix(stat["_lastTime"].(int64), 0)
	}
	if err != nil {
		return stat, lastTime, err
	}
	return stat, lastTime, nil
}

func (h *MackerelPlugin) saveValues(values map[string]interface{}, now time.Time) error {
	f, err := os.Create(h.Tempfilename())
	if err != nil {
		return err
	}
	defer f.Close()

	values["_lastTime"] = now.Unix()
	encoder := json.NewEncoder(f)
	err = encoder.Encode(values)
	if err != nil {
		return err
	}

	return nil
}

func (h *MackerelPlugin) calcDiff(value float64, now time.Time, lastValue float64, lastTime time.Time) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0, errors.New("Too long duration")
	}

	diff := (value - lastValue) * 60 / float64(diffTime)

	return diff, nil
}

func (h *MackerelPlugin) calcDiffUint32(value uint32, now time.Time, lastValue uint32, lastTime time.Time, lastDiff float64) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0, errors.New("Too long duration")
	}

	diff := float64((value-lastValue)*60) / float64(diffTime)

	if lastValue < value || diff < lastDiff*10 {
		return diff, nil
	}
	return 0.0, errors.New("Counter seems to be reseted.")

}

func (h *MackerelPlugin) calcDiffUint64(value uint64, now time.Time, lastValue uint64, lastTime time.Time, lastDiff float64) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0, errors.New("Too long duration")
	}

	diff := float64((value-lastValue)*60) / float64(diffTime)

	if lastValue < value || diff < lastDiff*10 {
		return diff, nil
	}
	return 0.0, errors.New("Counter seems to be reseted.")
}

func (h *MackerelPlugin) Tempfilename() string {
	return h.Tempfile
}

func (h *MackerelPlugin) OutputValues() {
	now := time.Now()
	stat, err := h.FetchMetrics()
	if err != nil {
		log.Fatalln("OutputValues: ", err)
	}

	lastStat, lastTime, err := h.fetchLastValues()
	if err != nil {
		log.Println("fetchLastValues (ignore):", err)
	}

	for key, graph := range h.GraphDefinition() {
		for _, metric := range graph.Metrics {
			var value interface{}
			value = stat[metric.Name]
			switch value.(type) {
			case string:
				switch metric.Type {
				case "uint32":
					value, _ = strconv.ParseUint(value.(string), 10, 32)
				case "uint64":
					value, _ = strconv.ParseUint(value.(string), 10, 64)
				default:
					value, _ = strconv.ParseFloat(value.(string), 64)
				}
			}

			if metric.Diff {
				_, ok := lastStat[metric.Name]
				if ok {
					var lastDiff float64
					if lastStat[".last_diff."+metric.Name] != nil {
						lastDiff = lastStat[".last_diff."+metric.Name].(float64)
					}
					switch metric.Type {
					case "uint32":
						value, err = h.calcDiffUint32(value.(uint32), now, toUint32(lastStat[metric.Name]), lastTime, lastDiff)
					case "uint64":
						value, err = h.calcDiffUint64(value.(uint64), now, toUint64(lastStat[metric.Name]), lastTime, lastDiff)
					default:
						value, err = h.calcDiff(value.(float64), now, lastStat[metric.Name].(float64), lastTime)
					}
					if err != nil {
						log.Println("OutputValues: ", err)
						continue
					} else {
						stat[".last_diff."+metric.Name] = value
					}
				} else {
					log.Printf("%s is not exist at last fetch\n", metric.Name)
				}
			}

			if metric.Scale != 0 {
				switch metric.Type {
				case "uint32":
					value = value.(uint32) * uint32(metric.Scale)
				case "uint64":
					value = value.(uint64) * uint64(metric.Scale)
				default:
					value = value.(float64) * metric.Scale
				}
			}

			h.printValue(os.Stdout, key+"."+metric.Name, value, now)
		}
	}

	err = h.saveValues(stat, now)
	if err != nil {
		log.Fatalf("saveValues: ", err)
	}

}

type GraphDef struct {
	Graphs map[string]Graphs `json:"graphs"`
}

func (h *MackerelPlugin) OutputDefinitions() {
	fmt.Println("# mackerel-agent-plugin")
	var graphs GraphDef
	graphs.Graphs = h.GraphDefinition()

	b, err := json.Marshal(graphs)
	if err != nil {
		log.Fatalln("OutputDefinitions: ", err)
	}
	fmt.Println(string(b))
}

func toUint32(value interface{}) uint32 {
	var ret uint32
	switch value.(type) {
	case uint32:
		ret = value.(uint32)
	case uint64:
		ret = uint32(value.(uint64))
	case float64:
		ret = uint32(value.(float64))
	}
	return ret
}

func toUint64(value interface{}) uint64 {
	var ret uint64
	switch value.(type) {
	case uint32:
		ret = uint64(value.(uint32))
	case uint64:
		ret = value.(uint64)
	case float64:
		ret = uint64(value.(float64))
	}
	return ret
}

func toFloat64(value interface{}) float64 {
	var ret float64
	switch value.(type) {
	case uint32:
		ret = float64(value.(uint32))
	case uint64:
		ret = float64(value.(uint64))
	case float64:
		ret = value.(float64)
	}
	return ret
}
<|endoftext|>


<commit_before>//manager a cluster of proxy
package manager

import (
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"
)

type Proxy struct {
	Link          string
	LastCheckOKTm time.Time
	LastCheckOK   bool
	TotalSecs     float64
	TotalWeight   float64
	Ticker        *time.Ticker
}

func (p *Proxy) AveSecs() float64 {
	return p.TotalSecs / p.TotalWeight
}

var proxies map[string]*Proxy
var proxyList []*Proxy

func init() {
	proxies = make(map[string]*Proxy)
	proxyList = make([]*Proxy, 0, 10)
}

func checkProxy(link string) (bool, float64) {
	log.Println("begin check:", link)
	start := time.Now()
	proxy, err := url.Parse(link)
	if err != nil {
		log.Println(err)
		return false, 0
	}
	client := &http.Client{
		Transport: &http.Transport{
			Dial: func(network, addr string) (net.Conn, error) {
				deadline := time.Now().Add(5 * time.Second)
				c, err := net.DialTimeout(network, addr, 5*time.Second)
				if err != nil {
					return nil, err
				}
				c.SetDeadline(deadline)
				return c, nil
			},
			DisableKeepAlives:     true,
			ResponseHeaderTimeout: 5 * time.Second,
			DisableCompression:    false,
			Proxy:                 http.ProxyURL(proxy),
		},
	}
	resp, err := client.Get("http://54.223.171.0:7183/check")
	if err != nil {
		log.Println(err)
		return false, 0
	}
	if resp == nil {
		log.Println("resp is nil")
		return false, 0
	}
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println(err)
		return false, 0
	}
	t := strings.Trim(string(b), " \n\t\r")
	log.Println(t)
	return strings.Contains(link, t), time.Now().Sub(start).Seconds()
}

func Register(link string) {
	if _, ok := proxies[link]; ok {
		return
	}
	alive, secs := checkProxy(link)
	if !alive {
		continue
	}
	p := &Proxy{
		Link:          link,
		LastCheckOKTm: time.Now(),
		LastCheckOK:   true,
		TotalSecs:     secs,
		TotalWeight:   1.0,
		Ticker:        time.NewTicker(time.Minute),
	}
	proxies[link] = p
	proxyList = append(proxyList, p)
	go func() {
		for _ = range p.Ticker.C {
			alive, secs := checkProxy(link)
			if alive {
				p.LastCheckOK = true
				p.LastCheckOKTm = time.Now()
				p.TotalSecs = p.TotalSecs*0.7 + secs
				p.TotalWeight = p.TotalWeight*0.7 + 1.0
			} else {
				p.LastCheckOK = false
				p.TotalSecs = p.TotalSecs*0.7 + 5.0
				p.TotalWeight = p.TotalWeight*0.7 + 1.0
			}
		}
	}()
}

func Select() *Proxy {
	for i := 0; i < len(proxyList) && i < 3; i++ {
		k := rand.Intn(len(proxyList))
		if time.Now().Sub(proxyList[k].LastCheckOKTm).Minutes() < 5 && proxyList[k].LastCheckOK && proxyList[k].AveSecs() < 2.0 {
			return proxyList[k]
		}
	}
	return nil
}

func HandleRegister(rw http.ResponseWriter, req *http.Request) {
	params := req.URL.Query()
	proxy := params.Get("proxy")
	Register(proxy)
	fmt.Fprint(rw, "ok")
}

func HandleSelect(rw http.ResponseWriter, req *http.Request) {
	p := Select()
	if p != nil {
		fmt.Fprint(rw, p.Link)
	}
}

func HandleCheck(rw http.ResponseWriter, req *http.Request) {
	tks := strings.Split(req.RemoteAddr, ":")
	fmt.Fprint(rw, tks[0])
}
<commit_msg>regular check in manager<commit_after>//manager a cluster of proxy
package manager

import (
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"
)

type Proxy struct {
	Link          string
	LastCheckOKTm time.Time
	LastCheckOK   bool
	TotalSecs     float64
	TotalWeight   float64
	Ticker        *time.Ticker
}

func (p *Proxy) AveSecs() float64 {
	return p.TotalSecs / p.TotalWeight
}

var proxies map[string]*Proxy
var proxyList []*Proxy

func init() {
	proxies = make(map[string]*Proxy)
	proxyList = make([]*Proxy, 0, 10)
}

func checkProxy(link string) (bool, float64) {
	log.Println("begin check:", link)
	start := time.Now()
	proxy, err := url.Parse(link)
	if err != nil {
		log.Println(err)
		return false, 0
	}
	client := &http.Client{
		Transport: &http.Transport{
			Dial: func(network, addr string) (net.Conn, error) {
				deadline := time.Now().Add(5 * time.Second)
				c, err := net.DialTimeout(network, addr, 5*time.Second)
				if err != nil {
					return nil, err
				}
				c.SetDeadline(deadline)
				return c, nil
			},
			DisableKeepAlives:     true,
			ResponseHeaderTimeout: 5 * time.Second,
			DisableCompression:    false,
			Proxy:                 http.ProxyURL(proxy),
		},
	}
	resp, err := client.Get("http://54.223.171.0:7183/check")
	if err != nil {
		log.Println(err)
		return false, 0
	}
	if resp == nil {
		log.Println("resp is nil")
		return false, 0
	}
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println(err)
		return false, 0
	}
	t := strings.Trim(string(b), " \n\t\r")
	log.Println(t)
	return strings.Contains(link, t), time.Now().Sub(start).Seconds()
}

func Register(link string) {
	if _, ok := proxies[link]; ok {
		return
	}
	alive, secs := checkProxy(link)
	if !alive {
		return
	}
	p := &Proxy{
		Link:          link,
		LastCheckOKTm: time.Now(),
		LastCheckOK:   true,
		TotalSecs:     secs,
		TotalWeight:   1.0,
		Ticker:        time.NewTicker(time.Minute),
	}
	proxies[link] = p
	proxyList = append(proxyList, p)
	go func() {
		for _ = range p.Ticker.C {
			alive, secs := checkProxy(link)
			if alive {
				p.LastCheckOK = true
				p.LastCheckOKTm = time.Now()
				p.TotalSecs = p.TotalSecs*0.7 + secs
				p.TotalWeight = p.TotalWeight*0.7 + 1.0
			} else {
				p.LastCheckOK = false
				p.TotalSecs = p.TotalSecs*0.7 + 5.0
				p.TotalWeight = p.TotalWeight*0.7 + 1.0
			}
		}
	}()
}

func Select() *Proxy {
	for i := 0; i < len(proxyList) && i < 3; i++ {
		k := rand.Intn(len(proxyList))
		if time.Now().Sub(proxyList[k].LastCheckOKTm).Minutes() < 5 && proxyList[k].LastCheckOK && proxyList[k].AveSecs() < 2.0 {
			return proxyList[k]
		}
	}
	return nil
}

func HandleRegister(rw http.ResponseWriter, req *http.Request) {
	params := req.URL.Query()
	proxy := params.Get("proxy")
	Register(proxy)
	fmt.Fprint(rw, "ok")
}

func HandleSelect(rw http.ResponseWriter, req *http.Request) {
	p := Select()
	if p != nil {
		fmt.Fprint(rw, p.Link)
	}
}

func HandleCheck(rw http.ResponseWriter, req *http.Request) {
	tks := strings.Split(req.RemoteAddr, ":")
	fmt.Fprint(rw, tks[0])
}
<|endoftext|>