<commit_before>// Copyright (c) 2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1beta1

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

var (
	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
	AddToScheme   = SchemeBuilder.AddToScheme
)

// GroupName is the group name used in this package.
const GroupName = "habitat.sh"

// SchemeGroupVersion is the group version used to register these objects.
var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1"}

// Resource takes an unqualified resource and returns a Group-qualified GroupResource.
func Resource(resource string) schema.GroupResource {
	return SchemeGroupVersion.WithResource(resource).GroupResource()
}

// addKnownTypes adds the set of types defined in this package to the supplied scheme.
func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&Habitat{},
		&HabitatList{},
	)

	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)

	return nil
}
<commit_msg>Change CRD version to v1beta1<commit_after>// Copyright (c) 2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1beta1

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
)

var (
	SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes)
	AddToScheme   = SchemeBuilder.AddToScheme
)

// GroupName is the group name used in this package.
const GroupName = "habitat.sh"

// SchemeGroupVersion is the group version used to register these objects.
var SchemeGroupVersion = schema.GroupVersion{Group: GroupName, Version: "v1beta1"}

// Resource takes an unqualified resource and returns a Group-qualified GroupResource.
func Resource(resource string) schema.GroupResource {
	return SchemeGroupVersion.WithResource(resource).GroupResource()
}

// addKnownTypes adds the set of types defined in this package to the supplied scheme.
func addKnownTypes(scheme *runtime.Scheme) error {
	scheme.AddKnownTypes(SchemeGroupVersion,
		&Habitat{},
		&HabitatList{},
	)

	metav1.AddToGroupVersion(scheme, SchemeGroupVersion)

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validation

import (
	"testing"

	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"

	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/kops/pkg/apis/kops"
)

func TestAWSValidateExternalCloudConfig(t *testing.T) {
	grid := []struct {
		Input          kops.ClusterSpec
		ExpectedErrors []string
	}{
		{
			Input: kops.ClusterSpec{
				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{},
			},
			ExpectedErrors: []string{"Forbidden::spec.externalCloudControllerManager"},
		},
		{
			Input: kops.ClusterSpec{
				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{},
				CloudConfig: &kops.CloudConfiguration{
					AWSEBSCSIDriver: &kops.AWSEBSCSIDriver{
						Enabled: fi.Bool(true),
					},
				},
			},
		},
		{
			Input: kops.ClusterSpec{
				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{},
				KubeControllerManager: &kops.KubeControllerManagerConfig{
					ExternalCloudVolumePlugin: "aws",
				},
			},
		},
	}
	for _, g := range grid {
		errs := awsValidateExternalCloudControllerManager(g.Input)

		testErrors(t, g.Input, errs, g.ExpectedErrors)
	}
}

func TestValidateInstanceGroupSpec(t *testing.T) {
	grid := []struct {
		Input          kops.InstanceGroupSpec
		ExpectedErrors []string
	}{
		{
			Input: kops.InstanceGroupSpec{
				AdditionalSecurityGroups: []string{},
			},
		},
		{
			Input: kops.InstanceGroupSpec{
				AdditionalSecurityGroups: []string{"sg-1234abcd"},
			},
		},
		{
			Input: kops.InstanceGroupSpec{
				AdditionalSecurityGroups: []string{"sg-1234abcd", ""},
			},
			ExpectedErrors: []string{"Invalid value::spec.additionalSecurityGroups[1]"},
		},
		{
			Input: kops.InstanceGroupSpec{
				AdditionalSecurityGroups: []string{" ", ""},
			},
			ExpectedErrors: []string{
				"Invalid value::spec.additionalSecurityGroups[0]",
				"Invalid value::spec.additionalSecurityGroups[1]",
			},
		},
		{
			Input: kops.InstanceGroupSpec{
				AdditionalSecurityGroups: []string{"--invalid"},
			},
			ExpectedErrors: []string{"Invalid value::spec.additionalSecurityGroups[0]"},
		},
		{
			Input: kops.InstanceGroupSpec{
				MachineType: "t2.micro",
			},
		},
		{
			Input: kops.InstanceGroupSpec{
				MachineType: "t2.invalidType",
			},
			ExpectedErrors: []string{"Invalid value::test-nodes.spec.machineType"},
		},
		{
			Input: kops.InstanceGroupSpec{
				MachineType: "m5.large",
				Image:       "k8s-1.9-debian-stretch-amd64-hvm-ebs-2018-03-11",
			},
			ExpectedErrors: []string{},
		},
		{
			Input: kops.InstanceGroupSpec{
				MachineType: "c5.large",
				Image:       "k8s-1.9-debian-stretch-amd64-hvm-ebs-2018-03-11",
			},
			ExpectedErrors: []string{},
		},
		{
			Input: kops.InstanceGroupSpec{
				SpotDurationInMinutes: fi.Int64(55),
			},
			ExpectedErrors: []string{
				"Unsupported value::test-nodes.spec.spotDurationInMinutes",
			},
		},
		{
			Input: kops.InstanceGroupSpec{
				SpotDurationInMinutes: fi.Int64(380),
			},
			ExpectedErrors: []string{
				"Unsupported value::test-nodes.spec.spotDurationInMinutes",
			},
		},
		{
			Input: kops.InstanceGroupSpec{
				SpotDurationInMinutes: fi.Int64(125),
			},
			ExpectedErrors: []string{
				"Unsupported value::test-nodes.spec.spotDurationInMinutes",
			},
		},
		{
			Input: kops.InstanceGroupSpec{
				SpotDurationInMinutes: fi.Int64(120),
			},
			ExpectedErrors: []string{},
		},
		{
			Input: kops.InstanceGroupSpec{
				InstanceInterruptionBehavior: fi.String("invalidValue"),
			},
			ExpectedErrors: []string{
				"Unsupported value::test-nodes.spec.instanceInterruptionBehavior",
			},
		},
		{
			Input: kops.InstanceGroupSpec{
				InstanceInterruptionBehavior: fi.String("terminate"),
			},
			ExpectedErrors: []string{},
		},
		{
			Input: kops.InstanceGroupSpec{
				InstanceInterruptionBehavior: fi.String("hibernate"),
			},
			ExpectedErrors: []string{},
		},
		{
			Input: kops.InstanceGroupSpec{
				InstanceInterruptionBehavior: fi.String("stop"),
			},
			ExpectedErrors: []string{},
		},
	}
	cloud := awsup.BuildMockAWSCloud("us-east-1", "abc")
	for _, g := range grid {
		ig := &kops.InstanceGroup{
			ObjectMeta: v1.ObjectMeta{
				Name: "test-nodes",
			},
			Spec: g.Input,
		}
		errs := awsValidateInstanceGroup(ig, cloud)

		testErrors(t, g.Input, errs, g.ExpectedErrors)
	}
}

func TestInstanceMetadataOptions(t *testing.T) {
	cloud := awsup.BuildMockAWSCloud("us-east-1", "abc")

	tests := []struct {
		ig       *kops.InstanceGroup
		expected []string
	}{
		{
			ig: &kops.InstanceGroup{
				ObjectMeta: v1.ObjectMeta{
					Name: "some-ig",
				},
				Spec: kops.InstanceGroupSpec{
					Role: "Node",
					InstanceMetadata: &kops.InstanceMetadataOptions{
						HTTPPutResponseHopLimit: fi.Int64(1),
						HTTPTokens:              fi.String("abc"),
					},
				},
			},
			expected: []string{"Unsupported value::spec.instanceMetadata.httpTokens"},
		},
		{
			ig: &kops.InstanceGroup{
				ObjectMeta: v1.ObjectMeta{
					Name: "some-ig",
				},
				Spec: kops.InstanceGroupSpec{
					Role: "Node",
					InstanceMetadata: &kops.InstanceMetadataOptions{
						HTTPPutResponseHopLimit: fi.Int64(-1),
						HTTPTokens:              fi.String("required"),
					},
				},
			},
			expected: []string{"Invalid value::spec.instanceMetadata.httpPutResponseHopLimit"},
		},
	}

	for _, test := range tests {
		errs := ValidateInstanceGroup(test.ig, cloud)
		testErrors(t, test.ig.ObjectMeta.Name, errs, test.expected)
	}
}

func TestLoadBalancerSubnets(t *testing.T) {
	cidr := "10.0.0.0/24"
	tests := []struct {
		lbType         *string
		class          *string
		clusterSubnets []string
		lbSubnets      []kops.LoadBalancerSubnetSpec
		expected       []string
	}{
		{ // valid (no privateIPv4Address)
			clusterSubnets: []string{"a", "b", "c"},
			lbSubnets: []kops.LoadBalancerSubnetSpec{
				{
					Name:               "a",
					PrivateIPv4Address: nil,
				},
				{
					Name:               "b",
					PrivateIPv4Address: nil,
				},
			},
		},
		{ // valid (with privateIPv4Address)
			clusterSubnets: []string{"a", "b", "c"},
			lbSubnets: []kops.LoadBalancerSubnetSpec{
				{
					Name:               "a",
					PrivateIPv4Address: fi.String("10.0.0.10"),
				},
				{
					Name:               "b",
					PrivateIPv4Address: nil,
				},
			},
		},
		{ // empty subnet name
			clusterSubnets: []string{"a", "b", "c"},
			lbSubnets: []kops.LoadBalancerSubnetSpec{
				{
					Name:               "",
					PrivateIPv4Address: nil,
				},
			},
			expected: []string{"Required value::spec.api.loadBalancer.subnets[0].name"},
		},
		{ // subnet not found
			clusterSubnets: []string{"a", "b", "c"},
			lbSubnets: []kops.LoadBalancerSubnetSpec{
				{
					Name:               "d",
					PrivateIPv4Address: nil,
				},
			},
			expected: []string{"Not found::spec.api.loadBalancer.subnets[0].name"},
		},
		{ // empty privateIPv4Address
			clusterSubnets: []string{"a", "b", "c"},
			lbSubnets: []kops.LoadBalancerSubnetSpec{
				{
					Name:               "a",
					PrivateIPv4Address: fi.String(""),
				},
			},
			expected: []string{"Required value::spec.api.loadBalancer.subnets[0].privateIPv4Address"},
		},
		{ // invalid privateIPv4Address
			clusterSubnets: []string{"a", "b", "c"},
			lbSubnets: []kops.LoadBalancerSubnetSpec{
				{
					Name:               "a",
					PrivateIPv4Address: fi.String("invalidip"),
				},
			},
			expected: []string{"Invalid value::spec.api.loadBalancer.subnets[0].privateIPv4Address"},
		},
		{ // privateIPv4Address not matching subnet cidr
			clusterSubnets: []string{"a", "b", "c"},
			lbSubnets: []kops.LoadBalancerSubnetSpec{
				{
					Name:               "a",
					PrivateIPv4Address: fi.String("11.0.0.10"),
				},
			},
			expected: []string{"Invalid value::spec.api.loadBalancer.subnets[0].privateIPv4Address"},
		},
		{ // invalid class
			class:          fi.String(string(kops.LoadBalancerClassClassic)),
			clusterSubnets: []string{"a", "b", "c"},
			lbSubnets: []kops.LoadBalancerSubnetSpec{
				{
					Name:               "a",
					PrivateIPv4Address: fi.String("10.0.0.10"),
				},
			},
			expected: []string{"Forbidden::spec.api.loadBalancer.subnets[0].privateIPv4Address"},
		},
		{ // invalid type
			lbType:         fi.String(string(kops.LoadBalancerTypePublic)),
			clusterSubnets: []string{"a", "b", "c"},
			lbSubnets: []kops.LoadBalancerSubnetSpec{
				{
					Name:               "a",
					PrivateIPv4Address: fi.String("10.0.0.10"),
				},
			},
			expected: []string{"Forbidden::spec.api.loadBalancer.subnets[0].privateIPv4Address"},
		},
	}

	for _, test := range tests {
		cluster := kops.Cluster{
			Spec: kops.ClusterSpec{
				API: &kops.AccessSpec{
					LoadBalancer: &kops.LoadBalancerAccessSpec{
						Class: kops.LoadBalancerClassNetwork,
						Type:  kops.LoadBalancerTypeInternal,
					},
				},
			},
		}
		if test.class != nil {
			cluster.Spec.API.LoadBalancer.Class = kops.LoadBalancerClass(*test.class)
		}
		if test.lbType != nil {
			cluster.Spec.API.LoadBalancer.Type = kops.LoadBalancerType(*test.lbType)
		}
		for _, s := range test.clusterSubnets {
			cluster.Spec.Subnets = append(cluster.Spec.Subnets, kops.ClusterSubnetSpec{
				Name: s,
				CIDR: cidr,
			})
		}
		cluster.Spec.API.LoadBalancer.Subnets = test.lbSubnets
		errs := awsValidateCluster(&cluster)
		testErrors(t, test, errs, test.expected)
	}
}
<commit_msg>Add unit tests<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package validation

import (
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ec2"
	"k8s.io/kops/cloudmock/aws/mockec2"

	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/awsup"

	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/kops/pkg/apis/kops"
)

func TestAWSValidateExternalCloudConfig(t *testing.T) {
	grid := []struct {
		Input          kops.ClusterSpec
		ExpectedErrors []string
	}{
		{
			Input: kops.ClusterSpec{
				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{},
			},
			ExpectedErrors: []string{"Forbidden::spec.externalCloudControllerManager"},
		},
		{
			Input: kops.ClusterSpec{
				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{},
				CloudConfig: &kops.CloudConfiguration{
					AWSEBSCSIDriver: &kops.AWSEBSCSIDriver{
						Enabled: fi.Bool(true),
					},
				},
			},
		},
		{
			Input: kops.ClusterSpec{
				ExternalCloudControllerManager: &kops.CloudControllerManagerConfig{},
				KubeControllerManager: &kops.KubeControllerManagerConfig{
					ExternalCloudVolumePlugin: "aws",
				},
			},
		},
	}
	for _, g := range grid {
		errs := awsValidateExternalCloudControllerManager(g.Input)

		testErrors(t, g.Input, errs, g.ExpectedErrors)
	}
}

func TestValidateInstanceGroupSpec(t *testing.T) {
	grid := []struct {
		Input          kops.InstanceGroupSpec
		ExpectedErrors []string
	}{
		{
			Input: kops.InstanceGroupSpec{
				AdditionalSecurityGroups: []string{},
			},
		},
		{
			Input: kops.InstanceGroupSpec{
				AdditionalSecurityGroups: []string{"sg-1234abcd"},
			},
		},
		{
			Input: kops.InstanceGroupSpec{
				AdditionalSecurityGroups: []string{"sg-1234abcd", ""},
			},
			ExpectedErrors: []string{"Invalid value::spec.additionalSecurityGroups[1]"},
		},
		{
			Input: kops.InstanceGroupSpec{
				AdditionalSecurityGroups: []string{" ", ""},
			},
			ExpectedErrors: []string{
				"Invalid value::spec.additionalSecurityGroups[0]",
				"Invalid value::spec.additionalSecurityGroups[1]",
			},
		},
		{
			Input: kops.InstanceGroupSpec{
				AdditionalSecurityGroups: []string{"--invalid"},
			},
			ExpectedErrors: []string{"Invalid value::spec.additionalSecurityGroups[0]"},
		},
		{
			Input: kops.InstanceGroupSpec{
				MachineType: "t2.micro",
				Image:       "ami-073c8c0760395aab8",
			},
		},
		{
			Input: kops.InstanceGroupSpec{
				MachineType: "t2.invalidType",
				Image:       "ami-073c8c0760395aab8",
			},
			ExpectedErrors: []string{"Invalid value::test-nodes.spec.machineType"},
		},
		{
			Input: kops.InstanceGroupSpec{
				MachineType: "m4.large",
				Image:       "ami-073c8c0760395aab8",
			},
			ExpectedErrors: []string{},
		},
		{
			Input: kops.InstanceGroupSpec{
				MachineType: "c5.large",
				Image:       "ami-073c8c0760395aab8",
			},
			ExpectedErrors: []string{},
		},
		{
			Input: kops.InstanceGroupSpec{
				MachineType: "a1.large",
				Image:       "ami-073c8c0760395aab8",
			},
			ExpectedErrors: []string{
				"Invalid value::test-nodes.spec.machineType",
			},
		},
		{
			Input: kops.InstanceGroupSpec{
				SpotDurationInMinutes: fi.Int64(55),
			},
			ExpectedErrors: []string{
				"Unsupported value::test-nodes.spec.spotDurationInMinutes",
			},
		},
		{
			Input: kops.InstanceGroupSpec{
				SpotDurationInMinutes: fi.Int64(380),
			},
			ExpectedErrors: []string{
				"Unsupported value::test-nodes.spec.spotDurationInMinutes",
			},
		},
		{
			Input: kops.InstanceGroupSpec{
				SpotDurationInMinutes: fi.Int64(125),
			},
			ExpectedErrors: []string{
				"Unsupported value::test-nodes.spec.spotDurationInMinutes",
			},
		},
		{
			Input: kops.InstanceGroupSpec{
				SpotDurationInMinutes: fi.Int64(120),
			},
			ExpectedErrors: []string{},
		},
		{
			Input: kops.InstanceGroupSpec{
				InstanceInterruptionBehavior: fi.String("invalidValue"),
			},
			ExpectedErrors: []string{
				"Unsupported value::test-nodes.spec.instanceInterruptionBehavior",
			},
		},
		{
			Input: kops.InstanceGroupSpec{
				InstanceInterruptionBehavior: fi.String("terminate"),
			},
			ExpectedErrors: []string{},
		},
		{
			Input: kops.InstanceGroupSpec{
				InstanceInterruptionBehavior: fi.String("hibernate"),
			},
			ExpectedErrors: []string{},
		},
		{
			Input: kops.InstanceGroupSpec{
				InstanceInterruptionBehavior: fi.String("stop"),
			},
			ExpectedErrors: []string{},
		},
	}
	cloud := awsup.BuildMockAWSCloud("us-east-1", "abc")
	mockEC2 := &mockec2.MockEC2{}
	cloud.MockEC2 = mockEC2

	mockEC2.Images = append(mockEC2.Images, &ec2.Image{
		CreationDate:   aws.String("2016-10-21T20:07:19.000Z"),
		ImageId:        aws.String("ami-073c8c0760395aab8"),
		Name:           aws.String("focal"),
		OwnerId:        aws.String(awsup.WellKnownAccountUbuntu),
		RootDeviceName: aws.String("/dev/xvda"),
		Architecture:   aws.String("x86_64"),
	})

	for _, g := range grid {
		ig := &kops.InstanceGroup{
			ObjectMeta: v1.ObjectMeta{
				Name: "test-nodes",
			},
			Spec: g.Input,
		}
		errs := awsValidateInstanceGroup(ig, cloud)

		testErrors(t, g.Input, errs, g.ExpectedErrors)
	}
}

func TestMixedInstancePolicies(t *testing.T) {
	grid := []struct {
		Input          kops.InstanceGroupSpec
		ExpectedErrors []string
	}{
		{
			Input: kops.InstanceGroupSpec{
				MachineType: "m4.large",
				Image:       "ami-073c8c0760395aab8",
				MixedInstancesPolicy: &kops.MixedInstancesPolicySpec{
					Instances: []string{
						"m4.large",
						"t3.medium",
						"c5.large",
					},
				},
			},
			ExpectedErrors: nil,
		},
		{
			Input: kops.InstanceGroupSpec{
				MachineType: "m4.large",
				Image:       "ami-073c8c0760395aab8",
				MixedInstancesPolicy: &kops.MixedInstancesPolicySpec{
					Instances: []string{
						"a1.large",
						"c4.large",
						"c5.large",
					},
				},
			},
			ExpectedErrors: []string{"Invalid value::spec.mixedInstancesPolicy.instances[0]"},
		},
		{
			Input: kops.InstanceGroupSpec{
				MachineType: "m4.large",
				Image:       "ami-073c8c0760395aab8",
				MixedInstancesPolicy: &kops.MixedInstancesPolicySpec{
					Instances: []string{
						"t3.medium",
						"c4.large",
						"c5.large",
					},
					OnDemandAboveBase: fi.Int64(231),
				},
			},
			ExpectedErrors: []string{"Invalid value::spec.mixedInstancesPolicy.onDemandAboveBase"},
		},
	}
	cloud := awsup.BuildMockAWSCloud("us-east-1", "abc")
	mockEC2 := &mockec2.MockEC2{}
	cloud.MockEC2 = mockEC2

	mockEC2.Images = append(mockEC2.Images, &ec2.Image{
		CreationDate:   aws.String("2016-10-21T20:07:19.000Z"),
		ImageId:        aws.String("ami-073c8c0760395aab8"),
		Name:           aws.String("focal"),
		OwnerId:        aws.String(awsup.WellKnownAccountUbuntu),
		RootDeviceName: aws.String("/dev/xvda"),
		Architecture:   aws.String("x86_64"),
	})

	for _, g := range grid {
		ig := &kops.InstanceGroup{
			ObjectMeta: v1.ObjectMeta{
				Name: "test-nodes",
			},
			Spec: g.Input,
		}
		errs := awsValidateInstanceGroup(ig, cloud)

		testErrors(t, g.Input, errs, g.ExpectedErrors)
	}

}

func TestInstanceMetadataOptions(t *testing.T) {
	cloud := awsup.BuildMockAWSCloud("us-east-1", "abc")

	tests := []struct {
		ig       *kops.InstanceGroup
		expected []string
	}{
		{
			ig: &kops.InstanceGroup{
				ObjectMeta: v1.ObjectMeta{
					Name: "some-ig",
				},
				Spec: kops.InstanceGroupSpec{
					Role: "Node",
					InstanceMetadata: &kops.InstanceMetadataOptions{
						HTTPPutResponseHopLimit: fi.Int64(1),
						HTTPTokens:              fi.String("abc"),
					},
				},
			},
			expected: []string{"Unsupported value::spec.instanceMetadata.httpTokens"},
		},
		{
			ig: &kops.InstanceGroup{
				ObjectMeta: v1.ObjectMeta{
					Name: "some-ig",
				},
				Spec: kops.InstanceGroupSpec{
					Role: "Node",
					InstanceMetadata: &kops.InstanceMetadataOptions{
						HTTPPutResponseHopLimit: fi.Int64(-1),
						HTTPTokens:              fi.String("required"),
					},
				},
			},
			expected: []string{"Invalid value::spec.instanceMetadata.httpPutResponseHopLimit"},
		},
	}

	for _, test := range tests {
		errs := ValidateInstanceGroup(test.ig, cloud)
		testErrors(t, test.ig.ObjectMeta.Name, errs, test.expected)
	}
}

func TestLoadBalancerSubnets(t *testing.T) {
	cidr := "10.0.0.0/24"
	tests := []struct {
		lbType         *string
		class          *string
		clusterSubnets []string
		lbSubnets      []kops.LoadBalancerSubnetSpec
		expected       []string
	}{
		{ // valid (no privateIPv4Address)
			clusterSubnets: []string{"a", "b", "c"},
			lbSubnets: []kops.LoadBalancerSubnetSpec{
				{
					Name:               "a",
					PrivateIPv4Address: nil,
				},
				{
					Name:               "b",
					PrivateIPv4Address: nil,
				},
			},
		},
		{ // valid (with privateIPv4Address)
			clusterSubnets: []string{"a", "b", "c"},
			lbSubnets: []kops.LoadBalancerSubnetSpec{
				{
					Name:               "a",
					PrivateIPv4Address: fi.String("10.0.0.10"),
				},
				{
					Name:               "b",
					PrivateIPv4Address: nil,
				},
			},
		},
		{ // empty subnet name
			clusterSubnets: []string{"a", "b", "c"},
			lbSubnets: []kops.LoadBalancerSubnetSpec{
				{
					Name:               "",
					PrivateIPv4Address: nil,
				},
			},
			expected: []string{"Required value::spec.api.loadBalancer.subnets[0].name"},
		},
		{ // subnet not found
			clusterSubnets: []string{"a", "b", "c"},
			lbSubnets: []kops.LoadBalancerSubnetSpec{
				{
					Name:               "d",
					PrivateIPv4Address: nil,
				},
			},
			expected: []string{"Not found::spec.api.loadBalancer.subnets[0].name"},
		},
		{ // empty privateIPv4Address
			clusterSubnets: []string{"a", "b", "c"},
			lbSubnets: []kops.LoadBalancerSubnetSpec{
				{
					Name:               "a",
					PrivateIPv4Address: fi.String(""),
				},
			},
			expected: []string{"Required value::spec.api.loadBalancer.subnets[0].privateIPv4Address"},
		},
		{ // invalid privateIPv4Address
			clusterSubnets: []string{"a", "b", "c"},
			lbSubnets: []kops.LoadBalancerSubnetSpec{
				{
					Name:               "a",
					PrivateIPv4Address: fi.String("invalidip"),
				},
			},
			expected: []string{"Invalid value::spec.api.loadBalancer.subnets[0].privateIPv4Address"},
		},
		{ // privateIPv4Address not matching subnet cidr
			clusterSubnets: []string{"a", "b", "c"},
			lbSubnets: []kops.LoadBalancerSubnetSpec{
				{
					Name:               "a",
					PrivateIPv4Address: fi.String("11.0.0.10"),
				},
			},
			expected: []string{"Invalid value::spec.api.loadBalancer.subnets[0].privateIPv4Address"},
		},
		{ // invalid class
			class:          fi.String(string(kops.LoadBalancerClassClassic)),
			clusterSubnets: []string{"a", "b", "c"},
			lbSubnets: []kops.LoadBalancerSubnetSpec{
				{
					Name:               "a",
					PrivateIPv4Address: fi.String("10.0.0.10"),
				},
			},
			expected: []string{"Forbidden::spec.api.loadBalancer.subnets[0].privateIPv4Address"},
		},
		{ // invalid type
			lbType:         fi.String(string(kops.LoadBalancerTypePublic)),
			clusterSubnets: []string{"a", "b", "c"},
			lbSubnets: []kops.LoadBalancerSubnetSpec{
				{
					Name:               "a",
					PrivateIPv4Address: fi.String("10.0.0.10"),
				},
			},
			expected: []string{"Forbidden::spec.api.loadBalancer.subnets[0].privateIPv4Address"},
		},
	}

	for _, test := range tests {
		cluster := kops.Cluster{
			Spec: kops.ClusterSpec{
				API: &kops.AccessSpec{
					LoadBalancer: &kops.LoadBalancerAccessSpec{
						Class: kops.LoadBalancerClassNetwork,
						Type:  kops.LoadBalancerTypeInternal,
					},
				},
			},
		}
		if test.class != nil {
			cluster.Spec.API.LoadBalancer.Class = kops.LoadBalancerClass(*test.class)
		}
		if test.lbType != nil {
			cluster.Spec.API.LoadBalancer.Type = kops.LoadBalancerType(*test.lbType)
		}
		for _, s := range test.clusterSubnets {
			cluster.Spec.Subnets = append(cluster.Spec.Subnets, kops.ClusterSubnetSpec{
				Name: s,
				CIDR: cidr,
			})
		}
		cluster.Spec.API.LoadBalancer.Subnets = test.lbSubnets
		errs := awsValidateCluster(&cluster)
		testErrors(t, test, errs, test.expected)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kms

import (
	"strings"

	"github.com/libopenstorage/secrets/vault"
	"github.com/libopenstorage/secrets/vault/utils"
	"github.com/pkg/errors"
	"github.com/rook/rook/pkg/clusterd"

	"github.com/hashicorp/vault/api"
)

const (
	kvVersionKey = "version"
	kvVersion1   = "kv"
	kvVersion2   = "kv-v2"
)

// vaultClient returns a vault client, also used in unit tests to mock the client
var vaultClient = newVaultClient

// newVaultClient returns a vault client, there is no need for any secretConfig validation
// Since this is called after an already validated call InitVault()
func newVaultClient(clusterdContext *clusterd.Context, namespace string, secretConfig map[string]string) (*api.Client, error) {
	// DefaultConfig uses the environment variables if present.
	config := api.DefaultConfig()

	// Always use a new map otherwise the map will mutate and subsequent calls will fail since the
	// TLS content has been altered by the TLS config in vaultClient()
	localSecretConfig := make(map[string]string)
	for k, v := range secretConfig {
		localSecretConfig[k] = v
	}

	// Convert map string to map interface
	c := make(map[string]interface{})
	for k, v := range localSecretConfig {
		c[k] = v
	}

	// Populate TLS config
	newConfigWithTLS, removeCertFiles, err := configTLS(clusterdContext, namespace, localSecretConfig)
	if err != nil {
		return nil, errors.Wrap(err, "failed to initialize vault tls configuration")
	}
	defer removeCertFiles()

	// Populate TLS config
	for key, value := range newConfigWithTLS {
		c[key] = string(value)
	}

	// Configure TLS
	if err := utils.ConfigureTLS(config, c); err != nil {
		return nil, err
	}

	// Initialize the vault client
	client, err := api.NewClient(config)
	if err != nil {
		return nil, err
	}

	// Set Vault address, was validated by ValidateConnectionDetails()
	err = client.SetAddress(strings.TrimSuffix(localSecretConfig[api.EnvVaultAddress], "\n"))
	if err != nil {
		return nil, err
	}

	// Set the namespace if the config has a namespace
	if backendPath := GetParam(secretConfig, api.EnvVaultNamespace); backendPath != "" {
		client.SetNamespace(secretConfig[api.EnvVaultNamespace])
	}

	// Configure the authentication method, either Token or Kubernetes.
	// Both return a token
	token, _, err := utils.Authenticate(client, c)
	if err != nil {
		return nil, errors.Wrap(err, "failed to get vault authentication token")
	}

	// Set the token if provided, token should be set by ValidateConnectionDetails() if applicable
	// api.NewClient() already looks up the token from the environment but we need to set it here and remove potential malformed tokens
	client.SetToken(token)

	return client, nil
}

func BackendVersion(clusterdContext *clusterd.Context, namespace string, secretConfig map[string]string) (string, error) {
	v1 := "v1"
	v2 := "v2"

	backendPath := GetParam(secretConfig, vault.VaultBackendPathKey)
	if backendPath == "" {
		backendPath = vault.DefaultBackendPath
	}

	backend := GetParam(secretConfig, vault.VaultBackendKey)
	switch backend {
	case kvVersion1, v1:
		logger.Info("vault kv secret engine version set to v1")
		return v1, nil
	case kvVersion2, v2:
		logger.Info("vault kv secret engine version set to v2")
		return v2, nil
	default:
		// Initialize Vault client
		vaultClient, err := vaultClient(clusterdContext, namespace, secretConfig)
		if err != nil {
			return "", errors.Wrap(err, "failed to initialize vault client")
		}

		mounts, err := vaultClient.Sys().ListMounts()
		if err != nil {
			return "", errors.Wrap(err, "failed to list vault system mounts")
		}

		for path, mount := range mounts {
			// path is represented as 'path/'
			if trimSlash(path) == trimSlash(backendPath) {
				version := mount.Options[kvVersionKey]
				if version == "2" {
					logger.Info("vault kv secret engine version auto-detected to v2")
					return v2, nil
				}
				logger.Info("vault kv secret engine version auto-detected to v1")
				return v1, nil
			}
		}
	}

	return "", errors.Errorf("secrets engine with mount path %q not found", backendPath)
}

func trimSlash(in string) string {
	return strings.Trim(in, "/")
}
<commit_msg>osd: clarify vault auth error message<commit_after>/*
Copyright 2021 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kms

import (
	"strings"

	"github.com/libopenstorage/secrets/vault"
	"github.com/libopenstorage/secrets/vault/utils"
	"github.com/pkg/errors"
	"github.com/rook/rook/pkg/clusterd"

	"github.com/hashicorp/vault/api"
)

const (
	kvVersionKey = "version"
	kvVersion1   = "kv"
	kvVersion2   = "kv-v2"
)

// vaultClient returns a vault client, also used in unit tests to mock the client
var vaultClient = newVaultClient

// newVaultClient returns a vault client, there is no need for any secretConfig validation
// Since this is called after an already validated call InitVault()
func newVaultClient(clusterdContext *clusterd.Context, namespace string, secretConfig map[string]string) (*api.Client, error) {
	// DefaultConfig uses the environment variables if present.
	config := api.DefaultConfig()

	// Always use a new map otherwise the map will mutate and subsequent calls will fail since the
	// TLS content has been altered by the TLS config in vaultClient()
	localSecretConfig := make(map[string]string)
	for k, v := range secretConfig {
		localSecretConfig[k] = v
	}

	// Convert map string to map interface
	c := make(map[string]interface{})
	for k, v := range localSecretConfig {
		c[k] = v
	}

	// Populate TLS config
	newConfigWithTLS, removeCertFiles, err := configTLS(clusterdContext, namespace, localSecretConfig)
	if err != nil {
		return nil, errors.Wrap(err, "failed to initialize vault tls configuration")
	}
	defer removeCertFiles()

	// Populate TLS config
	for key, value := range newConfigWithTLS {
		c[key] = string(value)
	}

	// Configure TLS
	if err := utils.ConfigureTLS(config, c); err != nil {
		return nil, err
	}

	// Initialize the vault client
	client, err := api.NewClient(config)
	if err != nil {
		return nil, err
	}

	// Set Vault address, was validated by ValidateConnectionDetails()
	err = client.SetAddress(strings.TrimSuffix(localSecretConfig[api.EnvVaultAddress], "\n"))
	if err != nil {
		return nil, err
	}

	// Set the namespace if the config has a namespace
	if backendPath := GetParam(secretConfig, api.EnvVaultNamespace); backendPath != "" {
		client.SetNamespace(secretConfig[api.EnvVaultNamespace])
	}

	// Configure the authentication method, either Token or Kubernetes.
	// Both return a token
	token, _, err := utils.Authenticate(client, c)
	if err != nil {
		if authType := GetParam(secretConfig, vault.AuthMethod); authType == vault.AuthMethodKubernetes {
			return nil, errors.Wrap(err, "failed to get vault authentication token for kubernetes authentication (missing Service Account?)")
		}
		return nil, errors.Wrap(err, "failed to get vault authentication token")
	}

	// Set the token if provided, token should be set by ValidateConnectionDetails() if applicable
	// api.NewClient() already looks up the token from the environment but we need to set it here and remove potential malformed tokens
	client.SetToken(token)

	return client, nil
}

func BackendVersion(clusterdContext *clusterd.Context, namespace string, secretConfig map[string]string) (string, error) {
	v1 := "v1"
	v2 := "v2"

	backendPath := GetParam(secretConfig, vault.VaultBackendPathKey)
	if backendPath == "" {
		backendPath = vault.DefaultBackendPath
	}

	backend := GetParam(secretConfig, vault.VaultBackendKey)
	switch backend {
	case kvVersion1, v1:
		logger.Info("vault kv secret engine version set to v1")
		return v1, nil
	case kvVersion2, v2:
		logger.Info("vault kv secret engine version set to v2")
		return v2, nil
	default:
		// Initialize Vault client
		vaultClient, err := vaultClient(clusterdContext, namespace, secretConfig)
		if err != nil {
			return "", errors.Wrap(err, "failed to initialize vault client")
		}

		mounts, err := vaultClient.Sys().ListMounts()
		if err != nil {
			return "", errors.Wrap(err, "failed to list vault system mounts")
		}

		for path, mount := range mounts {
			// path is represented as 'path/'
			if trimSlash(path) == trimSlash(backendPath) {
				version := mount.Options[kvVersionKey]
				if version == "2" {
					logger.Info("vault kv secret engine version auto-detected to v2")
					return v2, nil
				}
				logger.Info("vault kv secret engine version auto-detected to v1")
				return v1, nil
			}
		}
	}

	return "", errors.Errorf("secrets engine with mount path %q not found", backendPath)
}

func trimSlash(in string) string {
	return strings.Trim(in, "/")
}
<|endoftext|>


<commit_before>//go:build linux
// +build linux

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubelet

import (
	"fmt"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/klog/v2"
	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
	utilexec "k8s.io/utils/exec"
)

const (
	// KubeIPTablesHintChain is the chain whose existence in either iptables-legacy
	// or iptables-nft indicates which version of iptables the system is using
	KubeIPTablesHintChain utiliptables.Chain = "KUBE-IPTABLES-HINT"

	// KubeMarkMasqChain is the mark-for-masquerade chain
	// TODO: clean up this logic in kube-proxy
	KubeMarkMasqChain utiliptables.Chain = "KUBE-MARK-MASQ"

	// KubeMarkDropChain is the mark-for-drop chain
	KubeMarkDropChain utiliptables.Chain = "KUBE-MARK-DROP"

	// KubePostroutingChain is kubernetes postrouting rules
	KubePostroutingChain utiliptables.Chain = "KUBE-POSTROUTING"

	// KubeFirewallChain is kubernetes firewall rules
	KubeFirewallChain utiliptables.Chain = "KUBE-FIREWALL"
)

func (kl *Kubelet) initNetworkUtil() {
	exec := utilexec.New()
	iptClients := []utiliptables.Interface{
		utiliptables.New(exec, utiliptables.ProtocolIPv4),
		utiliptables.New(exec, utiliptables.ProtocolIPv6),
	}

	for i := range iptClients {
		iptClient := iptClients[i]
		if kl.syncNetworkUtil(iptClient) {
			klog.InfoS("Initialized protocol iptables rules.", "protocol", iptClient.Protocol())
			go iptClient.Monitor(
				utiliptables.Chain("KUBE-KUBELET-CANARY"),
				[]utiliptables.Table{utiliptables.TableMangle, utiliptables.TableNAT, utiliptables.TableFilter},
				func() { kl.syncNetworkUtil(iptClient) },
				1*time.Minute, wait.NeverStop,
			)
		} else {
			klog.InfoS("Failed to initialize protocol iptables rules; some functionality may be missing.", "protocol", iptClient.Protocol())
		}
	}
}

// syncNetworkUtil ensures the network utility are present on host.
// Network util includes:
//  1. In nat table, KUBE-MARK-DROP rule to mark connections for dropping
//     Marked connection will be drop on INPUT/OUTPUT Chain in filter table
//  2. In nat table, KUBE-MARK-MASQ rule to mark connections for SNAT
//     Marked connection will get SNAT on POSTROUTING Chain in nat table
func (kl *Kubelet) syncNetworkUtil(iptClient utiliptables.Interface) bool {
	// Setup KUBE-MARK-DROP rules
	dropMark := getIPTablesMark(kl.iptablesDropBit)
	if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkDropChain); err != nil {
		klog.ErrorS(err, "Failed to ensure that nat chain exists KUBE-MARK-DROP chain")
		return false
	}
	if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkDropChain, "-j", "MARK", "--or-mark", dropMark); err != nil {
		klog.ErrorS(err, "Failed to ensure marking rule for KUBE-MARK-DROP chain")
		return false
	}
	if _, err := iptClient.EnsureChain(utiliptables.TableFilter, KubeFirewallChain); err != nil {
		klog.ErrorS(err, "Failed to ensure that filter table exists KUBE-FIREWALL chain")
		return false
	}
	if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain,
		"-m", "comment", "--comment", "kubernetes firewall for dropping marked packets",
		"-m", "mark", "--mark", fmt.Sprintf("%s/%s", dropMark, dropMark),
		"-j", "DROP"); err != nil {
		klog.ErrorS(err, "Failed to ensure rule to drop packet marked by the KUBE-MARK-DROP in KUBE-FIREWALL chain")
		return false
	}

	// drop all non-local packets to localhost if they're not part of an existing
	// forwarded connection. See #90259
	if !iptClient.IsIPv6() { // ipv6 doesn't have this issue
		if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain,
			"-m", "comment", "--comment", "block incoming localnet connections",
			"--dst", "127.0.0.0/8",
			"!", "--src", "127.0.0.0/8",
			"-m", "conntrack",
			"!", "--ctstate", "RELATED,ESTABLISHED,DNAT",
			"-j", "DROP"); err != nil {
			klog.ErrorS(err, "Failed to ensure rule to drop invalid localhost packets in filter table KUBE-FIREWALL chain")
			return false
		}
	}

	if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainOutput, "-j", string(KubeFirewallChain)); err != nil {
		klog.ErrorS(err, "Failed to ensure that filter table  from OUTPUT chain jumps to KUBE-FIREWALL chain")
		return false
	}
	if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainInput, "-j", string(KubeFirewallChain)); err != nil {
		klog.ErrorS(err, "Failed to ensure that filter table INPUT chain jumps to KUBE-FIREWALL chain")
		return false
	}

	// Setup KUBE-MARK-MASQ rules
	masqueradeMark := getIPTablesMark(kl.iptablesMasqueradeBit)
	if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkMasqChain); err != nil {
		klog.ErrorS(err, "Failed to ensure that nat table exists KUBE-MARK-MASQ chain")
		return false
	}
	if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubePostroutingChain); err != nil {
		klog.ErrorS(err, "Failed to ensure that nat table exists kube POSTROUTING chain")
		return false
	}
	if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkMasqChain, "-j", "MARK", "--or-mark", masqueradeMark); err != nil {
		klog.ErrorS(err, "Failed to ensure marking rule for KUBE-MARK-MASQ chain")
		return false
	}
	if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableNAT, utiliptables.ChainPostrouting,
		"-m", "comment", "--comment", "kubernetes postrouting rules", "-j", string(KubePostroutingChain)); err != nil {
		klog.ErrorS(err, "Failed to ensure that nat table from POSTROUTING chain jumps to KUBE-POSTROUTING chain")
		return false
	}

	// Set up KUBE-POSTROUTING to unmark and masquerade marked packets
	// NB: THIS MUST MATCH the corresponding code in the iptables and ipvs
	// modes of kube-proxy
	if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain,
		"-m", "mark", "!", "--mark", fmt.Sprintf("%s/%s", masqueradeMark, masqueradeMark),
		"-j", "RETURN"); err != nil {
		klog.ErrorS(err, "Failed to ensure filtering rule for KUBE-POSTROUTING chain")
		return false
	}
	// Clear the mark to avoid re-masquerading if the packet re-traverses the network stack.
	// We know the mark bit is currently set so we can use --xor-mark to clear it (without needing
	// to Sprintf another bitmask).
	if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain,
		"-j", "MARK", "--xor-mark", masqueradeMark); err != nil {
		klog.ErrorS(err, "Failed to ensure unmarking rule for KUBE-POSTROUTING chain")
		return false
	}
	masqRule := []string{
		"-m", "comment", "--comment", "kubernetes service traffic requiring SNAT",
		"-j", "MASQUERADE",
	}
	if iptClient.HasRandomFully() {
		masqRule = append(masqRule, "--random-fully")
	}
	if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain, masqRule...); err != nil {
		klog.ErrorS(err, "Failed to ensure SNAT rule for packets marked by KUBE-MARK-MASQ chain in nat table KUBE-POSTROUTING chain")
		return false
	}

	// Create hint chain so other components can see whether we are using iptables-legacy
	// or iptables-nft.
	if _, err := iptClient.EnsureChain(utiliptables.TableMangle, KubeIPTablesHintChain); err != nil {
		klog.ErrorS(err, "Failed to ensure that iptables hint chain exists")
		return false
	}

	return true
}

// getIPTablesMark returns the fwmark given the bit
func getIPTablesMark(bit int) string {
	value := 1 << uint(bit)
	return fmt.Sprintf("%#08x", value)
}
<commit_msg>Clean up kubelet iptables error messages<commit_after>//go:build linux
// +build linux

/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubelet

import (
	"fmt"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/klog/v2"
	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
	utilexec "k8s.io/utils/exec"
)

const (
	// KubeIPTablesHintChain is the chain whose existence in either iptables-legacy
	// or iptables-nft indicates which version of iptables the system is using
	KubeIPTablesHintChain utiliptables.Chain = "KUBE-IPTABLES-HINT"

	// KubeMarkMasqChain is the mark-for-masquerade chain
	// TODO: clean up this logic in kube-proxy
	KubeMarkMasqChain utiliptables.Chain = "KUBE-MARK-MASQ"

	// KubeMarkDropChain is the mark-for-drop chain
	KubeMarkDropChain utiliptables.Chain = "KUBE-MARK-DROP"

	// KubePostroutingChain is kubernetes postrouting rules
	KubePostroutingChain utiliptables.Chain = "KUBE-POSTROUTING"

	// KubeFirewallChain is kubernetes firewall rules
	KubeFirewallChain utiliptables.Chain = "KUBE-FIREWALL"
)

func (kl *Kubelet) initNetworkUtil() {
	exec := utilexec.New()
	iptClients := []utiliptables.Interface{
		utiliptables.New(exec, utiliptables.ProtocolIPv4),
		utiliptables.New(exec, utiliptables.ProtocolIPv6),
	}

	for i := range iptClients {
		iptClient := iptClients[i]
		if kl.syncNetworkUtil(iptClient) {
			klog.InfoS("Initialized iptables rules.", "protocol", iptClient.Protocol())
			go iptClient.Monitor(
				utiliptables.Chain("KUBE-KUBELET-CANARY"),
				[]utiliptables.Table{utiliptables.TableMangle, utiliptables.TableNAT, utiliptables.TableFilter},
				func() { kl.syncNetworkUtil(iptClient) },
				1*time.Minute, wait.NeverStop,
			)
		} else {
			klog.InfoS("Failed to initialize iptables rules; some functionality may be missing.", "protocol", iptClient.Protocol())
		}
	}
}

// syncNetworkUtil ensures the network utility are present on host.
// Network util includes:
//  1. In nat table, KUBE-MARK-DROP rule to mark connections for dropping
//     Marked connection will be drop on INPUT/OUTPUT Chain in filter table
//  2. In nat table, KUBE-MARK-MASQ rule to mark connections for SNAT
//     Marked connection will get SNAT on POSTROUTING Chain in nat table
func (kl *Kubelet) syncNetworkUtil(iptClient utiliptables.Interface) bool {
	// Setup KUBE-MARK-DROP rules
	dropMark := getIPTablesMark(kl.iptablesDropBit)
	if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkDropChain); err != nil {
		klog.ErrorS(err, "Failed to ensure that KUBE-MARK-DROP chain exists")
		return false
	}
	if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkDropChain, "-j", "MARK", "--or-mark", dropMark); err != nil {
		klog.ErrorS(err, "Failed to ensure that KUBE-MARK-DROP rule exists")
		return false
	}
	if _, err := iptClient.EnsureChain(utiliptables.TableFilter, KubeFirewallChain); err != nil {
		klog.ErrorS(err, "Failed to ensure that KUBE-FIREWALL chain exists")
		return false
	}
	if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain,
		"-m", "comment", "--comment", "kubernetes firewall for dropping marked packets",
		"-m", "mark", "--mark", fmt.Sprintf("%s/%s", dropMark, dropMark),
		"-j", "DROP"); err != nil {
		klog.ErrorS(err, "Failed to ensure that KUBE-FIREWALL rule exists")
		return false
	}

	// drop all non-local packets to localhost if they're not part of an existing
	// forwarded connection. See #90259
	if !iptClient.IsIPv6() { // ipv6 doesn't have this issue
		if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableFilter, KubeFirewallChain,
			"-m", "comment", "--comment", "block incoming localnet connections",
			"--dst", "127.0.0.0/8",
			"!", "--src", "127.0.0.0/8",
			"-m", "conntrack",
			"!", "--ctstate", "RELATED,ESTABLISHED,DNAT",
			"-j", "DROP"); err != nil {
			klog.ErrorS(err, "Failed to ensure rule to drop invalid localhost packets exists")
			return false
		}
	}

	if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainOutput, "-j", string(KubeFirewallChain)); err != nil {
		klog.ErrorS(err, "Failed to ensure that OUTPUT chain jumps to KUBE-FIREWALL")
		return false
	}
	if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableFilter, utiliptables.ChainInput, "-j", string(KubeFirewallChain)); err != nil {
		klog.ErrorS(err, "Failed to ensure that INPUT chain jumps to KUBE-FIREWALL")
		return false
	}

	// Setup KUBE-MARK-MASQ rules
	masqueradeMark := getIPTablesMark(kl.iptablesMasqueradeBit)
	if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubeMarkMasqChain); err != nil {
		klog.ErrorS(err, "Failed to ensure that KUBE-MARK-MASQ chain exists")
		return false
	}
	if _, err := iptClient.EnsureChain(utiliptables.TableNAT, KubePostroutingChain); err != nil {
		klog.ErrorS(err, "Failed to ensure that KUBE-POSTROUTING chain exists")
		return false
	}
	if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubeMarkMasqChain, "-j", "MARK", "--or-mark", masqueradeMark); err != nil {
		klog.ErrorS(err, "Failed to ensure that KUBE-MARK-MASQ rule exists")
		return false
	}
	if _, err := iptClient.EnsureRule(utiliptables.Prepend, utiliptables.TableNAT, utiliptables.ChainPostrouting,
		"-m", "comment", "--comment", "kubernetes postrouting rules", "-j", string(KubePostroutingChain)); err != nil {
		klog.ErrorS(err, "Failed to ensure that POSTROUTING chain jumps to KUBE-POSTROUTING")
		return false
	}

	// Set up KUBE-POSTROUTING to unmark and masquerade marked packets
	// NB: THIS MUST MATCH the corresponding code in the iptables and ipvs
	// modes of kube-proxy
	if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain,
		"-m", "mark", "!", "--mark", fmt.Sprintf("%s/%s", masqueradeMark, masqueradeMark),
		"-j", "RETURN"); err != nil {
		klog.ErrorS(err, "Failed to ensure first masquerading rule exists")
		return false
	}
	// Clear the mark to avoid re-masquerading if the packet re-traverses the network stack.
	// We know the mark bit is currently set so we can use --xor-mark to clear it (without needing
	// to Sprintf another bitmask).
	if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain,
		"-j", "MARK", "--xor-mark", masqueradeMark); err != nil {
		klog.ErrorS(err, "Failed to ensure second masquerading rule exists")
		return false
	}
	masqRule := []string{
		"-m", "comment", "--comment", "kubernetes service traffic requiring SNAT",
		"-j", "MASQUERADE",
	}
	if iptClient.HasRandomFully() {
		masqRule = append(masqRule, "--random-fully")
	}
	if _, err := iptClient.EnsureRule(utiliptables.Append, utiliptables.TableNAT, KubePostroutingChain, masqRule...); err != nil {
		klog.ErrorS(err, "Failed to ensure third masquerading rule exists")
		return false
	}

	// Create hint chain so other components can see whether we are using iptables-legacy
	// or iptables-nft.
	if _, err := iptClient.EnsureChain(utiliptables.TableMangle, KubeIPTablesHintChain); err != nil {
		klog.ErrorS(err, "Failed to ensure that iptables hint chain exists")
		return false
	}

	return true
}

// getIPTablesMark returns the fwmark given the bit
func getIPTablesMark(bit int) string {
	value := 1 << uint(bit)
	return fmt.Sprintf("%#08x", value)
}
<|endoftext|>


<commit_before>package resourcecollector

import (
	"strings"

	rbacv1 "k8s.io/api/rbac/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apiserver/pkg/authentication/serviceaccount"
)

// Checks if the subject is in the specified namespace
func (r *ResourceCollector) subjectInNamespace(subject *rbacv1.Subject, namespace string) (bool, error) {
	switch subject.Kind {
	case rbacv1.ServiceAccountKind:
		// skip check for default service account
		if subject.Name == "default" {
			return false, nil
		}
		// check if user has an access to sa in namespace
		_, err := r.coreOps.GetServiceAccount(subject.Name, namespace)
		if err != nil && !apierrors.IsNotFound(err) {
			if apierrors.IsForbidden(err) {
				return false, nil
			}
			return false, err
		}
		return true, nil
	case rbacv1.UserKind:
		// For User we need to parse the username to get the namespace
		userNamespace, _, err := serviceaccount.SplitUsername(subject.Name)
		if err != nil {
			return false, nil
		}
		if userNamespace == namespace {
			return true, nil
		}
	case rbacv1.GroupKind:
		// For Group  we need to parse the username to get the namespace
		groupNamespace := strings.TrimPrefix(subject.Name, serviceaccount.ServiceAccountUsernamePrefix)
		if groupNamespace == namespace {
			return true, nil
		}
	}
	return false, nil
}

func (r *ResourceCollector) clusterRoleBindingToBeCollected(
	labelSelectors map[string]string,
	object runtime.Unstructured,
	namespace string,
) (bool, error) {
	var crb rbacv1.ClusterRoleBinding
	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), &crb); err != nil {
		return false, err
	}
	if val, ok := crb.Labels["kubernetes.io/bootstrapping"]; ok && val == "rbac-defaults" {
		return false, nil
	}
	// Check if there is a subject for the namespace which is requested
	for _, subject := range crb.Subjects {
		collect, err := r.subjectInNamespace(&subject, namespace)
		if err != nil || collect {
			return collect, err
		}
	}
	return false, nil
}

func (r *ResourceCollector) clusterRoleToBeCollected(
	labelSelectors map[string]string,
	object runtime.Unstructured,
	crbs *rbacv1.ClusterRoleBindingList,
	namespace string,
) (bool, error) {
	metadata, err := meta.Accessor(object)
	if err != nil {
		return false, err
	}
	if val, ok := metadata.GetLabels()["kubernetes.io/bootstrapping"]; ok && val == "rbac-defaults" {
		return false, nil
	}

	name := metadata.GetName()
	// Find the corresponding ClusterRoleBinding and see if it belongs to the requested namespace
	for _, crb := range crbs.Items {
		if crb.RoleRef.Name == name {
			for _, subject := range crb.Subjects {
				collect, err := r.subjectInNamespace(&subject, namespace)
				if err != nil || collect {
					return collect, err
				}
			}
		}
	}
	return false, nil
}

func (r *ResourceCollector) prepareClusterRoleBindingForCollection(
	object runtime.Unstructured,
	namespaces []string,
) error {
	var crb rbacv1.ClusterRoleBinding
	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), &crb); err != nil {
		return err
	}
	subjectsToCollect := make([]rbacv1.Subject, 0)
	// Check if there is a subject for the namespace which is requested
	for _, subject := range crb.Subjects {
		for _, ns := range namespaces {
			collect, err := r.subjectInNamespace(&subject, ns)
			if err != nil {
				return err
			}

			if collect {
				subjectsToCollect = append(subjectsToCollect, subject)
			}
		}
	}
	crb.Subjects = subjectsToCollect
	o, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&crb)
	if err != nil {
		return err
	}
	object.SetUnstructuredContent(o)

	return nil
}

func (r *ResourceCollector) prepareClusterRoleBindingForApply(
	object runtime.Unstructured,
	namespaceMappings map[string]string,
) error {
	var crb rbacv1.ClusterRoleBinding
	err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), &crb)
	if err != nil {
		return err
	}
	crb.Subjects, err = r.updateSubjects(crb.Subjects, namespaceMappings)
	if err != nil {
		return err
	}
	o, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&crb)
	if err != nil {
		return err
	}
	object.SetUnstructuredContent(o)

	return nil
}

func (r *ResourceCollector) updateSubjects(
	subjects []rbacv1.Subject,
	namespaceMappings map[string]string,
) ([]rbacv1.Subject, error) {
	subjectsToApply := make([]rbacv1.Subject, 0)
	for sourceNamespace, destNamespace := range namespaceMappings {
		// Check if there is a subject for the namespace which is requested
		for _, subject := range subjects {
			collect, err := r.subjectInNamespace(&subject, sourceNamespace)
			if err != nil {
				return nil, err
			}
			if !collect {
				continue
			}

			switch subject.Kind {
			case rbacv1.UserKind:
				_, username, err := serviceaccount.SplitUsername(subject.Name)
				if err != nil {
					return nil, err
				}
				// Regnerate the Username for the destination namespace
				subject.Name = serviceaccount.MakeUsername(destNamespace, username)
			case rbacv1.GroupKind:
				// Regnerate the Group name for the destination namespace
				subject.Name = serviceaccount.MakeNamespaceGroupName(destNamespace)
			case rbacv1.ServiceAccountKind:
				// Update the Namespace in the spec for ServiceAccounts
				subject.Namespace = destNamespace
			}
			subjectsToApply = append(subjectsToApply, subject)
		}
	}
	return subjectsToApply, nil
}

func (r *ResourceCollector) mergeAndUpdateClusterRoleBinding(
	object runtime.Unstructured,
) error {
	var newCRB rbacv1.ClusterRoleBinding
	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), &newCRB); err != nil {
		return err
	}

	currentCRB, err := r.rbacOps.GetClusterRoleBinding(newCRB.Name)
	if err != nil {
		if apierrors.IsNotFound(err) {
			_, err = r.rbacOps.CreateClusterRoleBinding(&newCRB)
		}
		return err
	}

	// Map which will help eliminate duplicate subjects since the subject string
	// will be unique for different subjects
	updatedSubjects := make(map[string]rbacv1.Subject)

	// First add the current subjects to the map
	for _, subject := range currentCRB.Subjects {
		updatedSubjects[subject.String()] = subject
	}
	// Then add the new subjects to be merged
	for _, subject := range newCRB.Subjects {
		updatedSubjects[subject.String()] = subject
	}

	// Then get the updated list of subjects from the map to apply
	currentCRB.Subjects = make([]rbacv1.Subject, 0)
	for _, subject := range updatedSubjects {
		currentCRB.Subjects = append(currentCRB.Subjects, subject)
	}

	_, err = r.rbacOps.UpdateClusterRoleBinding(currentCRB)
	return err
}
<commit_msg>revert clsuterolebinding collection based on user access to SA<commit_after>package resourcecollector

import (
	"strings"

	rbacv1 "k8s.io/api/rbac/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apiserver/pkg/authentication/serviceaccount"
)

// Checks if the subject is in the specified namespace
func (r *ResourceCollector) subjectInNamespace(subject *rbacv1.Subject, namespace string) (bool, error) {
	switch subject.Kind {
	case rbacv1.ServiceAccountKind:
		// skip check for default service account
		if subject.Name == "default" {
			return false, nil
		}
		if subject.Namespace == namespace {
			return true, nil
		}
	case rbacv1.UserKind:
		// For User we need to parse the username to get the namespace
		userNamespace, _, err := serviceaccount.SplitUsername(subject.Name)
		if err != nil {
			return false, nil
		}
		if userNamespace == namespace {
			return true, nil
		}
	case rbacv1.GroupKind:
		// For Group  we need to parse the username to get the namespace
		groupNamespace := strings.TrimPrefix(subject.Name, serviceaccount.ServiceAccountUsernamePrefix)
		if groupNamespace == namespace {
			return true, nil
		}
	}
	return false, nil
}

func (r *ResourceCollector) clusterRoleBindingToBeCollected(
	labelSelectors map[string]string,
	object runtime.Unstructured,
	namespace string,
) (bool, error) {
	var crb rbacv1.ClusterRoleBinding
	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), &crb); err != nil {
		return false, err
	}
	if val, ok := crb.Labels["kubernetes.io/bootstrapping"]; ok && val == "rbac-defaults" {
		return false, nil
	}
	// Check if there is a subject for the namespace which is requested
	for _, subject := range crb.Subjects {
		collect, err := r.subjectInNamespace(&subject, namespace)
		if err != nil || collect {
			return collect, err
		}
	}
	return false, nil
}

func (r *ResourceCollector) clusterRoleToBeCollected(
	labelSelectors map[string]string,
	object runtime.Unstructured,
	crbs *rbacv1.ClusterRoleBindingList,
	namespace string,
) (bool, error) {
	metadata, err := meta.Accessor(object)
	if err != nil {
		return false, err
	}
	if val, ok := metadata.GetLabels()["kubernetes.io/bootstrapping"]; ok && val == "rbac-defaults" {
		return false, nil
	}

	name := metadata.GetName()
	// Find the corresponding ClusterRoleBinding and see if it belongs to the requested namespace
	for _, crb := range crbs.Items {
		if crb.RoleRef.Name == name {
			for _, subject := range crb.Subjects {
				collect, err := r.subjectInNamespace(&subject, namespace)
				if err != nil || collect {
					return collect, err
				}
			}
		}
	}
	return false, nil
}

func (r *ResourceCollector) prepareClusterRoleBindingForCollection(
	object runtime.Unstructured,
	namespaces []string,
) error {
	var crb rbacv1.ClusterRoleBinding
	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), &crb); err != nil {
		return err
	}
	subjectsToCollect := make([]rbacv1.Subject, 0)
	// Check if there is a subject for the namespace which is requested
	for _, subject := range crb.Subjects {
		for _, ns := range namespaces {
			collect, err := r.subjectInNamespace(&subject, ns)
			if err != nil {
				return err
			}

			if collect {
				subjectsToCollect = append(subjectsToCollect, subject)
			}
		}
	}
	crb.Subjects = subjectsToCollect
	o, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&crb)
	if err != nil {
		return err
	}
	object.SetUnstructuredContent(o)

	return nil
}

func (r *ResourceCollector) prepareClusterRoleBindingForApply(
	object runtime.Unstructured,
	namespaceMappings map[string]string,
) error {
	var crb rbacv1.ClusterRoleBinding
	err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), &crb)
	if err != nil {
		return err
	}
	crb.Subjects, err = r.updateSubjects(crb.Subjects, namespaceMappings)
	if err != nil {
		return err
	}
	o, err := runtime.DefaultUnstructuredConverter.ToUnstructured(&crb)
	if err != nil {
		return err
	}
	object.SetUnstructuredContent(o)

	return nil
}

func (r *ResourceCollector) updateSubjects(
	subjects []rbacv1.Subject,
	namespaceMappings map[string]string,
) ([]rbacv1.Subject, error) {
	subjectsToApply := make([]rbacv1.Subject, 0)
	for sourceNamespace, destNamespace := range namespaceMappings {
		// Check if there is a subject for the namespace which is requested
		for _, subject := range subjects {
			collect, err := r.subjectInNamespace(&subject, sourceNamespace)
			if err != nil {
				return nil, err
			}
			if !collect {
				continue
			}

			switch subject.Kind {
			case rbacv1.UserKind:
				_, username, err := serviceaccount.SplitUsername(subject.Name)
				if err != nil {
					return nil, err
				}
				// Regnerate the Username for the destination namespace
				subject.Name = serviceaccount.MakeUsername(destNamespace, username)
			case rbacv1.GroupKind:
				// Regnerate the Group name for the destination namespace
				subject.Name = serviceaccount.MakeNamespaceGroupName(destNamespace)
			case rbacv1.ServiceAccountKind:
				// Update the Namespace in the spec for ServiceAccounts
				subject.Namespace = destNamespace
			}
			subjectsToApply = append(subjectsToApply, subject)
		}
	}
	return subjectsToApply, nil
}

func (r *ResourceCollector) mergeAndUpdateClusterRoleBinding(
	object runtime.Unstructured,
) error {
	var newCRB rbacv1.ClusterRoleBinding
	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(object.UnstructuredContent(), &newCRB); err != nil {
		return err
	}

	currentCRB, err := r.rbacOps.GetClusterRoleBinding(newCRB.Name)
	if err != nil {
		if apierrors.IsNotFound(err) {
			_, err = r.rbacOps.CreateClusterRoleBinding(&newCRB)
		}
		return err
	}

	// Map which will help eliminate duplicate subjects since the subject string
	// will be unique for different subjects
	updatedSubjects := make(map[string]rbacv1.Subject)

	// First add the current subjects to the map
	for _, subject := range currentCRB.Subjects {
		updatedSubjects[subject.String()] = subject
	}
	// Then add the new subjects to be merged
	for _, subject := range newCRB.Subjects {
		updatedSubjects[subject.String()] = subject
	}

	// Then get the updated list of subjects from the map to apply
	currentCRB.Subjects = make([]rbacv1.Subject, 0)
	for _, subject := range updatedSubjects {
		currentCRB.Subjects = append(currentCRB.Subjects, subject)
	}

	_, err = r.rbacOps.UpdateClusterRoleBinding(currentCRB)
	return err
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
	"reflect"
	"strings"
	"testing"

	"k8s.io/kubernetes/pkg/api/resource"
	"k8s.io/kubernetes/pkg/labels"
)

func TestConversionError(t *testing.T) {
	var i int
	var s string
	i = 3
	s = "foo"
	c := ConversionError{
		In: &i, Out: &s,
		Message: "Can't make x into y, silly",
	}
	var e error
	e = &c // ensure it implements error
	msg := e.Error()
	t.Logf("Message is %v", msg)
	for _, part := range []string{"3", "int", "string", "Can't"} {
		if !strings.Contains(msg, part) {
			t.Errorf("didn't find %v", part)
		}
	}
}

func TestSemantic(t *testing.T) {
	table := []struct {
		a, b        interface{}
		shouldEqual bool
	}{
		{resource.MustParse("0"), resource.Quantity{}, true},
		{resource.Quantity{}, resource.MustParse("0"), true},
		{resource.Quantity{}, resource.MustParse("1m"), false},
		{
			resource.NewQuantity(5, resource.BinarySI),
			resource.NewQuantity(5, resource.DecimalSI),
			true,
		},
		{resource.MustParse("2m"), resource.MustParse("1m"), false},
	}

	for index, item := range table {
		if e, a := item.shouldEqual, Semantic.DeepEqual(item.a, item.b); e != a {
			t.Errorf("case[%d], expected %v, got %v.", index, e, a)
		}
	}
}

func TestIsStandardResource(t *testing.T) {
	testCases := []struct {
		input  string
		output bool
	}{
		{"cpu", true},
		{"memory", true},
		{"disk", false},
		{"blah", false},
		{"x.y.z", false},
	}
	for i, tc := range testCases {
		if IsStandardResourceName(tc.input) != tc.output {
			t.Errorf("case[%d], expected: %t, got: %t", i, tc.output, !tc.output)
		}
	}
}

func TestAddToNodeAddresses(t *testing.T) {
	testCases := []struct {
		existing []NodeAddress
		toAdd    []NodeAddress
		expected []NodeAddress
	}{
		{
			existing: []NodeAddress{},
			toAdd:    []NodeAddress{},
			expected: []NodeAddress{},
		},
		{
			existing: []NodeAddress{},
			toAdd: []NodeAddress{
				{Type: NodeExternalIP, Address: "1.1.1.1"},
				{Type: NodeHostName, Address: "localhost"},
			},
			expected: []NodeAddress{
				{Type: NodeExternalIP, Address: "1.1.1.1"},
				{Type: NodeHostName, Address: "localhost"},
			},
		},
		{
			existing: []NodeAddress{},
			toAdd: []NodeAddress{
				{Type: NodeExternalIP, Address: "1.1.1.1"},
				{Type: NodeExternalIP, Address: "1.1.1.1"},
			},
			expected: []NodeAddress{
				{Type: NodeExternalIP, Address: "1.1.1.1"},
			},
		},
		{
			existing: []NodeAddress{
				{Type: NodeExternalIP, Address: "1.1.1.1"},
				{Type: NodeInternalIP, Address: "10.1.1.1"},
			},
			toAdd: []NodeAddress{
				{Type: NodeExternalIP, Address: "1.1.1.1"},
				{Type: NodeHostName, Address: "localhost"},
			},
			expected: []NodeAddress{
				{Type: NodeExternalIP, Address: "1.1.1.1"},
				{Type: NodeInternalIP, Address: "10.1.1.1"},
				{Type: NodeHostName, Address: "localhost"},
			},
		},
	}

	for i, tc := range testCases {
		AddToNodeAddresses(&tc.existing, tc.toAdd...)
		if !Semantic.DeepEqual(tc.expected, tc.existing) {
			t.Errorf("case[%d], expected: %v, got: %v", i, tc.expected, tc.existing)
		}
	}
}

func TestGetAccessModesFromString(t *testing.T) {
	modes := GetAccessModesFromString("ROX")
	if !containsAccessMode(modes, ReadOnlyMany) {
		t.Errorf("Expected mode %s, but got %+v", ReadOnlyMany, modes)
	}

	modes = GetAccessModesFromString("ROX,RWX")
	if !containsAccessMode(modes, ReadOnlyMany) {
		t.Errorf("Expected mode %s, but got %+v", ReadOnlyMany, modes)
	}
	if !containsAccessMode(modes, ReadWriteMany) {
		t.Errorf("Expected mode %s, but got %+v", ReadWriteMany, modes)
	}

	modes = GetAccessModesFromString("RWO,ROX,RWX")
	if !containsAccessMode(modes, ReadOnlyMany) {
		t.Errorf("Expected mode %s, but got %+v", ReadOnlyMany, modes)
	}
	if !containsAccessMode(modes, ReadWriteMany) {
		t.Errorf("Expected mode %s, but got %+v", ReadWriteMany, modes)
	}
}

func TestRemoveDuplicateAccessModes(t *testing.T) {
	modes := []PersistentVolumeAccessMode{
		ReadWriteOnce, ReadOnlyMany, ReadOnlyMany, ReadOnlyMany,
	}
	modes = removeDuplicateAccessModes(modes)
	if len(modes) != 2 {
		t.Errorf("Expected 2 distinct modes in set but found %v", len(modes))
	}
}

func TestNodeSelectorRequirementsAsSelector(t *testing.T) {
	matchExpressions := []NodeSelectorRequirement{{
		Key:      "foo",
		Operator: NodeSelectorOpIn,
		Values:   []string{"bar", "baz"},
	}}
	mustParse := func(s string) labels.Selector {
		out, e := labels.Parse(s)
		if e != nil {
			panic(e)
		}
		return out
	}
	tc := []struct {
		in        []NodeSelectorRequirement
		out       labels.Selector
		expectErr bool
	}{
		{in: nil, out: labels.Nothing()},
		{in: []NodeSelectorRequirement{}, out: labels.Nothing()},
		{
			in:  matchExpressions,
			out: mustParse("foo in (baz,bar)"),
		},
		{
			in: []NodeSelectorRequirement{{
				Key:      "foo",
				Operator: NodeSelectorOpExists,
				Values:   []string{"bar", "baz"},
			}},
			expectErr: true,
		},
		{
			in: []NodeSelectorRequirement{{
				Key:      "foo",
				Operator: NodeSelectorOpGt,
				Values:   []string{"1.1"},
			}},
			out: mustParse("foo>1.1"),
		},
		{
			in: []NodeSelectorRequirement{{
				Key:      "bar",
				Operator: NodeSelectorOpLt,
				Values:   []string{"7.1"},
			}},
			out: mustParse("bar<7.1"),
		},
	}

	for i, tc := range tc {
		out, err := NodeSelectorRequirementsAsSelector(tc.in)
		if err == nil && tc.expectErr {
			t.Errorf("[%v]expected error but got none.", i)
		}
		if err != nil && !tc.expectErr {
			t.Errorf("[%v]did not expect error but got: %v", i, err)
		}
		if !reflect.DeepEqual(out, tc.out) {
			t.Errorf("[%v]expected:\n\t%+v\nbut got:\n\t%+v", i, tc.out, out)
		}
	}
}

func TestGetAffinityFromPod(t *testing.T) {
	testCases := []struct {
		pod       *Pod
		expectErr bool
	}{
		{
			pod:       &Pod{},
			expectErr: false,
		},
		{
			pod: &Pod{
				ObjectMeta: ObjectMeta{
					Annotations: map[string]string{
						AffinityAnnotationKey: `
						{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
							"nodeSelectorTerms": [{
								"matchExpressions": [{
									"key": "foo",
									"operator": "In",
									"values": ["value1", "value2"]
								}]
							}]
						}}}`,
					},
				},
			},
			expectErr: false,
		},
		{
			pod: &Pod{
				ObjectMeta: ObjectMeta{
					Annotations: map[string]string{
						AffinityAnnotationKey: `
						{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
							"nodeSelectorTerms": [{
								"matchExpressions": [{
									"key": "foo",
						`,
					},
				},
			},
			expectErr: true,
		},
	}

	for i, tc := range testCases {
		_, err := GetAffinityFromPodAnnotations(tc.pod.Annotations)
		if err == nil && tc.expectErr {
			t.Errorf("[%v]expected error but got none.", i)
		}
		if err != nil && !tc.expectErr {
			t.Errorf("[%v]did not expect error but got: %v", i, err)
		}
	}
}
<commit_msg>selector: make sure value of GT and LT is integer<commit_after>/*
Copyright 2015 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
	"reflect"
	"strings"
	"testing"

	"k8s.io/kubernetes/pkg/api/resource"
	"k8s.io/kubernetes/pkg/labels"
)

func TestConversionError(t *testing.T) {
	var i int
	var s string
	i = 3
	s = "foo"
	c := ConversionError{
		In: &i, Out: &s,
		Message: "Can't make x into y, silly",
	}
	var e error
	e = &c // ensure it implements error
	msg := e.Error()
	t.Logf("Message is %v", msg)
	for _, part := range []string{"3", "int", "string", "Can't"} {
		if !strings.Contains(msg, part) {
			t.Errorf("didn't find %v", part)
		}
	}
}

func TestSemantic(t *testing.T) {
	table := []struct {
		a, b        interface{}
		shouldEqual bool
	}{
		{resource.MustParse("0"), resource.Quantity{}, true},
		{resource.Quantity{}, resource.MustParse("0"), true},
		{resource.Quantity{}, resource.MustParse("1m"), false},
		{
			resource.NewQuantity(5, resource.BinarySI),
			resource.NewQuantity(5, resource.DecimalSI),
			true,
		},
		{resource.MustParse("2m"), resource.MustParse("1m"), false},
	}

	for index, item := range table {
		if e, a := item.shouldEqual, Semantic.DeepEqual(item.a, item.b); e != a {
			t.Errorf("case[%d], expected %v, got %v.", index, e, a)
		}
	}
}

func TestIsStandardResource(t *testing.T) {
	testCases := []struct {
		input  string
		output bool
	}{
		{"cpu", true},
		{"memory", true},
		{"disk", false},
		{"blah", false},
		{"x.y.z", false},
	}
	for i, tc := range testCases {
		if IsStandardResourceName(tc.input) != tc.output {
			t.Errorf("case[%d], expected: %t, got: %t", i, tc.output, !tc.output)
		}
	}
}

func TestAddToNodeAddresses(t *testing.T) {
	testCases := []struct {
		existing []NodeAddress
		toAdd    []NodeAddress
		expected []NodeAddress
	}{
		{
			existing: []NodeAddress{},
			toAdd:    []NodeAddress{},
			expected: []NodeAddress{},
		},
		{
			existing: []NodeAddress{},
			toAdd: []NodeAddress{
				{Type: NodeExternalIP, Address: "1.1.1.1"},
				{Type: NodeHostName, Address: "localhost"},
			},
			expected: []NodeAddress{
				{Type: NodeExternalIP, Address: "1.1.1.1"},
				{Type: NodeHostName, Address: "localhost"},
			},
		},
		{
			existing: []NodeAddress{},
			toAdd: []NodeAddress{
				{Type: NodeExternalIP, Address: "1.1.1.1"},
				{Type: NodeExternalIP, Address: "1.1.1.1"},
			},
			expected: []NodeAddress{
				{Type: NodeExternalIP, Address: "1.1.1.1"},
			},
		},
		{
			existing: []NodeAddress{
				{Type: NodeExternalIP, Address: "1.1.1.1"},
				{Type: NodeInternalIP, Address: "10.1.1.1"},
			},
			toAdd: []NodeAddress{
				{Type: NodeExternalIP, Address: "1.1.1.1"},
				{Type: NodeHostName, Address: "localhost"},
			},
			expected: []NodeAddress{
				{Type: NodeExternalIP, Address: "1.1.1.1"},
				{Type: NodeInternalIP, Address: "10.1.1.1"},
				{Type: NodeHostName, Address: "localhost"},
			},
		},
	}

	for i, tc := range testCases {
		AddToNodeAddresses(&tc.existing, tc.toAdd...)
		if !Semantic.DeepEqual(tc.expected, tc.existing) {
			t.Errorf("case[%d], expected: %v, got: %v", i, tc.expected, tc.existing)
		}
	}
}

func TestGetAccessModesFromString(t *testing.T) {
	modes := GetAccessModesFromString("ROX")
	if !containsAccessMode(modes, ReadOnlyMany) {
		t.Errorf("Expected mode %s, but got %+v", ReadOnlyMany, modes)
	}

	modes = GetAccessModesFromString("ROX,RWX")
	if !containsAccessMode(modes, ReadOnlyMany) {
		t.Errorf("Expected mode %s, but got %+v", ReadOnlyMany, modes)
	}
	if !containsAccessMode(modes, ReadWriteMany) {
		t.Errorf("Expected mode %s, but got %+v", ReadWriteMany, modes)
	}

	modes = GetAccessModesFromString("RWO,ROX,RWX")
	if !containsAccessMode(modes, ReadOnlyMany) {
		t.Errorf("Expected mode %s, but got %+v", ReadOnlyMany, modes)
	}
	if !containsAccessMode(modes, ReadWriteMany) {
		t.Errorf("Expected mode %s, but got %+v", ReadWriteMany, modes)
	}
}

func TestRemoveDuplicateAccessModes(t *testing.T) {
	modes := []PersistentVolumeAccessMode{
		ReadWriteOnce, ReadOnlyMany, ReadOnlyMany, ReadOnlyMany,
	}
	modes = removeDuplicateAccessModes(modes)
	if len(modes) != 2 {
		t.Errorf("Expected 2 distinct modes in set but found %v", len(modes))
	}
}

func TestNodeSelectorRequirementsAsSelector(t *testing.T) {
	matchExpressions := []NodeSelectorRequirement{{
		Key:      "foo",
		Operator: NodeSelectorOpIn,
		Values:   []string{"bar", "baz"},
	}}
	mustParse := func(s string) labels.Selector {
		out, e := labels.Parse(s)
		if e != nil {
			panic(e)
		}
		return out
	}
	tc := []struct {
		in        []NodeSelectorRequirement
		out       labels.Selector
		expectErr bool
	}{
		{in: nil, out: labels.Nothing()},
		{in: []NodeSelectorRequirement{}, out: labels.Nothing()},
		{
			in:  matchExpressions,
			out: mustParse("foo in (baz,bar)"),
		},
		{
			in: []NodeSelectorRequirement{{
				Key:      "foo",
				Operator: NodeSelectorOpExists,
				Values:   []string{"bar", "baz"},
			}},
			expectErr: true,
		},
		{
			in: []NodeSelectorRequirement{{
				Key:      "foo",
				Operator: NodeSelectorOpGt,
				Values:   []string{"1"},
			}},
			out: mustParse("foo>1"),
		},
		{
			in: []NodeSelectorRequirement{{
				Key:      "bar",
				Operator: NodeSelectorOpLt,
				Values:   []string{"7"},
			}},
			out: mustParse("bar<7"),
		},
	}

	for i, tc := range tc {
		out, err := NodeSelectorRequirementsAsSelector(tc.in)
		if err == nil && tc.expectErr {
			t.Errorf("[%v]expected error but got none.", i)
		}
		if err != nil && !tc.expectErr {
			t.Errorf("[%v]did not expect error but got: %v", i, err)
		}
		if !reflect.DeepEqual(out, tc.out) {
			t.Errorf("[%v]expected:\n\t%+v\nbut got:\n\t%+v", i, tc.out, out)
		}
	}
}

func TestGetAffinityFromPod(t *testing.T) {
	testCases := []struct {
		pod       *Pod
		expectErr bool
	}{
		{
			pod:       &Pod{},
			expectErr: false,
		},
		{
			pod: &Pod{
				ObjectMeta: ObjectMeta{
					Annotations: map[string]string{
						AffinityAnnotationKey: `
						{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
							"nodeSelectorTerms": [{
								"matchExpressions": [{
									"key": "foo",
									"operator": "In",
									"values": ["value1", "value2"]
								}]
							}]
						}}}`,
					},
				},
			},
			expectErr: false,
		},
		{
			pod: &Pod{
				ObjectMeta: ObjectMeta{
					Annotations: map[string]string{
						AffinityAnnotationKey: `
						{"nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": {
							"nodeSelectorTerms": [{
								"matchExpressions": [{
									"key": "foo",
						`,
					},
				},
			},
			expectErr: true,
		},
	}

	for i, tc := range testCases {
		_, err := GetAffinityFromPodAnnotations(tc.pod.Annotations)
		if err == nil && tc.expectErr {
			t.Errorf("[%v]expected error but got none.", i)
		}
		if err != nil && !tc.expectErr {
			t.Errorf("[%v]did not expect error but got: %v", i, err)
		}
	}
}
<|endoftext|>


<commit_before>/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package etcdutil

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"
	"time"

	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/etcdserver/etcdhttp"
	"github.com/coreos/etcd/pkg/testutil"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/rafthttp"
)

const (
	tickDuration   = 10 * time.Millisecond
	clusterName    = "etcd"
	requestTimeout = 2 * time.Second
)

var (
	electionTicks = 10
)

func newLocalListener(t *testing.T) net.Listener {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	return l
}

func StartNewEtcdServer(t *testing.T, name string) *member {
	m := MustNewMember(t, name)
	m.Launch()
	return m
}

type member struct {
	etcdserver.ServerConfig
	PeerListeners, ClientListeners []net.Listener

	raftHandler *testutil.PauseableHandler
	s           *etcdserver.EtcdServer
	hss         []*httptest.Server
}

func MustNewMember(t *testing.T, name string) *member {
	var err error
	m := &member{}

	pln := newLocalListener(t)
	m.PeerListeners = []net.Listener{pln}
	m.PeerURLs, err = types.NewURLs([]string{"http://" + pln.Addr().String()})
	if err != nil {
		t.Fatal(err)
	}

	cln := newLocalListener(t)
	m.ClientListeners = []net.Listener{cln}
	m.ClientURLs, err = types.NewURLs([]string{"http://" + cln.Addr().String()})
	if err != nil {
		t.Fatal(err)
	}

	m.Name = name

	m.DataDir, err = ioutil.TempDir(os.TempDir(), "etcd")
	if err != nil {
		t.Fatal(err)
	}
	clusterStr := fmt.Sprintf("%s=http://%s", name, pln.Addr().String())
	m.Cluster, err = etcdserver.NewClusterFromString(clusterName, clusterStr)
	if err != nil {
		t.Fatal(err)
	}
	m.NewCluster = true
	m.Transport = mustNewTransport(t)
	m.ElectionTimeoutTicks = electionTicks
	return m
}

// Launch starts a member based on ServerConfig, PeerListeners
// and ClientListeners.
func (m *member) Launch() error {
	var err error
	if m.s, err = etcdserver.NewServer(&m.ServerConfig); err != nil {
		return fmt.Errorf("failed to initialize the etcd server: %v", err)
	}
	m.s.Ticker = time.Tick(tickDuration)
	m.s.SyncTicker = time.Tick(500 * time.Millisecond)
	m.s.Start()

	m.raftHandler = &testutil.PauseableHandler{Next: etcdhttp.NewPeerHandler(m.s.Cluster, m.s.RaftHandler())}

	for _, ln := range m.PeerListeners {
		hs := &httptest.Server{
			Listener: ln,
			Config:   &http.Server{Handler: m.raftHandler},
		}
		hs.Start()
		m.hss = append(m.hss, hs)
	}
	for _, ln := range m.ClientListeners {
		hs := &httptest.Server{
			Listener: ln,
			Config:   &http.Server{Handler: etcdhttp.NewClientHandler(m.s)},
		}
		hs.Start()
		m.hss = append(m.hss, hs)
	}
	return nil
}

func (m *member) URL() string { return m.ClientURLs[0].String() }

// Terminate stops the member and removes the data dir.
func (m *member) Terminate(t *testing.T) {
	m.s.Stop()
	for _, hs := range m.hss {
		hs.CloseClientConnections()
		hs.Close()
	}
	if err := os.RemoveAll(m.ServerConfig.DataDir); err != nil {
		t.Fatal(err)
	}
}
func mustNewTransport(t *testing.T) *http.Transport {
	tr, err := transport.NewTimeoutTransport(transport.TLSInfo{}, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
	if err != nil {
		t.Fatal(err)
	}
	return tr
}
<commit_msg>track etcd cluster change<commit_after>/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package etcdutil

import (
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"
	"time"

	"github.com/coreos/etcd/etcdserver"
	"github.com/coreos/etcd/etcdserver/etcdhttp"
	"github.com/coreos/etcd/pkg/testutil"
	"github.com/coreos/etcd/pkg/transport"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/rafthttp"
)

const (
	tickDuration   = 10 * time.Millisecond
	clusterName    = "etcd"
	requestTimeout = 2 * time.Second
)

var (
	electionTicks = 10
)

func newLocalListener(t *testing.T) net.Listener {
	l, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	return l
}

func StartNewEtcdServer(t *testing.T, name string) *member {
	m := MustNewMember(t, name)
	m.Launch()
	return m
}

type member struct {
	etcdserver.ServerConfig
	PeerListeners, ClientListeners []net.Listener

	raftHandler *testutil.PauseableHandler
	s           *etcdserver.EtcdServer
	hss         []*httptest.Server
}

func MustNewMember(t *testing.T, name string) *member {
	var err error
	m := &member{}

	pln := newLocalListener(t)
	m.PeerListeners = []net.Listener{pln}
	m.PeerURLs, err = types.NewURLs([]string{"http://" + pln.Addr().String()})
	if err != nil {
		t.Fatal(err)
	}

	cln := newLocalListener(t)
	m.ClientListeners = []net.Listener{cln}
	m.ClientURLs, err = types.NewURLs([]string{"http://" + cln.Addr().String()})
	if err != nil {
		t.Fatal(err)
	}

	m.Name = name

	m.DataDir, err = ioutil.TempDir(os.TempDir(), "etcd")
	if err != nil {
		t.Fatal(err)
	}
	clusterStr := fmt.Sprintf("%s=http://%s", name, pln.Addr().String())
	m.Cluster, err = etcdserver.NewClusterFromString(clusterName, clusterStr)
	if err != nil {
		t.Fatal(err)
	}
	m.NewCluster = true
	m.Transport = mustNewTransport(t)
	m.ElectionTicks = electionTicks
	m.TickMs = uint(tickDuration / time.Millisecond)
	return m
}

// Launch starts a member based on ServerConfig, PeerListeners
// and ClientListeners.
func (m *member) Launch() error {
	var err error
	if m.s, err = etcdserver.NewServer(&m.ServerConfig); err != nil {
		return fmt.Errorf("failed to initialize the etcd server: %v", err)
	}
	m.s.SyncTicker = time.Tick(500 * time.Millisecond)
	m.s.Start()

	m.raftHandler = &testutil.PauseableHandler{Next: etcdhttp.NewPeerHandler(m.s.Cluster, m.s.RaftHandler())}

	for _, ln := range m.PeerListeners {
		hs := &httptest.Server{
			Listener: ln,
			Config:   &http.Server{Handler: m.raftHandler},
		}
		hs.Start()
		m.hss = append(m.hss, hs)
	}
	for _, ln := range m.ClientListeners {
		hs := &httptest.Server{
			Listener: ln,
			Config:   &http.Server{Handler: etcdhttp.NewClientHandler(m.s)},
		}
		hs.Start()
		m.hss = append(m.hss, hs)
	}
	return nil
}

func (m *member) URL() string { return m.ClientURLs[0].String() }

// Terminate stops the member and removes the data dir.
func (m *member) Terminate(t *testing.T) {
	m.s.Stop()
	for _, hs := range m.hss {
		hs.CloseClientConnections()
		hs.Close()
	}
	if err := os.RemoveAll(m.ServerConfig.DataDir); err != nil {
		t.Fatal(err)
	}
}

func mustNewTransport(t *testing.T) *http.Transport {
	tr, err := transport.NewTimeoutTransport(transport.TLSInfo{}, rafthttp.ConnReadTimeout, rafthttp.ConnWriteTimeout)
	if err != nil {
		t.Fatal(err)
	}
	return tr
}
<|endoftext|>