<commit_before>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.

package server

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"net/url"
	"sync/atomic"
	"testing"
	"time"

	"golang.org/x/net/context"

	"github.com/cockroachdb/cockroach/pkg/base"
	"github.com/cockroachdb/cockroach/pkg/build"
	"github.com/cockroachdb/cockroach/pkg/roachpb"
	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
)

func stubURL(target **url.URL, stubURL *url.URL) func() {
	realURL := *target
	*target = stubURL
	return func() {
		*target = realURL
	}
}

func TestCheckVersion(t *testing.T) {
	defer leaktest.AfterTest(t)()

	updateChecks := int32(0)
	uuid := ""
	version := ""

	recorder := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		atomic.AddInt32(&updateChecks, 1)
		uuid = r.URL.Query().Get("uuid")
		version = r.URL.Query().Get("version")
	}))
	u, err := url.Parse(recorder.URL)
	if err != nil {
		t.Fatal(err)
	}
	defer stubURL(&updatesURL, u)()

	s, _, _ := serverutils.StartServer(t, base.TestServerArgs{})
	s.(*TestServer).checkForUpdates(time.Minute)
	recorder.Close()
	s.Stopper().Stop()

	if expected, actual := int32(1), atomic.LoadInt32(&updateChecks); actual != expected {
		t.Fatalf("expected %v update checks, got %v", expected, actual)
	}

	if expected, actual := s.(*TestServer).node.ClusterID.String(), uuid; expected != actual {
		t.Errorf("expected uuid %v, got %v", expected, actual)
	}

	if expected, actual := build.GetInfo().Tag, version; expected != actual {
		t.Errorf("expected version tag %v, got %v", expected, actual)
	}
}

func TestReportUsage(t *testing.T) {
	defer leaktest.AfterTest(t)()

	usageReports := int32(0)
	uuid := ""
	reported := reportingInfo{}

	recorder := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		atomic.AddInt32(&usageReports, 1)
		uuid = r.URL.Query().Get("uuid")
		if err := json.NewDecoder(r.Body).Decode(&reported); err != nil {
			t.Fatal(err)
		}
	}))
	u, err := url.Parse(recorder.URL)
	if err != nil {
		t.Fatal(err)
	}
	defer stubURL(&reportingURL, u)()

	params := base.TestServerArgs{
		StoreSpecs: []base.StoreSpec{
			base.DefaultTestStoreSpec,
			base.DefaultTestStoreSpec,
		},
	}
	s, _, _ := serverutils.StartServer(t, params)
	ts := s.(*TestServer)

	if err := ts.WaitForInitialSplits(); err != nil {
		t.Fatal(err)
	}

	node := ts.node.recorder.GetStatusSummary()
	ts.reportUsage(context.TODO())

	ts.Stopper().Stop() // stopper will wait for the update/report loop to finish too.
	recorder.Close()

	keyCounts := make(map[roachpb.StoreID]int)
	rangeCounts := make(map[roachpb.StoreID]int)
	totalKeys := 0
	totalRanges := 0

	for _, store := range node.StoreStatuses {
		if keys, ok := store.Metrics["keycount"]; ok {
			totalKeys += int(keys)
			keyCounts[store.Desc.StoreID] = int(keys)
		} else {
			t.Fatal("keycount not in metrics")
		}
		if replicas, ok := store.Metrics["replicas"]; ok {
			totalRanges += int(replicas)
			rangeCounts[store.Desc.StoreID] = int(replicas)
		} else {
			t.Fatal("replicas not in metrics")
		}
	}

	if expected, actual := int32(1), atomic.LoadInt32(&usageReports); expected != actual {
		t.Fatalf("expected %v reports, got %v", expected, actual)
	}
	if expected, actual := ts.node.ClusterID.String(), uuid; expected != actual {
		t.Errorf("expected cluster id %v got %v", expected, actual)
	}
	if expected, actual := ts.node.Descriptor.NodeID, reported.Node.NodeID; expected != actual {
		t.Errorf("expected node id %v got %v", expected, actual)
	}
	if minExpected, actual := totalKeys, reported.Node.KeyCount; minExpected > actual {
		t.Errorf("expected node keys at least %v got %v", minExpected, actual)
	}
	if minExpected, actual := totalRanges, reported.Node.RangeCount; minExpected > actual {
		t.Errorf("expected node ranges at least %v got %v", minExpected, actual)
	}
	if minExpected, actual := len(params.StoreSpecs), len(reported.Stores); minExpected > actual {
		t.Errorf("expected at least %v stores got %v", minExpected, actual)
	}

	for _, store := range reported.Stores {
		if minExpected, actual := keyCounts[store.StoreID], store.KeyCount; minExpected > actual {
			t.Errorf("expected at least %v keys in store %v got %v", minExpected, store.StoreID, actual)
		}
		if minExpected, actual := rangeCounts[store.StoreID], store.RangeCount; minExpected > actual {
			t.Errorf("expected at least %v ranges in store %v got %v", minExpected, store.StoreID, actual)
		}
	}

}
<commit_msg>server: De-flake TestReportUsage by retrying non-deterministic check<commit_after>// Copyright 2016 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.

package server

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"net/url"
	"sync/atomic"
	"testing"
	"time"

	"golang.org/x/net/context"

	"github.com/cockroachdb/cockroach/pkg/base"
	"github.com/cockroachdb/cockroach/pkg/build"
	"github.com/cockroachdb/cockroach/pkg/roachpb"
	"github.com/cockroachdb/cockroach/pkg/testutils"
	"github.com/cockroachdb/cockroach/pkg/testutils/serverutils"
	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
	"github.com/pkg/errors"
)

func stubURL(target **url.URL, stubURL *url.URL) func() {
	realURL := *target
	*target = stubURL
	return func() {
		*target = realURL
	}
}

func TestCheckVersion(t *testing.T) {
	defer leaktest.AfterTest(t)()

	updateChecks := int32(0)
	uuid := ""
	version := ""

	recorder := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		atomic.AddInt32(&updateChecks, 1)
		uuid = r.URL.Query().Get("uuid")
		version = r.URL.Query().Get("version")
	}))
	u, err := url.Parse(recorder.URL)
	if err != nil {
		t.Fatal(err)
	}
	defer stubURL(&updatesURL, u)()

	s, _, _ := serverutils.StartServer(t, base.TestServerArgs{})
	s.(*TestServer).checkForUpdates(time.Minute)
	recorder.Close()
	s.Stopper().Stop()

	if expected, actual := int32(1), atomic.LoadInt32(&updateChecks); actual != expected {
		t.Fatalf("expected %v update checks, got %v", expected, actual)
	}

	if expected, actual := s.(*TestServer).node.ClusterID.String(), uuid; expected != actual {
		t.Errorf("expected uuid %v, got %v", expected, actual)
	}

	if expected, actual := build.GetInfo().Tag, version; expected != actual {
		t.Errorf("expected version tag %v, got %v", expected, actual)
	}
}

func TestReportUsage(t *testing.T) {
	defer leaktest.AfterTest(t)()

	usageReports := int32(0)
	uuid := ""
	reported := reportingInfo{}

	recorder := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		defer r.Body.Close()
		atomic.AddInt32(&usageReports, 1)
		uuid = r.URL.Query().Get("uuid")
		if err := json.NewDecoder(r.Body).Decode(&reported); err != nil {
			t.Fatal(err)
		}
	}))
	u, err := url.Parse(recorder.URL)
	if err != nil {
		t.Fatal(err)
	}
	defer stubURL(&reportingURL, u)()

	params := base.TestServerArgs{
		StoreSpecs: []base.StoreSpec{
			base.DefaultTestStoreSpec,
			base.DefaultTestStoreSpec,
		},
	}
	s, _, _ := serverutils.StartServer(t, params)
	ts := s.(*TestServer)

	if err := ts.WaitForInitialSplits(); err != nil {
		t.Fatal(err)
	}

	var expectedUsageReports int32
	testutils.SucceedsSoon(t, func() error {
		expectedUsageReports++

		node := ts.node.recorder.GetStatusSummary()
		ts.reportUsage(context.TODO())

		keyCounts := make(map[roachpb.StoreID]int)
		rangeCounts := make(map[roachpb.StoreID]int)
		totalKeys := 0
		totalRanges := 0

		for _, store := range node.StoreStatuses {
			if keys, ok := store.Metrics["keycount"]; ok {
				totalKeys += int(keys)
				keyCounts[store.Desc.StoreID] = int(keys)
			} else {
				t.Fatal("keycount not in metrics")
			}
			if replicas, ok := store.Metrics["replicas"]; ok {
				totalRanges += int(replicas)
				rangeCounts[store.Desc.StoreID] = int(replicas)
			} else {
				t.Fatal("replicas not in metrics")
			}
		}

		if expected, actual := expectedUsageReports, atomic.LoadInt32(&usageReports); expected != actual {
			t.Fatalf("expected %v reports, got %v", expected, actual)
		}
		if expected, actual := ts.node.ClusterID.String(), uuid; expected != actual {
			return errors.Errorf("expected cluster id %v got %v", expected, actual)
		}
		if expected, actual := ts.node.Descriptor.NodeID, reported.Node.NodeID; expected != actual {
			return errors.Errorf("expected node id %v got %v", expected, actual)
		}
		if minExpected, actual := totalKeys, reported.Node.KeyCount; minExpected > actual {
			return errors.Errorf("expected node keys at least %v got %v", minExpected, actual)
		}
		if minExpected, actual := totalRanges, reported.Node.RangeCount; minExpected > actual {
			return errors.Errorf("expected node ranges at least %v got %v", minExpected, actual)
		}
		if minExpected, actual := len(params.StoreSpecs), len(reported.Stores); minExpected > actual {
			return errors.Errorf("expected at least %v stores got %v", minExpected, actual)
		}

		for _, store := range reported.Stores {
			if minExpected, actual := keyCounts[store.StoreID], store.KeyCount; minExpected > actual {
				return errors.Errorf("expected at least %v keys in store %v got %v", minExpected, store.StoreID, actual)
			}
			if minExpected, actual := rangeCounts[store.StoreID], store.RangeCount; minExpected > actual {
				return errors.Errorf("expected at least %v ranges in store %v got %v", minExpected, store.StoreID, actual)
			}
		}
		return nil
	})

	ts.Stopper().Stop() // stopper will wait for the update/report loop to finish too.
	recorder.Close()
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package manila

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/gophercloud/gophercloud/openstack/sharedfilesystems/apiversions"
	"k8s.io/cloud-provider-openstack/pkg/share/manila/shareoptions"
)

const (
	minimumManilaVersion = "2.21"
)

var (
	microversionRegexp = regexp.MustCompile("^\\d+\\.\\d+$")
)

func splitMicroversion(microversion string) (major, minor int) {
	if err := validateMicroversion(microversion); err != nil {
		return
	}

	parts := strings.Split(microversion, ".")
	major, _ = strconv.Atoi(parts[0])
	minor, _ = strconv.Atoi(parts[1])

	return
}

func validateMicroversion(microversion string) error {
	if !microversionRegexp.MatchString(microversion) {
		return fmt.Errorf("Invalid microversion format in %q", microversion)
	}

	return nil
}

func compareVersionsLessThan(a, b string) bool {
	aMaj, aMin := splitMicroversion(a)
	bMaj, bMin := splitMicroversion(b)

	return aMaj < bMaj || (aMaj == bMaj && aMin < bMin)
}

// NewManilaV2Client Creates Manila v2 client
// Authenticates to the Manila service with credentials passed in env variables
func NewManilaV2Client(osOptions *shareoptions.OpenStackOptions) (*gophercloud.ServiceClient, error) {
	// Authenticate

	provider, err := openstack.AuthenticatedClient(*osOptions.ToAuthOptions())
	if err != nil {
		return nil, fmt.Errorf("failed to authenticate Manila v2 client: %v", err)
	}

	client, err := openstack.NewSharedFileSystemV2(provider, gophercloud.EndpointOpts{Region: osOptions.OSRegionName})
	if err != nil {
		return nil, fmt.Errorf("failed to create Manila v2 client: %v", err)
	}

	// Check client's and server's versions for compatibility

	client.Microversion = minimumManilaVersion

	serverVersion, err := apiversions.Get(client, "v2").Extract()
	if err != nil {
		return nil, fmt.Errorf("failed to get Manila v2 API microversions: %v", err)
	}

	if err = validateMicroversion(serverVersion.MinVersion); err != nil {
		return nil, fmt.Errorf("server's minimum microversion is invalid: %v", err)
	}

	if err = validateMicroversion(serverVersion.Version); err != nil {
		return nil, fmt.Errorf("server's maximum microversion is invalid: %v", err)
	}

	if compareVersionsLessThan(client.Microversion, serverVersion.MinVersion) {
		return nil, fmt.Errorf("client's microversion %s is lower than server's minimum microversion %s", client.Microversion, serverVersion.MinVersion)
	}

	if compareVersionsLessThan(serverVersion.Version, client.Microversion) {
		return nil, fmt.Errorf("client's microversion %s is higher than server's highest supported microversion %s", client.Microversion, serverVersion.Version)
	}

	return client, nil
}
<commit_msg>manila: client: add support for trustee authentication<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package manila

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/gophercloud/gophercloud/openstack/sharedfilesystems/apiversions"
	gophercloudutils "github.com/gophercloud/gophercloud/openstack/utils"
	"k8s.io/cloud-provider-openstack/pkg/share/manila/shareoptions"
)

const (
	minimumManilaVersion = "2.21"
)

var (
	microversionRegexp = regexp.MustCompile("^\\d+\\.\\d+$")
)

func splitMicroversion(microversion string) (major, minor int) {
	if err := validateMicroversion(microversion); err != nil {
		return
	}

	parts := strings.Split(microversion, ".")
	major, _ = strconv.Atoi(parts[0])
	minor, _ = strconv.Atoi(parts[1])

	return
}

func validateMicroversion(microversion string) error {
	if !microversionRegexp.MatchString(microversion) {
		return fmt.Errorf("Invalid microversion format in %q", microversion)
	}

	return nil
}

func compareVersionsLessThan(a, b string) bool {
	aMaj, aMin := splitMicroversion(a)
	bMaj, bMin := splitMicroversion(b)

	return aMaj < bMaj || (aMaj == bMaj && aMin < bMin)
}

// NewManilaV2Client Creates Manila v2 client
// Authenticates to the Manila service with credentials passed in env variables
func NewManilaV2Client(osOptions *shareoptions.OpenStackOptions) (*gophercloud.ServiceClient, error) {
	// Authenticate

	provider, err := openstack.NewClient(osOptions.OSAuthURL)
	if err != nil {
		return nil, fmt.Errorf("failed to create Keystone client: %v", err)
	}

	const (
		v2 = "v2.0"
		v3 = "v3"
	)

	chosenVersion, _, err := gophercloudutils.ChooseVersion(provider, []*gophercloudutils.Version{
		{ID: v2, Priority: 20, Suffix: "/v2.0/"},
		{ID: v3, Priority: 30, Suffix: "/v3/"},
	})

	switch chosenVersion.ID {
	case v2:
		if osOptions.OSTrustID != "" {
			return nil, fmt.Errorf("Keystone %s does not support trustee authentication", chosenVersion.ID)
		}

		err = openstack.AuthenticateV2(provider, *osOptions.ToAuthOptions(), gophercloud.EndpointOpts{})
	case v3:
		err = openstack.AuthenticateV3(provider, *osOptions.ToAuthOptionsExt(), gophercloud.EndpointOpts{})
	default:
		return nil, fmt.Errorf("unrecognized Keystone version: %s", chosenVersion.ID)
	}

	if err != nil {
		return nil, fmt.Errorf("failed to authenticate with Keystone: %v", err)
	}

	client, err := openstack.NewSharedFileSystemV2(provider, gophercloud.EndpointOpts{Region: osOptions.OSRegionName})
	if err != nil {
		return nil, fmt.Errorf("failed to create Manila v2 client: %v", err)
	}

	// Check client's and server's versions for compatibility

	client.Microversion = minimumManilaVersion

	serverVersion, err := apiversions.Get(client, "v2").Extract()
	if err != nil {
		return nil, fmt.Errorf("failed to get Manila v2 API microversions: %v", err)
	}

	if err = validateMicroversion(serverVersion.MinVersion); err != nil {
		return nil, fmt.Errorf("server's minimum microversion is invalid: %v", err)
	}

	if err = validateMicroversion(serverVersion.Version); err != nil {
		return nil, fmt.Errorf("server's maximum microversion is invalid: %v", err)
	}

	if compareVersionsLessThan(client.Microversion, serverVersion.MinVersion) {
		return nil, fmt.Errorf("client's microversion %s is lower than server's minimum microversion %s", client.Microversion, serverVersion.MinVersion)
	}

	if compareVersionsLessThan(serverVersion.Version, client.Microversion) {
		return nil, fmt.Errorf("client's microversion %s is higher than server's highest supported microversion %s", client.Microversion, serverVersion.Version)
	}

	return client, nil
}
<|endoftext|>


<commit_before>package system

import (
	"bytes"

	"golang.org/x/sys/unix"
)

const (
	// Value is larger than the maximum size allowed
	E2BIG unix.Errno = unix.E2BIG

	// Operation not supported
	EOPNOTSUPP unix.Errno = unix.EOPNOTSUPP
)

// Lgetxattr retrieves the value of the extended attribute identified by attr
// and associated with the given path in the file system.
// It will returns a nil slice and nil error if the xattr is not set.
func Lgetxattr(path string, attr string) ([]byte, error) {
	// Start with a 128 length byte array
	dest := make([]byte, 128)
	sz, errno := unix.Lgetxattr(path, attr, dest)

	switch {
	case errno == unix.ENODATA:
		return nil, nil
	case errno == unix.ERANGE:
		// 128 byte array might just not be good enough. A dummy buffer is used
		// to get the real size of the xattrs on disk
		sz, errno = unix.Lgetxattr(path, attr, []byte{})
		if errno != nil {
			return nil, errno
		}
		dest = make([]byte, sz)
		sz, errno = unix.Lgetxattr(path, attr, dest)
		if errno != nil {
			return nil, errno
		}
	case errno != nil:
		return nil, errno
	}
	return dest[:sz], nil
}

// Lsetxattr sets the value of the extended attribute identified by attr
// and associated with the given path in the file system.
func Lsetxattr(path string, attr string, data []byte, flags int) error {
	return unix.Lsetxattr(path, attr, data, flags)
}

// Llistxattr lists extended attributes associated with the given path
// in the file system.
func Llistxattr(path string) ([]string, error) {
	var dest []byte

	for {
		sz, err := unix.Llistxattr(path, dest)
		if err != nil {
			return nil, err
		}

		if sz > len(dest) {
			dest = make([]byte, sz)
		} else {
			dest = dest[:sz]
			break
		}
	}

	var attrs []string
	for _, token := range bytes.Split(dest, []byte{0}) {
		if len(token) > 0 {
			attrs = append(attrs, string(token))
		}
	}

	return attrs, nil
}
<commit_msg>pkg/system: handle changed size case<commit_after>package system

import (
	"bytes"

	"golang.org/x/sys/unix"
)

const (
	// Value is larger than the maximum size allowed
	E2BIG unix.Errno = unix.E2BIG

	// Operation not supported
	EOPNOTSUPP unix.Errno = unix.EOPNOTSUPP
)

// Lgetxattr retrieves the value of the extended attribute identified by attr
// and associated with the given path in the file system.
// Returns a []byte slice if the xattr is set and nil otherwise.
func Lgetxattr(path string, attr string) ([]byte, error) {
	// Start with a 128 length byte array
	dest := make([]byte, 128)
	sz, errno := unix.Lgetxattr(path, attr, dest)

	for errno == unix.ERANGE {
		// Buffer too small, use zero-sized buffer to get the actual size
		sz, errno = unix.Lgetxattr(path, attr, []byte{})
		if errno != nil {
			return nil, errno
		}
		dest = make([]byte, sz)
		sz, errno = unix.Lgetxattr(path, attr, dest)
	}

	switch {
	case errno == unix.ENODATA:
		return nil, nil
	case errno != nil:
		return nil, errno
	}

	return dest[:sz], nil
}

// Lsetxattr sets the value of the extended attribute identified by attr
// and associated with the given path in the file system.
func Lsetxattr(path string, attr string, data []byte, flags int) error {
	return unix.Lsetxattr(path, attr, data, flags)
}

// Llistxattr lists extended attributes associated with the given path
// in the file system.
func Llistxattr(path string) ([]string, error) {
	var dest []byte

	for {
		sz, err := unix.Llistxattr(path, dest)
		if err != nil {
			return nil, err
		}

		if sz > len(dest) {
			dest = make([]byte, sz)
		} else {
			dest = dest[:sz]
			break
		}
	}

	var attrs []string
	for _, token := range bytes.Split(dest, []byte{0}) {
		if len(token) > 0 {
			attrs = append(attrs, string(token))
		}
	}

	return attrs, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package webhook

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"net/url"

	"github.com/hashicorp/golang-lru"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/client-go/rest"
)

const (
	defaultCacheSize = 200
)

// ClientConfig defines parameters required for creating a hook client.
type ClientConfig struct {
	Name     string
	URL      string
	CABundle []byte
	Service  *ClientConfigService
}

// ClientConfigService defines service discovery parameters of the webhook.
type ClientConfigService struct {
	Name      string
	Namespace string
	Path      string
	Port      int32
}

// ClientManager builds REST clients to talk to webhooks. It caches the clients
// to avoid duplicate creation.
type ClientManager struct {
	authInfoResolver     AuthenticationInfoResolver
	serviceResolver      ServiceResolver
	negotiatedSerializer runtime.NegotiatedSerializer
	cache                *lru.Cache
}

// NewClientManager creates a clientManager.
func NewClientManager(gvs []schema.GroupVersion, addToSchemaFuncs ...func(s *runtime.Scheme) error) (ClientManager, error) {
	cache, err := lru.New(defaultCacheSize)
	if err != nil {
		return ClientManager{}, err
	}
	hookScheme := runtime.NewScheme()
	for _, addToSchemaFunc := range addToSchemaFuncs {
		if err := addToSchemaFunc(hookScheme); err != nil {
			return ClientManager{}, err
		}
	}
	return ClientManager{
		cache: cache,
		negotiatedSerializer: serializer.NegotiatedSerializerWrapper(runtime.SerializerInfo{
			Serializer: serializer.NewCodecFactory(hookScheme).LegacyCodec(gvs...),
		}),
	}, nil
}

// SetAuthenticationInfoResolverWrapper sets the
// AuthenticationInfoResolverWrapper.
func (cm *ClientManager) SetAuthenticationInfoResolverWrapper(wrapper AuthenticationInfoResolverWrapper) {
	if wrapper != nil {
		cm.authInfoResolver = wrapper(cm.authInfoResolver)
	}
}

// SetAuthenticationInfoResolver sets the AuthenticationInfoResolver.
func (cm *ClientManager) SetAuthenticationInfoResolver(resolver AuthenticationInfoResolver) {
	cm.authInfoResolver = resolver
}

// SetServiceResolver sets the ServiceResolver.
func (cm *ClientManager) SetServiceResolver(sr ServiceResolver) {
	if sr != nil {
		cm.serviceResolver = sr
	}
}

// Validate checks if ClientManager is properly set up.
func (cm *ClientManager) Validate() error {
	var errs []error
	if cm.negotiatedSerializer == nil {
		errs = append(errs, fmt.Errorf("the clientManager requires a negotiatedSerializer"))
	}
	if cm.serviceResolver == nil {
		errs = append(errs, fmt.Errorf("the clientManager requires a serviceResolver"))
	}
	if cm.authInfoResolver == nil {
		errs = append(errs, fmt.Errorf("the clientManager requires an authInfoResolver"))
	}
	return utilerrors.NewAggregate(errs)
}

// HookClient get a RESTClient from the cache, or constructs one based on the
// webhook configuration.
func (cm *ClientManager) HookClient(cc ClientConfig) (*rest.RESTClient, error) {
	ccWithNoName := cc
	ccWithNoName.Name = ""
	cacheKey, err := json.Marshal(ccWithNoName)
	if err != nil {
		return nil, err
	}
	if client, ok := cm.cache.Get(string(cacheKey)); ok {
		return client.(*rest.RESTClient), nil
	}

	complete := func(cfg *rest.Config) (*rest.RESTClient, error) {
		// Combine CAData from the config with any existing CA bundle provided
		if len(cfg.TLSClientConfig.CAData) > 0 {
			cfg.TLSClientConfig.CAData = append(cfg.TLSClientConfig.CAData, '\n')
		}
		cfg.TLSClientConfig.CAData = append(cfg.TLSClientConfig.CAData, cc.CABundle...)

		cfg.ContentConfig.NegotiatedSerializer = cm.negotiatedSerializer
		cfg.ContentConfig.ContentType = runtime.ContentTypeJSON
		client, err := rest.UnversionedRESTClientFor(cfg)
		if err == nil {
			cm.cache.Add(string(cacheKey), client)
		}
		return client, err
	}

	if cc.Service != nil {
		restConfig, err := cm.authInfoResolver.ClientConfigForService(cc.Service.Name, cc.Service.Namespace)
		if err != nil {
			return nil, err
		}
		cfg := rest.CopyConfig(restConfig)
		serverName := cc.Service.Name + "." + cc.Service.Namespace + ".svc"
		host := serverName + ":443"
		cfg.Host = "https://" + host
		cfg.APIPath = cc.Service.Path
		// Set the server name if not already set
		if len(cfg.TLSClientConfig.ServerName) == 0 {
			cfg.TLSClientConfig.ServerName = serverName
		}

		delegateDialer := cfg.Dial
		if delegateDialer == nil {
			var d net.Dialer
			delegateDialer = d.DialContext
		}
		cfg.Dial = func(ctx context.Context, network, addr string) (net.Conn, error) {
			if addr == host {
				port := cc.Service.Port
				if port == 0 {
					port = 443
				}
				u, err := cm.serviceResolver.ResolveEndpoint(cc.Service.Namespace, cc.Service.Name, port)
				if err != nil {
					return nil, err
				}
				addr = u.Host
			}
			return delegateDialer(ctx, network, addr)
		}

		return complete(cfg)
	}

	if cc.URL == "" {
		return nil, &ErrCallingWebhook{WebhookName: cc.Name, Reason: errors.New("webhook configuration must have either service or URL")}
	}

	u, err := url.Parse(cc.URL)
	if err != nil {
		return nil, &ErrCallingWebhook{WebhookName: cc.Name, Reason: fmt.Errorf("Unparsable URL: %v", err)}
	}

	restConfig, err := cm.authInfoResolver.ClientConfigFor(u.Host)
	if err != nil {
		return nil, err
	}

	cfg := rest.CopyConfig(restConfig)
	cfg.Host = u.Scheme + "://" + u.Host
	cfg.APIPath = u.Path

	return complete(cfg)
}
<commit_msg>Use http/1.1 in apiserver->webhook clients<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package webhook

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"net/url"

	"github.com/hashicorp/golang-lru"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/client-go/rest"
)

const (
	defaultCacheSize = 200
)

// ClientConfig defines parameters required for creating a hook client.
type ClientConfig struct {
	Name     string
	URL      string
	CABundle []byte
	Service  *ClientConfigService
}

// ClientConfigService defines service discovery parameters of the webhook.
type ClientConfigService struct {
	Name      string
	Namespace string
	Path      string
	Port      int32
}

// ClientManager builds REST clients to talk to webhooks. It caches the clients
// to avoid duplicate creation.
type ClientManager struct {
	authInfoResolver     AuthenticationInfoResolver
	serviceResolver      ServiceResolver
	negotiatedSerializer runtime.NegotiatedSerializer
	cache                *lru.Cache
}

// NewClientManager creates a clientManager.
func NewClientManager(gvs []schema.GroupVersion, addToSchemaFuncs ...func(s *runtime.Scheme) error) (ClientManager, error) {
	cache, err := lru.New(defaultCacheSize)
	if err != nil {
		return ClientManager{}, err
	}
	hookScheme := runtime.NewScheme()
	for _, addToSchemaFunc := range addToSchemaFuncs {
		if err := addToSchemaFunc(hookScheme); err != nil {
			return ClientManager{}, err
		}
	}
	return ClientManager{
		cache: cache,
		negotiatedSerializer: serializer.NegotiatedSerializerWrapper(runtime.SerializerInfo{
			Serializer: serializer.NewCodecFactory(hookScheme).LegacyCodec(gvs...),
		}),
	}, nil
}

// SetAuthenticationInfoResolverWrapper sets the
// AuthenticationInfoResolverWrapper.
func (cm *ClientManager) SetAuthenticationInfoResolverWrapper(wrapper AuthenticationInfoResolverWrapper) {
	if wrapper != nil {
		cm.authInfoResolver = wrapper(cm.authInfoResolver)
	}
}

// SetAuthenticationInfoResolver sets the AuthenticationInfoResolver.
func (cm *ClientManager) SetAuthenticationInfoResolver(resolver AuthenticationInfoResolver) {
	cm.authInfoResolver = resolver
}

// SetServiceResolver sets the ServiceResolver.
func (cm *ClientManager) SetServiceResolver(sr ServiceResolver) {
	if sr != nil {
		cm.serviceResolver = sr
	}
}

// Validate checks if ClientManager is properly set up.
func (cm *ClientManager) Validate() error {
	var errs []error
	if cm.negotiatedSerializer == nil {
		errs = append(errs, fmt.Errorf("the clientManager requires a negotiatedSerializer"))
	}
	if cm.serviceResolver == nil {
		errs = append(errs, fmt.Errorf("the clientManager requires a serviceResolver"))
	}
	if cm.authInfoResolver == nil {
		errs = append(errs, fmt.Errorf("the clientManager requires an authInfoResolver"))
	}
	return utilerrors.NewAggregate(errs)
}

// HookClient get a RESTClient from the cache, or constructs one based on the
// webhook configuration.
func (cm *ClientManager) HookClient(cc ClientConfig) (*rest.RESTClient, error) {
	ccWithNoName := cc
	ccWithNoName.Name = ""
	cacheKey, err := json.Marshal(ccWithNoName)
	if err != nil {
		return nil, err
	}
	if client, ok := cm.cache.Get(string(cacheKey)); ok {
		return client.(*rest.RESTClient), nil
	}

	complete := func(cfg *rest.Config) (*rest.RESTClient, error) {
		// Combine CAData from the config with any existing CA bundle provided
		if len(cfg.TLSClientConfig.CAData) > 0 {
			cfg.TLSClientConfig.CAData = append(cfg.TLSClientConfig.CAData, '\n')
		}
		cfg.TLSClientConfig.CAData = append(cfg.TLSClientConfig.CAData, cc.CABundle...)

		// Use http/1.1 instead of http/2.
		// This is a workaround for http/2-enabled clients not load-balancing concurrent requests to multiple backends.
		// See http://issue.k8s.io/75791 for details.
		cfg.NextProtos = []string{"http/1.1"}

		cfg.ContentConfig.NegotiatedSerializer = cm.negotiatedSerializer
		cfg.ContentConfig.ContentType = runtime.ContentTypeJSON
		client, err := rest.UnversionedRESTClientFor(cfg)
		if err == nil {
			cm.cache.Add(string(cacheKey), client)
		}
		return client, err
	}

	if cc.Service != nil {
		restConfig, err := cm.authInfoResolver.ClientConfigForService(cc.Service.Name, cc.Service.Namespace)
		if err != nil {
			return nil, err
		}
		cfg := rest.CopyConfig(restConfig)
		serverName := cc.Service.Name + "." + cc.Service.Namespace + ".svc"
		host := serverName + ":443"
		cfg.Host = "https://" + host
		cfg.APIPath = cc.Service.Path
		// Set the server name if not already set
		if len(cfg.TLSClientConfig.ServerName) == 0 {
			cfg.TLSClientConfig.ServerName = serverName
		}

		delegateDialer := cfg.Dial
		if delegateDialer == nil {
			var d net.Dialer
			delegateDialer = d.DialContext
		}
		cfg.Dial = func(ctx context.Context, network, addr string) (net.Conn, error) {
			if addr == host {
				port := cc.Service.Port
				if port == 0 {
					port = 443
				}
				u, err := cm.serviceResolver.ResolveEndpoint(cc.Service.Namespace, cc.Service.Name, port)
				if err != nil {
					return nil, err
				}
				addr = u.Host
			}
			return delegateDialer(ctx, network, addr)
		}

		return complete(cfg)
	}

	if cc.URL == "" {
		return nil, &ErrCallingWebhook{WebhookName: cc.Name, Reason: errors.New("webhook configuration must have either service or URL")}
	}

	u, err := url.Parse(cc.URL)
	if err != nil {
		return nil, &ErrCallingWebhook{WebhookName: cc.Name, Reason: fmt.Errorf("Unparsable URL: %v", err)}
	}

	restConfig, err := cm.authInfoResolver.ClientConfigFor(u.Host)
	if err != nil {
		return nil, err
	}

	cfg := rest.CopyConfig(restConfig)
	cfg.Host = u.Scheme + "://" + u.Host
	cfg.APIPath = u.Path

	return complete(cfg)
}
<|endoftext|>


<commit_before>package mempool

import (
	"bytes"
	"container/list"
	"sync"
	"sync/atomic"
	"time"

	"github.com/tendermint/go-clist"
	. "github.com/tendermint/go-common"
	"github.com/tendermint/tendermint/proxy"
	"github.com/tendermint/tendermint/types"
	tmsp "github.com/tendermint/tmsp/types"
)

/*

The mempool pushes new txs onto the proxyAppConn.
It gets a stream of (req, res) tuples from the proxy.
The memool stores good txs in a concurrent linked-list.

Multiple concurrent go-routines can traverse this linked-list
safely by calling .NextWait() on each element.

So we have several go-routines:
1. Consensus calling Update() and Reap() synchronously
2. Many mempool reactor's peer routines calling CheckTx()
3. Many mempool reactor's peer routines traversing the txs linked list
4. Another goroutine calling GarbageCollectTxs() periodically

To manage these goroutines, there are three methods of locking.
1. Mutations to the linked-list is protected by an internal mtx (CList is goroutine-safe)
2. Mutations to the linked-list elements are atomic
3. CheckTx() calls can be paused upon Update() and Reap(), protected by .proxyMtx

Garbage collection of old elements from mempool.txs is handlde via
the DetachPrev() call, which makes old elements not reachable by
peer broadcastTxRoutine() automatically garbage collected.

TODO: Better handle tmsp client errors. (make it automatically handle connection errors)

*/

const cacheSize = 100000

type Mempool struct {
	proxyMtx      sync.Mutex
	proxyAppConn  proxy.AppConn
	txs           *clist.CList    // concurrent linked-list of good txs
	counter       int64           // simple incrementing counter
	height        int             // the last block Update()'d to
	rechecking    int32           // for re-checking filtered txs on Update()
	recheckCursor *clist.CElement // next expected response
	recheckEnd    *clist.CElement // re-checking stops here

	// Keep a cache of already-seen txs.
	// This reduces the pressure on the proxyApp.
	cacheMap  map[string]struct{}
	cacheList *list.List
}

func NewMempool(proxyAppConn proxy.AppConn) *Mempool {
	mempool := &Mempool{
		proxyAppConn:  proxyAppConn,
		txs:           clist.New(),
		counter:       0,
		height:        0,
		rechecking:    0,
		recheckCursor: nil,
		recheckEnd:    nil,

		cacheMap:  make(map[string]struct{}, cacheSize),
		cacheList: list.New(),
	}
	proxyAppConn.SetResponseCallback(mempool.resCb)
	return mempool
}

// Return the first element of mem.txs for peer goroutines to call .NextWait() on.
// Blocks until txs has elements.
func (mem *Mempool) TxsFrontWait() *clist.CElement {
	return mem.txs.FrontWait()
}

// Try a new transaction in the mempool.
// Potentially blocking if we're blocking on Update() or Reap().
// cb: A callback from the CheckTx command.
//     It gets called from another goroutine.
// CONTRACT: Either cb will get called, or err returned.
func (mem *Mempool) CheckTx(tx types.Tx, cb func(*tmsp.Response)) (err error) {
	mem.proxyMtx.Lock()
	defer mem.proxyMtx.Unlock()

	// CACHE
	if _, exists := mem.cacheMap[string(tx)]; exists {
		if cb != nil {
			cb(&tmsp.Response{
				Code: tmsp.CodeType_BadNonce, // TODO or duplicate tx
				Log:  "Duplicate transaction (ignored)",
			})
		}
		return nil
	}
	if mem.cacheList.Len() >= cacheSize {
		popped := mem.cacheList.Front()
		poppedTx := popped.Value.(types.Tx)
		delete(mem.cacheMap, string(poppedTx))
		mem.cacheList.Remove(popped)
	}
	mem.cacheMap[string(tx)] = struct{}{}
	mem.cacheList.PushBack(tx)
	// END CACHE

	// NOTE: proxyAppConn may error if tx buffer is full
	if err = mem.proxyAppConn.Error(); err != nil {
		return err
	}
	reqRes := mem.proxyAppConn.CheckTxAsync(tx)
	if cb != nil {
		reqRes.SetCallback(cb)
	}

	return nil
}

// TMSP callback function
func (mem *Mempool) resCb(req *tmsp.Request, res *tmsp.Response) {
	if mem.recheckCursor == nil {
		mem.resCbNormal(req, res)
	} else {
		mem.resCbRecheck(req, res)
	}
}

func (mem *Mempool) resCbNormal(req *tmsp.Request, res *tmsp.Response) {
	switch res.Type {
	case tmsp.MessageType_CheckTx:
		if res.Code == tmsp.CodeType_OK {
			mem.counter++
			memTx := &mempoolTx{
				counter: mem.counter,
				height:  int64(mem.height),
				tx:      req.Data,
			}
			mem.txs.PushBack(memTx)
		} else {
			// ignore bad transaction
			// TODO: handle other retcodes
		}
	default:
		// ignore other messages
	}
}

func (mem *Mempool) resCbRecheck(req *tmsp.Request, res *tmsp.Response) {
	switch res.Type {
	case tmsp.MessageType_CheckTx:
		memTx := mem.recheckCursor.Value.(*mempoolTx)
		if !bytes.Equal(req.Data, memTx.tx) {
			PanicSanity(Fmt("Unexpected tx response from proxy during recheck\n"+
				"Expected %X, got %X", req.Data, memTx.tx))
		}
		if res.Code == tmsp.CodeType_OK {
			// Good, nothing to do.
		} else {
			// Tx became invalidated due to newly committed block.
			mem.txs.Remove(mem.recheckCursor)
			mem.recheckCursor.DetachPrev()
		}
		if mem.recheckCursor == mem.recheckEnd {
			mem.recheckCursor = nil
		} else {
			mem.recheckCursor = mem.recheckCursor.Next()
		}
		if mem.recheckCursor == nil {
			// Done!
			atomic.StoreInt32(&mem.rechecking, 0)
		}
	default:
		// ignore other messages
	}
}

// Get the valid transactions remaining
// If maxTxs is 0, there is no cap.
func (mem *Mempool) Reap(maxTxs int) []types.Tx {
	mem.proxyMtx.Lock()
	defer mem.proxyMtx.Unlock()

	for atomic.LoadInt32(&mem.rechecking) > 0 {
		// TODO: Something better?
		time.Sleep(time.Millisecond * 10)
	}

	txs := mem.collectTxs(maxTxs)
	return txs
}

// maxTxs: 0 means uncapped
func (mem *Mempool) collectTxs(maxTxs int) []types.Tx {
	if maxTxs == 0 {
		maxTxs = mem.txs.Len()
	}
	txs := make([]types.Tx, 0, MinInt(mem.txs.Len(), maxTxs))
	for e := mem.txs.Front(); e != nil && len(txs) < maxTxs; e = e.Next() {
		memTx := e.Value.(*mempoolTx)
		txs = append(txs, memTx.tx)
	}
	return txs
}

// Tell mempool that these txs were committed.
// Mempool will discard these txs.
// NOTE: this should be called *after* block is committed by consensus.
func (mem *Mempool) Update(height int, txs []types.Tx) {
	mem.proxyMtx.Lock()
	defer mem.proxyMtx.Unlock()

	// First, create a lookup map of txns in new txs.
	txsMap := make(map[string]struct{})
	for _, tx := range txs {
		txsMap[string(tx)] = struct{}{}
	}

	// Set height
	mem.height = height
	// Remove transactions that are already in txs.
	goodTxs := mem.filterTxs(txsMap)
	// Recheck mempool txs
	if config.GetBool("mempool_recheck") {
		mem.recheckTxs(goodTxs)
		// At this point, mem.txs are being rechecked.
		// mem.recheckCursor re-scans mem.txs and possibly removes some txs.
		// Before mem.Reap(), we should wait for mem.recheckCursor to be nil.
	}
}

func (mem *Mempool) filterTxs(blockTxsMap map[string]struct{}) []types.Tx {
	goodTxs := make([]types.Tx, 0, mem.txs.Len())
	for e := mem.txs.Front(); e != nil; e = e.Next() {
		memTx := e.Value.(*mempoolTx)
		if _, ok := blockTxsMap[string(memTx.tx)]; ok {
			// Remove the tx since already in block.
			mem.txs.Remove(e)
			e.DetachPrev()
			continue
		}
		// Good tx!
		goodTxs = append(goodTxs, memTx.tx)
	}
	return goodTxs
}

// NOTE: pass in goodTxs because mem.txs can mutate concurrently.
func (mem *Mempool) recheckTxs(goodTxs []types.Tx) {
	if len(goodTxs) == 0 {
		return
	}
	atomic.StoreInt32(&mem.rechecking, 1)
	mem.recheckCursor = mem.txs.Front()
	mem.recheckEnd = mem.txs.Back()

	// Push txs to proxyAppConn
	// NOTE: resCb() may be called concurrently.
	for _, tx := range goodTxs {
		mem.proxyAppConn.CheckTxAsync(tx)
	}
	mem.proxyAppConn.FlushAsync()
}

//--------------------------------------------------------------------------------

// A transaction that successfully ran
type mempoolTx struct {
	counter int64    // a simple incrementing counter
	height  int64    // height that this tx had been validated in
	tx      types.Tx //
}

func (memTx *mempoolTx) Height() int {
	return int(atomic.LoadInt64(&memTx.height))
}
<commit_msg>Fix CheckTx/Update race condition<commit_after>package mempool

import (
	"bytes"
	"container/list"
	"sync"
	"sync/atomic"
	"time"

	"github.com/tendermint/go-clist"
	. "github.com/tendermint/go-common"
	"github.com/tendermint/tendermint/proxy"
	"github.com/tendermint/tendermint/types"
	tmsp "github.com/tendermint/tmsp/types"
)

/*

The mempool pushes new txs onto the proxyAppConn.
It gets a stream of (req, res) tuples from the proxy.
The memool stores good txs in a concurrent linked-list.

Multiple concurrent go-routines can traverse this linked-list
safely by calling .NextWait() on each element.

So we have several go-routines:
1. Consensus calling Update() and Reap() synchronously
2. Many mempool reactor's peer routines calling CheckTx()
3. Many mempool reactor's peer routines traversing the txs linked list
4. Another goroutine calling GarbageCollectTxs() periodically

To manage these goroutines, there are three methods of locking.
1. Mutations to the linked-list is protected by an internal mtx (CList is goroutine-safe)
2. Mutations to the linked-list elements are atomic
3. CheckTx() calls can be paused upon Update() and Reap(), protected by .proxyMtx

Garbage collection of old elements from mempool.txs is handlde via
the DetachPrev() call, which makes old elements not reachable by
peer broadcastTxRoutine() automatically garbage collected.

TODO: Better handle tmsp client errors. (make it automatically handle connection errors)

*/

const cacheSize = 100000

type Mempool struct {
	proxyMtx      sync.Mutex
	proxyAppConn  proxy.AppConn
	txs           *clist.CList    // concurrent linked-list of good txs
	counter       int64           // simple incrementing counter
	height        int             // the last block Update()'d to
	rechecking    int32           // for re-checking filtered txs on Update()
	recheckCursor *clist.CElement // next expected response
	recheckEnd    *clist.CElement // re-checking stops here

	// Keep a cache of already-seen txs.
	// This reduces the pressure on the proxyApp.
	cacheMap  map[string]struct{}
	cacheList *list.List
}

func NewMempool(proxyAppConn proxy.AppConn) *Mempool {
	mempool := &Mempool{
		proxyAppConn:  proxyAppConn,
		txs:           clist.New(),
		counter:       0,
		height:        0,
		rechecking:    0,
		recheckCursor: nil,
		recheckEnd:    nil,

		cacheMap:  make(map[string]struct{}, cacheSize),
		cacheList: list.New(),
	}
	proxyAppConn.SetResponseCallback(mempool.resCb)
	return mempool
}

// Return the first element of mem.txs for peer goroutines to call .NextWait() on.
// Blocks until txs has elements.
func (mem *Mempool) TxsFrontWait() *clist.CElement {
	return mem.txs.FrontWait()
}

// Try a new transaction in the mempool.
// Potentially blocking if we're blocking on Update() or Reap().
// cb: A callback from the CheckTx command.
//     It gets called from another goroutine.
// CONTRACT: Either cb will get called, or err returned.
func (mem *Mempool) CheckTx(tx types.Tx, cb func(*tmsp.Response)) (err error) {
	mem.proxyMtx.Lock()
	defer mem.proxyMtx.Unlock()

	// CACHE
	if _, exists := mem.cacheMap[string(tx)]; exists {
		if cb != nil {
			cb(&tmsp.Response{
				Code: tmsp.CodeType_BadNonce, // TODO or duplicate tx
				Log:  "Duplicate transaction (ignored)",
			})
		}
		return nil
	}
	if mem.cacheList.Len() >= cacheSize {
		popped := mem.cacheList.Front()
		poppedTx := popped.Value.(types.Tx)
		delete(mem.cacheMap, string(poppedTx))
		mem.cacheList.Remove(popped)
	}
	mem.cacheMap[string(tx)] = struct{}{}
	mem.cacheList.PushBack(tx)
	// END CACHE

	// NOTE: proxyAppConn may error if tx buffer is full
	if err = mem.proxyAppConn.Error(); err != nil {
		return err
	}
	reqRes := mem.proxyAppConn.CheckTxAsync(tx)
	if cb != nil {
		reqRes.SetCallback(cb)
	}

	return nil
}

// TMSP callback function
func (mem *Mempool) resCb(req *tmsp.Request, res *tmsp.Response) {
	if mem.recheckCursor == nil {
		mem.resCbNormal(req, res)
	} else {
		mem.resCbRecheck(req, res)
	}
}

func (mem *Mempool) resCbNormal(req *tmsp.Request, res *tmsp.Response) {
	switch res.Type {
	case tmsp.MessageType_CheckTx:
		if res.Code == tmsp.CodeType_OK {
			mem.counter++
			memTx := &mempoolTx{
				counter: mem.counter,
				height:  int64(mem.height),
				tx:      req.Data,
			}
			mem.txs.PushBack(memTx)
		} else {
			// ignore bad transaction
			// TODO: handle other retcodes
		}
	default:
		// ignore other messages
	}
}

func (mem *Mempool) resCbRecheck(req *tmsp.Request, res *tmsp.Response) {
	switch res.Type {
	case tmsp.MessageType_CheckTx:
		memTx := mem.recheckCursor.Value.(*mempoolTx)
		if !bytes.Equal(req.Data, memTx.tx) {
			PanicSanity(Fmt("Unexpected tx response from proxy during recheck\n"+
				"Expected %X, got %X", req.Data, memTx.tx))
		}
		if res.Code == tmsp.CodeType_OK {
			// Good, nothing to do.
		} else {
			// Tx became invalidated due to newly committed block.
			mem.txs.Remove(mem.recheckCursor)
			mem.recheckCursor.DetachPrev()
		}
		if mem.recheckCursor == mem.recheckEnd {
			mem.recheckCursor = nil
		} else {
			mem.recheckCursor = mem.recheckCursor.Next()
		}
		if mem.recheckCursor == nil {
			// Done!
			atomic.StoreInt32(&mem.rechecking, 0)
		}
	default:
		// ignore other messages
	}
}

// Get the valid transactions remaining
// If maxTxs is 0, there is no cap.
func (mem *Mempool) Reap(maxTxs int) []types.Tx {
	mem.proxyMtx.Lock()
	defer mem.proxyMtx.Unlock()

	for atomic.LoadInt32(&mem.rechecking) > 0 {
		// TODO: Something better?
		time.Sleep(time.Millisecond * 10)
	}

	txs := mem.collectTxs(maxTxs)
	return txs
}

// maxTxs: 0 means uncapped
func (mem *Mempool) collectTxs(maxTxs int) []types.Tx {
	if maxTxs == 0 {
		maxTxs = mem.txs.Len()
	}
	txs := make([]types.Tx, 0, MinInt(mem.txs.Len(), maxTxs))
	for e := mem.txs.Front(); e != nil && len(txs) < maxTxs; e = e.Next() {
		memTx := e.Value.(*mempoolTx)
		txs = append(txs, memTx.tx)
	}
	return txs
}

// Tell mempool that these txs were committed.
// Mempool will discard these txs.
// NOTE: this should be called *after* block is committed by consensus.
func (mem *Mempool) Update(height int, txs []types.Tx) {
	mem.proxyMtx.Lock()
	defer mem.proxyMtx.Unlock()
	mem.proxyAppConn.FlushSync() // To flush async resCb calls e.g. from CheckTx

	// First, create a lookup map of txns in new txs.
	txsMap := make(map[string]struct{})
	for _, tx := range txs {
		txsMap[string(tx)] = struct{}{}
	}

	// Set height
	mem.height = height
	// Remove transactions that are already in txs.
	goodTxs := mem.filterTxs(txsMap)
	// Recheck mempool txs
	if config.GetBool("mempool_recheck") {
		mem.recheckTxs(goodTxs)
		// At this point, mem.txs are being rechecked.
		// mem.recheckCursor re-scans mem.txs and possibly removes some txs.
		// Before mem.Reap(), we should wait for mem.recheckCursor to be nil.
	}
}

func (mem *Mempool) filterTxs(blockTxsMap map[string]struct{}) []types.Tx {
	goodTxs := make([]types.Tx, 0, mem.txs.Len())
	for e := mem.txs.Front(); e != nil; e = e.Next() {
		memTx := e.Value.(*mempoolTx)
		if _, ok := blockTxsMap[string(memTx.tx)]; ok {
			// Remove the tx since already in block.
			mem.txs.Remove(e)
			e.DetachPrev()
			continue
		}
		// Good tx!
		goodTxs = append(goodTxs, memTx.tx)
	}
	return goodTxs
}

// NOTE: pass in goodTxs because mem.txs can mutate concurrently.
func (mem *Mempool) recheckTxs(goodTxs []types.Tx) {
	if len(goodTxs) == 0 {
		return
	}
	atomic.StoreInt32(&mem.rechecking, 1)
	mem.recheckCursor = mem.txs.Front()
	mem.recheckEnd = mem.txs.Back()

	// Push txs to proxyAppConn
	// NOTE: resCb() may be called concurrently.
	for _, tx := range goodTxs {
		mem.proxyAppConn.CheckTxAsync(tx)
	}
	mem.proxyAppConn.FlushAsync()
}

//--------------------------------------------------------------------------------

// A transaction that successfully ran
type mempoolTx struct {
	counter int64    // a simple incrementing counter
	height  int64    // height that this tx had been validated in
	tx      types.Tx //
}

func (memTx *mempoolTx) Height() int {
	return int(atomic.LoadInt64(&memTx.height))
}
<|endoftext|>


<commit_before>// +build !daemon

package ogo

/* {{{ import
 */
import (
	"fmt"
	"net/http"
	"os"
	"path/filepath"
	"runtime"

	"github.com/Odinman/ogo/bind"
	"github.com/Odinman/ogo/graceful"
	"github.com/VividCortex/godaemon"
	"github.com/nightlyone/lockfile"
	"github.com/zenazn/goji"
	"github.com/zenazn/goji/web/middleware"
)

/* }}} */

/* {{{ func Run()
 * Run ogo application.
 */
func Run() {
	defer func() {
		if err := recover(); err != nil {
			WriteMsg("App crashed with error:", err)
			for i := 1; ; i++ {
				_, file, line, ok := runtime.Caller(i)
				if !ok {
					break
				}
				WriteMsg(file, line)
			}
			//panic要输出到console
			fmt.Println("App crashed with error:", err)
		}
	}()
	if Env.Daemonize {
		godaemon.MakeDaemon(&godaemon.DaemonAttr{})
	}
	//check&write pidfile, added by odin
	dir := filepath.Dir(Env.PidFile)
	if _, err := os.Stat(dir); err != nil {
		if os.IsNotExist(err) {
			//mkdir
			if err := os.Mkdir(dir, 0755); err != nil {
				panic(err)
			}
		}
	}
	if l, err := lockfile.New(Env.PidFile); err == nil {
		if le := l.TryLock(); le != nil {
			panic(le)
		}
	} else {
		panic(err)
	}

	Debugger.Debug("will run http server")

	// 废除一些goji默认的middleware
	goji.Abandon(middleware.Logger)
	goji.Abandon(middleware.AutomaticOptions)

	//增加自定义的middleware
	goji.Use(EnvInit)
	goji.Use(Defer)
	goji.Use(Authentication)

	// in goji appengine mode (tags --appengine)
	goji.Serve()

	// socket listen
	bind.WithFlag()
	listener := bind.Default()
	Debugger.Warn("Starting Ogo on", listener.Addr())

	graceful.HandleSignals()
	bind.Ready()
	graceful.PreHook(func() { Debugger.Warn("Goji received signal, gracefully stopping") })
	graceful.PostHook(func() { Debugger.Warn("Goji stopped") })

	err := graceful.Serve(listener, http.DefaultServeMux)

	if err != nil {
		Debugger.Critical(err.Error())
	}

	graceful.Wait()
}

/* }}} */
<commit_msg>bugfix<commit_after>// +build !daemon

package ogo

/* {{{ import
 */
import (
	"fmt"
	"net/http"
	"os"
	"path/filepath"
	"runtime"

	"github.com/Odinman/ogo/bind"
	"github.com/Odinman/ogo/graceful"
	"github.com/VividCortex/godaemon"
	"github.com/nightlyone/lockfile"
	"github.com/zenazn/goji"
	"github.com/zenazn/goji/web/middleware"
)

/* }}} */

/* {{{ func Run()
 * Run ogo application.
 */
func Run() {
	defer func() {
		if err := recover(); err != nil {
			WriteMsg("App crashed with error:", err)
			for i := 1; ; i++ {
				_, file, line, ok := runtime.Caller(i)
				if !ok {
					break
				}
				WriteMsg(file, line)
			}
			//panic要输出到console
			fmt.Println("App crashed with error:", err)
		}
	}()
	if Env.Daemonize {
		godaemon.MakeDaemon(&godaemon.DaemonAttr{})
	}
	//check&write pidfile, added by odin
	dir := filepath.Dir(Env.PidFile)
	if _, err := os.Stat(dir); err != nil {
		if os.IsNotExist(err) {
			//mkdir
			if err := os.Mkdir(dir, 0755); err != nil {
				panic(err)
			}
		}
	}
	if l, err := lockfile.New(Env.PidFile); err == nil {
		if le := l.TryLock(); le != nil {
			panic(le)
		}
	} else {
		panic(err)
	}

	Debugger.Debug("will run http server")

	// 废除一些goji默认的middleware
	goji.Abandon(middleware.Logger)
	goji.Abandon(middleware.AutomaticOptions)

	//增加自定义的middleware
	goji.Use(EnvInit)
	goji.Use(Defer)
	goji.Use(Authentication)

	// in goji appengine mode (tags --appengine)
	goji.Serve()

	// socket listen
	bind.WithFlag()
	listener := bind.Default()
	Debugger.Warn("Starting Ogo on: %s", listener.Addr().String())

	graceful.HandleSignals()
	bind.Ready()
	graceful.PreHook(func() { Debugger.Warn("Received signal, gracefully stopping") })
	graceful.PostHook(func() { Debugger.Warn("Stopped") })

	err := graceful.Serve(listener, http.DefaultServeMux)

	if err != nil {
		Debugger.Critical(err.Error())
	}

	graceful.Wait()
}

/* }}} */
<|endoftext|>


<commit_before>package test

import (
	"strings"
	"unicode/utf8"

	"github.com/jaguilar/vt100"
)

// FromLines generates a VT100 from content text.
// Each line must have the same number of runes.
func FromLines(s string) *vt100.VT100 {
	return FromLinesAndFormats(s, nil)
}

// FromLinesAndFormats generates a *VT100 whose state is set according
// to s (for content) and a (for attributes).
//
// Dimensions are set to the width of s' first line and the height of the
// number of lines in s.
//
// If a is nil, the default attributes are used.
func FromLinesAndFormats(s string, a [][]vt100.Format) *vt100.VT100 {
	lines := strings.Split(s, "\n")
	v := vt100.NewVT100(len(lines), utf8.RuneCountInString(lines[0]))
	for y := 0; y < v.Height; y++ {
		x := 0
		for _, r := range lines[y] {
			v.Content[y][x] = r
			if a != nil {
				v.Format[y][x] = a[y][x]
			}
			x++
		}
	}
	return v
}
<commit_msg>commit reverted generate.go<commit_after><|endoftext|>


<commit_before>package http

import (
	"bytes"
	"context"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
)

var (
	testString = "test string"
)

func TestResponseEncoder(t *testing.T) {
	cases := []struct {
		name        string
		contentType string
		acceptType  string
		encoderType string
	}{
		{"no ct, no at", "", "", "*json.Encoder"},
		{"no ct, at json", "", "application/json", "*json.Encoder"},
		{"no ct, at xml", "", "application/xml", "*xml.Encoder"},
		{"no ct, at gob", "", "application/gob", "*gob.Encoder"},
		{"no ct, at html", "", "text/html", "*http.textEncoder"},
		{"no ct, at plain", "", "text/plain", "*http.textEncoder"},
		{"ct json", "application/json", "application/gob", "*json.Encoder"},
		{"ct +json", "+json", "application/gob", "*json.Encoder"},
		{"ct xml", "application/xml", "application/gob", "*xml.Encoder"},
		{"ct +xml", "+xml", "application/gob", "*xml.Encoder"},
		{"ct gob", "application/gob", "application/xml", "*gob.Encoder"},
		{"ct +gob", "+gob", "application/xml", "*gob.Encoder"},
		{"ct html", "text/html", "application/gob", "*http.textEncoder"},
		{"ct +html", "+html", "application/gob", "*http.textEncoder"},
		{"ct plain", "text/plain", "application/gob", "*http.textEncoder"},
		{"ct +txt", "+txt", "application/gob", "*http.textEncoder"},
		{"no ct, at json with params", "", "application/json; charset=utf-8", "*json.Encoder"},
		{"no ct, at xml with params", "", "application/xml; charset=utf-8", "*xml.Encoder"},
		{"no ct, at gob with params", "", "application/gob; charset=utf-8", "*gob.Encoder"},
		{"no ct, at html with params", "", "text/html; charset=utf-8", "*http.textEncoder"},
		{"no ct, at plain with params", "", "text/plain; charset=utf-8", "*http.textEncoder"},
		{"ct json with params", "application/json; charset=utf-8", "application/gob", "*json.Encoder"},
		{"ct +json with params", "+json; charset=utf-8", "application/gob", "*json.Encoder"},
		{"ct xml with params", "application/xml; charset=utf-8", "application/gob", "*xml.Encoder"},
		{"ct +xml with params", "+xml; charset=utf-8", "application/gob", "*xml.Encoder"},
		{"ct gob with params", "application/gob; charset=utf-8", "application/xml", "*gob.Encoder"},
		{"ct +gob with params", "+gob; charset=utf-8", "application/xml", "*gob.Encoder"},
		{"ct html with params", "text/html; charset=utf-8", "application/gob", "*http.textEncoder"},
		{"ct +html with params", "+html; charset=utf-8", "application/gob", "*http.textEncoder"},
		{"ct plain with params", "text/plain; charset=utf-8", "application/gob", "*http.textEncoder"},
		{"ct +txt with params", "+txt; charset=utf-8", "application/gob", "*http.textEncoder"},
	}

	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			ctx := context.Background()
			ctx = context.WithValue(ctx, AcceptTypeKey, c.acceptType)
			ctx = context.WithValue(ctx, ContentTypeKey, c.contentType)
			w := httptest.NewRecorder()
			encoder := ResponseEncoder(ctx, w)
			if c.encoderType != fmt.Sprintf("%T", encoder) {
				t.Errorf("got encoder type %s, expected %s", fmt.Sprintf("%T", encoder), c.encoderType)
			}
		})
	}
}

func TestResponseDecoder(t *testing.T) {
	cases := []struct {
		contentType string
		decoderType string
	}{
		{"application/json", "*json.Decoder"},
		{"+json", "*json.Decoder"},
		{"application/xml", "*xml.Decoder"},
		{"+xml", "*xml.Decoder"},
		{"application/gob", "*gob.Decoder"},
		{"+gob", "*gob.Decoder"},
		{"text/html", "*http.textDecoder"},
		{"+html", "*http.textDecoder"},
		{"text/plain", "*http.textDecoder"},
		{"+txt", "*http.textDecoder"},
	}

	for _, c := range cases {
		t.Run(c.contentType, func(t *testing.T) {
			r := &http.Response{
				Header: map[string][]string{
					"Content-Type": {c.contentType},
				},
			}
			decoder := ResponseDecoder(r)
			if c.decoderType != fmt.Sprintf("%T", decoder) {
				t.Errorf("got decoder type %s, expected %s", fmt.Sprintf("%T", decoder), c.decoderType)
			}
		})
	}
}

func TestTextEncoder_Encode(t *testing.T) {
	cases := []struct {
		name  string
		value interface{}
		error error
	}{
		{"string", testString, nil},
		{"*string", &testString, nil},
		{"[]byte", []byte(testString), nil},
		{"other", 123, fmt.Errorf("can't encode int as content/type")},
	}

	buffer := bytes.Buffer{}
	encoder := newTextEncoder(&buffer, "content/type")

	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			buffer.Reset()
			err := encoder.Encode(c.value)
			if c.error != nil {
				if err == nil || c.error.Error() != err.Error() {
					t.Errorf("got error %q, expected %q", err, c.error)
				}
			} else {
				if err != nil {
					t.Errorf("got error %q, expected <nil>", err)
				}
				if buffer.String() != testString {
					t.Errorf("got string %s, expected %s", buffer.String(), testString)
				}
			}
		})
	}
}

func TestTextPlainDecoder_Decode_String(t *testing.T) {
	decoder := makeTextDecoder()

	var value string
	err := decoder.Decode(&value)
	if err != nil {
		t.Errorf("got error %q, expected <nil>", err)
	}
	if testString != value {
		t.Errorf("got string %s, expected %s", value, testString)
	}
}

func TestTextPlainDecoder_Decode_Bytes(t *testing.T) {
	decoder := makeTextDecoder()

	var value []byte
	err := decoder.Decode(&value)
	if err != nil {
		t.Errorf("got error %q, expected <nil>", err)
	}
	if testString != string(value) {
		t.Errorf("got string %s, expected %s", value, testString)
	}
}

func TestTextPlainDecoder_Decode_Other(t *testing.T) {
	decoder := makeTextDecoder()

	expectedErr := fmt.Errorf("can't decode content/type to *int")

	var value int
	err := decoder.Decode(&value)
	if err == nil || err.Error() != expectedErr.Error() {
		t.Errorf("got error %q, expectedErr %q", err, expectedErr)
	}
}

func makeTextDecoder() Decoder {
	buffer := bytes.Buffer{}
	buffer.WriteString(testString)
	return newTextDecoder(&buffer, "content/type")
}
<commit_msg>Add tests for http.ResponseDecoder() (#2862)<commit_after>package http

import (
	"bytes"
	"context"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
)

var (
	testString = "test string"
)

func TestResponseEncoder(t *testing.T) {
	cases := []struct {
		name        string
		contentType string
		acceptType  string
		encoderType string
	}{
		{"no ct, no at", "", "", "*json.Encoder"},
		{"no ct, at json", "", "application/json", "*json.Encoder"},
		{"no ct, at xml", "", "application/xml", "*xml.Encoder"},
		{"no ct, at gob", "", "application/gob", "*gob.Encoder"},
		{"no ct, at html", "", "text/html", "*http.textEncoder"},
		{"no ct, at plain", "", "text/plain", "*http.textEncoder"},
		{"ct json", "application/json", "application/gob", "*json.Encoder"},
		{"ct +json", "+json", "application/gob", "*json.Encoder"},
		{"ct xml", "application/xml", "application/gob", "*xml.Encoder"},
		{"ct +xml", "+xml", "application/gob", "*xml.Encoder"},
		{"ct gob", "application/gob", "application/xml", "*gob.Encoder"},
		{"ct +gob", "+gob", "application/xml", "*gob.Encoder"},
		{"ct html", "text/html", "application/gob", "*http.textEncoder"},
		{"ct +html", "+html", "application/gob", "*http.textEncoder"},
		{"ct plain", "text/plain", "application/gob", "*http.textEncoder"},
		{"ct +txt", "+txt", "application/gob", "*http.textEncoder"},
		{"no ct, at json with params", "", "application/json; charset=utf-8", "*json.Encoder"},
		{"no ct, at xml with params", "", "application/xml; charset=utf-8", "*xml.Encoder"},
		{"no ct, at gob with params", "", "application/gob; charset=utf-8", "*gob.Encoder"},
		{"no ct, at html with params", "", "text/html; charset=utf-8", "*http.textEncoder"},
		{"no ct, at plain with params", "", "text/plain; charset=utf-8", "*http.textEncoder"},
		{"ct json with params", "application/json; charset=utf-8", "application/gob", "*json.Encoder"},
		{"ct +json with params", "+json; charset=utf-8", "application/gob", "*json.Encoder"},
		{"ct xml with params", "application/xml; charset=utf-8", "application/gob", "*xml.Encoder"},
		{"ct +xml with params", "+xml; charset=utf-8", "application/gob", "*xml.Encoder"},
		{"ct gob with params", "application/gob; charset=utf-8", "application/xml", "*gob.Encoder"},
		{"ct +gob with params", "+gob; charset=utf-8", "application/xml", "*gob.Encoder"},
		{"ct html with params", "text/html; charset=utf-8", "application/gob", "*http.textEncoder"},
		{"ct +html with params", "+html; charset=utf-8", "application/gob", "*http.textEncoder"},
		{"ct plain with params", "text/plain; charset=utf-8", "application/gob", "*http.textEncoder"},
		{"ct +txt with params", "+txt; charset=utf-8", "application/gob", "*http.textEncoder"},
	}

	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			ctx := context.Background()
			ctx = context.WithValue(ctx, AcceptTypeKey, c.acceptType)
			ctx = context.WithValue(ctx, ContentTypeKey, c.contentType)
			w := httptest.NewRecorder()
			encoder := ResponseEncoder(ctx, w)
			if c.encoderType != fmt.Sprintf("%T", encoder) {
				t.Errorf("got encoder type %s, expected %s", fmt.Sprintf("%T", encoder), c.encoderType)
			}
		})
	}
}

func TestResponseDecoder(t *testing.T) {
	cases := []struct {
		contentType string
		decoderType string
	}{
		{"application/json", "*json.Decoder"},
		{"+json", "*json.Decoder"},
		{"application/xml", "*xml.Decoder"},
		{"+xml", "*xml.Decoder"},
		{"application/gob", "*gob.Decoder"},
		{"+gob", "*gob.Decoder"},
		{"text/html", "*http.textDecoder"},
		{"+html", "*http.textDecoder"},
		{"text/plain", "*http.textDecoder"},
		{"+txt", "*http.textDecoder"},
		{"application/json; charset=utf-8", "*json.Decoder"},
		{"+json; charset=utf-8", "*json.Decoder"},
		{"application/xml; charset=utf-8", "*xml.Decoder"},
		{"+xml; charset=utf-8", "*xml.Decoder"},
		{"application/gob; charset=utf-8", "*gob.Decoder"},
		{"+gob; charset=utf-8", "*gob.Decoder"},
		{"text/html; charset=utf-8", "*http.textDecoder"},
		{"+html; charset=utf-8", "*http.textDecoder"},
		{"text/plain; charset=utf-8", "*http.textDecoder"},
		{"+txt; charset=utf-8", "*http.textDecoder"},
	}

	for _, c := range cases {
		t.Run(c.contentType, func(t *testing.T) {
			r := &http.Response{
				Header: map[string][]string{
					"Content-Type": {c.contentType},
				},
			}
			decoder := ResponseDecoder(r)
			if c.decoderType != fmt.Sprintf("%T", decoder) {
				t.Errorf("got decoder type %s, expected %s", fmt.Sprintf("%T", decoder), c.decoderType)
			}
		})
	}
}

func TestTextEncoder_Encode(t *testing.T) {
	cases := []struct {
		name  string
		value interface{}
		error error
	}{
		{"string", testString, nil},
		{"*string", &testString, nil},
		{"[]byte", []byte(testString), nil},
		{"other", 123, fmt.Errorf("can't encode int as content/type")},
	}

	buffer := bytes.Buffer{}
	encoder := newTextEncoder(&buffer, "content/type")

	for _, c := range cases {
		t.Run(c.name, func(t *testing.T) {
			buffer.Reset()
			err := encoder.Encode(c.value)
			if c.error != nil {
				if err == nil || c.error.Error() != err.Error() {
					t.Errorf("got error %q, expected %q", err, c.error)
				}
			} else {
				if err != nil {
					t.Errorf("got error %q, expected <nil>", err)
				}
				if buffer.String() != testString {
					t.Errorf("got string %s, expected %s", buffer.String(), testString)
				}
			}
		})
	}
}

func TestTextPlainDecoder_Decode_String(t *testing.T) {
	decoder := makeTextDecoder()

	var value string
	err := decoder.Decode(&value)
	if err != nil {
		t.Errorf("got error %q, expected <nil>", err)
	}
	if testString != value {
		t.Errorf("got string %s, expected %s", value, testString)
	}
}

func TestTextPlainDecoder_Decode_Bytes(t *testing.T) {
	decoder := makeTextDecoder()

	var value []byte
	err := decoder.Decode(&value)
	if err != nil {
		t.Errorf("got error %q, expected <nil>", err)
	}
	if testString != string(value) {
		t.Errorf("got string %s, expected %s", value, testString)
	}
}

func TestTextPlainDecoder_Decode_Other(t *testing.T) {
	decoder := makeTextDecoder()

	expectedErr := fmt.Errorf("can't decode content/type to *int")

	var value int
	err := decoder.Decode(&value)
	if err == nil || err.Error() != expectedErr.Error() {
		t.Errorf("got error %q, expectedErr %q", err, expectedErr)
	}
}

func makeTextDecoder() Decoder {
	buffer := bytes.Buffer{}
	buffer.WriteString(testString)
	return newTextDecoder(&buffer, "content/type")
}
<|endoftext|>


<commit_before>// Package test contains simple test helpers that should not
// have any dependencies on horizon's packages.  think constants,
// custom matchers, generic helpers etc.
package test

import (
	"bytes"
	"log"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"

	"github.com/Sirupsen/logrus"
	"github.com/jmoiron/sqlx"
	hlog "github.com/stellar/horizon/log"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"golang.org/x/net/context"
)

//go:generate go get github.com/jteeuwen/go-bindata/go-bindata
//go:generate go-bindata -pkg test scenarios

var (
	coreDB    *sqlx.DB
	horizonDB *sqlx.DB
)

const (
	// DefaultTestDatabaseURL is the default postgres connection string for
	// horizon's test database.
	DefaultTestDatabaseURL = "postgres://localhost:5432/horizon_test?sslmode=disable"

	// DefaultTestStellarCoreDatabaseURL is the default postgres connection string
	// for horizon's test stellar core database.
	DefaultTestStellarCoreDatabaseURL = "postgres://localhost:5432/stellar-core_test?sslmode=disable"
)

// StaticMockServer is a test helper that records it's last request
type StaticMockServer struct {
	*httptest.Server
	LastRequest *http.Request
}

// T provides a common set of functionality for each test in horizon
type T struct {
	T          *testing.T
	Assert     *assert.Assertions
	Require    *require.Assertions
	Ctx        context.Context
	HorizonDB  *sqlx.DB
	CoreDB     *sqlx.DB
	Logger     *hlog.Entry
	LogMetrics *hlog.Metrics
	LogBuffer  *bytes.Buffer
}

// Context provides a context suitable for testing in tests that do not create
// a full App instance (in which case your tests should be using the app's
// context).  This context has a logger bound to it suitable for testing.
func Context() context.Context {
	return hlog.Set(context.Background(), testLogger)
}

// ContextWithLogBuffer returns a context and a buffer into which the new, bound
// logger will write into.  This method allows you to inspect what data was
// logged more easily in your tests.
func ContextWithLogBuffer() (context.Context, *bytes.Buffer) {
	output := new(bytes.Buffer)
	l, _ := hlog.New()
	l.Logger.Out = output
	l.Logger.Formatter.(*logrus.TextFormatter).DisableColors = true
	l.Logger.Level = logrus.DebugLevel

	ctx := hlog.Set(context.Background(), l)
	return ctx, output

}

// Database returns a connection to the horizon test database
func Database() *sqlx.DB {
	if horizonDB != nil {
		return horizonDB
	}
	horizonDB = OpenDatabase(DatabaseURL())
	return horizonDB
}

// DatabaseURL returns the database connection the url any test
// use when connecting to the history/horizon database
func DatabaseURL() string {
	databaseURL := os.Getenv("DATABASE_URL")

	if databaseURL == "" {
		databaseURL = DefaultTestDatabaseURL
	}

	return databaseURL
}

// LoadScenario populates the test databases with pre-created scenarios.  Each
// scenario is in the scenarios subfolder of this package and are a pair of
// sql files, one per database.
func LoadScenario(scenarioName string) {
	loadScenario(scenarioName, true)
}

// LoadScenarioWithoutHorizon populates the test Stellar core database a with
// pre-created scenario.  Unlike `LoadScenario`, this
func LoadScenarioWithoutHorizon(scenarioName string) {
	loadScenario(scenarioName, false)
}

// OpenDatabase opens a database, panicing if it cannot
func OpenDatabase(dsn string) *sqlx.DB {
	db, err := sqlx.Open("postgres", dsn)

	if err != nil {
		log.Panic(err)
	}

	return db
}

// Start initializes a new test helper object and conceptually "starts" a new
// test
func Start(t *testing.T) *T {
	result := &T{}

	result.T = t
	result.LogBuffer = new(bytes.Buffer)
	result.Logger, result.LogMetrics = hlog.New()
	result.Logger.Logger.Out = result.LogBuffer
	result.Logger.Logger.Formatter.(*logrus.TextFormatter).DisableColors = true
	result.Logger.Logger.Level = logrus.DebugLevel

	result.Ctx = hlog.Set(context.Background(), result.Logger)
	result.HorizonDB = Database()
	result.CoreDB = StellarCoreDatabase()
	result.Assert = assert.New(t)
	result.Require = require.New(t)

	return result
}

// StellarCoreDatabase returns a connection to the stellar core test database
func StellarCoreDatabase() *sqlx.DB {
	if coreDB != nil {
		return coreDB
	}
	coreDB = OpenDatabase(StellarCoreDatabaseURL())
	return coreDB
}

// StellarCoreDatabaseURL returns the database connection the url any test
// use when connecting to the stellar-core database
func StellarCoreDatabaseURL() string {
	databaseURL := os.Getenv("STELLAR_CORE_DATABASE_URL")

	if databaseURL == "" {
		databaseURL = DefaultTestStellarCoreDatabaseURL
	}

	return databaseURL
}
<commit_msg>Ensure pq is included when the test helper is included<commit_after>// Package test contains simple test helpers that should not
// have any dependencies on horizon's packages.  think constants,
// custom matchers, generic helpers etc.
package test

import (
	"bytes"
	"log"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"

	"github.com/Sirupsen/logrus"
	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"
	hlog "github.com/stellar/horizon/log"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"golang.org/x/net/context"
)

//go:generate go get github.com/jteeuwen/go-bindata/go-bindata
//go:generate go-bindata -pkg test scenarios

var (
	coreDB    *sqlx.DB
	horizonDB *sqlx.DB
)

const (
	// DefaultTestDatabaseURL is the default postgres connection string for
	// horizon's test database.
	DefaultTestDatabaseURL = "postgres://localhost:5432/horizon_test?sslmode=disable"

	// DefaultTestStellarCoreDatabaseURL is the default postgres connection string
	// for horizon's test stellar core database.
	DefaultTestStellarCoreDatabaseURL = "postgres://localhost:5432/stellar-core_test?sslmode=disable"
)

// StaticMockServer is a test helper that records it's last request
type StaticMockServer struct {
	*httptest.Server
	LastRequest *http.Request
}

// T provides a common set of functionality for each test in horizon
type T struct {
	T          *testing.T
	Assert     *assert.Assertions
	Require    *require.Assertions
	Ctx        context.Context
	HorizonDB  *sqlx.DB
	CoreDB     *sqlx.DB
	Logger     *hlog.Entry
	LogMetrics *hlog.Metrics
	LogBuffer  *bytes.Buffer
}

// Context provides a context suitable for testing in tests that do not create
// a full App instance (in which case your tests should be using the app's
// context).  This context has a logger bound to it suitable for testing.
func Context() context.Context {
	return hlog.Set(context.Background(), testLogger)
}

// ContextWithLogBuffer returns a context and a buffer into which the new, bound
// logger will write into.  This method allows you to inspect what data was
// logged more easily in your tests.
func ContextWithLogBuffer() (context.Context, *bytes.Buffer) {
	output := new(bytes.Buffer)
	l, _ := hlog.New()
	l.Logger.Out = output
	l.Logger.Formatter.(*logrus.TextFormatter).DisableColors = true
	l.Logger.Level = logrus.DebugLevel

	ctx := hlog.Set(context.Background(), l)
	return ctx, output

}

// Database returns a connection to the horizon test database
func Database() *sqlx.DB {
	if horizonDB != nil {
		return horizonDB
	}
	horizonDB = OpenDatabase(DatabaseURL())
	return horizonDB
}

// DatabaseURL returns the database connection the url any test
// use when connecting to the history/horizon database
func DatabaseURL() string {
	databaseURL := os.Getenv("DATABASE_URL")

	if databaseURL == "" {
		databaseURL = DefaultTestDatabaseURL
	}

	return databaseURL
}

// LoadScenario populates the test databases with pre-created scenarios.  Each
// scenario is in the scenarios subfolder of this package and are a pair of
// sql files, one per database.
func LoadScenario(scenarioName string) {
	loadScenario(scenarioName, true)
}

// LoadScenarioWithoutHorizon populates the test Stellar core database a with
// pre-created scenario.  Unlike `LoadScenario`, this
func LoadScenarioWithoutHorizon(scenarioName string) {
	loadScenario(scenarioName, false)
}

// OpenDatabase opens a database, panicing if it cannot
func OpenDatabase(dsn string) *sqlx.DB {
	db, err := sqlx.Open("postgres", dsn)

	if err != nil {
		log.Panic(err)
	}

	return db
}

// Start initializes a new test helper object and conceptually "starts" a new
// test
func Start(t *testing.T) *T {
	result := &T{}

	result.T = t
	result.LogBuffer = new(bytes.Buffer)
	result.Logger, result.LogMetrics = hlog.New()
	result.Logger.Logger.Out = result.LogBuffer
	result.Logger.Logger.Formatter.(*logrus.TextFormatter).DisableColors = true
	result.Logger.Logger.Level = logrus.DebugLevel

	result.Ctx = hlog.Set(context.Background(), result.Logger)
	result.HorizonDB = Database()
	result.CoreDB = StellarCoreDatabase()
	result.Assert = assert.New(t)
	result.Require = require.New(t)

	return result
}

// StellarCoreDatabase returns a connection to the stellar core test database
func StellarCoreDatabase() *sqlx.DB {
	if coreDB != nil {
		return coreDB
	}
	coreDB = OpenDatabase(StellarCoreDatabaseURL())
	return coreDB
}

// StellarCoreDatabaseURL returns the database connection the url any test
// use when connecting to the stellar-core database
func StellarCoreDatabaseURL() string {
	databaseURL := os.Getenv("STELLAR_CORE_DATABASE_URL")

	if databaseURL == "" {
		databaseURL = DefaultTestStellarCoreDatabaseURL
	}

	return databaseURL
}
<|endoftext|>


<commit_before>package httpevents

import (
	"bufio"
	"net"
	"net/http"
	"sync"

	"github.com/segmentio/events"
)

// NewHandler wraps the HTTP handler and returns a new handler which logs all
// requests with the default logger.
func NewHandler(handler http.Handler) http.Handler {
	return NewHandlerWith(events.DefaultLogger, handler)
}

// NewHandlerWith wraps the HTTP handler and returns a new handler which logs all
// requests with logger.
//
// Panics from handler are intercepted and trigger a 500 response if no response
// header was sent yet. The panic is not slienced tho and is propagated to the
// parent handler.
func NewHandlerWith(logger *events.Logger, handler http.Handler) http.Handler {
	return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
		var laddr string

		if value, ok := req.Context().Value(http.LocalAddrContextKey).(net.Addr); ok {
			laddr = value.String()
		}

		w := responseWriterPool.Get().(*responseWriter)
		// We capture all the values we need from req in case the object
		// gets modified by the handler.
		w.ResponseWriter = res
		w.logger = logger
		w.request.reset(req, laddr)

		// If the handler panics we want to make sure we report the issue in the
		// access log, while also ensuring that a response is going to be sent
		// down to the client.
		// We don't silence the panic here tho and instead we forward it back to
		// the parent handler which may need to be aware that a panic occurred.
		defer func() {
			err := recover()

			if err != nil {
				w.WriteHeader(http.StatusInternalServerError)
			}

			w.ResponseWriter = nil
			w.logger = nil
			w.wroteHeader = false
			w.request.release()
			responseWriterPool.Put(w)

			if err != nil {
				panic(err)
			}
		}()

		// The request is forwarded to the handler, if it never calls the
		// writer's WriteHeader method we force the call with "200 OK" status
		// to match the default behavior of the net/http package (and also make
		// sure an access log will be written).
		handler.ServeHTTP(w, req)
		w.WriteHeader(http.StatusOK)
	})
}

type responseWriter struct {
	http.ResponseWriter
	logger *events.Logger
	request
	wroteHeader bool
}

func (w *responseWriter) WriteHeader(status int) {
	w.log(1, status)

	if !w.wroteHeader {
		w.wroteHeader = true
		w.ResponseWriter.WriteHeader(status)
	}
}

func (w *responseWriter) Write(b []byte) (int, error) {
	if !w.wroteHeader {
		w.WriteHeader(http.StatusOK)
	}
	return w.ResponseWriter.Write(b)
}

func (w *responseWriter) Hijack() (conn net.Conn, rw *bufio.ReadWriter, err error) {
	if conn, rw, err = w.ResponseWriter.(http.Hijacker).Hijack(); err == nil {
		w.log(1, http.StatusSwitchingProtocols)
	}
	return
}

func (w *responseWriter) log(depth int, status int) {
	if logger := w.logger; logger != nil {
		w.logger = nil
		w.request.status = status
		w.request.statusText = http.StatusText(status)
		w.request.log(logger, w.ResponseWriter.Header(), depth+1)
	}
}

var responseWriterPool = sync.Pool{
	New: func() interface{} { return &responseWriter{} },
}
<commit_msg>Fix typo: "slienced" is a misspelling of "silenced" (#32)<commit_after>package httpevents

import (
	"bufio"
	"net"
	"net/http"
	"sync"

	"github.com/segmentio/events"
)

// NewHandler wraps the HTTP handler and returns a new handler which logs all
// requests with the default logger.
func NewHandler(handler http.Handler) http.Handler {
	return NewHandlerWith(events.DefaultLogger, handler)
}

// NewHandlerWith wraps the HTTP handler and returns a new handler which logs all
// requests with logger.
//
// Panics from handler are intercepted and trigger a 500 response if no response
// header was sent yet. The panic is not silenced tho and is propagated to the
// parent handler.
func NewHandlerWith(logger *events.Logger, handler http.Handler) http.Handler {
	return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
		var laddr string

		if value, ok := req.Context().Value(http.LocalAddrContextKey).(net.Addr); ok {
			laddr = value.String()
		}

		w := responseWriterPool.Get().(*responseWriter)
		// We capture all the values we need from req in case the object
		// gets modified by the handler.
		w.ResponseWriter = res
		w.logger = logger
		w.request.reset(req, laddr)

		// If the handler panics we want to make sure we report the issue in the
		// access log, while also ensuring that a response is going to be sent
		// down to the client.
		// We don't silence the panic here tho and instead we forward it back to
		// the parent handler which may need to be aware that a panic occurred.
		defer func() {
			err := recover()

			if err != nil {
				w.WriteHeader(http.StatusInternalServerError)
			}

			w.ResponseWriter = nil
			w.logger = nil
			w.wroteHeader = false
			w.request.release()
			responseWriterPool.Put(w)

			if err != nil {
				panic(err)
			}
		}()

		// The request is forwarded to the handler, if it never calls the
		// writer's WriteHeader method we force the call with "200 OK" status
		// to match the default behavior of the net/http package (and also make
		// sure an access log will be written).
		handler.ServeHTTP(w, req)
		w.WriteHeader(http.StatusOK)
	})
}

type responseWriter struct {
	http.ResponseWriter
	logger *events.Logger
	request
	wroteHeader bool
}

func (w *responseWriter) WriteHeader(status int) {
	w.log(1, status)

	if !w.wroteHeader {
		w.wroteHeader = true
		w.ResponseWriter.WriteHeader(status)
	}
}

func (w *responseWriter) Write(b []byte) (int, error) {
	if !w.wroteHeader {
		w.WriteHeader(http.StatusOK)
	}
	return w.ResponseWriter.Write(b)
}

func (w *responseWriter) Hijack() (conn net.Conn, rw *bufio.ReadWriter, err error) {
	if conn, rw, err = w.ResponseWriter.(http.Hijacker).Hijack(); err == nil {
		w.log(1, http.StatusSwitchingProtocols)
	}
	return
}

func (w *responseWriter) log(depth int, status int) {
	if logger := w.logger; logger != nil {
		w.logger = nil
		w.request.status = status
		w.request.statusText = http.StatusText(status)
		w.request.log(logger, w.ResponseWriter.Header(), depth+1)
	}
}

var responseWriterPool = sync.Pool{
	New: func() interface{} { return &responseWriter{} },
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: Apache-2.0
// Copyright 2018-2020 Authors of Cilium

package k8sTest

import (
	"fmt"
	"strings"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
)

var _ = Describe("K8sCLI", func() {
	SkipContextIf(func() bool {
		return helpers.DoesNotRunOnGKE() && helpers.DoesNotRunOnEKS()
	}, "CLI", func() {
		var kubectl *helpers.Kubectl
		var ciliumFilename string

		BeforeAll(func() {
			kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

			ciliumFilename = helpers.TimestampFilename("cilium.yaml")
			DeployCiliumAndDNS(kubectl, ciliumFilename)
			ExpectCiliumReady(kubectl)
		})

		AfterAll(func() {
			UninstallCiliumFromManifest(kubectl, ciliumFilename)
		})

		JustAfterEach(func() {
			kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
		})

		Context("Identity CLI testing", func() {
			const (
				manifestYAML = "test-cli.yaml"
				fooID        = "foo"
				fooSHA       = "a83c739e630049e46b9ac6883dc2682b31bf8472b09c8bb81d87092a51d14ddf"
				fooNode      = "k8s1"
				// These labels are automatically added to all pods in the default namespace.
				defaultLabels = "k8s:io.cilium.k8s.policy.cluster=default " +
					"k8s:io.cilium.k8s.policy.serviceaccount=default k8s:io.kubernetes.pod.namespace=default"
			)

			var (
				cliManifest string
				ciliumPod   string
				err         error
				identity    int64
			)

			BeforeAll(func() {
				cliManifest = helpers.ManifestGet(kubectl.BasePath(), manifestYAML)
				res := kubectl.ApplyDefault(cliManifest)
				res.ExpectSuccess("Unable to apply %s", cliManifest)
				err = kubectl.WaitforPods(helpers.DefaultNamespace, "-l id", helpers.HelperTimeout)
				Expect(err).Should(BeNil(), "The pods were not ready after timeout")

				ciliumPod, err = kubectl.GetCiliumPodOnNode(fooNode)
				Expect(err).Should(BeNil())

				err := kubectl.WaitForCEPIdentity(helpers.DefaultNamespace, fooID)
				Expect(err).Should(BeNil())

				ep, err := kubectl.GetCiliumEndpoint(helpers.DefaultNamespace, fooID)
				Expect(err).Should(BeNil(), fmt.Sprintf("Unable to get CEP for pod %s", fooID))
				identity = ep.Identity.ID
			})

			AfterAll(func() {
				_ = kubectl.Delete(cliManifest)
				ExpectAllPodsTerminated(kubectl)
			})

			It("Test labelsSHA256", func() {
				cmd := fmt.Sprintf("cilium identity get %d -o json", identity)
				res := kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, cmd)
				res.ExpectSuccess()
				out, err := res.Filter("{[0].labelsSHA256}")
				Expect(err).Should(BeNil(), "Error getting SHA from identity")
				Expect(out.String()).Should(Equal(fooSHA))
			})

			It("Test identity list", func() {
				By("Testing 'cilium identity list' for an endpoint's identity")
				cmd := fmt.Sprintf("cilium identity list k8s:id=%s %s", fooID, defaultLabels)
				res := kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, cmd)
				res.ExpectSuccess(fmt.Sprintf("Unable to get identity list output for label k8s:id=%s %s", fooID, defaultLabels))

				resSingleOut := res.SingleOut()
				containsIdentity := strings.Contains(resSingleOut, fmt.Sprintf("%d", identity))
				Expect(containsIdentity).To(BeTrue(), "Identity %d of endpoint %s not in 'cilium identity list' output", identity, resSingleOut)

				By("Testing 'cilium identity list' for reserved identities")
				res = kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, "cilium identity list")
				res.ExpectSuccess("Unable to get identity list output")
				resSingleOut = res.SingleOut()

				reservedIdentities := []string{"health", "host", "world", "init"}
				for _, id := range reservedIdentities {
					By("Checking that reserved identity '%s' is in 'cilium identity list' output", id)
					containsReservedIdentity := strings.Contains(resSingleOut, id)
					Expect(containsReservedIdentity).To(BeTrue(), "Reserved identity '%s' not in 'cilium identity list' output", id)
				}
			})

			It("Test cilium bpf metrics list", func() {
				demoManifest := helpers.ManifestGet(kubectl.BasePath(), "demo-named-port.yaml")
				app1Service := "app1-service"
				l3L4DenyPolicy := helpers.ManifestGet(kubectl.BasePath(), "l3-l4-policy-deny.yaml")

				namespaceForTest := helpers.GenerateNamespaceForTest("")
				kubectl.NamespaceDelete(namespaceForTest)
				kubectl.NamespaceCreate(namespaceForTest).ExpectSuccess("could not create namespace")
				kubectl.Apply(helpers.ApplyOptions{FilePath: demoManifest, Namespace: namespaceForTest}).ExpectSuccess("could not create resource")

				err := kubectl.WaitforPods(namespaceForTest, "-l zgroup=testapp", helpers.HelperTimeout)
				Expect(err).To(BeNil(),
					"testapp pods are not ready after timeout in namespace %q", namespaceForTest)

				_, err = kubectl.CiliumPolicyAction(
					namespaceForTest, l3L4DenyPolicy, helpers.KubectlApply, helpers.HelperTimeout)
				Expect(err).Should(BeNil(), "Cannot apply L3 Deny Policy")

				ciliumPodK8s1, err := kubectl.GetCiliumPodOnNode(helpers.K8s1)
				ExpectWithOffset(2, err).Should(BeNil(), "Cannot get cilium pod on k8s1")
				ciliumPodK8s2, err := kubectl.GetCiliumPodOnNode(helpers.K8s2)
				ExpectWithOffset(2, err).Should(BeNil(), "Cannot get cilium pod on k8s2")

				countBeforeK8s1, _ := helpers.GetBPFPacketsCount(kubectl, ciliumPodK8s1, "Policy denied by denylist", "ingress")
				countBeforeK8s2, _ := helpers.GetBPFPacketsCount(kubectl, ciliumPodK8s2, "Policy denied by denylist", "ingress")

				appPods := helpers.GetAppPods([]string{helpers.App2}, namespaceForTest, kubectl, "id")

				clusterIP, _, err := kubectl.GetServiceHostPort(namespaceForTest, app1Service)
				Expect(err).To(BeNil(), "Cannot get service in %q namespace", namespaceForTest)

				res := kubectl.ExecPodCmd(
					namespaceForTest, appPods[helpers.App2],
					helpers.CurlFail("http://%s/public", clusterIP))
				res.ExpectFail("Unexpected connection from %q to 'http://%s/public'",
					appPods[helpers.App2], clusterIP)

				countAfterK8s1, _ := helpers.GetBPFPacketsCount(kubectl, ciliumPodK8s1, "Policy denied by denylist", "ingress")
				countAfterK8s2, _ := helpers.GetBPFPacketsCount(kubectl, ciliumPodK8s2, "Policy denied by denylist", "ingress")

				Expect((countAfterK8s1 + countAfterK8s2) - (countBeforeK8s1 + countBeforeK8s2)).To(Equal(3))

				_, err = kubectl.CiliumPolicyAction(
					namespaceForTest, l3L4DenyPolicy, helpers.KubectlDelete, helpers.HelperTimeout)
				Expect(err).Should(BeNil(), "Cannot delete L3 Policy")
			})
		})

		Context("stdout/stderr testing", func() {
			var (
				ciliumPod string
				err       error
			)

			BeforeAll(func() {
				ciliumPod, err = kubectl.GetCiliumPodOnNode("k8s1")
				Expect(err).Should(BeNil())
			})

			It("Root command help should print to stdout", func() {
				res := kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, "cilium help")
				Expect(res.Stdout()).Should(ContainSubstring("Use \"cilium [command] --help\" for more information about a command."))
			})

			It("Subcommand help should print to stdout", func() {
				res := kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, "cilium help bpf")
				Expect(res.Stdout()).Should(ContainSubstring("Use \"cilium bpf [command] --help\" for more information about a command."))
			})

			It("Failed subcommand should print help to stdout", func() {
				res := kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, "cilium endpoint confi 173")
				Expect(res.Stdout()).Should(ContainSubstring("Use \"cilium endpoint [command] --help\" for more information about a command."))
			})
		})
	})
})
<commit_msg>test: Delete the test namespace in CLI test<commit_after>// SPDX-License-Identifier: Apache-2.0
// Copyright 2018-2020 Authors of Cilium

package k8sTest

import (
	"fmt"
	"strings"

	. "github.com/cilium/cilium/test/ginkgo-ext"
	"github.com/cilium/cilium/test/helpers"

	. "github.com/onsi/gomega"
)

var _ = Describe("K8sCLI", func() {
	SkipContextIf(func() bool {
		return helpers.DoesNotRunOnGKE() && helpers.DoesNotRunOnEKS()
	}, "CLI", func() {
		var kubectl *helpers.Kubectl
		var ciliumFilename string

		BeforeAll(func() {
			kubectl = helpers.CreateKubectl(helpers.K8s1VMName(), logger)

			ciliumFilename = helpers.TimestampFilename("cilium.yaml")
			DeployCiliumAndDNS(kubectl, ciliumFilename)
			ExpectCiliumReady(kubectl)
		})

		AfterAll(func() {
			UninstallCiliumFromManifest(kubectl, ciliumFilename)
		})

		JustAfterEach(func() {
			kubectl.ValidateNoErrorsInLogs(CurrentGinkgoTestDescription().Duration)
		})

		Context("Identity CLI testing", func() {
			const (
				manifestYAML = "test-cli.yaml"
				fooID        = "foo"
				fooSHA       = "a83c739e630049e46b9ac6883dc2682b31bf8472b09c8bb81d87092a51d14ddf"
				fooNode      = "k8s1"
				// These labels are automatically added to all pods in the default namespace.
				defaultLabels = "k8s:io.cilium.k8s.policy.cluster=default " +
					"k8s:io.cilium.k8s.policy.serviceaccount=default k8s:io.kubernetes.pod.namespace=default"
			)

			var (
				cliManifest string
				ciliumPod   string
				err         error
				identity    int64
			)

			BeforeAll(func() {
				cliManifest = helpers.ManifestGet(kubectl.BasePath(), manifestYAML)
				res := kubectl.ApplyDefault(cliManifest)
				res.ExpectSuccess("Unable to apply %s", cliManifest)
				err = kubectl.WaitforPods(helpers.DefaultNamespace, "-l id", helpers.HelperTimeout)
				Expect(err).Should(BeNil(), "The pods were not ready after timeout")

				ciliumPod, err = kubectl.GetCiliumPodOnNode(fooNode)
				Expect(err).Should(BeNil())

				err := kubectl.WaitForCEPIdentity(helpers.DefaultNamespace, fooID)
				Expect(err).Should(BeNil())

				ep, err := kubectl.GetCiliumEndpoint(helpers.DefaultNamespace, fooID)
				Expect(err).Should(BeNil(), fmt.Sprintf("Unable to get CEP for pod %s", fooID))
				identity = ep.Identity.ID
			})

			AfterAll(func() {
				_ = kubectl.Delete(cliManifest)
				ExpectAllPodsTerminated(kubectl)
			})

			It("Test labelsSHA256", func() {
				cmd := fmt.Sprintf("cilium identity get %d -o json", identity)
				res := kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, cmd)
				res.ExpectSuccess()
				out, err := res.Filter("{[0].labelsSHA256}")
				Expect(err).Should(BeNil(), "Error getting SHA from identity")
				Expect(out.String()).Should(Equal(fooSHA))
			})

			It("Test identity list", func() {
				By("Testing 'cilium identity list' for an endpoint's identity")
				cmd := fmt.Sprintf("cilium identity list k8s:id=%s %s", fooID, defaultLabels)
				res := kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, cmd)
				res.ExpectSuccess(fmt.Sprintf("Unable to get identity list output for label k8s:id=%s %s", fooID, defaultLabels))

				resSingleOut := res.SingleOut()
				containsIdentity := strings.Contains(resSingleOut, fmt.Sprintf("%d", identity))
				Expect(containsIdentity).To(BeTrue(), "Identity %d of endpoint %s not in 'cilium identity list' output", identity, resSingleOut)

				By("Testing 'cilium identity list' for reserved identities")
				res = kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, "cilium identity list")
				res.ExpectSuccess("Unable to get identity list output")
				resSingleOut = res.SingleOut()

				reservedIdentities := []string{"health", "host", "world", "init"}
				for _, id := range reservedIdentities {
					By("Checking that reserved identity '%s' is in 'cilium identity list' output", id)
					containsReservedIdentity := strings.Contains(resSingleOut, id)
					Expect(containsReservedIdentity).To(BeTrue(), "Reserved identity '%s' not in 'cilium identity list' output", id)
				}
			})

			It("Test cilium bpf metrics list", func() {
				demoManifest := helpers.ManifestGet(kubectl.BasePath(), "demo-named-port.yaml")
				app1Service := "app1-service"
				l3L4DenyPolicy := helpers.ManifestGet(kubectl.BasePath(), "l3-l4-policy-deny.yaml")

				namespaceForTest := helpers.GenerateNamespaceForTest("")
				kubectl.NamespaceDelete(namespaceForTest)
				kubectl.NamespaceCreate(namespaceForTest).ExpectSuccess("could not create namespace")
				kubectl.Apply(helpers.ApplyOptions{FilePath: demoManifest, Namespace: namespaceForTest}).ExpectSuccess("could not create resource")

				err := kubectl.WaitforPods(namespaceForTest, "-l zgroup=testapp", helpers.HelperTimeout)
				Expect(err).To(BeNil(),
					"testapp pods are not ready after timeout in namespace %q", namespaceForTest)

				_, err = kubectl.CiliumPolicyAction(
					namespaceForTest, l3L4DenyPolicy, helpers.KubectlApply, helpers.HelperTimeout)
				Expect(err).Should(BeNil(), "Cannot apply L3 Deny Policy")

				ciliumPodK8s1, err := kubectl.GetCiliumPodOnNode(helpers.K8s1)
				ExpectWithOffset(2, err).Should(BeNil(), "Cannot get cilium pod on k8s1")
				ciliumPodK8s2, err := kubectl.GetCiliumPodOnNode(helpers.K8s2)
				ExpectWithOffset(2, err).Should(BeNil(), "Cannot get cilium pod on k8s2")

				countBeforeK8s1, _ := helpers.GetBPFPacketsCount(kubectl, ciliumPodK8s1, "Policy denied by denylist", "ingress")
				countBeforeK8s2, _ := helpers.GetBPFPacketsCount(kubectl, ciliumPodK8s2, "Policy denied by denylist", "ingress")

				appPods := helpers.GetAppPods([]string{helpers.App2}, namespaceForTest, kubectl, "id")

				clusterIP, _, err := kubectl.GetServiceHostPort(namespaceForTest, app1Service)
				Expect(err).To(BeNil(), "Cannot get service in %q namespace", namespaceForTest)

				res := kubectl.ExecPodCmd(
					namespaceForTest, appPods[helpers.App2],
					helpers.CurlFail("http://%s/public", clusterIP))
				res.ExpectFail("Unexpected connection from %q to 'http://%s/public'",
					appPods[helpers.App2], clusterIP)

				countAfterK8s1, _ := helpers.GetBPFPacketsCount(kubectl, ciliumPodK8s1, "Policy denied by denylist", "ingress")
				countAfterK8s2, _ := helpers.GetBPFPacketsCount(kubectl, ciliumPodK8s2, "Policy denied by denylist", "ingress")

				Expect((countAfterK8s1 + countAfterK8s2) - (countBeforeK8s1 + countBeforeK8s2)).To(Equal(3))

				_, err = kubectl.CiliumPolicyAction(
					namespaceForTest, l3L4DenyPolicy, helpers.KubectlDelete, helpers.HelperTimeout)
				Expect(err).Should(BeNil(), "Cannot delete L3 Policy")

				kubectl.NamespaceDelete(namespaceForTest)
			})
		})

		Context("stdout/stderr testing", func() {
			var (
				ciliumPod string
				err       error
			)

			BeforeAll(func() {
				ciliumPod, err = kubectl.GetCiliumPodOnNode("k8s1")
				Expect(err).Should(BeNil())
			})

			It("Root command help should print to stdout", func() {
				res := kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, "cilium help")
				Expect(res.Stdout()).Should(ContainSubstring("Use \"cilium [command] --help\" for more information about a command."))
			})

			It("Subcommand help should print to stdout", func() {
				res := kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, "cilium help bpf")
				Expect(res.Stdout()).Should(ContainSubstring("Use \"cilium bpf [command] --help\" for more information about a command."))
			})

			It("Failed subcommand should print help to stdout", func() {
				res := kubectl.ExecPodCmd(helpers.CiliumNamespace, ciliumPod, "cilium endpoint confi 173")
				Expect(res.Stdout()).Should(ContainSubstring("Use \"cilium endpoint [command] --help\" for more information about a command."))
			})
		})
	})
})
<|endoftext|>