<commit_before>package servicecatalog_test

import (
	"fmt"
	"os"
	"testing"
	"time"

	appTypes "github.com/kyma-project/kyma/components/application-operator/pkg/apis/applicationconnector/v1alpha1"
	appClient "github.com/kyma-project/kyma/components/application-operator/pkg/client/clientset/versioned"

	mappingTypes "github.com/kyma-project/kyma/components/application-broker/pkg/apis/applicationconnector/v1alpha1"
	mappingClient "github.com/kyma-project/kyma/components/application-broker/pkg/client/clientset/versioned"

	scc "github.com/kubernetes-incubator/service-catalog/pkg/client/clientset_generated/clientset"

	corev1 "github.com/kubernetes/client-go/kubernetes/typed/core/v1"
	"github.com/kyma-project/kyma/tests/acceptance/pkg/repeat"
	"github.com/pkg/errors"
	"github.com/pmorie/go-open-service-broker-client/v2"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	apimerr "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/rand"
	restclient "k8s.io/client-go/rest"
)

const (
	helmBrokerURLEnvName        = "HELM_BROKER_URL"
	applicationBrokerURLEnvName = "APPLICATION_BROKER_URL"
	releaseNamespaceEnvName     = "RELEASE_NAMESPACE"
)

func TestBrokerHasIstioRbacAuthorizationRules(t *testing.T) {
	for testName, brokerURL := range map[string]string{
		"Helm Broker":        os.Getenv(helmBrokerURLEnvName),
		"Application Broker": os.Getenv(applicationBrokerURLEnvName),
		// "<next broker>": os.Getenv("<broker url env name>"),
	} {
		t.Run(testName, func(t *testing.T) {
			repeat.FuncAtMost(t, func() error {
				isForbidden, err := isCatalogForbidden(brokerURL)
				if err != nil {
					return errors.Wrap(err, "while getting catalog")
				}
				if !isForbidden {
					return fmt.Errorf("%s catalog response must be forbidden", testName)
				}
				return nil
			}, 5*time.Second)
		})
	}
}

func TestServiceCatalogContainsABServiceClasses(t *testing.T) {
	// given
	k8sConfig, err := restclient.InClusterConfig()
	require.NoError(t, err)
	aClient, err := appClient.NewForConfig(k8sConfig)
	require.NoError(t, err)
	mClient, err := mappingClient.NewForConfig(k8sConfig)
	require.NoError(t, err)
	k8sClient, err := corev1.NewForConfig(k8sConfig)
	require.NoError(t, err)
	releaseNS := os.Getenv(releaseNamespaceEnvName)
	name := fmt.Sprintf("test-acc-app-%s", rand.String(4))
	fixApp := fixApplication(name)

	broker := brokerURL{
		namespace: fmt.Sprintf("test-acc-ns-broker-%s", rand.String(4)),
		prefix:    "ab-ns-for-",
	}
	var brokerServices []v2.Service

	t.Log("Creating Application")
	app, err := aClient.ApplicationconnectorV1alpha1().Applications().Create(fixApp)
	require.NoError(t, err)

	t.Logf("Creating Namespace %s", broker.namespace)
	_, err = k8sClient.Namespaces().Create(fixNamespace(broker.namespace))
	require.NoError(t, err)

	defer func() {
		if t.Failed() {
			testDetailsReport(t, brokerServices, broker.namespace)
		}
		err = k8sClient.Namespaces().Delete(broker.namespace, &metav1.DeleteOptions{})
		assert.NoError(t, err)
		err = aClient.ApplicationconnectorV1alpha1().Applications().Delete(app.Name, &metav1.DeleteOptions{
			GracePeriodSeconds: new(int64), // zero
		})
		assert.NoError(t, err)
	}()

	// when
	t.Log("Creating ApplicationMapping")
	_, err = mClient.ApplicationconnectorV1alpha1().ApplicationMappings(broker.namespace).Create(fixApplicationMapping(name))
	require.NoError(t, err)

	// then
	repeat.FuncAtMost(t, func() error {
		brokerServices, err = getCatalogForBroker(broker.buildURL(releaseNS))
		if err != nil {
			return errors.Wrap(err, "while getting catalog")
		}
		for _, svc := range brokerServices {
			if svc.ID == fixApp.Spec.Services[0].ID {
				return nil
			}
		}
		return fmt.Errorf("%s catalog response should contains service with ID: %s", broker, fixApp.Spec.Services[0].ID)
	}, time.Second*90)

	awaitCatalogContainsServiceClasses(t, broker.namespace, timeoutPerAssert, brokerServices)
}

func TestServiceCatalogResourcesAreCleanUp(t *testing.T) {
	// given
	k8sConfig, err := restclient.InClusterConfig()
	require.NoError(t, err)

	aClient, err := appClient.NewForConfig(k8sConfig)
	require.NoError(t, err)

	mClient, err := mappingClient.NewForConfig(k8sConfig)
	require.NoError(t, err)

	k8sClient, err := corev1.NewForConfig(k8sConfig)
	require.NoError(t, err)

	scClient, err := scc.NewForConfig(k8sConfig)
	require.NoError(t, err)

	name := fmt.Sprintf("test-acc-app-%s", rand.String(4))
	namespace := fmt.Sprintf("test-acc-ns-broker-%s", rand.String(4))

	t.Log("Creating Application")
	app, err := aClient.ApplicationconnectorV1alpha1().Applications().Create(fixApplication(name))
	require.NoError(t, err)

	t.Logf("Creating Namespace %s", namespace)
	_, err = k8sClient.Namespaces().Create(fixNamespace(namespace))
	require.NoError(t, err)

	t.Log("Creating ApplicationMapping")
	am, err := mClient.ApplicationconnectorV1alpha1().ApplicationMappings(namespace).Create(fixApplicationMapping(name))
	require.NoError(t, err)

	// when
	t.Logf("Removing Namespace %s", namespace)
	err = k8sClient.Namespaces().Delete(namespace, &metav1.DeleteOptions{})
	assert.NoError(t, err)

	t.Log("Removing Application")
	err = aClient.ApplicationconnectorV1alpha1().Applications().Delete(app.Name, &metav1.DeleteOptions{})
	assert.NoError(t, err)

	t.Log("Assert namespace is removed")
	repeat.FuncAtMost(t, func() error {
		_, err := k8sClient.Namespaces().Get(namespace, metav1.GetOptions{})
		if apimerr.IsNotFound(err) {
			return nil
		}
		return fmt.Errorf("%s is not removed", namespace)
	}, time.Second*90)

	// then
	t.Log("Check resources are removed")
	_, err = aClient.ApplicationconnectorV1alpha1().Applications().Get(app.Name, metav1.GetOptions{})
	assert.True(t, apimerr.IsNotFound(err))

	_, err = mClient.ApplicationconnectorV1alpha1().ApplicationMappings(namespace).Get(am.Name, metav1.GetOptions{})
	assert.True(t, apimerr.IsNotFound(err))

	listServices, err := k8sClient.Services(namespace).List(metav1.ListOptions{})
	assert.Empty(t, listServices.Items)

	listServiceBrokers, err := scClient.ServicecatalogV1beta1().ServiceBrokers(namespace).List(metav1.ListOptions{})
	assert.Empty(t, listServiceBrokers.Items)
}

type brokerURL struct {
	prefix    string
	namespace string
}

func (b *brokerURL) buildURL(releaseNS string) string {
	return fmt.Sprintf("http://%s%s.%s.svc.cluster.local", b.prefix, b.namespace, releaseNS)
}

func fixApplication(name string) *appTypes.Application {
	return &appTypes.Application{
		TypeMeta: metav1.TypeMeta{
			Kind:       "Application",
			APIVersion: "applicationconnector.kyma-project.io/v1alpha1",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
		},
		Spec: appTypes.ApplicationSpec{
			Description:      "Application used by acceptance test",
			AccessLabel:      "fix-access",
			SkipInstallation: true,
			Services: []appTypes.Service{
				{
					ID:   "id-00000-1234-test",
					Name: "provider-4951",
					Labels: map[string]string{
						"connected-app": name,
					},
					ProviderDisplayName: "provider",
					DisplayName:         name,
					Description:         "Application Service Class used by acceptance test",
					Tags:                []string{},
					Entries: []appTypes.Entry{
						{
							Type:        "API",
							AccessLabel: "acc-label",
							GatewayUrl:  "http://promotions-gateway.production.svc.cluster.local/",
						},
					},
				},
			},
		},
	}
}

func fixApplicationMapping(name string) *mappingTypes.ApplicationMapping {
	return &mappingTypes.ApplicationMapping{
		TypeMeta: metav1.TypeMeta{
			Kind:       "ApplicationMapping",
			APIVersion: "applicationconnector.kyma-project.io/v1alpha1",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
		},
	}
}
<commit_msg>Disable ServiceCatalog Application broker test because of Istio RBAC blocks direct call (#4014)<commit_after>package servicecatalog_test

import (
	"fmt"
	"os"
	"testing"
	"time"

	appTypes "github.com/kyma-project/kyma/components/application-operator/pkg/apis/applicationconnector/v1alpha1"
	appClient "github.com/kyma-project/kyma/components/application-operator/pkg/client/clientset/versioned"

	mappingTypes "github.com/kyma-project/kyma/components/application-broker/pkg/apis/applicationconnector/v1alpha1"
	mappingClient "github.com/kyma-project/kyma/components/application-broker/pkg/client/clientset/versioned"

	scc "github.com/kubernetes-incubator/service-catalog/pkg/client/clientset_generated/clientset"

	corev1 "github.com/kubernetes/client-go/kubernetes/typed/core/v1"
	"github.com/kyma-project/kyma/tests/acceptance/pkg/repeat"
	"github.com/pkg/errors"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	apimerr "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/rand"
	restclient "k8s.io/client-go/rest"
)

const (
	helmBrokerURLEnvName        = "HELM_BROKER_URL"
	applicationBrokerURLEnvName = "APPLICATION_BROKER_URL"
	releaseNamespaceEnvName     = "RELEASE_NAMESPACE"
)

func TestBrokerHasIstioRbacAuthorizationRules(t *testing.T) {
	for testName, brokerURL := range map[string]string{
		"Helm Broker":        os.Getenv(helmBrokerURLEnvName),
		"Application Broker": os.Getenv(applicationBrokerURLEnvName),
		// "<next broker>": os.Getenv("<broker url env name>"),
	} {
		t.Run(testName, func(t *testing.T) {
			repeat.FuncAtMost(t, func() error {
				isForbidden, err := isCatalogForbidden(brokerURL)
				if err != nil {
					return errors.Wrap(err, "while getting catalog")
				}
				if !isForbidden {
					return fmt.Errorf("%s catalog response must be forbidden", testName)
				}
				return nil
			}, 5*time.Second)
		})
	}
}

func TestServiceCatalogResourcesAreCleanUp(t *testing.T) {
	// given
	k8sConfig, err := restclient.InClusterConfig()
	require.NoError(t, err)

	aClient, err := appClient.NewForConfig(k8sConfig)
	require.NoError(t, err)

	mClient, err := mappingClient.NewForConfig(k8sConfig)
	require.NoError(t, err)

	k8sClient, err := corev1.NewForConfig(k8sConfig)
	require.NoError(t, err)

	scClient, err := scc.NewForConfig(k8sConfig)
	require.NoError(t, err)

	name := fmt.Sprintf("test-acc-app-%s", rand.String(4))
	namespace := fmt.Sprintf("test-acc-ns-broker-%s", rand.String(4))

	t.Log("Creating Application")
	app, err := aClient.ApplicationconnectorV1alpha1().Applications().Create(fixApplication(name))
	require.NoError(t, err)

	t.Logf("Creating Namespace %s", namespace)
	_, err = k8sClient.Namespaces().Create(fixNamespace(namespace))
	require.NoError(t, err)

	t.Log("Creating ApplicationMapping")
	am, err := mClient.ApplicationconnectorV1alpha1().ApplicationMappings(namespace).Create(fixApplicationMapping(name))
	require.NoError(t, err)

	// when
	t.Logf("Removing Namespace %s", namespace)
	err = k8sClient.Namespaces().Delete(namespace, &metav1.DeleteOptions{})
	assert.NoError(t, err)

	t.Log("Removing Application")
	err = aClient.ApplicationconnectorV1alpha1().Applications().Delete(app.Name, &metav1.DeleteOptions{})
	assert.NoError(t, err)

	t.Log("Assert namespace is removed")
	repeat.FuncAtMost(t, func() error {
		_, err := k8sClient.Namespaces().Get(namespace, metav1.GetOptions{})
		if apimerr.IsNotFound(err) {
			return nil
		}
		return fmt.Errorf("%s is not removed", namespace)
	}, time.Second*90)

	// then
	t.Log("Check resources are removed")
	_, err = aClient.ApplicationconnectorV1alpha1().Applications().Get(app.Name, metav1.GetOptions{})
	assert.True(t, apimerr.IsNotFound(err))

	_, err = mClient.ApplicationconnectorV1alpha1().ApplicationMappings(namespace).Get(am.Name, metav1.GetOptions{})
	assert.True(t, apimerr.IsNotFound(err))

	listServices, err := k8sClient.Services(namespace).List(metav1.ListOptions{})
	assert.Empty(t, listServices.Items)

	listServiceBrokers, err := scClient.ServicecatalogV1beta1().ServiceBrokers(namespace).List(metav1.ListOptions{})
	assert.Empty(t, listServiceBrokers.Items)
}

type brokerURL struct {
	prefix    string
	namespace string
}

func (b *brokerURL) buildURL(releaseNS string) string {
	return fmt.Sprintf("http://%s%s.%s.svc.cluster.local", b.prefix, b.namespace, releaseNS)
}

func fixApplication(name string) *appTypes.Application {
	return &appTypes.Application{
		TypeMeta: metav1.TypeMeta{
			Kind:       "Application",
			APIVersion: "applicationconnector.kyma-project.io/v1alpha1",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
		},
		Spec: appTypes.ApplicationSpec{
			Description:      "Application used by acceptance test",
			AccessLabel:      "fix-access",
			SkipInstallation: true,
			Services: []appTypes.Service{
				{
					ID:   "id-00000-1234-test",
					Name: "provider-4951",
					Labels: map[string]string{
						"connected-app": name,
					},
					ProviderDisplayName: "provider",
					DisplayName:         name,
					Description:         "Application Service Class used by acceptance test",
					Tags:                []string{},
					Entries: []appTypes.Entry{
						{
							Type:        "API",
							AccessLabel: "acc-label",
							GatewayUrl:  "http://promotions-gateway.production.svc.cluster.local/",
						},
					},
				},
			},
		},
	}
}

func fixApplicationMapping(name string) *mappingTypes.ApplicationMapping {
	return &mappingTypes.ApplicationMapping{
		TypeMeta: metav1.TypeMeta{
			Kind:       "ApplicationMapping",
			APIVersion: "applicationconnector.kyma-project.io/v1alpha1",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
		},
	}
}
<|endoftext|>


<commit_before>// Package prioritybitmap implements a set of integers ordered by attached
// priorities.
package prioritybitmap

import (
	"sync"

	"github.com/anacrolix/missinggo/bitmap"
	"github.com/anacrolix/missinggo/iter"
	"github.com/anacrolix/missinggo/orderedmap"
)

var (
	bitSets = sync.Pool{
		New: func() interface{} {
			return make(map[int]struct{}, 1)
		},
	}
)

// Maintains set of ints ordered by priority.
type PriorityBitmap struct {
	mu sync.Mutex
	// From priority to singleton or set of bit indices.
	om orderedmap.OrderedMap
	// From bit index to priority
	priorities map[int]int
}

var _ bitmap.Interface = (*PriorityBitmap)(nil)

func (me *PriorityBitmap) Contains(bit int) bool {
	_, ok := me.priorities[bit]
	return ok
}

func (me *PriorityBitmap) Len() int {
	return len(me.priorities)
}

func (me *PriorityBitmap) Clear() {
	me.om = nil
	me.priorities = nil
}

func (me *PriorityBitmap) deleteBit(bit int) (priority int, ok bool) {
	priority, ok = me.priorities[bit]
	if !ok {
		return
	}
	switch v := me.om.Get(priority).(type) {
	case int:
		if v != bit {
			panic("invariant broken")
		}
	case map[int]struct{}:
		if _, ok := v[bit]; !ok {
			panic("invariant broken")
		}
		delete(v, bit)
		if len(v) != 0 {
			return
		}
		bitSets.Put(v)
	}
	me.om.Unset(priority)
	if me.om.Len() == 0 {
		me.om = nil
	}
	return
}

func bitLess(l, r interface{}) bool {
	return l.(int) < r.(int)
}

func (me *PriorityBitmap) lazyInit() {
	me.om = orderedmap.New(func(l, r interface{}) bool {
		return l.(int) < r.(int)
	})
	me.priorities = make(map[int]int)
}

// Returns true if the priority is changed, or the bit wasn't present.
func (me *PriorityBitmap) Set(bit int, priority int) bool {
	if p, ok := me.priorities[bit]; ok && p == priority {
		return false
	}
	if oldPriority, deleted := me.deleteBit(bit); deleted && oldPriority == priority {
		panic("should have already returned")
	}
	if me.priorities == nil {
		me.priorities = make(map[int]int)
	}
	me.priorities[bit] = priority
	if me.om == nil {
		me.om = orderedmap.New(bitLess)
	}
	_v, ok := me.om.GetOk(priority)
	if !ok {
		// No other bits with this priority, set it to a lone int.
		me.om.Set(priority, bit)
		return true
	}
	switch v := _v.(type) {
	case int:
		newV := bitSets.Get().(map[int]struct{})
		newV[v] = struct{}{}
		newV[bit] = struct{}{}
		me.om.Set(priority, newV)
	case map[int]struct{}:
		v[bit] = struct{}{}
	default:
		panic(v)
	}
	return true
}

func (me *PriorityBitmap) Remove(bit int) bool {
	me.mu.Lock()
	defer me.mu.Unlock()
	if _, ok := me.deleteBit(bit); !ok {
		return false
	}
	delete(me.priorities, bit)
	if len(me.priorities) == 0 {
		me.priorities = nil
	}
	if me.om != nil && me.om.Len() == 0 {
		me.om = nil
	}
	return true
}

func (me *PriorityBitmap) Iter(f iter.Callback) {
	me.IterTyped(func(i int) bool {
		return f(i)
	})
}

func (me *PriorityBitmap) IterTyped(_f func(i bitmap.BitIndex) bool) bool {
	me.mu.Lock()
	defer me.mu.Unlock()
	if me == nil || me.om == nil {
		return true
	}
	f := func(i int) bool {
		me.mu.Unlock()
		defer me.mu.Lock()
		return _f(i)
	}
	return iter.All(func(key interface{}) bool {
		value := me.om.Get(key)
		switch v := value.(type) {
		case int:
			return f(v)
		case map[int]struct{}:
			for i := range v {
				if !f(i) {
					return false
				}
			}
		}
		return true
	}, me.om.Iter)
}

func (me *PriorityBitmap) IsEmpty() bool {
	if me.om == nil {
		return true
	}
	return me.om.Len() == 0
}

// ok is false if the bit is not set.
func (me *PriorityBitmap) GetPriority(bit int) (prio int, ok bool) {
	prio, ok = me.priorities[bit]
	return
}
<commit_msg>prioritybitmap: Remove internal mutex<commit_after>// Package prioritybitmap implements a set of integers ordered by attached
// priorities.
package prioritybitmap

import (
	"sync"

	"github.com/anacrolix/missinggo/bitmap"
	"github.com/anacrolix/missinggo/iter"
	"github.com/anacrolix/missinggo/orderedmap"
)

var (
	bitSets = sync.Pool{
		New: func() interface{} {
			return make(map[int]struct{}, 1)
		},
	}
)

// Maintains set of ints ordered by priority.
type PriorityBitmap struct {
	// From priority to singleton or set of bit indices.
	om orderedmap.OrderedMap
	// From bit index to priority
	priorities map[int]int
}

var _ bitmap.Interface = (*PriorityBitmap)(nil)

func (me *PriorityBitmap) Contains(bit int) bool {
	_, ok := me.priorities[bit]
	return ok
}

func (me *PriorityBitmap) Len() int {
	return len(me.priorities)
}

func (me *PriorityBitmap) Clear() {
	me.om = nil
	me.priorities = nil
}

func (me *PriorityBitmap) deleteBit(bit int) (priority int, ok bool) {
	priority, ok = me.priorities[bit]
	if !ok {
		return
	}
	switch v := me.om.Get(priority).(type) {
	case int:
		if v != bit {
			panic("invariant broken")
		}
	case map[int]struct{}:
		if _, ok := v[bit]; !ok {
			panic("invariant broken")
		}
		delete(v, bit)
		if len(v) != 0 {
			return
		}
		bitSets.Put(v)
	}
	me.om.Unset(priority)
	if me.om.Len() == 0 {
		me.om = nil
	}
	return
}

func bitLess(l, r interface{}) bool {
	return l.(int) < r.(int)
}

func (me *PriorityBitmap) lazyInit() {
	me.om = orderedmap.New(func(l, r interface{}) bool {
		return l.(int) < r.(int)
	})
	me.priorities = make(map[int]int)
}

// Returns true if the priority is changed, or the bit wasn't present.
func (me *PriorityBitmap) Set(bit int, priority int) bool {
	if p, ok := me.priorities[bit]; ok && p == priority {
		return false
	}
	if oldPriority, deleted := me.deleteBit(bit); deleted && oldPriority == priority {
		panic("should have already returned")
	}
	if me.priorities == nil {
		me.priorities = make(map[int]int)
	}
	me.priorities[bit] = priority
	if me.om == nil {
		me.om = orderedmap.New(bitLess)
	}
	_v, ok := me.om.GetOk(priority)
	if !ok {
		// No other bits with this priority, set it to a lone int.
		me.om.Set(priority, bit)
		return true
	}
	switch v := _v.(type) {
	case int:
		newV := bitSets.Get().(map[int]struct{})
		newV[v] = struct{}{}
		newV[bit] = struct{}{}
		me.om.Set(priority, newV)
	case map[int]struct{}:
		v[bit] = struct{}{}
	default:
		panic(v)
	}
	return true
}

func (me *PriorityBitmap) Remove(bit int) bool {
	if _, ok := me.deleteBit(bit); !ok {
		return false
	}
	delete(me.priorities, bit)
	if len(me.priorities) == 0 {
		me.priorities = nil
	}
	if me.om != nil && me.om.Len() == 0 {
		me.om = nil
	}
	return true
}

func (me *PriorityBitmap) Iter(f iter.Callback) {
	me.IterTyped(func(i int) bool {
		return f(i)
	})
}

func (me *PriorityBitmap) IterTyped(_f func(i bitmap.BitIndex) bool) bool {
	if me == nil || me.om == nil {
		return true
	}
	f := func(i int) bool {
		return _f(i)
	}
	return iter.All(func(key interface{}) bool {
		value := me.om.Get(key)
		switch v := value.(type) {
		case int:
			return f(v)
		case map[int]struct{}:
			for i := range v {
				if !f(i) {
					return false
				}
			}
		}
		return true
	}, me.om.Iter)
}

func (me *PriorityBitmap) IsEmpty() bool {
	if me.om == nil {
		return true
	}
	return me.om.Len() == 0
}

// ok is false if the bit is not set.
func (me *PriorityBitmap) GetPriority(bit int) (prio int, ok bool) {
	prio, ok = me.priorities[bit]
	return
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package acls

import (
	"v.io/v23/context"
	"v.io/v23/security"
	"v.io/v23/security/access"
	"v.io/x/lib/vlog"

	"v.io/x/ref/profiles/internal/rpc"
)

// hierarchicalAuthorizer contains the state needed to implement
// hierarchical authorization in the Authorize method.
type hierarchicalAuthorizer struct {
	rootDir, childDir string
	get               TAMGetter
}

// TAMGetter defines an abstract interface that a customer of
// NewHierarchicalAuthorizer can use to obtain the PermissionsAuthorizer
// instances that it needs to construct a hierarchicalAuthorizer.
type TAMGetter interface {
	// TAMForPath has two successful outcomes: either returning a valid
	// Permissions object or a boolean status true indicating that the
	// Permissions object is intentionally not present. Finally, it returns an
	// error if anything has gone wrong.
	TAMForPath(path string) (access.Permissions, bool, error)
}

func mkRootAuth(rootTam access.Permissions) (security.Authorizer, error) {
	rootAuth, err := access.PermissionsAuthorizer(rootTam, access.TypicalTagType())
	if err != nil {
		vlog.Errorf("Successfully obtained an AccessList from the filesystem but PermissionsAuthorizer couldn't use it: %v", err)
		return nil, err
	}
	return rootAuth, nil
}

// NewHierarchicalAuthorizer creates a new hierarchicalAuthorizer: one
// that implements a "root" like concept: admin rights at the root of
// a server can invoke RPCs regardless of permissions set on child objects.
func NewHierarchicalAuthorizer(rootDir, childDir string, get TAMGetter) (security.Authorizer, error) {
	return &hierarchicalAuthorizer{
		rootDir:  rootDir,
		childDir: childDir,
		get:      get,
	}, nil
}

func (ha *hierarchicalAuthorizer) Authorize(ctx *context.T) error {
	rootPerms, intentionallyEmpty, err := ha.get.TAMForPath(ha.rootDir)
	if err != nil {
		return err
	} else if intentionallyEmpty {
		vlog.VI(2).Infof("TAMForPath(%s) is intentionally empty", ha.rootDir)
		return defaultAuthorizer(ctx)
	}

	// We are at the root so exit early.
	if ha.rootDir == ha.childDir {
		a, err := mkRootAuth(rootPerms)
		if err != nil {
			return err
		}
		return a.Authorize(ctx)
	}

	// This is not fatal: the childDir may not exist if we are invoking
	// a Create() method so we only use the root Permissions.
	childPerms, intentionallyEmpty, err := ha.get.TAMForPath(ha.childDir)
	if err != nil {
		return err
	} else if intentionallyEmpty {
		a, err := mkRootAuth(rootPerms)
		if err != nil {
			return err
		}
		return a.Authorize(ctx)
	}

	childAuth, err := access.PermissionsAuthorizer(childPerms, access.TypicalTagType())
	if err != nil {
		vlog.Errorf("Successfully obtained a Permissions from the filesystem but PermissionsAuthorizer couldn't use it: %v", err)
		return err
	}

	childErr := childAuth.Authorize(ctx)
	if childErr == nil {
		return nil
	}

	// Maybe the invoking principal can invoke this method because
	// it has root permissions.
	names, _ := security.RemoteBlessingNames(ctx)
	if len(names) > 0 && rootPerms[string(access.Admin)].Includes(names...) {
		return nil
	}

	return childErr
}

// defaultAuthorizer implements an authorization policy that requires one end
// of the RPC to have a blessing that makes it a delegate of the other.
// TODO(rjkroege): Remove this when the defaultAuthorizer becomes public.
func defaultAuthorizer(ctx *context.T) error {
	var (
		localNames             = security.LocalBlessingNames(ctx)
		remoteNames, remoteErr = security.RemoteBlessingNames(ctx)
	)
	// Authorize if any element in localNames is a "delegate of" (i.e., has been
	// blessed by) any element in remoteNames, OR vice-versa.
	for _, l := range localNames {
		if security.BlessingPattern(l).MatchedBy(remoteNames...) {
			// l is a delegate of an element in remote.
			return nil
		}
	}
	for _, r := range remoteNames {
		if security.BlessingPattern(r).MatchedBy(localNames...) {
			// r is a delegate of an element in localNames.
			return nil
		}
	}

	return rpc.NewErrInvalidBlessings(nil, remoteNames, remoteErr, localNames)
}
<commit_msg>services/mgmt/lib/acls: additional hierarchical authorizer changes<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package acls

import (
	"v.io/v23/context"
	"v.io/v23/security"
	"v.io/v23/security/access"
	"v.io/x/lib/vlog"

	"v.io/x/ref/profiles/internal/rpc"
)

// hierarchicalAuthorizer contains the state needed to implement
// hierarchical authorization in the Authorize method.
type hierarchicalAuthorizer struct {
	rootDir, childDir string
	get               TAMGetter
}

// TAMGetter defines an abstract interface that a customer of
// NewHierarchicalAuthorizer can use to obtain the PermissionsAuthorizer
// instances that it needs to construct a hierarchicalAuthorizer.
type TAMGetter interface {
	// TAMForPath has two successful outcomes: either returning a valid
	// Permissions object or a boolean status true indicating that the
	// Permissions object is intentionally not present. Finally, it returns an
	// error if anything has gone wrong.
	TAMForPath(path string) (access.Permissions, bool, error)
}

func mkRootAuth(rootTam access.Permissions) (security.Authorizer, error) {
	rootAuth, err := access.PermissionsAuthorizer(rootTam, access.TypicalTagType())
	if err != nil {
		vlog.Errorf("Successfully obtained an AccessList from the filesystem but PermissionsAuthorizer couldn't use it: %v", err)
		return nil, err
	}
	return rootAuth, nil
}

// NewHierarchicalAuthorizer creates a new hierarchicalAuthorizer: one
// that implements a "root" like concept: admin rights at the root of
// a server can invoke RPCs regardless of permissions set on child objects.
func NewHierarchicalAuthorizer(rootDir, childDir string, get TAMGetter) (security.Authorizer, error) {
	return &hierarchicalAuthorizer{
		rootDir:  rootDir,
		childDir: childDir,
		get:      get,
	}, nil
}

func (ha *hierarchicalAuthorizer) Authorize(ctx *context.T) error {
	rootPerms, intentionallyEmpty, err := ha.get.TAMForPath(ha.rootDir)
	if err != nil {
		return err
	} else if intentionallyEmpty {
		vlog.VI(2).Infof("TAMForPath(%s) is intentionally empty", ha.rootDir)
		return defaultAuthorizer(ctx)
	}

	// We are at the root so exit early.
	if ha.rootDir == ha.childDir {
		a, err := mkRootAuth(rootPerms)
		if err != nil {
			return err
		}
		return adminCheckAuth(ctx, a, rootPerms)
	}

	// This is not fatal: the childDir may not exist if we are invoking
	// a Create() method so we only use the root Permissions.
	childPerms, intentionallyEmpty, err := ha.get.TAMForPath(ha.childDir)
	if err != nil {
		return err
	} else if intentionallyEmpty {
		a, err := mkRootAuth(rootPerms)
		if err != nil {
			return err
		}
		return adminCheckAuth(ctx, a, rootPerms)
	}

	childAuth, err := access.PermissionsAuthorizer(childPerms, access.TypicalTagType())
	if err != nil {
		vlog.Errorf("Successfully obtained a Permissions from the filesystem but PermissionsAuthorizer couldn't use it: %v", err)
		return err
	}
	return adminCheckAuth(ctx, childAuth, rootPerms)
}

func adminCheckAuth(ctx *context.T, auth security.Authorizer, perms access.Permissions) error {
	err := auth.Authorize(ctx)
	if err == nil {
		return nil
	}

	// Maybe the invoking principal can invoke this method because
	// it has Admin permissions.
	names, _ := security.RemoteBlessingNames(ctx)
	if len(names) > 0 && perms[string(access.Admin)].Includes(names...) {
		return nil
	}

	return err
}

// defaultAuthorizer implements an authorization policy that requires one end
// of the RPC to have a blessing that makes it a delegate of the other.
// TODO(rjkroege): Remove this when the defaultAuthorizer becomes public.
func defaultAuthorizer(ctx *context.T) error {
	var (
		localNames             = security.LocalBlessingNames(ctx)
		remoteNames, remoteErr = security.RemoteBlessingNames(ctx)
	)
	// Authorize if any element in localNames is a "delegate of" (i.e., has been
	// blessed by) any element in remoteNames, OR vice-versa.
	for _, l := range localNames {
		if security.BlessingPattern(l).MatchedBy(remoteNames...) {
			// l is a delegate of an element in remote.
			return nil
		}
	}
	for _, r := range remoteNames {
		if security.BlessingPattern(r).MatchedBy(localNames...) {
			// r is a delegate of an element in localNames.
			return nil
		}
	}

	return rpc.NewErrInvalidBlessings(nil, remoteNames, remoteErr, localNames)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"log"
	"time"

	"go.opentelemetry.io/otel/api/global"
	"go.opentelemetry.io/otel/exporters/otlp"
	sdktrace "go.opentelemetry.io/otel/sdk/trace"
	"google.golang.org/grpc"
)

func main() {
	exp, err := otlp.NewExporter(otlp.WithInsecure(),
		otlp.WithGRPCDialOption(grpc.WithBlock()))
	if err != nil {
		log.Fatalf("Failed to create the collector exporter: %v", err)
	}
	if err = exp.GetLastConnectError(); err != nil {
		log.Fatalf("Failed to create the collector exporter: %v", err)
	}

	defer func() {

		_ = exp.Stop()
	}()

	tp, _ := sdktrace.NewProvider(
		sdktrace.WithConfig(sdktrace.Config{DefaultSampler: sdktrace.AlwaysSample()}),
		sdktrace.WithBatcher(exp, // add following two options to ensure flush
			sdktrace.WithScheduleDelayMillis(5),
			sdktrace.WithMaxExportBatchSize(1),
		))
	if err != nil {
		log.Fatalf("error creating trace provider: %v\n", err)
	}

	global.SetTraceProvider(tp)
	tracer := global.Tracer("test-tracer")

	// Then use the OpenTelemetry tracing library, like we normally would.
	ctx, span := tracer.Start(context.Background(), "CollectorExporter-Example")
	defer span.End()

	for i := 0; i < 10; i++ {
		_, iSpan := tracer.Start(ctx, fmt.Sprintf("Sample-%d", i))
		<-time.After(6 * time.Second)
		iSpan.End()
	}

	// exp.ExportSpan(ctx, span)
}
<commit_msg>Update main.go<commit_after>package main

import (
	"context"
	"fmt"
	"log"
	"time"

	"go.opentelemetry.io/otel/api/global"
	"go.opentelemetry.io/otel/exporters/otlp"
	sdktrace "go.opentelemetry.io/otel/sdk/trace"
	"google.golang.org/grpc"
)

func main() {
	exp, err := otlp.NewExporter(otlp.WithInsecure(),
		otlp.WithGRPCDialOption(grpc.WithBlock()))
	if err != nil {
		log.Fatalf("Failed to create the collector exporter: %v", err)
	}
	defer func() {
		_ = exp.Stop()
	}()

	tp, _ := sdktrace.NewProvider(
		sdktrace.WithConfig(sdktrace.Config{DefaultSampler: sdktrace.AlwaysSample()}),
		sdktrace.WithBatcher(exp, // add following two options to ensure flush
			sdktrace.WithScheduleDelayMillis(5),
			sdktrace.WithMaxExportBatchSize(1),
		))
	if err != nil {
		log.Fatalf("error creating trace provider: %v\n", err)
	}

	global.SetTraceProvider(tp)
	tracer := global.Tracer("test-tracer")

	// Then use the OpenTelemetry tracing library, like we normally would.
	ctx, span := tracer.Start(context.Background(), "CollectorExporter-Example")
	defer span.End()

	for i := 0; i < 10; i++ {
		_, iSpan := tracer.Start(ctx, fmt.Sprintf("Sample-%d", i))
		<-time.After(6 * time.Second)
		iSpan.End()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"

	"github.com/skatsuta/nand2tetris/projects/07/vmtranslator/vmtranslator"
)

var (
	appName = "vmtranslator"
	usage   = "Usage: %s [-h | --help] path"
)

func init() {
	flag.Usage = func() {
		printErr(usage, appName)
	}
}

func main() {
	flag.Parse()

	// check whether one argument is passed
	args := flag.Args()
	if len(args) != 1 {
		flag.Usage()
		return
	}

	path := args[0]

	if e := convert(path); e != nil {
		printErr("%v", e)
		return
	}
}

// printErr prints an formatted error message in os.Stderr.
func printErr(format string, args ...interface{}) {
	_, _ = fmt.Fprintf(os.Stderr, format+"\n", args...)
}

// convert converts files in path to one .asm file.
func convert(path string) error {
	// check whether the given path is valid
	info, err := os.Stat(path)
	if err != nil {
		return fmt.Errorf("invalid path is given: %s", path)
	}

	// prepare an output .asm file
	opath := outpath(path, info.IsDir())
	out, err := os.Create(opath)
	if err != nil {
		return fmt.Errorf("cannot create %s", opath)
	}

	vmt := vmtranslator.New(out)
	// walk throuth path and run conversion
	if e := filepath.Walk(path, vmt.Run); e != nil {
		return fmt.Errorf("failed to convert: %v", e)
	}

	return nil
}

// outpath returns an output file path.
// This function expects the suffix of the path to be ".vm" if it is a file.
func outpath(path string, isDir bool) string {
	if !isDir {
		// file.vm => file.asm
		return path[:len(path)-2] + "asm"
	}

	filename := filepath.Base(path)
	return filepath.Join(path, filename+".asm")
}
<commit_msg>07/main: use VMTranslator.Close() to close the output file<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"path/filepath"

	"github.com/skatsuta/nand2tetris/projects/07/vmtranslator/vmtranslator"
)

var (
	appName = "vmtranslator"
	usage   = "Usage: %s [-h | --help] path"
)

func init() {
	flag.Usage = func() {
		printErr(usage, appName)
	}
}

func main() {
	flag.Parse()

	// check whether one argument is passed
	args := flag.Args()
	if len(args) != 1 {
		flag.Usage()
		return
	}

	path := args[0]

	if e := convert(path); e != nil {
		printErr("%v", e)
		return
	}
}

// printErr prints an formatted error message in os.Stderr.
func printErr(format string, args ...interface{}) {
	_, _ = fmt.Fprintf(os.Stderr, format+"\n", args...)
}

// convert converts files in path to one .asm file.
func convert(path string) error {
	// check whether the given path is valid
	info, err := os.Stat(path)
	if err != nil {
		return fmt.Errorf("invalid path is given: %s", path)
	}

	// prepare an output .asm file
	opath := outpath(path, info.IsDir())
	out, err := os.Create(opath)
	if err != nil {
		return fmt.Errorf("cannot create %s", opath)
	}

	vmt := vmtranslator.New(out)
	defer func() {
		_ = vmt.Close()
	}()

	// walk throuth path and run conversion
	if e := filepath.Walk(path, vmt.Run); e != nil {
		return fmt.Errorf("failed to convert: %v", e)
	}

	return nil
}

// outpath returns an output file path.
// This function expects the suffix of the path to be ".vm" if it is a file.
func outpath(path string, isDir bool) string {
	if !isDir {
		// file.vm => file.asm
		return path[:len(path)-2] + "asm"
	}

	filename := filepath.Base(path)
	return filepath.Join(path, filename+".asm")
}
<|endoftext|>


<commit_before>package main

import(
	"runtime"
	"time"
	"fmt"
	"os"
	"strconv"
)

func Switch_1(n int) {
	c := make(chan bool)
	go func(){
		for i := 0; i < n; i++ {
			// switch
			runtime.Gosched()
        }

		c <- true
    }()

	<-c
}

func Switch_1000(n int) {
	nCoro := 1000
	c := make(chan bool, nCoro)
	for i := 0; i < nCoro; i++ {
		go func(){
			for i := 0; i < n / nCoro; i++ {
				// switch
				runtime.Gosched()
			}

			c <- true
		}()
	}

	for i := 0; i < nCoro; i++ {
		<-c
	}
}

func Channel_0(n int) {
	c := make(chan bool)
	go func(){
		for i := 0; i < n; i++ {
			c <- true
        }
    }()

	for i := 0; i < n; i++ {
		<-c
	}
}

func Channel_1(n int) {
	c := make(chan bool, 1)
	go func(){
		for i := 0; i < n; i++ {
			c <- true
        }
    }()

	for i := 0; i < n; i++ {
		<-c
	}
}

func Channel_N(n int) {
	c := make(chan bool, n)
	go func(){
		for i := 0; i < n; i++ {
			c <- true
        }
    }()

	for i := 0; i < n; i++ {
		<-c
	}
}

func main() {
	var n int64 = 10000000
	var N int = int(n)

	nThreads, _ := strconv.Atoi(os.Args[1])
	runtime.GOMAXPROCS(nThreads)

	fmt.Printf("Thread: %d\n", runtime.GOMAXPROCS(0));

	t := time.Now()
	Switch_1(N)
	nanos := time.Since(t).Nanoseconds()
	fmt.Printf("%s    %d    %d ns/op    %d w/s\n", "Switch_1", n, nanos/n, 100000 * n / nanos);

	t = time.Now()
	Switch_1000(N)
	nanos = time.Since(t).Nanoseconds()
	fmt.Printf("%s    %d    %d ns/op    %d w/s\n", "Switch_1000", n, nanos/n, 100000 * n / nanos);

	t = time.Now()
	Channel_0(N)
	nanos = time.Since(t).Nanoseconds()
	fmt.Printf("%s    %d    %d ns/op    %d w/s\n", "Channel_0", n, nanos/n, 100000 * n / nanos);

	t = time.Now()
	Channel_1(N)
	nanos = time.Since(t).Nanoseconds()
	fmt.Printf("%s    %d    %d ns/op    %d w/s\n", "Channel_1", n, nanos/n, 100000 * n / nanos);

	t = time.Now()
	Channel_N(N)
	nanos = time.Since(t).Nanoseconds()
	fmt.Printf("%s    %d    %d ns/op    %d w/s\n", "Channel_N", n, nanos/n, 100000 * n / nanos);
}
<commit_msg>统一golang测试代码<commit_after>package main

import(
	"runtime"
	"time"
	"fmt"
	"os"
	"strconv"
	"sync/atomic"
)

func Switch_1(n int) {
	c := make(chan bool)
	go func(){
		for i := 0; i < n; i++ {
			// switch
			runtime.Gosched()
        }

		c <- true
    }()

	<-c
}

func Switch_1000(n int) {
	nCoro := 1000
	var done int64 = 0
	c := make(chan bool, 1)
	for i := 0; i < nCoro; i++ {
		go func(){
			for i := 0; i < n / nCoro; i++ {
				// switch
				runtime.Gosched()
			}

			if atomic.AddInt64(&done, 1) == int64(nCoro) {
				c <- true
			}
		}()
	}

	<-c
}

func Channel_0(n int) {
	c := make(chan bool)
	go func(){
		for i := 0; i < n; i++ {
			c <- true
        }
    }()

	for i := 0; i < n; i++ {
		<-c
	}
}

func Channel_1(n int) {
	c := make(chan bool, 1)
	go func(){
		for i := 0; i < n; i++ {
			c <- true
        }
    }()

	for i := 0; i < n; i++ {
		<-c
	}
}

func Channel_N(n int) {
	c := make(chan bool, n)
	go func(){
		for i := 0; i < n; i++ {
			c <- true
        }
    }()

	for i := 0; i < n; i++ {
		<-c
	}
}

func main() {
	var n int64 = 10000000
	var N int = int(n)

	if len(os.Args) > 1 {
		nThreads, _ := strconv.Atoi(os.Args[1])
		runtime.GOMAXPROCS(nThreads)
	}

	fmt.Printf("Thread: %d\n", runtime.GOMAXPROCS(0));

	t := time.Now()
	Switch_1(N)
	nanos := time.Since(t).Nanoseconds()
	fmt.Printf("%s    %d    %d ns/op    %d w/s\n", "Switch_1", n, nanos/n, 100000 * n / nanos);

	t = time.Now()
	Switch_1000(N)
	nanos = time.Since(t).Nanoseconds()
	fmt.Printf("%s    %d    %d ns/op    %d w/s\n", "Switch_1000", n, nanos/n, 100000 * n / nanos);

	t = time.Now()
	Channel_0(N)
	nanos = time.Since(t).Nanoseconds()
	fmt.Printf("%s    %d    %d ns/op    %d w/s\n", "Channel_0", n, nanos/n, 100000 * n / nanos);

	t = time.Now()
	Channel_1(N)
	nanos = time.Since(t).Nanoseconds()
	fmt.Printf("%s    %d    %d ns/op    %d w/s\n", "Channel_1", n, nanos/n, 100000 * n / nanos);

	t = time.Now()
	Channel_N(N)
	nanos = time.Since(t).Nanoseconds()
	fmt.Printf("%s    %d    %d ns/op    %d w/s\n", "Channel_N", n, nanos/n, 100000 * n / nanos);
}
<|endoftext|>


<commit_before>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"fmt"

	"github.com/cilium/cilium/test/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	log "github.com/sirupsen/logrus"
)

var _ = Describe("K8sPolicyTest", func() {

	var demoPath string
	var initialized bool
	var kubectl *helpers.Kubectl
	var l3Policy, l7Policy string
	var logger *log.Entry
	var path string
	var podFilter string

	initialize := func() {
		if initialized == true {
			return
		}

		logger = log.WithFields(log.Fields{"testName": "K8sPolicyTest"})
		logger.Info("Starting")
		kubectl = helpers.CreateKubectl("k8s1", logger)
		podFilter = "k8s:zgroup=testapp"

		//Manifest paths
		demoPath = fmt.Sprintf("%s/demo.yaml", kubectl.ManifestsPath())
		l3Policy = fmt.Sprintf("%s/l3_l4_policy.yaml", kubectl.ManifestsPath())
		l7Policy = fmt.Sprintf("%s/l7_policy.yaml", kubectl.ManifestsPath())

		path = fmt.Sprintf("%s/cilium_ds.yaml", kubectl.ManifestsPath())
		kubectl.Apply(path)
		status, err := kubectl.WaitforPods("kube-system", "-l k8s-app=cilium", 300)
		Expect(status).Should(BeTrue())
		Expect(err).Should(BeNil())
		initialized = true
	}

	BeforeEach(func() {
		initialize()
		// kubectl.CiliumPolicyDeleteAll("kube-system")
		kubectl.Apply(demoPath)
		_, err := kubectl.WaitforPods("default", "-l zgroup=testapp", 300)
		Expect(err).Should(BeNil())
	})

	AfterEach(func() {
		if CurrentGinkgoTestDescription().Failed {
			ciliumPod, _ := kubectl.GetCiliumPodOnNode("kube-system", "k8s1")
			kubectl.CiliumReport("kube-system", ciliumPod, []string{
				"cilium bpf tunnel list",
				"cilium endpoint list"})
		}

		kubectl.Delete(demoPath)
		// TO make sure that are not in place
		kubectl.Delete(l3Policy)
		kubectl.Delete(l7Policy)
	})

	waitUntilEndpointUpdates := func(pod string, eps map[string]int64, min int) error {
		body := func() bool {
			updated := 0
			newEps := kubectl.CiliumEndpointPolicyVersion(pod)
			for k, v := range newEps {
				if eps[k] < v {
					logger.Infof("Endpoint %s had version %d now %d", k, eps[k], v)
					updated++
				}
			}
			return updated >= min
		}
		err := helpers.WithTimeout(body, "No new version applied", &helpers.TimeoutConfig{Timeout: 100})
		return err
	}

	getAppPods := func() map[string]string {
		appPods := make(map[string]string)
		apps := []string{"app1", "app2", "app3"}
		for _, v := range apps {
			res, err := kubectl.GetPodNames("default", fmt.Sprintf("id=%s", v))
			Expect(err).Should(BeNil())
			appPods[v] = res[0]
			logger.Infof("PolicyRulesTest: pod='%s' assigned to '%s'", res[0], v)
		}
		return appPods
	}

	It("PolicyEnforcement Changes", func() {
		//This is a small test that check that everything is working in k8s. Full monkey testing
		// is in runtime/Policies
		ciliumPod, err := kubectl.GetCiliumPodOnNode("kube-system", "k8s1")
		Expect(err).Should(BeNil())

		status := kubectl.CiliumExec(ciliumPod, "cilium config PolicyEnforcement=default")
		status.ExpectSuccess()
		helpers.Sleep(5)
		kubectl.CiliumEndpointWait(ciliumPod)

		epsStatus := helpers.WithTimeout(func() bool {
			endpoints, err := kubectl.CiliumEndpointsListByTag(ciliumPod, podFilter)
			if err != nil {
				return false
			}
			return endpoints.AreReady()
		}, "Could not get endpoints", &helpers.TimeoutConfig{Timeout: 100})
		Expect(epsStatus).Should(BeNil())

		endpoints, err := kubectl.CiliumEndpointsListByTag(ciliumPod, podFilter)
		Expect(err).Should(BeNil())
		Expect(endpoints.AreReady()).Should(BeTrue())
		policyStatus := endpoints.GetPolicyStatus()
		Expect(policyStatus["enabled"]).Should(Equal(0))
		Expect(policyStatus["disabled"]).Should(Equal(4))

		By("Set PolicyEnforcement to always")

		status = kubectl.CiliumExec(ciliumPod, "cilium config PolicyEnforcement=always")
		status.ExpectSuccess()
		kubectl.CiliumEndpointWait(ciliumPod)

		endpoints, err = kubectl.CiliumEndpointsListByTag(ciliumPod, podFilter)
		Expect(err).Should(BeNil())
		Expect(endpoints.AreReady()).Should(BeTrue())
		policyStatus = endpoints.GetPolicyStatus()
		Expect(policyStatus["enabled"]).Should(Equal(4))
		Expect(policyStatus["disabled"]).Should(Equal(0))

		By("Return PolicyEnforcement to default")
		status = kubectl.CiliumExec(ciliumPod, "cilium config PolicyEnforcement=default")
		status.ExpectSuccess()
		kubectl.CiliumEndpointWait(ciliumPod)

		endpoints, err = kubectl.CiliumEndpointsListByTag(ciliumPod, podFilter)
		Expect(err).Should(BeNil())
		Expect(endpoints.AreReady()).Should(BeTrue())
		policyStatus = endpoints.GetPolicyStatus()
		Expect(policyStatus["enabled"]).Should(Equal(0))
		Expect(policyStatus["disabled"]).Should(Equal(4))
	}, 500)

	It("Policies", func() {
		appPods := getAppPods()
		clusterIP, err := kubectl.Get("default", "svc").Filter(
			"{.items[?(@.metadata.name == \"app1-service\")].spec.clusterIP}")
		logger.Infof("PolicyRulesTest: cluster service ip '%s'", clusterIP)
		Expect(err).Should(BeNil())

		ciliumPod, err := kubectl.GetCiliumPodOnNode("kube-system", "k8s1")
		Expect(err).Should(BeNil())

		status := kubectl.CiliumExec(ciliumPod, "cilium config PolicyEnforcement=default")
		status.ExpectSuccess()
		kubectl.CiliumEndpointWait(ciliumPod)

		By("Testing L3/L4 rules")

		eps := kubectl.CiliumEndpointPolicyVersion(ciliumPod)
		_, err = kubectl.CiliumImportPolicy("kube-system", l3Policy, 300)
		Expect(err).Should(BeNil())

		err = waitUntilEndpointUpdates(ciliumPod, eps, 4)
		Expect(err).Should(BeNil())
		epsStatus := helpers.WithTimeout(func() bool {
			endpoints, err := kubectl.CiliumEndpointsListByTag(ciliumPod, podFilter)
			if err != nil {
				return false
			}
			return endpoints.AreReady()
		}, "Could not get endpoints", &helpers.TimeoutConfig{Timeout: 100})

		Expect(epsStatus).Should(BeNil())
		appPods = getAppPods()

		endpoints, err := kubectl.CiliumEndpointsListByTag(ciliumPod, podFilter)
		policyStatus := endpoints.GetPolicyStatus()
		Expect(policyStatus["enabled"]).Should(Equal(2))
		Expect(policyStatus["disabled"]).Should(Equal(2))

		trace := kubectl.CiliumExec(ciliumPod, fmt.Sprintf(
			"cilium policy trace --src-k8s-pod default:%s --dst-k8s-pod default:%s --dport 80",
			appPods["app2"], appPods["app1"]))
		trace.ExpectSuccess(trace.Output().String())
		Expect(trace.Output().String()).Should(ContainSubstring("Final verdict: ALLOWED"))

		trace = kubectl.CiliumExec(ciliumPod, fmt.Sprintf(
			"cilium policy trace --src-k8s-pod default:%s --dst-k8s-pod default:%s",
			appPods["app3"], appPods["app1"]))
		trace.ExpectSuccess()
		Expect(trace.Output().String()).Should(ContainSubstring("Final verdict: DENIED"))

		_, err = kubectl.Exec(
			"default", appPods["app2"], fmt.Sprintf("curl http://%s/public", clusterIP))
		Expect(err).Should(BeNil())

		_, err = kubectl.Exec(
			"default", appPods["app3"], fmt.Sprintf("curl --fail -s http://%s/public", clusterIP))
		Expect(err).Should(HaveOccurred())

		eps = kubectl.CiliumEndpointPolicyVersion(ciliumPod)
		status = kubectl.Delete(l3Policy)
		status.ExpectSuccess()
		kubectl.CiliumEndpointWait(ciliumPod)

		//Only 1 endpoint is affected by L7 rule
		err = waitUntilEndpointUpdates(ciliumPod, eps, 4)
		Expect(err).Should(BeNil())

		By("Testing L7 Policy")
		//All Monkey testing in this section is on runtime

		eps = kubectl.CiliumEndpointPolicyVersion(ciliumPod)
		_, err = kubectl.CiliumImportPolicy("kube-system", l7Policy, 300)
		Expect(err).Should(BeNil())
		err = waitUntilEndpointUpdates(ciliumPod, eps, 4)
		Expect(err).Should(BeNil())

		appPods = getAppPods()

		_, err = kubectl.Exec(
			"default", appPods["app2"], fmt.Sprintf("curl http://%s/public", clusterIP))
		Expect(err).Should(BeNil())

		msg, err := kubectl.Exec(
			"default", appPods["app2"], fmt.Sprintf("curl --fail -s http://%s/private", clusterIP))
		Expect(err).Should(HaveOccurred(), msg)

		_, err = kubectl.Exec(
			"default", appPods["app3"], fmt.Sprintf("curl -s --fail http://%s/public", clusterIP))
		Expect(err).Should(HaveOccurred())

		msg, err = kubectl.Exec(
			"default", appPods["app3"], fmt.Sprintf("curl --fail -s http://%s/private", clusterIP))
		Expect(err).Should(HaveOccurred(), msg)

		eps = kubectl.CiliumEndpointPolicyVersion(ciliumPod)
		status = kubectl.Delete(l7Policy)
		status.ExpectSuccess()

		//Only 1 endpoint is affected by L7 rule
		err = waitUntilEndpointUpdates(ciliumPod, eps, 4)
		Expect(err).Should(BeNil())

		_, err = kubectl.Exec(
			"default", appPods["app3"], fmt.Sprintf("curl -s --fail http://%s/public", clusterIP))
		Expect(err).Should(BeNil())
	}, 500)

})
<commit_msg>Ginkgo - Add Combined output on k8st.Policies<commit_after>// Copyright 2017 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package k8sTest

import (
	"fmt"

	"github.com/cilium/cilium/test/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	log "github.com/sirupsen/logrus"
)

var _ = Describe("K8sPolicyTest", func() {

	var demoPath string
	var initialized bool
	var kubectl *helpers.Kubectl
	var l3Policy, l7Policy string
	var logger *log.Entry
	var path string
	var podFilter string

	initialize := func() {
		if initialized == true {
			return
		}

		logger = log.WithFields(log.Fields{"testName": "K8sPolicyTest"})
		logger.Info("Starting")
		kubectl = helpers.CreateKubectl("k8s1", logger)
		podFilter = "k8s:zgroup=testapp"

		//Manifest paths
		demoPath = fmt.Sprintf("%s/demo.yaml", kubectl.ManifestsPath())
		l3Policy = fmt.Sprintf("%s/l3_l4_policy.yaml", kubectl.ManifestsPath())
		l7Policy = fmt.Sprintf("%s/l7_policy.yaml", kubectl.ManifestsPath())

		path = fmt.Sprintf("%s/cilium_ds.yaml", kubectl.ManifestsPath())
		kubectl.Apply(path)
		status, err := kubectl.WaitforPods("kube-system", "-l k8s-app=cilium", 300)
		Expect(status).Should(BeTrue())
		Expect(err).Should(BeNil())
		initialized = true
	}

	BeforeEach(func() {
		initialize()
		// kubectl.CiliumPolicyDeleteAll("kube-system")
		kubectl.Apply(demoPath)
		_, err := kubectl.WaitforPods("default", "-l zgroup=testapp", 300)
		Expect(err).Should(BeNil())
	})

	AfterEach(func() {
		if CurrentGinkgoTestDescription().Failed {
			ciliumPod, _ := kubectl.GetCiliumPodOnNode("kube-system", "k8s1")
			kubectl.CiliumReport("kube-system", ciliumPod, []string{
				"cilium bpf tunnel list",
				"cilium endpoint list"})
		}

		kubectl.Delete(demoPath)
		// TO make sure that are not in place
		kubectl.Delete(l3Policy)
		kubectl.Delete(l7Policy)
	})

	waitUntilEndpointUpdates := func(pod string, eps map[string]int64, min int) error {
		body := func() bool {
			updated := 0
			newEps := kubectl.CiliumEndpointPolicyVersion(pod)
			for k, v := range newEps {
				if eps[k] < v {
					logger.Infof("Endpoint %s had version %d now %d", k, eps[k], v)
					updated++
				}
			}
			return updated >= min
		}
		err := helpers.WithTimeout(body, "No new version applied", &helpers.TimeoutConfig{Timeout: 100})
		return err
	}

	getAppPods := func() map[string]string {
		appPods := make(map[string]string)
		apps := []string{"app1", "app2", "app3"}
		for _, v := range apps {
			res, err := kubectl.GetPodNames("default", fmt.Sprintf("id=%s", v))
			Expect(err).Should(BeNil())
			appPods[v] = res[0]
			logger.Infof("PolicyRulesTest: pod='%s' assigned to '%s'", res[0], v)
		}
		return appPods
	}

	It("PolicyEnforcement Changes", func() {
		//This is a small test that check that everything is working in k8s. Full monkey testing
		// is in runtime/Policies
		ciliumPod, err := kubectl.GetCiliumPodOnNode("kube-system", "k8s1")
		Expect(err).Should(BeNil())

		status := kubectl.CiliumExec(ciliumPod, "cilium config PolicyEnforcement=default")
		status.ExpectSuccess()
		helpers.Sleep(5)
		kubectl.CiliumEndpointWait(ciliumPod)

		epsStatus := helpers.WithTimeout(func() bool {
			endpoints, err := kubectl.CiliumEndpointsListByTag(ciliumPod, podFilter)
			if err != nil {
				return false
			}
			return endpoints.AreReady()
		}, "Could not get endpoints", &helpers.TimeoutConfig{Timeout: 100})
		Expect(epsStatus).Should(BeNil())

		endpoints, err := kubectl.CiliumEndpointsListByTag(ciliumPod, podFilter)
		Expect(err).Should(BeNil())
		Expect(endpoints.AreReady()).Should(BeTrue())
		policyStatus := endpoints.GetPolicyStatus()
		Expect(policyStatus["enabled"]).Should(Equal(0))
		Expect(policyStatus["disabled"]).Should(Equal(4))

		By("Set PolicyEnforcement to always")

		status = kubectl.CiliumExec(ciliumPod, "cilium config PolicyEnforcement=always")
		status.ExpectSuccess()
		kubectl.CiliumEndpointWait(ciliumPod)

		endpoints, err = kubectl.CiliumEndpointsListByTag(ciliumPod, podFilter)
		Expect(err).Should(BeNil())
		Expect(endpoints.AreReady()).Should(BeTrue())
		policyStatus = endpoints.GetPolicyStatus()
		Expect(policyStatus["enabled"]).Should(Equal(4))
		Expect(policyStatus["disabled"]).Should(Equal(0))

		By("Return PolicyEnforcement to default")
		status = kubectl.CiliumExec(ciliumPod, "cilium config PolicyEnforcement=default")
		status.ExpectSuccess()
		kubectl.CiliumEndpointWait(ciliumPod)

		endpoints, err = kubectl.CiliumEndpointsListByTag(ciliumPod, podFilter)
		Expect(err).Should(BeNil())
		Expect(endpoints.AreReady()).Should(BeTrue())
		policyStatus = endpoints.GetPolicyStatus()
		Expect(policyStatus["enabled"]).Should(Equal(0))
		Expect(policyStatus["disabled"]).Should(Equal(4))
	}, 500)

	It("Policies", func() {
		appPods := getAppPods()
		clusterIP, err := kubectl.Get("default", "svc").Filter(
			"{.items[?(@.metadata.name == \"app1-service\")].spec.clusterIP}")
		logger.Infof("PolicyRulesTest: cluster service ip '%s'", clusterIP)
		Expect(err).Should(BeNil())

		ciliumPod, err := kubectl.GetCiliumPodOnNode("kube-system", "k8s1")
		Expect(err).Should(BeNil())

		status := kubectl.CiliumExec(ciliumPod, "cilium config PolicyEnforcement=default")
		status.ExpectSuccess()
		kubectl.CiliumEndpointWait(ciliumPod)

		By("Testing L3/L4 rules")

		eps := kubectl.CiliumEndpointPolicyVersion(ciliumPod)
		_, err = kubectl.CiliumImportPolicy("kube-system", l3Policy, 300)
		Expect(err).Should(BeNil())

		err = waitUntilEndpointUpdates(ciliumPod, eps, 4)
		Expect(err).Should(BeNil())
		epsStatus := helpers.WithTimeout(func() bool {
			endpoints, err := kubectl.CiliumEndpointsListByTag(ciliumPod, podFilter)
			if err != nil {
				return false
			}
			return endpoints.AreReady()
		}, "Could not get endpoints", &helpers.TimeoutConfig{Timeout: 100})

		Expect(epsStatus).Should(BeNil())
		appPods = getAppPods()

		endpoints, err := kubectl.CiliumEndpointsListByTag(ciliumPod, podFilter)
		policyStatus := endpoints.GetPolicyStatus()
		Expect(policyStatus["enabled"]).Should(Equal(2))
		Expect(policyStatus["disabled"]).Should(Equal(2))

		trace := kubectl.CiliumExec(ciliumPod, fmt.Sprintf(
			"cilium policy trace --src-k8s-pod default:%s --dst-k8s-pod default:%s --dport 80",
			appPods["app2"], appPods["app1"]))
		trace.ExpectSuccess(trace.CombineOutput().String())
		Expect(trace.Output().String()).Should(ContainSubstring("Final verdict: ALLOWED"))

		trace = kubectl.CiliumExec(ciliumPod, fmt.Sprintf(
			"cilium policy trace --src-k8s-pod default:%s --dst-k8s-pod default:%s",
			appPods["app3"], appPods["app1"]))
		trace.ExpectSuccess(trace.CombineOutput().String())
		Expect(trace.Output().String()).Should(ContainSubstring("Final verdict: DENIED"))

		_, err = kubectl.Exec(
			"default", appPods["app2"], fmt.Sprintf("curl http://%s/public", clusterIP))
		Expect(err).Should(BeNil())

		_, err = kubectl.Exec(
			"default", appPods["app3"], fmt.Sprintf("curl --fail -s http://%s/public", clusterIP))
		Expect(err).Should(HaveOccurred())

		eps = kubectl.CiliumEndpointPolicyVersion(ciliumPod)
		status = kubectl.Delete(l3Policy)
		status.ExpectSuccess()
		kubectl.CiliumEndpointWait(ciliumPod)

		//Only 1 endpoint is affected by L7 rule
		err = waitUntilEndpointUpdates(ciliumPod, eps, 4)
		Expect(err).Should(BeNil())

		By("Testing L7 Policy")
		//All Monkey testing in this section is on runtime

		eps = kubectl.CiliumEndpointPolicyVersion(ciliumPod)
		_, err = kubectl.CiliumImportPolicy("kube-system", l7Policy, 300)
		Expect(err).Should(BeNil())
		err = waitUntilEndpointUpdates(ciliumPod, eps, 4)
		Expect(err).Should(BeNil())

		appPods = getAppPods()

		_, err = kubectl.Exec(
			"default", appPods["app2"], fmt.Sprintf("curl http://%s/public", clusterIP))
		Expect(err).Should(BeNil())

		msg, err := kubectl.Exec(
			"default", appPods["app2"], fmt.Sprintf("curl --fail -s http://%s/private", clusterIP))
		Expect(err).Should(HaveOccurred(), msg)

		_, err = kubectl.Exec(
			"default", appPods["app3"], fmt.Sprintf("curl -s --fail http://%s/public", clusterIP))
		Expect(err).Should(HaveOccurred())

		msg, err = kubectl.Exec(
			"default", appPods["app3"], fmt.Sprintf("curl --fail -s http://%s/private", clusterIP))
		Expect(err).Should(HaveOccurred(), msg)

		eps = kubectl.CiliumEndpointPolicyVersion(ciliumPod)
		status = kubectl.Delete(l7Policy)
		status.ExpectSuccess()

		//Only 1 endpoint is affected by L7 rule
		err = waitUntilEndpointUpdates(ciliumPod, eps, 4)
		Expect(err).Should(BeNil())

		_, err = kubectl.Exec(
			"default", appPods["app3"], fmt.Sprintf("curl -s --fail http://%s/public", clusterIP))
		Expect(err).Should(BeNil())
	}, 500)

})
<|endoftext|>


<commit_before>// Package main provides ...
package basenode

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"os"

	"code.google.com/p/go-uuid/uuid"
	log "github.com/cihub/seelog"
)

var config = &Config{}
var host string
var port string

func init() {
	flag.StringVar(&host, "host", "localhost", "Server host/ip")
	flag.StringVar(&port, "port", "8282", "Server port")
}

func NewConfig() *Config {
	var config = &Config{}

	config.Host = host
	config.Port = port

	return config
}

func SetConfig(c *Config) {

	configFromFile := readConfigFromFile()
	c.Merge(configFromFile)

	if c.Uuid == "" {
		c.Uuid = uuid.New()
	}

	// Load logger
	logger, err := log.LoggerFromConfigAsFile("../logconfig.xml")
	if err != nil {
		panic(err)
	}
	log.ReplaceLogger(logger)

	config = c
	fmt.Println("config:", config)
	saveConfigToFile()
}

func saveConfigToFile() {
	configFile, err := os.Create("config.json")
	if err != nil {
		log.Error("creating config file", err.Error())
	}

	log.Info("Save config: ", config)
	var out bytes.Buffer
	b, err := json.MarshalIndent(config, "", "\t")
	if err != nil {
		log.Error("error marshal json", err)
	}
	json.Indent(&out, b, "", "\t")
	out.WriteTo(configFile)
}

func readConfigFromFile() *Config {
	configFile, err := os.Open("config.json")
	if err != nil {
		log.Error("opening config file", err.Error())
	}

	config := &Config{}
	jsonParser := json.NewDecoder(configFile)
	if err = jsonParser.Decode(&config); err != nil {
		log.Error("parsing config file", err.Error())
	}

	return config
}

type Config struct {
	Host string
	Port string
	Uuid string
}

func (c *Config) GetUuid() string {
	return c.Uuid
}

func (c *Config) Merge(c2 *Config) {

	if c.Host != c2.Host && c.Host == "localhost" {
		c.Host = c2.Host
	}
	if c.Port != c2.Port && c.Port == "8282" {
		c.Port = c2.Port
	}

	c.Uuid = c2.Uuid

}
<commit_msg>Fixed config prio again<commit_after>// Package main provides ...
package basenode

import (
	"bytes"
	"encoding/json"
	"flag"
	"os"

	"code.google.com/p/go-uuid/uuid"
	log "github.com/cihub/seelog"
)

var config = &Config{}
var host string
var port string

func init() {
	flag.StringVar(&host, "host", "localhost", "Server host/ip")
	flag.StringVar(&port, "port", "8282", "Server port")
}

func NewConfig() *Config {
	var config = &Config{}

	config.Host = host
	config.Port = port

	return config
}

func SetConfig(c *Config) {

	configFromFile := readConfigFromFile()
	c.Merge(configFromFile)

	if c.Uuid == "" {
		c.Uuid = uuid.New()
	}

	// Load logger
	logger, err := log.LoggerFromConfigAsFile("../logconfig.xml")
	if err != nil {
		panic(err)
	}
	log.ReplaceLogger(logger)

	config = c

	saveConfigToFile()
}

func saveConfigToFile() {
	configFile, err := os.Create("config.json")
	if err != nil {
		log.Error("creating config file", err.Error())
	}

	log.Info("Save config: ", config)
	var out bytes.Buffer
	b, err := json.MarshalIndent(config, "", "\t")
	if err != nil {
		log.Error("error marshal json", err)
	}
	json.Indent(&out, b, "", "\t")
	out.WriteTo(configFile)
}

func readConfigFromFile() *Config {
	configFile, err := os.Open("config.json")
	if err != nil {
		log.Error("opening config file", err.Error())
	}

	config := &Config{}
	jsonParser := json.NewDecoder(configFile)
	if err = jsonParser.Decode(&config); err != nil {
		log.Error("parsing config file", err.Error())
	}

	return config
}

type Config struct {
	Host string
	Port string
	Uuid string
}

func (c *Config) GetUuid() string {
	return c.Uuid
}

func (c *Config) Merge(c2 *Config) {

	if c.Host != c2.Host && c.Host == "localhost" && c2.Host != "" {
		c.Host = c2.Host
	}
	if c.Port != c2.Port && c.Port == "8282" && c2.Port != "" {
		c.Port = c2.Port
	}

	c.Uuid = c2.Uuid

}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package x509

import (
	"crypto/dsa"
	"crypto/ecdsa"
	"crypto/rsa"
	"encoding/asn1"
	"encoding/json"

	"time"

	"github.com/asaskevich/govalidator"
	"github.com/zmap/zgrab/ztools/keys"
	"github.com/zmap/zgrab/ztools/x509/pkix"
)

type auxKeyUsage struct {
	DigitalSignature  bool   `json:"digital_signature,omitempty"`
	ContentCommitment bool   `json:"content_commitment,omitempty"`
	KeyEncipherment   bool   `json:"key_encipherment,omitempty"`
	DataEncipherment  bool   `json:"data_encipherment,omitempty"`
	KeyAgreement      bool   `json:"key_agreement,omitempty"`
	CertificateSign   bool   `json:"certificate_sign,omitempty"`
	CRLSign           bool   `json:"crl_sign,omitempty"`
	EncipherOnly      bool   `json:"encipher_only,omitempty"`
	DecipherOnly      bool   `json:"decipher_only,omitempty"`
	Value             uint32 `json:"value"`
}

// MarshalJSON implements the json.Marshaler interface
func (k KeyUsage) MarshalJSON() ([]byte, error) {
	var enc auxKeyUsage
	enc.Value = uint32(k)
	if k&KeyUsageDigitalSignature > 0 {
		enc.DigitalSignature = true
	}
	if k&KeyUsageContentCommitment > 0 {
		enc.ContentCommitment = true
	}
	if k&KeyUsageKeyEncipherment > 0 {
		enc.KeyEncipherment = true
	}
	if k&KeyUsageDataEncipherment > 0 {
		enc.DataEncipherment = true
	}
	if k&KeyUsageKeyAgreement > 0 {
		enc.KeyAgreement = true
	}
	if k&KeyUsageCertSign > 0 {
		enc.CertificateSign = true
	}
	if k&KeyUsageCRLSign > 0 {
		enc.CRLSign = true
	}
	if k&KeyUsageEncipherOnly > 0 {
		enc.EncipherOnly = true
	}
	if k&KeyUsageDecipherOnly > 0 {
		enc.DecipherOnly = true
	}
	return json.Marshal(&enc)
}

// UnmarshalJSON implements the json.Unmarshler interface
func (k *KeyUsage) UnmarshalJSON(b []byte) error {
	var aux auxKeyUsage
	if err := json.Unmarshal(b, &aux); err != nil {
		return err
	}
	// TODO: validate the flags match
	v := int(aux.Value)
	*k = KeyUsage(v)
	return nil
}

type auxSignatureAlgorithm struct {
	Name string      `json:"name,omitempty"`
	OID  pkix.AuxOID `json:"oid"`
}

// MarshalJSON implements the json.Marshaler interface
func (s *SignatureAlgorithm) MarshalJSON() ([]byte, error) {
	aux := auxSignatureAlgorithm{
		Name: s.String(),
	}
	for _, val := range signatureAlgorithmDetails {
		if val.algo == *s {
			aux.OID = make([]int, len(val.oid))
			for idx := range val.oid {
				aux.OID[idx] = val.oid[idx]
			}
		}
	}
	return json.Marshal(&aux)
}

// UnmarshalJSON implements the json.Unmarshler interface
func (s *SignatureAlgorithm) UnmarshalJSON(b []byte) error {
	var aux auxSignatureAlgorithm
	if err := json.Unmarshal(b, &aux); err != nil {
		return err
	}
	*s = UnknownSignatureAlgorithm
	oid := asn1.ObjectIdentifier(aux.OID.AsSlice())
	for _, val := range signatureAlgorithmDetails {
		if val.oid.Equal(oid) {
			*s = val.algo
			break
		}
	}
	return nil
}

type auxPublicKeyAlgorithm struct {
	Name string      `json:"name,omitempty"`
	OID  pkix.AuxOID `json:"oid"`
}

// MarshalJSON implements the json.Marshaler interface
func (p *PublicKeyAlgorithm) MarshalJSON() ([]byte, error) {
	aux := auxPublicKeyAlgorithm{
		Name: p.String(),
	}
	return json.Marshal(&aux)
}

// UnmarshalJSON implements the json.Unmarshaler interface
func (p *PublicKeyAlgorithm) UnmarshalJSON(b []byte) error {
	var aux auxPublicKeyAlgorithm
	if err := json.Unmarshal(b, &aux); err != nil {
		return err
	}
	panic("unimplemented")
}

type auxValidity struct {
	Start          string `json:"start"`
	End            string `json:"end"`
	ValidityPeriod int    `json:"length"`
}

func (v *validity) MarshalJSON() ([]byte, error) {
	aux := auxValidity{
		Start:          v.NotBefore.UTC().Format(time.RFC3339),
		End:            v.NotAfter.UTC().Format(time.RFC3339),
		ValidityPeriod: int(v.NotAfter.Sub(v.NotBefore).Seconds()),
	}
	return json.Marshal(&aux)
}

func (v *validity) UnmarshalJSON(b []byte) error {
	var aux auxValidity
	if err := json.Unmarshal(b, &aux); err != nil {
		return err
	}
	var err error
	if v.NotBefore, err = time.Parse(time.RFC3339, aux.Start); err != nil {
		return err
	}
	if v.NotAfter, err = time.Parse(time.RFC3339, aux.End); err != nil {
		return err
	}

	return nil
}

type jsonSubjectKeyInfo struct {
	KeyAlgorithm    PublicKeyAlgorithm     `json:"key_algorithm"`
	RSAPublicKey    *keys.RSAPublicKey     `json:"rsa_public_key,omitempty"`
	DSAPublicKey    interface{}            `json:"dsa_public_key,omitempty"`
	ECDSAPublicKey  interface{}            `json:"ecdsa_public_key,omitempty"`
	SPKIFingerprint CertificateFingerprint `json:"fingerprint_sha256"`
}

type jsonSignature struct {
	SignatureAlgorithm SignatureAlgorithm `json:"signature_algorithm"`
	Value              []byte             `json:"value"`
	Valid              bool               `json:"valid"`
	SelfSigned         bool               `json:"self_signed"`
}

type fullValidity struct {
	validity
	ValidityPeriod int
}

type jsonCertificate struct {
	Version                   int                          `json:"version"`
	SerialNumber              string                       `json:"serial_number"`
	SignatureAlgorithm        SignatureAlgorithm           `json:"signature_algorithm"`
	Issuer                    pkix.Name                    `json:"issuer"`
	IssuerDN                  string                       `json:"issuer_dn,omitempty"`
	Validity                  fullValidity                 `json:"validity"`
	Subject                   pkix.Name                    `json:"subject"`
	SubjectDN                 string                       `json:"subject_dn,omitempty"`
	SubjectKeyInfo            jsonSubjectKeyInfo           `json:"subject_key_info"`
	Extensions                *CertificateExtensions       `json:"extensions,omitempty"`
	UnknownExtensions         UnknownCertificateExtensions `json:"unknown_extensions,omitempty"`
	Signature                 jsonSignature                `json:"signature"`
	FingerprintMD5            CertificateFingerprint       `json:"fingerprint_md5"`
	FingerprintSHA1           CertificateFingerprint       `json:"fingerprint_sha1"`
	FingerprintSHA256         CertificateFingerprint       `json:"fingerprint_sha256"`
	SPKISubjectFingerprint    CertificateFingerprint       `json:"spki_subject_fingerprint"`
	TBSCertificateFingerprint CertificateFingerprint       `json:"tbs_fingerprint"`
	ValidationLevel           CertValidationLevel          `json:"validation_level"`
	Names                     []string                     `json:"names"`
}

func (c *Certificate) MarshalJSON() ([]byte, error) {
	// Fill out the certificate
	jc := new(jsonCertificate)
	jc.Version = c.Version
	jc.SerialNumber = c.SerialNumber.String()
	jc.SignatureAlgorithm = c.SignatureAlgorithm
	jc.Issuer = c.Issuer
	jc.IssuerDN = c.Issuer.String()
	jc.Validity.NotBefore = c.NotBefore
	jc.Validity.NotAfter = c.NotAfter
	jc.Validity.ValidityPeriod = c.ValidityPeriod
	jc.Subject = c.Subject
	jc.SubjectDN = c.Subject.String()
	jc.SubjectKeyInfo.KeyAlgorithm = c.PublicKeyAlgorithm

	// Include all subject names, DNS names there are
	for _, obj := range c.Subject.Names {

		switch name := obj.Value.(type) {
		case string:

			flag := true

			if name[0] == '*' {
				flag = govalidator.IsURL(name[2:])
			} else {
				flag = govalidator.IsURL(name)
			}

			// Check that this is actually a url and not something else
			if flag {
				jc.Names = append(jc.Names, name)
			}
		}
	}

	for _, name := range c.DNSNames {

		flag := true

		if name[0] == '*' {
			flag = govalidator.IsURL(name[2:])
		} else {
			flag = govalidator.IsURL(name)
		}

		if flag {
			jc.Names = append(jc.Names, name)
		}
	}

	// Pull out the key
	keyMap := make(map[string]interface{})

	switch key := c.PublicKey.(type) {
	case *rsa.PublicKey:
		rsaKey := new(keys.RSAPublicKey)
		rsaKey.PublicKey = key
		jc.SubjectKeyInfo.RSAPublicKey = rsaKey
	case *dsa.PublicKey:
		keyMap["p"] = key.P.Bytes()
		keyMap["q"] = key.Q.Bytes()
		keyMap["g"] = key.G.Bytes()
		keyMap["y"] = key.Y.Bytes()
		jc.SubjectKeyInfo.DSAPublicKey = keyMap
	case *ecdsa.PublicKey:
		params := key.Params()
		keyMap["p"] = params.P.Bytes()
		keyMap["n"] = params.N.Bytes()
		keyMap["b"] = params.B.Bytes()
		keyMap["gx"] = params.Gx.Bytes()
		keyMap["gy"] = params.Gy.Bytes()
		keyMap["x"] = key.X.Bytes()
		keyMap["y"] = key.Y.Bytes()
		jc.SubjectKeyInfo.ECDSAPublicKey = keyMap
	case *AugmentedECDSA:
		pub := key.Pub

		keyMap["pub"] = key.Raw.Bytes

		params := pub.Params()
		keyMap["p"] = params.P.Bytes()
		keyMap["n"] = params.N.Bytes()
		keyMap["b"] = params.B.Bytes()
		keyMap["gx"] = params.Gx.Bytes()
		keyMap["gy"] = params.Gy.Bytes()
		keyMap["x"] = pub.X.Bytes()
		keyMap["y"] = pub.Y.Bytes()

		keyMap["asn1_oid"] = c.SignatureAlgorithmOID

		jc.SubjectKeyInfo.ECDSAPublicKey = keyMap
	}

	jc.Extensions, jc.UnknownExtensions = c.jsonifyExtensions()

	// TODO: Handle the fact this might not match
	jc.Signature.SignatureAlgorithm = jc.SignatureAlgorithm
	jc.Signature.Value = c.Signature
	jc.Signature.Valid = c.validSignature
	if c.Subject.CommonName == c.Issuer.CommonName {
		jc.Signature.SelfSigned = true
	}
	jc.FingerprintMD5 = c.FingerprintMD5
	jc.FingerprintSHA1 = c.FingerprintSHA1
	jc.FingerprintSHA256 = c.FingerprintSHA256
	jc.SPKISubjectFingerprint = c.SPKISubjectFingerprint
	jc.TBSCertificateFingerprint = c.TBSCertificateFingerprint
	jc.ValidationLevel = c.ValidationLevel

	return json.Marshal(jc)
}
<commit_msg>Added bounds checking to avoid empty subject fields.<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package x509

import (
	"crypto/dsa"
	"crypto/ecdsa"
	"crypto/rsa"
	"encoding/asn1"
	"encoding/json"

	"time"

	"github.com/asaskevich/govalidator"
	"github.com/zmap/zgrab/ztools/keys"
	"github.com/zmap/zgrab/ztools/x509/pkix"
)

type auxKeyUsage struct {
	DigitalSignature  bool   `json:"digital_signature,omitempty"`
	ContentCommitment bool   `json:"content_commitment,omitempty"`
	KeyEncipherment   bool   `json:"key_encipherment,omitempty"`
	DataEncipherment  bool   `json:"data_encipherment,omitempty"`
	KeyAgreement      bool   `json:"key_agreement,omitempty"`
	CertificateSign   bool   `json:"certificate_sign,omitempty"`
	CRLSign           bool   `json:"crl_sign,omitempty"`
	EncipherOnly      bool   `json:"encipher_only,omitempty"`
	DecipherOnly      bool   `json:"decipher_only,omitempty"`
	Value             uint32 `json:"value"`
}

// MarshalJSON implements the json.Marshaler interface
func (k KeyUsage) MarshalJSON() ([]byte, error) {
	var enc auxKeyUsage
	enc.Value = uint32(k)
	if k&KeyUsageDigitalSignature > 0 {
		enc.DigitalSignature = true
	}
	if k&KeyUsageContentCommitment > 0 {
		enc.ContentCommitment = true
	}
	if k&KeyUsageKeyEncipherment > 0 {
		enc.KeyEncipherment = true
	}
	if k&KeyUsageDataEncipherment > 0 {
		enc.DataEncipherment = true
	}
	if k&KeyUsageKeyAgreement > 0 {
		enc.KeyAgreement = true
	}
	if k&KeyUsageCertSign > 0 {
		enc.CertificateSign = true
	}
	if k&KeyUsageCRLSign > 0 {
		enc.CRLSign = true
	}
	if k&KeyUsageEncipherOnly > 0 {
		enc.EncipherOnly = true
	}
	if k&KeyUsageDecipherOnly > 0 {
		enc.DecipherOnly = true
	}
	return json.Marshal(&enc)
}

// UnmarshalJSON implements the json.Unmarshler interface
func (k *KeyUsage) UnmarshalJSON(b []byte) error {
	var aux auxKeyUsage
	if err := json.Unmarshal(b, &aux); err != nil {
		return err
	}
	// TODO: validate the flags match
	v := int(aux.Value)
	*k = KeyUsage(v)
	return nil
}

type auxSignatureAlgorithm struct {
	Name string      `json:"name,omitempty"`
	OID  pkix.AuxOID `json:"oid"`
}

// MarshalJSON implements the json.Marshaler interface
func (s *SignatureAlgorithm) MarshalJSON() ([]byte, error) {
	aux := auxSignatureAlgorithm{
		Name: s.String(),
	}
	for _, val := range signatureAlgorithmDetails {
		if val.algo == *s {
			aux.OID = make([]int, len(val.oid))
			for idx := range val.oid {
				aux.OID[idx] = val.oid[idx]
			}
		}
	}
	return json.Marshal(&aux)
}

// UnmarshalJSON implements the json.Unmarshler interface
func (s *SignatureAlgorithm) UnmarshalJSON(b []byte) error {
	var aux auxSignatureAlgorithm
	if err := json.Unmarshal(b, &aux); err != nil {
		return err
	}
	*s = UnknownSignatureAlgorithm
	oid := asn1.ObjectIdentifier(aux.OID.AsSlice())
	for _, val := range signatureAlgorithmDetails {
		if val.oid.Equal(oid) {
			*s = val.algo
			break
		}
	}
	return nil
}

type auxPublicKeyAlgorithm struct {
	Name string      `json:"name,omitempty"`
	OID  pkix.AuxOID `json:"oid"`
}

// MarshalJSON implements the json.Marshaler interface
func (p *PublicKeyAlgorithm) MarshalJSON() ([]byte, error) {
	aux := auxPublicKeyAlgorithm{
		Name: p.String(),
	}
	return json.Marshal(&aux)
}

// UnmarshalJSON implements the json.Unmarshaler interface
func (p *PublicKeyAlgorithm) UnmarshalJSON(b []byte) error {
	var aux auxPublicKeyAlgorithm
	if err := json.Unmarshal(b, &aux); err != nil {
		return err
	}
	panic("unimplemented")
}

type auxValidity struct {
	Start          string `json:"start"`
	End            string `json:"end"`
	ValidityPeriod int    `json:"length"`
}

func (v *validity) MarshalJSON() ([]byte, error) {
	aux := auxValidity{
		Start:          v.NotBefore.UTC().Format(time.RFC3339),
		End:            v.NotAfter.UTC().Format(time.RFC3339),
		ValidityPeriod: int(v.NotAfter.Sub(v.NotBefore).Seconds()),
	}
	return json.Marshal(&aux)
}

func (v *validity) UnmarshalJSON(b []byte) error {
	var aux auxValidity
	if err := json.Unmarshal(b, &aux); err != nil {
		return err
	}
	var err error
	if v.NotBefore, err = time.Parse(time.RFC3339, aux.Start); err != nil {
		return err
	}
	if v.NotAfter, err = time.Parse(time.RFC3339, aux.End); err != nil {
		return err
	}

	return nil
}

type jsonSubjectKeyInfo struct {
	KeyAlgorithm    PublicKeyAlgorithm     `json:"key_algorithm"`
	RSAPublicKey    *keys.RSAPublicKey     `json:"rsa_public_key,omitempty"`
	DSAPublicKey    interface{}            `json:"dsa_public_key,omitempty"`
	ECDSAPublicKey  interface{}            `json:"ecdsa_public_key,omitempty"`
	SPKIFingerprint CertificateFingerprint `json:"fingerprint_sha256"`
}

type jsonSignature struct {
	SignatureAlgorithm SignatureAlgorithm `json:"signature_algorithm"`
	Value              []byte             `json:"value"`
	Valid              bool               `json:"valid"`
	SelfSigned         bool               `json:"self_signed"`
}

type fullValidity struct {
	validity
	ValidityPeriod int
}

type jsonCertificate struct {
	Version                   int                          `json:"version"`
	SerialNumber              string                       `json:"serial_number"`
	SignatureAlgorithm        SignatureAlgorithm           `json:"signature_algorithm"`
	Issuer                    pkix.Name                    `json:"issuer"`
	IssuerDN                  string                       `json:"issuer_dn,omitempty"`
	Validity                  fullValidity                 `json:"validity"`
	Subject                   pkix.Name                    `json:"subject"`
	SubjectDN                 string                       `json:"subject_dn,omitempty"`
	SubjectKeyInfo            jsonSubjectKeyInfo           `json:"subject_key_info"`
	Extensions                *CertificateExtensions       `json:"extensions,omitempty"`
	UnknownExtensions         UnknownCertificateExtensions `json:"unknown_extensions,omitempty"`
	Signature                 jsonSignature                `json:"signature"`
	FingerprintMD5            CertificateFingerprint       `json:"fingerprint_md5"`
	FingerprintSHA1           CertificateFingerprint       `json:"fingerprint_sha1"`
	FingerprintSHA256         CertificateFingerprint       `json:"fingerprint_sha256"`
	SPKISubjectFingerprint    CertificateFingerprint       `json:"spki_subject_fingerprint"`
	TBSCertificateFingerprint CertificateFingerprint       `json:"tbs_fingerprint"`
	ValidationLevel           CertValidationLevel          `json:"validation_level"`
	Names                     []string                     `json:"names"`
}

func (c *Certificate) MarshalJSON() ([]byte, error) {
	// Fill out the certificate
	jc := new(jsonCertificate)
	jc.Version = c.Version
	jc.SerialNumber = c.SerialNumber.String()
	jc.SignatureAlgorithm = c.SignatureAlgorithm
	jc.Issuer = c.Issuer
	jc.IssuerDN = c.Issuer.String()
	jc.Validity.NotBefore = c.NotBefore
	jc.Validity.NotAfter = c.NotAfter
	jc.Validity.ValidityPeriod = c.ValidityPeriod
	jc.Subject = c.Subject
	jc.SubjectDN = c.Subject.String()
	jc.SubjectKeyInfo.KeyAlgorithm = c.PublicKeyAlgorithm

	// Include all subject names, DNS names there are
	for _, obj := range c.Subject.Names {

		switch name := obj.Value.(type) {
		case string:

			flag := false

			if len(name) > 2 && name[0] == '*' {
				flag = govalidator.IsURL(name[2:])
			} else {
				flag = govalidator.IsURL(name)
			}

			// Check that this is actually a url and not something else
			if flag {
				jc.Names = append(jc.Names, name)
			}
		}
	}

	for _, name := range c.DNSNames {

		flag := false

		if len(name) > 2 && name[0] == '*' {
			flag = govalidator.IsURL(name[2:])
		} else {
			flag = govalidator.IsURL(name)
		}

		if flag {
			jc.Names = append(jc.Names, name)
		}
	}

	// Pull out the key
	keyMap := make(map[string]interface{})

	switch key := c.PublicKey.(type) {
	case *rsa.PublicKey:
		rsaKey := new(keys.RSAPublicKey)
		rsaKey.PublicKey = key
		jc.SubjectKeyInfo.RSAPublicKey = rsaKey
	case *dsa.PublicKey:
		keyMap["p"] = key.P.Bytes()
		keyMap["q"] = key.Q.Bytes()
		keyMap["g"] = key.G.Bytes()
		keyMap["y"] = key.Y.Bytes()
		jc.SubjectKeyInfo.DSAPublicKey = keyMap
	case *ecdsa.PublicKey:
		params := key.Params()
		keyMap["p"] = params.P.Bytes()
		keyMap["n"] = params.N.Bytes()
		keyMap["b"] = params.B.Bytes()
		keyMap["gx"] = params.Gx.Bytes()
		keyMap["gy"] = params.Gy.Bytes()
		keyMap["x"] = key.X.Bytes()
		keyMap["y"] = key.Y.Bytes()
		jc.SubjectKeyInfo.ECDSAPublicKey = keyMap
	case *AugmentedECDSA:
		pub := key.Pub

		keyMap["pub"] = key.Raw.Bytes

		params := pub.Params()
		keyMap["p"] = params.P.Bytes()
		keyMap["n"] = params.N.Bytes()
		keyMap["b"] = params.B.Bytes()
		keyMap["gx"] = params.Gx.Bytes()
		keyMap["gy"] = params.Gy.Bytes()
		keyMap["x"] = pub.X.Bytes()
		keyMap["y"] = pub.Y.Bytes()

		keyMap["asn1_oid"] = c.SignatureAlgorithmOID

		jc.SubjectKeyInfo.ECDSAPublicKey = keyMap
	}

	jc.Extensions, jc.UnknownExtensions = c.jsonifyExtensions()

	// TODO: Handle the fact this might not match
	jc.Signature.SignatureAlgorithm = jc.SignatureAlgorithm
	jc.Signature.Value = c.Signature
	jc.Signature.Valid = c.validSignature
	if c.Subject.CommonName == c.Issuer.CommonName {
		jc.Signature.SelfSigned = true
	}
	jc.FingerprintMD5 = c.FingerprintMD5
	jc.FingerprintSHA1 = c.FingerprintSHA1
	jc.FingerprintSHA256 = c.FingerprintSHA256
	jc.SPKISubjectFingerprint = c.SPKISubjectFingerprint
	jc.TBSCertificateFingerprint = c.TBSCertificateFingerprint
	jc.ValidationLevel = c.ValidationLevel

	return json.Marshal(jc)
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"github.com/mxk/go-imap/imap"
)

// IMAP date format (rfc3501)
const IMAPDateFormat = "_2-Jan-2006"

// IMAPSource provides support for traversing mails of an IMAP mailbox.
type IMAPSource struct {
	*IMAPConnection
	callbackFunc      IMAPSourceCallback
	deletePlainCopies bool
	minAge            time.Duration
}

// IMAPSourceCallback is the type for the IMAPSource callback parameter
type IMAPSourceCallback func(imap.FlagSet, *time.Time, imap.Literal) error

// The duration of a day
const Day = 24 * time.Hour

// NewIMAPSource returns a new IMAPSource instance.
func NewIMAPSource(deletePlainCopies bool, minAgeInDays time.Duration) *IMAPSource {
	return &IMAPSource{
		IMAPConnection:    NewIMAPConnection(),
		deletePlainCopies: deletePlainCopies,
		minAge:            minAgeInDays * Day,
	}
}

// Iterate loops through the given mailbox, filters the results by the currently
// static search filter and invokes the callback for each message.
func (w *IMAPSource) Iterate(mailbox string, callbackFunc IMAPSourceCallback) error {
	w.callbackFunc = callbackFunc
	logger.Debugf("selecting mailbox '%s'", mailbox)
	_, err := w.conn.Select(mailbox, false /* read-write */)
	if err != nil {
		logger.Errorf("failed to select mailbox: %s", err)
		return err
	}
	date := time.Now().Add(-w.minAge)
	dateStr := date.Format(IMAPDateFormat)
	searchFilter := ("UNDELETED SEEN UNFLAGGED (NOT HEADER X-Lemoncrypt \"\") " +
		"(OR SENTBEFORE " + dateStr + " BEFORE " + dateStr + ")")
	logger.Debugf("searching for: %s", searchFilter)
	cmd, err := imap.Wait(w.conn.Search(searchFilter))
	if err != nil {
		logger.Errorf("search failed: %s", err)
		return err
	}
	logger.Debugf("found %d result sets", len(cmd.Data))
	for idx, rsp := range cmd.Data {
		results := rsp.SearchResults()
		logger.Debugf("result set #%d contains %d results", idx, len(results))
		if len(results) == 0 {
			continue
		}
		_ = w.fetchIDs(results)
	}
	logger.Debugf("finally removing mail marked for deletion")
	_, err = imap.Wait(w.conn.Expunge(nil))
	if err != nil {
		logger.Errorf("failed to remove mail marked for deletion: %s", err)
	}
	return err
}

// fetchIDs downloads the messages with the given IDs and invokes the callback for
// each message.
func (w *IMAPSource) fetchIDs(ids []uint32) error {
	for _, id := range ids {
		err := w.fetchID(id)
		if err != nil {
			logger.Errorf("fetchID failed: %s", err)
			return err
		}
	}
	return nil
}

// fetchID downloads the message with the given ID and invokes the callback
// for the message.
func (w *IMAPSource) fetchID(id uint32) error {
	set, _ := imap.NewSeqSet("")
	set.AddNum(id)
	cmd, err := imap.Wait(w.conn.Fetch(set, "RFC822", "UID", "FLAGS", "INTERNALDATE"))
	if err != nil {
		logger.Errorf("FETCH failed: %s", err)
		return err
	}
	for _, rsp := range cmd.Data {
		_ = w.handleMessage(rsp)
	}

	logger.Debugf("FETCH completed without errors")
	return err

}

// handleMessage processes one message, invokes the callback and deletes it on
// success.
func (w *IMAPSource) handleMessage(rsp *imap.Response) error {
	msgInfo := rsp.MessageInfo()
	err := w.invokeMessageCallback(msgInfo)
	if err != nil {
		return err
	}
	uid := imap.AsNumber(msgInfo.Attrs["UID"])
	return w.deleteMessage(uid)
}

// deleteMessage marks the message with the given UID for deletion.
func (w *IMAPSource) deleteMessage(uid uint32) error {
	if !w.deletePlainCopies {
		return nil
	}

	logger.Debugf("marking message uid=%d for deletion", uid)
	set, _ := imap.NewSeqSet("")
	set.AddNum(uid)
	_, err := imap.Wait(w.conn.UIDStore(set, "+FLAGS", "(\\Deleted)"))
	if err != nil {
		logger.Errorf("failed to mark uid=%d for deletion: %s", uid, err)
		return err
	}
	return nil
}

// invokeMessageCallback extracts the relevant data from the passed FETCH response
// and invokes the user-provided callback.
func (w *IMAPSource) invokeMessageCallback(msgInfo *imap.MessageInfo) error {
	logger.Debugf("handling mail uid=%d", msgInfo.Attrs["UID"])
	flags := imap.AsFlagSet(msgInfo.Attrs["FLAGS"])
	idate := imap.AsDateTime(msgInfo.Attrs["INTERNALDATE"])
	mailBytes := imap.AsBytes(msgInfo.Attrs["RFC822"])
	mailLiteral := imap.NewLiteral(mailBytes)
	logger.Debugf("invoking message transformer")
	err := w.callbackFunc(flags, &idate, mailLiteral)
	if err == nil {
		logger.Debugf("message transformation successful")
	} else {
		logger.Warningf("message transformation failed: %s", err)
	}
	return err
}
<commit_msg>imap: pre-fetch next message for performance reasons<commit_after>package main

import (
	"time"

	"github.com/mxk/go-imap/imap"
)

// IMAP date format (rfc3501)
const IMAPDateFormat = "_2-Jan-2006"

// IMAPSource provides support for traversing mails of an IMAP mailbox.
type IMAPSource struct {
	*IMAPConnection
	callbackFunc      IMAPSourceCallback
	deletePlainCopies bool
	minAge            time.Duration
}

// IMAPSourceCallback is the type for the IMAPSource callback parameter
type IMAPSourceCallback func(imap.FlagSet, *time.Time, imap.Literal) error

// The duration of a day
const Day = 24 * time.Hour

// NewIMAPSource returns a new IMAPSource instance.
func NewIMAPSource(deletePlainCopies bool, minAgeInDays time.Duration) *IMAPSource {
	return &IMAPSource{
		IMAPConnection:    NewIMAPConnection(),
		deletePlainCopies: deletePlainCopies,
		minAge:            minAgeInDays * Day,
	}
}

// Iterate loops through the given mailbox, filters the results by the currently
// static search filter and invokes the callback for each message.
func (w *IMAPSource) Iterate(mailbox string, callbackFunc IMAPSourceCallback) error {
	w.callbackFunc = callbackFunc
	logger.Debugf("selecting mailbox '%s'", mailbox)
	_, err := w.conn.Select(mailbox, false /* read-write */)
	if err != nil {
		logger.Errorf("failed to select mailbox: %s", err)
		return err
	}
	date := time.Now().Add(-w.minAge)
	dateStr := date.Format(IMAPDateFormat)
	searchFilter := ("UNDELETED SEEN UNFLAGGED (NOT HEADER X-Lemoncrypt \"\") " +
		"(OR SENTBEFORE " + dateStr + " BEFORE " + dateStr + ")")
	logger.Debugf("searching for: %s", searchFilter)
	cmd, err := imap.Wait(w.conn.Search(searchFilter))
	if err != nil {
		logger.Errorf("search failed: %s", err)
		return err
	}
	logger.Debugf("found %d result sets", len(cmd.Data))
	for idx, rsp := range cmd.Data {
		results := rsp.SearchResults()
		logger.Debugf("result set #%d contains %d results", idx, len(results))
		if len(results) == 0 {
			continue
		}
		_ = w.fetchIDs(results)
	}
	logger.Debugf("finally removing mail marked for deletion")
	_, err = imap.Wait(w.conn.Expunge(nil))
	if err != nil {
		logger.Errorf("failed to remove mail marked for deletion: %s", err)
	}
	return err
}

// fetchIDs downloads the messages with the given IDs and invokes the callback for
// each message.
// In order to improve performance, we always prefetch one message so that the IMAP
// server action and network transfer can happen while we handle the previous message.
func (w *IMAPSource) fetchIDs(ids []uint32) error {
	var cur *imap.Command
	for idx, id := range ids {
		if idx == 0 {
			var err error
			cur, err = w.startFetch(id)
			if err != nil {
				logger.Errorf("pre-fetch request failed: %s", err)
				return err
			}
			// don't start working on the first message just yet,
			// we pre-fetch the second message in the next loop
			//before we start working.
			continue
		}
		next, err := w.startFetch(id)
		if err != nil {
			logger.Errorf("fetch request failed: %s", err)
			return err
		}
		err = w.handleFetchResult(cur)
		if err != nil {
			logger.Errorf("fetching failed: %s", err)
			return err
		}
		cur = next
	}
	// as our loop is one FETCH ahead, we still have to handle the last message here:
	if cur == nil {
		// loop didn't run
		return nil
	}
	err := w.handleFetchResult(cur)
	if err != nil {
		logger.Errorf("fetching failed in last loop: %s", err)
		return err
	}
	return nil
}

func (w *IMAPSource) startFetch(id uint32) (*imap.Command, error) {
	set, _ := imap.NewSeqSet("")
	set.AddNum(id)
	return w.conn.Fetch(set, "RFC822", "UID", "FLAGS", "INTERNALDATE")
}

func (w *IMAPSource) handleFetchResult(cmd *imap.Command) error {
	cmd, err := imap.Wait(cmd, nil)
	if err != nil {
		return err
	}
	for _, rsp := range cmd.Data {
		// we only fetch one message at a time, so this loop should
		// only run once
		err := w.handleMessage(rsp)
		if err != nil {
			logger.Errorf("handleMessage failed: %s", err)
			return err
		}
	}
	return nil
}

// handleMessage processes one message, invokes the callback and deletes it on
// success.
func (w *IMAPSource) handleMessage(rsp *imap.Response) error {
	msgInfo := rsp.MessageInfo()
	err := w.invokeMessageCallback(msgInfo)
	if err != nil {
		return err
	}
	uid := imap.AsNumber(msgInfo.Attrs["UID"])
	return w.deleteMessage(uid)
}

// deleteMessage marks the message with the given UID for deletion.
func (w *IMAPSource) deleteMessage(uid uint32) error {
	if !w.deletePlainCopies {
		return nil
	}

	logger.Debugf("marking message uid=%d for deletion", uid)
	set, _ := imap.NewSeqSet("")
	set.AddNum(uid)
	_, err := imap.Wait(w.conn.UIDStore(set, "+FLAGS", "(\\Deleted)"))
	if err != nil {
		logger.Errorf("failed to mark uid=%d for deletion: %s", uid, err)
		return err
	}
	return nil
}

// invokeMessageCallback extracts the relevant data from the passed FETCH response
// and invokes the user-provided callback.
func (w *IMAPSource) invokeMessageCallback(msgInfo *imap.MessageInfo) error {
	logger.Debugf("handling mail uid=%d", msgInfo.Attrs["UID"])
	flags := imap.AsFlagSet(msgInfo.Attrs["FLAGS"])
	idate := imap.AsDateTime(msgInfo.Attrs["INTERNALDATE"])
	mailBytes := imap.AsBytes(msgInfo.Attrs["RFC822"])
	mailLiteral := imap.NewLiteral(mailBytes)
	logger.Debugf("invoking message transformer")
	err := w.callbackFunc(flags, &idate, mailLiteral)
	if err == nil {
		logger.Debugf("message transformation successful")
	} else {
		logger.Warningf("message transformation failed: %s", err)
	}
	return err
}
<|endoftext|>