<commit_before>package identify

import (
	"context"
	"testing"
	"time"

	blhost "github.com/libp2p/go-libp2p-blankhost"
	"github.com/libp2p/go-libp2p-core/network"
	"github.com/libp2p/go-libp2p-core/peer"
	swarmt "github.com/libp2p/go-libp2p-swarm/testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestFastDisconnect(t *testing.T) {
	// This test checks to see if we correctly abort sending an identify
	// response if the peer disconnects before we handle the request.
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	target := blhost.NewBlankHost(swarmt.GenSwarm(t))
	defer target.Close()
	ids, err := NewIDService(target)
	require.NoError(t, err)
	defer ids.Close()

	sync := make(chan struct{})
	target.SetStreamHandler(ID, func(s network.Stream) {
		// Wait till the stream is set up on both sides.
		select {
		case <-sync:
		case <-ctx.Done():
			return
		}

		// Kill the connection, and make sure we're completely disconnected.
		s.Conn().Close()
		assert.Eventually(t,
			func() bool { return target.Network().Connectedness(s.Conn().RemotePeer()) != network.Connected },
			2*time.Second,
			time.Millisecond,
		)
		// Now try to handle the response.
		// This should not block indefinitely, or panic, or anything like that.
		//
		// However, if we have a bug, that _could_ happen.
		ids.sendIdentifyResp(s)

		// Ok, allow the outer test to continue.
		select {
		case <-sync:
		case <-ctx.Done():
			return
		}
	})

	source := blhost.NewBlankHost(swarmt.GenSwarm(t))
	defer source.Close()

	// only connect to the first address, to make sure we only end up with one connection
	require.NoError(t, source.Connect(ctx, peer.AddrInfo{ID: target.ID(), Addrs: target.Addrs()[:1]}))
	s, err := source.NewStream(ctx, target.ID(), ID)
	require.NoError(t, err)
	select {
	case sync <- struct{}{}:
	case <-ctx.Done():
		t.Fatal(ctx.Err())
	}
	s.Reset()
	select {
	case sync <- struct{}{}:
	case <-ctx.Done():
		t.Fatal(ctx.Err())
	}
	// double-check to make sure we didn't actually timeout somewhere.
	require.NoError(t, ctx.Err())
}
<commit_msg>close all connections in the TestFastDisconnect test<commit_after>package identify

import (
	"context"
	"testing"
	"time"

	blhost "github.com/libp2p/go-libp2p-blankhost"
	"github.com/libp2p/go-libp2p-core/network"
	"github.com/libp2p/go-libp2p-core/peer"
	swarmt "github.com/libp2p/go-libp2p-swarm/testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestFastDisconnect(t *testing.T) {
	// This test checks to see if we correctly abort sending an identify
	// response if the peer disconnects before we handle the request.
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	target := blhost.NewBlankHost(swarmt.GenSwarm(t))
	defer target.Close()
	ids, err := NewIDService(target)
	require.NoError(t, err)
	defer ids.Close()

	sync := make(chan struct{})
	target.SetStreamHandler(ID, func(s network.Stream) {
		// Wait till the stream is set up on both sides.
		select {
		case <-sync:
		case <-ctx.Done():
			return
		}

		// Kill the connection, and make sure we're completely disconnected.
		assert.Eventually(t,
			func() bool {
				for _, conn := range target.Network().ConnsToPeer(s.Conn().RemotePeer()) {
					conn.Close()
				}
				return target.Network().Connectedness(s.Conn().RemotePeer()) != network.Connected
			},
			2*time.Second,
			time.Millisecond,
		)
		// Now try to handle the response.
		// This should not block indefinitely, or panic, or anything like that.
		//
		// However, if we have a bug, that _could_ happen.
		ids.sendIdentifyResp(s)

		// Ok, allow the outer test to continue.
		select {
		case <-sync:
		case <-ctx.Done():
			return
		}
	})

	source := blhost.NewBlankHost(swarmt.GenSwarm(t))
	defer source.Close()

	// only connect to the first address, to make sure we only end up with one connection
	require.NoError(t, source.Connect(ctx, peer.AddrInfo{ID: target.ID(), Addrs: target.Addrs()}))
	s, err := source.NewStream(ctx, target.ID(), ID)
	require.NoError(t, err)
	select {
	case sync <- struct{}{}:
	case <-ctx.Done():
		t.Fatal(ctx.Err())
	}
	s.Reset()
	select {
	case sync <- struct{}{}:
	case <-ctx.Done():
		t.Fatal(ctx.Err())
	}
	// double-check to make sure we didn't actually timeout somewhere.
	require.NoError(t, ctx.Err())
}
<|endoftext|>


<commit_before>package blockothertenantresources

import (
	"fmt"

	"os"
	"strings"
	"testing"

	"github.com/onsi/gomega"
	v1 "k8s.io/api/rbac/v1"
	apiextensionspkg "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
	"k8s.io/cli-runtime/pkg/genericclioptions"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"sigs.k8s.io/kind/pkg/cluster"
	"sigs.k8s.io/multi-tenancy/benchmarks/kubectl-mtb/test/utils/unittestutils"
)

var (
	testClient *unittestutils.TestClient
	tenantConfig *rest.Config
	tenantClient *kubernetes.Clientset
	clusterExists bool
	saNamespace = "default"
	tenantName = "tenant1"
	tenantAdminNamespaceName = "tenant1admin"
	tenantNamespaceName = "tenantnamespace1"
	actualTenantNamespaceName = "t1-ns1"
	saName = "tenant1-admin"
	apiExtensions *apiextensionspkg.Clientset
	g *gomega.GomegaWithT
)

type TestFunction func(t *testing.T) (bool, bool)

func TestMain(m *testing.M) {
	// Create kind instance
	kind := &unittestutils.KindCluster{}

	// Tenant setup function
	setUp := func() error {
		provider := cluster.NewProvider()

		// List the clusters available
		clusterList, err := provider.List()
		clusters := strings.Join(clusterList, " ")

		// Checks if the main cluster (test) is running
		if strings.Contains(clusters, "kubectl-mtb-suite") {
			clusterExists = true
		} else {
			clusterExists = false
			err := kind.CreateCluster()
			if err != nil {
				return err
			}
		}

		kubecfgFlags := genericclioptions.NewConfigFlags(false)

		// Create the K8s clientSet
		cfg, err := kubecfgFlags.ToRESTConfig()
		k8sClient, err := kubernetes.NewForConfig(cfg)
		if err != nil {
			return err
		}
		rest := k8sClient.CoreV1().RESTClient()
		apiExtensions, err = apiextensionspkg.NewForConfig(cfg)

		// Initialize testclient
		testClient = unittestutils.TestNewClient("unittests", k8sClient, apiExtensions, rest, cfg)
		tenantConfig := testClient.Config
		tenantConfig.Impersonate.UserName = "system:serviceaccount:" + saNamespace  + ":" + saName
		tenantClient, _ = kubernetes.NewForConfig(tenantConfig)
		testClient.Namespace = actualTenantNamespaceName
		testClient.ServiceAccount = unittestutils.ServiceAccountObj(saName, saNamespace)
		return nil
	}

	//exec setUp function
	err := setUp()

	if err != nil {
		g.Expect(err).NotTo(gomega.HaveOccurred())
		os.Exit(1)
	}

	// exec test and this returns an exit code to pass to os
	retCode := m.Run()

	tearDown := func() error {
		var err error
		if !clusterExists {
			err := kind.DeleteCluster()
			if err != nil {
				return err
			}
		}
		return err
	}

	// exec tearDown function
	err = tearDown()
	if err != nil {
		g.Expect(err).NotTo(gomega.HaveOccurred())
	}

	os.Exit(retCode)
}

func CreateTenants(t *testing.T, g *gomega.GomegaWithT) {
	err := unittestutils.CreateCrds(testClient)
	if err != nil {
		g.Expect(err).NotTo(gomega.HaveOccurred())
	}

	unittestutils.ServiceAccounts = append(unittestutils.ServiceAccounts, unittestutils.ServiceAccountObj(saName, saNamespace))
	unittestutils.Tenants = append(unittestutils.Tenants, unittestutils.TenantObj(tenantName,  unittestutils.ServiceAccountObj(saName, saNamespace), tenantAdminNamespaceName))
	unittestutils.Tenantnamespaces = append(unittestutils.Tenantnamespaces, unittestutils.TenantNamespaceObj(tenantNamespaceName, tenantAdminNamespaceName, actualTenantNamespaceName))

	fmt.Println("Creating tenants")
	unittestutils.CreateTenant(t, g)
}

func TestBenchmark(t *testing.T) {
	defer func() {
		testClient.DeletePolicy()
		testClient.DeleteRole()
	}()

	g = gomega.NewGomegaWithT(t)
	// test to create tenants

	if !clusterExists {
		CreateTenants(t, g)
	}

	if !unittestutils.CheckNamespaceExist(actualTenantNamespaceName, testClient.K8sClient) {
		CreateTenants(t, g)
	}

	tests := []struct {
		testName     string
		testFunction TestFunction
		preRun       bool
		run          bool
	}{}

	for _, tc := range tests {
		fmt.Println("Running test: ", tc.testName)
		preRun, run := tc.testFunction(t)
		g.Expect(preRun).To(gomega.Equal(tc.preRun))
		g.Expect(run).To(gomega.Equal(tc.run))
	}
}
<commit_msg>removed extra code<commit_after>package blockothertenantresources<|endoftext|>


<commit_before>package iiif

import (
	"testing"

	"github.com/uoregon-libraries/gopkg/assert"
)

func TestSizeTypeFull(t *testing.T) {
	s := StringToSize("full")
	assert.True(s.Valid(), "s.Valid()", t)
	assert.Equal(STFull, s.Type, "s.Type == STFull", t)
}

func TestSizeTypeScaleWidth(t *testing.T) {
	s := StringToSize("125,")
	assert.True(s.Valid(), "s.Valid()", t)
	assert.Equal(STScaleToWidth, s.Type, "s.Type == STScaleToWidth", t)
	assert.Equal(125, s.W, "s.W", t)
	assert.Equal(0, s.H, "s.H", t)
}

func TestSizeTypeScaleHeight(t *testing.T) {
	s := StringToSize(",250")
	assert.True(s.Valid(), "s.Valid()", t)
	assert.Equal(STScaleToHeight, s.Type, "s.Type == STScaleToHeight", t)
	assert.Equal(0, s.W, "s.W", t)
	assert.Equal(250, s.H, "s.H", t)
}

func TestSizeTypePercent(t *testing.T) {
	s := StringToSize("pct:41.6")
	assert.True(s.Valid(), "s.Valid()", t)
	assert.Equal(STScalePercent, s.Type, "s.Type == STScalePercent", t)
	assert.Equal(41.6, s.Percent, "s.Percent", t)
}

func TestSizeTypeExact(t *testing.T) {
	s := StringToSize("125,250")
	assert.True(s.Valid(), "s.Valid()", t)
	assert.Equal(STExact, s.Type, "s.Type == STExact", t)
	assert.Equal(125, s.W, "s.W", t)
	assert.Equal(250, s.H, "s.H", t)
}

func TestSizeTypeBestFit(t *testing.T) {
	s := StringToSize("!25,50")
	assert.True(s.Valid(), "s.Valid()", t)
	assert.Equal(STBestFit, s.Type, "s.Type == STBestFit", t)
	assert.Equal(25, s.W, "s.W", t)
	assert.Equal(50, s.H, "s.H", t)
}

func TestInvalidSizes(t *testing.T) {
	s := Size{}
	assert.True(!s.Valid(), "!s.Valid()", t)
	s = StringToSize(",0")
	assert.True(!s.Valid(), "!s.Valid()", t)
	s = StringToSize("0,")
	assert.True(!s.Valid(), "!s.Valid()", t)
	s = StringToSize("0,100")
	assert.True(!s.Valid(), "!s.Valid()", t)
	s = StringToSize("100,0")
	assert.True(!s.Valid(), "!s.Valid()", t)
	s = StringToSize("!0,100")
	assert.True(!s.Valid(), "!s.Valid()", t)
	s = StringToSize("!100,0")
	assert.True(!s.Valid(), "!s.Valid()", t)
	s = StringToSize("pct:0")
	assert.True(!s.Valid(), "!s.Valid()", t)
}
<commit_msg>iiif: add test for Size.GetResize<commit_after>package iiif

import (
	"image"
	"testing"

	"github.com/uoregon-libraries/gopkg/assert"
)

func TestSizeTypeFull(t *testing.T) {
	s := StringToSize("full")
	assert.True(s.Valid(), "s.Valid()", t)
	assert.Equal(STFull, s.Type, "s.Type == STFull", t)
}

func TestSizeTypeScaleWidth(t *testing.T) {
	s := StringToSize("125,")
	assert.True(s.Valid(), "s.Valid()", t)
	assert.Equal(STScaleToWidth, s.Type, "s.Type == STScaleToWidth", t)
	assert.Equal(125, s.W, "s.W", t)
	assert.Equal(0, s.H, "s.H", t)
}

func TestSizeTypeScaleHeight(t *testing.T) {
	s := StringToSize(",250")
	assert.True(s.Valid(), "s.Valid()", t)
	assert.Equal(STScaleToHeight, s.Type, "s.Type == STScaleToHeight", t)
	assert.Equal(0, s.W, "s.W", t)
	assert.Equal(250, s.H, "s.H", t)
}

func TestSizeTypePercent(t *testing.T) {
	s := StringToSize("pct:41.6")
	assert.True(s.Valid(), "s.Valid()", t)
	assert.Equal(STScalePercent, s.Type, "s.Type == STScalePercent", t)
	assert.Equal(41.6, s.Percent, "s.Percent", t)
}

func TestSizeTypeExact(t *testing.T) {
	s := StringToSize("125,250")
	assert.True(s.Valid(), "s.Valid()", t)
	assert.Equal(STExact, s.Type, "s.Type == STExact", t)
	assert.Equal(125, s.W, "s.W", t)
	assert.Equal(250, s.H, "s.H", t)
}

func TestSizeTypeBestFit(t *testing.T) {
	s := StringToSize("!25,50")
	assert.True(s.Valid(), "s.Valid()", t)
	assert.Equal(STBestFit, s.Type, "s.Type == STBestFit", t)
	assert.Equal(25, s.W, "s.W", t)
	assert.Equal(50, s.H, "s.H", t)
}

func TestInvalidSizes(t *testing.T) {
	s := Size{}
	assert.True(!s.Valid(), "!s.Valid()", t)
	s = StringToSize(",0")
	assert.True(!s.Valid(), "!s.Valid()", t)
	s = StringToSize("0,")
	assert.True(!s.Valid(), "!s.Valid()", t)
	s = StringToSize("0,100")
	assert.True(!s.Valid(), "!s.Valid()", t)
	s = StringToSize("100,0")
	assert.True(!s.Valid(), "!s.Valid()", t)
	s = StringToSize("!0,100")
	assert.True(!s.Valid(), "!s.Valid()", t)
	s = StringToSize("!100,0")
	assert.True(!s.Valid(), "!s.Valid()", t)
	s = StringToSize("pct:0")
	assert.True(!s.Valid(), "!s.Valid()", t)
}

func TestGetResize(t *testing.T) {
	s := Size{Type: STFull}
	source := image.Rect(0, 0, 600, 1200)
	scale := s.GetResize(source)
	assert.Equal(scale.Dx(), source.Dx(), "full resize Dx", t)
	assert.Equal(scale.Dy(), source.Dy(), "full resize Dy", t)

	s.Type = STScaleToWidth
	s.W = 90
	scale = s.GetResize(source)
	assert.Equal(scale.Dx(), 90, "scale-to-width Dx", t)
	assert.Equal(scale.Dy(), 180, "scale-to-width Dy", t)

	s.Type = STScaleToHeight
	s.H = 90
	scale = s.GetResize(source)
	assert.Equal(scale.Dx(), 45, "scale-to-height Dx", t)
	assert.Equal(scale.Dy(), 90, "scale-to-height Dy", t)

	s.Type = STScalePercent
	s.Percent = 100 * 2.0 / 3.0
	scale = s.GetResize(source)
	assert.Equal(scale.Dx(), 400, "scale-to-pct Dx", t)
	assert.Equal(scale.Dy(), 800, "scale-to-pct Dy", t)

	s.Type = STExact
	s.W = 95
	s.H = 100
	scale = s.GetResize(source)
	assert.Equal(scale.Dx(), 95, "scale-to-exact Dx", t)
	assert.Equal(scale.Dy(), 100, "scale-to-exact Dy", t)

	s.Type = STBestFit
	scale = s.GetResize(source)
	assert.Equal(scale.Dx(), 50, "scale-to-pct Dx", t)
	assert.Equal(scale.Dy(), 100, "scale-to-pct Dy", t)
}
<|endoftext|>


<commit_before>// Copyright (c) 2020 The Jaeger Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tlscfg

import (
	"crypto/tls"
	"crypto/x509"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
	"go.uber.org/zap/zaptest/observer"
)

const (
	serverCert = "./testdata/example-server-cert.pem"
	serverKey  = "./testdata/example-server-key.pem"
	clientCert = "./testdata/example-client-cert.pem"
	clientKey  = "./testdata/example-client-key.pem"

	caCert    = "./testdata/example-CA-cert.pem"
	badCaCert = "./testdata/bad-CA-cert.txt"
)

func TestReload(t *testing.T) {
	// copy certs to temp so we can modify them
	certFile, err := ioutil.TempFile("", "cert.crt")
	require.NoError(t, err)
	defer os.Remove(certFile.Name())
	certData, err := ioutil.ReadFile(serverCert)
	require.NoError(t, err)
	_, err = certFile.Write(certData)
	require.NoError(t, err)
	certFile.Close()

	keyFile, err := ioutil.TempFile("", "key.crt")
	require.NoError(t, err)
	defer os.Remove(keyFile.Name())
	keyData, err := ioutil.ReadFile(serverKey)
	require.NoError(t, err)
	_, err = keyFile.Write(keyData)
	require.NoError(t, err)
	keyFile.Close()

	zcore, logObserver := observer.New(zapcore.InfoLevel)
	logger := zap.New(zcore)
	opts := Options{
		CAPath:       caCert,
		ClientCAPath: caCert,
		CertPath:     certFile.Name(),
		KeyPath:      keyFile.Name(),
	}
	watcher, err := newCertWatcher(opts, logger)
	require.NoError(t, err)
	assert.NotNil(t, watcher.certificate())
	defer watcher.Close()

	certPool := x509.NewCertPool()
	require.NoError(t, err)
	go watcher.watchChangesLoop(certPool, certPool)
	cert, err := tls.LoadX509KeyPair(serverCert, serverKey)
	require.NoError(t, err)
	assert.Equal(t, &cert, watcher.certificate())

	// update the content with client certs
	certData, err = ioutil.ReadFile(clientCert)
	require.NoError(t, err)
	err = syncWrite(certFile.Name(), certData, 0644)
	require.NoError(t, err)
	keyData, err = ioutil.ReadFile(clientKey)
	require.NoError(t, err)
	err = syncWrite(keyFile.Name(), keyData, 0644)
	require.NoError(t, err)

	waitUntil(func() bool {
		return logObserver.FilterField(zap.String("certificate", certFile.Name())).Len() > 0
	}, 100, time.Millisecond*200)
	assert.True(t, logObserver.FilterField(zap.String("certificate", certFile.Name())).Len() > 0)

	cert, err = tls.LoadX509KeyPair(filepath.Clean(clientCert), clientKey)
	require.NoError(t, err)
	assert.Equal(t, &cert, watcher.certificate())
}

func TestReload_ca_certs(t *testing.T) {
	// copy certs to temp so we can modify them
	caFile, err := ioutil.TempFile("", "cert.crt")
	require.NoError(t, err)
	defer os.Remove(caFile.Name())
	caData, err := ioutil.ReadFile(caCert)
	require.NoError(t, err)
	_, err = caFile.Write(caData)
	require.NoError(t, err)
	caFile.Close()

	clientCaFile, err := ioutil.TempFile("", "key.crt")
	require.NoError(t, err)
	defer os.Remove(clientCaFile.Name())
	clientCaData, err := ioutil.ReadFile(caCert)
	require.NoError(t, err)
	_, err = clientCaFile.Write(clientCaData)
	require.NoError(t, err)
	clientCaFile.Close()

	zcore, logObserver := observer.New(zapcore.InfoLevel)
	logger := zap.New(zcore)
	opts := Options{
		CAPath:       caFile.Name(),
		ClientCAPath: clientCaFile.Name(),
	}
	watcher, err := newCertWatcher(opts, logger)
	require.NoError(t, err)
	defer watcher.Close()

	certPool := x509.NewCertPool()
	require.NoError(t, err)
	go watcher.watchChangesLoop(certPool, certPool)

	// update the content with client certs
	caData, err = ioutil.ReadFile(caCert)
	require.NoError(t, err)
	err = syncWrite(caFile.Name(), caData, 0644)
	require.NoError(t, err)
	clientCaData, err = ioutil.ReadFile(caCert)
	require.NoError(t, err)
	err = syncWrite(clientCaFile.Name(), clientCaData, 0644)
	require.NoError(t, err)

	waitUntil(func() bool {
		return logObserver.FilterField(zap.String("certificate", caFile.Name())).Len() > 0
	}, 100, time.Millisecond*200)
	assert.True(t, logObserver.FilterField(zap.String("certificate", caFile.Name())).Len() > 0)

	waitUntil(func() bool {
		return logObserver.FilterField(zap.String("certificate", clientCaFile.Name())).Len() > 0
	}, 100, time.Millisecond*200)
	assert.True(t, logObserver.FilterField(zap.String("certificate", clientCaFile.Name())).Len() > 0)

}

func TestReload_err_cert_update(t *testing.T) {
	// copy certs to temp so we can modify them
	certFile, err := ioutil.TempFile("", "cert.crt")
	require.NoError(t, err)
	defer os.Remove(certFile.Name())
	certData, err := ioutil.ReadFile(serverCert)
	require.NoError(t, err)
	_, err = certFile.Write(certData)
	require.NoError(t, err)
	certFile.Close()

	keyFile, err := ioutil.TempFile("", "key.crt")
	require.NoError(t, err)
	defer os.Remove(keyFile.Name())
	keyData, err := ioutil.ReadFile(serverKey)
	require.NoError(t, err)
	_, err = keyFile.Write(keyData)
	require.NoError(t, err)
	keyFile.Close()

	zcore, logObserver := observer.New(zapcore.InfoLevel)
	logger := zap.New(zcore)
	opts := Options{
		CAPath:       caCert,
		ClientCAPath: caCert,
		CertPath:     certFile.Name(),
		KeyPath:      keyFile.Name(),
	}
	watcher, err := newCertWatcher(opts, logger)
	require.NoError(t, err)
	assert.NotNil(t, watcher.certificate())
	defer watcher.Close()

	certPool := x509.NewCertPool()
	require.NoError(t, err)
	go watcher.watchChangesLoop(certPool, certPool)
	serverCert, err := tls.LoadX509KeyPair(filepath.Clean(serverCert), filepath.Clean(serverKey))
	require.NoError(t, err)
	assert.Equal(t, &serverCert, watcher.certificate())

	// update the content with client certs
	certData, err = ioutil.ReadFile(badCaCert)
	require.NoError(t, err)
	err = syncWrite(certFile.Name(), certData, 0644)
	require.NoError(t, err)
	keyData, err = ioutil.ReadFile(clientKey)
	require.NoError(t, err)
	err = syncWrite(keyFile.Name(), keyData, 0644)
	require.NoError(t, err)

	waitUntil(func() bool {
		return logObserver.FilterMessage("Failed to load certificate").Len() > 0
	}, 100, time.Millisecond*200)
	assert.True(t, logObserver.FilterField(zap.String("certificate", certFile.Name())).Len() > 0)
	assert.Equal(t, &serverCert, watcher.certificate())
}

func TestReload_err_watch(t *testing.T) {
	opts := Options{
		CAPath: "doesnotexists",
	}
	watcher, err := newCertWatcher(opts, zap.NewNop())
	require.Error(t, err)
	assert.Contains(t, err.Error(), "no such file or directory")
	assert.Nil(t, watcher)
}

func TestAddCertsToWatch_err(t *testing.T) {
	watcher, err := fsnotify.NewWatcher()
	require.NoError(t, err)
	defer watcher.Close()

	tests := []struct {
		opts Options
	}{
		{
			opts: Options{
				CAPath: "doesnotexists",
			},
		},
		{
			opts: Options{
				CAPath:       caCert,
				ClientCAPath: "doesnotexists",
			},
		},
		{
			opts: Options{
				CAPath:       caCert,
				ClientCAPath: caCert,
				CertPath:     "doesnotexists",
			},
		},
		{
			opts: Options{
				CAPath:       caCert,
				ClientCAPath: caCert,
				CertPath:     serverCert,
				KeyPath:      "doesnotexists",
			},
		},
	}
	for _, test := range tests {
		err := addCertsToWatch(watcher, test.opts)
		require.Error(t, err)
		assert.Contains(t, err.Error(), "no such file or directory")
	}
}

func TestAddCertsToWatch_remove_ca(t *testing.T) {
	caFile, err := ioutil.TempFile("", "ca.crt")
	require.NoError(t, err)
	defer os.Remove(caFile.Name())
	caData, err := ioutil.ReadFile(caCert)
	require.NoError(t, err)
	_, err = caFile.Write(caData)
	require.NoError(t, err)
	caFile.Close()

	clientCaFile, err := ioutil.TempFile("", "clientCa.crt")
	require.NoError(t, err)
	defer os.Remove(clientCaFile.Name())
	clientCaData, err := ioutil.ReadFile(caCert)
	require.NoError(t, err)
	_, err = clientCaFile.Write(clientCaData)
	require.NoError(t, err)
	clientCaFile.Close()

	zcore, logObserver := observer.New(zapcore.InfoLevel)
	logger := zap.New(zcore)
	opts := Options{
		CAPath:       caFile.Name(),
		ClientCAPath: clientCaFile.Name(),
	}
	watcher, err := newCertWatcher(opts, logger)
	require.NoError(t, err)
	defer watcher.Close()

	certPool := x509.NewCertPool()
	require.NoError(t, err)
	go watcher.watchChangesLoop(certPool, certPool)

	require.NoError(t, os.Remove(caFile.Name()))
	require.NoError(t, os.Remove(clientCaFile.Name()))
	waitUntil(func() bool {
		return logObserver.FilterMessage("Certificate has been removed, using the last known version").Len() > 0
	}, 100, time.Millisecond*100)
	assert.True(t, logObserver.FilterMessage("Certificate has been removed, using the last known version").FilterField(zap.String("certificate", caFile.Name())).Len() > 0)
	assert.True(t, logObserver.FilterMessage("Certificate has been removed, using the last known version").FilterField(zap.String("certificate", clientCaFile.Name())).Len() > 0)
}

func waitUntil(f func() bool, iterations int, sleepInterval time.Duration) {
	for i := 0; i < iterations; i++ {
		if f() {
			return
		}
		time.Sleep(sleepInterval)
	}
}

// syncWrite ensures data is written to the given filename and flushed to disk.
// This ensures that any watchers looking for file system changes can be reliably alerted.
func syncWrite(filename string, data []byte, perm os.FileMode) error {
	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC|os.O_SYNC, perm)
	if err != nil {
		return err
	}
	defer f.Close()
	if _, err = f.Write(data); err != nil {
		return err
	}
	return f.Sync()
}
<commit_msg>Fix flaky TestAddCertsToWatch_remove_ca test (#2610)<commit_after>// Copyright (c) 2020 The Jaeger Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tlscfg

import (
	"crypto/tls"
	"crypto/x509"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"go.uber.org/zap"
	"go.uber.org/zap/zapcore"
	"go.uber.org/zap/zaptest/observer"
)

const (
	serverCert = "./testdata/example-server-cert.pem"
	serverKey  = "./testdata/example-server-key.pem"
	clientCert = "./testdata/example-client-cert.pem"
	clientKey  = "./testdata/example-client-key.pem"

	caCert    = "./testdata/example-CA-cert.pem"
	badCaCert = "./testdata/bad-CA-cert.txt"
)

func TestReload(t *testing.T) {
	// copy certs to temp so we can modify them
	certFile, err := ioutil.TempFile("", "cert.crt")
	require.NoError(t, err)
	defer os.Remove(certFile.Name())
	certData, err := ioutil.ReadFile(serverCert)
	require.NoError(t, err)
	_, err = certFile.Write(certData)
	require.NoError(t, err)
	certFile.Close()

	keyFile, err := ioutil.TempFile("", "key.crt")
	require.NoError(t, err)
	defer os.Remove(keyFile.Name())
	keyData, err := ioutil.ReadFile(serverKey)
	require.NoError(t, err)
	_, err = keyFile.Write(keyData)
	require.NoError(t, err)
	keyFile.Close()

	zcore, logObserver := observer.New(zapcore.InfoLevel)
	logger := zap.New(zcore)
	opts := Options{
		CAPath:       caCert,
		ClientCAPath: caCert,
		CertPath:     certFile.Name(),
		KeyPath:      keyFile.Name(),
	}
	watcher, err := newCertWatcher(opts, logger)
	require.NoError(t, err)
	assert.NotNil(t, watcher.certificate())
	defer watcher.Close()

	certPool := x509.NewCertPool()
	require.NoError(t, err)
	go watcher.watchChangesLoop(certPool, certPool)
	cert, err := tls.LoadX509KeyPair(serverCert, serverKey)
	require.NoError(t, err)
	assert.Equal(t, &cert, watcher.certificate())

	// update the content with client certs
	certData, err = ioutil.ReadFile(clientCert)
	require.NoError(t, err)
	err = syncWrite(certFile.Name(), certData, 0644)
	require.NoError(t, err)
	keyData, err = ioutil.ReadFile(clientKey)
	require.NoError(t, err)
	err = syncWrite(keyFile.Name(), keyData, 0644)
	require.NoError(t, err)

	waitUntil(func() bool {
		return logObserver.FilterField(zap.String("certificate", certFile.Name())).Len() > 0
	}, 100, time.Millisecond*200)
	assert.True(t, logObserver.FilterField(zap.String("certificate", certFile.Name())).Len() > 0)

	cert, err = tls.LoadX509KeyPair(filepath.Clean(clientCert), clientKey)
	require.NoError(t, err)
	assert.Equal(t, &cert, watcher.certificate())
}

func TestReload_ca_certs(t *testing.T) {
	// copy certs to temp so we can modify them
	caFile, err := ioutil.TempFile("", "cert.crt")
	require.NoError(t, err)
	defer os.Remove(caFile.Name())
	caData, err := ioutil.ReadFile(caCert)
	require.NoError(t, err)
	_, err = caFile.Write(caData)
	require.NoError(t, err)
	caFile.Close()

	clientCaFile, err := ioutil.TempFile("", "key.crt")
	require.NoError(t, err)
	defer os.Remove(clientCaFile.Name())
	clientCaData, err := ioutil.ReadFile(caCert)
	require.NoError(t, err)
	_, err = clientCaFile.Write(clientCaData)
	require.NoError(t, err)
	clientCaFile.Close()

	zcore, logObserver := observer.New(zapcore.InfoLevel)
	logger := zap.New(zcore)
	opts := Options{
		CAPath:       caFile.Name(),
		ClientCAPath: clientCaFile.Name(),
	}
	watcher, err := newCertWatcher(opts, logger)
	require.NoError(t, err)
	defer watcher.Close()

	certPool := x509.NewCertPool()
	require.NoError(t, err)
	go watcher.watchChangesLoop(certPool, certPool)

	// update the content with client certs
	caData, err = ioutil.ReadFile(caCert)
	require.NoError(t, err)
	err = syncWrite(caFile.Name(), caData, 0644)
	require.NoError(t, err)
	clientCaData, err = ioutil.ReadFile(caCert)
	require.NoError(t, err)
	err = syncWrite(clientCaFile.Name(), clientCaData, 0644)
	require.NoError(t, err)

	waitUntil(func() bool {
		return logObserver.FilterField(zap.String("certificate", caFile.Name())).Len() > 0
	}, 100, time.Millisecond*200)
	assert.True(t, logObserver.FilterField(zap.String("certificate", caFile.Name())).Len() > 0)

	waitUntil(func() bool {
		return logObserver.FilterField(zap.String("certificate", clientCaFile.Name())).Len() > 0
	}, 100, time.Millisecond*200)
	assert.True(t, logObserver.FilterField(zap.String("certificate", clientCaFile.Name())).Len() > 0)

}

func TestReload_err_cert_update(t *testing.T) {
	// copy certs to temp so we can modify them
	certFile, err := ioutil.TempFile("", "cert.crt")
	require.NoError(t, err)
	defer os.Remove(certFile.Name())
	certData, err := ioutil.ReadFile(serverCert)
	require.NoError(t, err)
	_, err = certFile.Write(certData)
	require.NoError(t, err)
	certFile.Close()

	keyFile, err := ioutil.TempFile("", "key.crt")
	require.NoError(t, err)
	defer os.Remove(keyFile.Name())
	keyData, err := ioutil.ReadFile(serverKey)
	require.NoError(t, err)
	_, err = keyFile.Write(keyData)
	require.NoError(t, err)
	keyFile.Close()

	zcore, logObserver := observer.New(zapcore.InfoLevel)
	logger := zap.New(zcore)
	opts := Options{
		CAPath:       caCert,
		ClientCAPath: caCert,
		CertPath:     certFile.Name(),
		KeyPath:      keyFile.Name(),
	}
	watcher, err := newCertWatcher(opts, logger)
	require.NoError(t, err)
	assert.NotNil(t, watcher.certificate())
	defer watcher.Close()

	certPool := x509.NewCertPool()
	require.NoError(t, err)
	go watcher.watchChangesLoop(certPool, certPool)
	serverCert, err := tls.LoadX509KeyPair(filepath.Clean(serverCert), filepath.Clean(serverKey))
	require.NoError(t, err)
	assert.Equal(t, &serverCert, watcher.certificate())

	// update the content with client certs
	certData, err = ioutil.ReadFile(badCaCert)
	require.NoError(t, err)
	err = syncWrite(certFile.Name(), certData, 0644)
	require.NoError(t, err)
	keyData, err = ioutil.ReadFile(clientKey)
	require.NoError(t, err)
	err = syncWrite(keyFile.Name(), keyData, 0644)
	require.NoError(t, err)

	waitUntil(func() bool {
		return logObserver.FilterMessage("Failed to load certificate").Len() > 0
	}, 100, time.Millisecond*200)
	assert.True(t, logObserver.FilterField(zap.String("certificate", certFile.Name())).Len() > 0)
	assert.Equal(t, &serverCert, watcher.certificate())
}

func TestReload_err_watch(t *testing.T) {
	opts := Options{
		CAPath: "doesnotexists",
	}
	watcher, err := newCertWatcher(opts, zap.NewNop())
	require.Error(t, err)
	assert.Contains(t, err.Error(), "no such file or directory")
	assert.Nil(t, watcher)
}

func TestAddCertsToWatch_err(t *testing.T) {
	watcher, err := fsnotify.NewWatcher()
	require.NoError(t, err)
	defer watcher.Close()

	tests := []struct {
		opts Options
	}{
		{
			opts: Options{
				CAPath: "doesnotexists",
			},
		},
		{
			opts: Options{
				CAPath:       caCert,
				ClientCAPath: "doesnotexists",
			},
		},
		{
			opts: Options{
				CAPath:       caCert,
				ClientCAPath: caCert,
				CertPath:     "doesnotexists",
			},
		},
		{
			opts: Options{
				CAPath:       caCert,
				ClientCAPath: caCert,
				CertPath:     serverCert,
				KeyPath:      "doesnotexists",
			},
		},
	}
	for _, test := range tests {
		err := addCertsToWatch(watcher, test.opts)
		require.Error(t, err)
		assert.Contains(t, err.Error(), "no such file or directory")
	}
}

func TestAddCertsToWatch_remove_ca(t *testing.T) {
	caFile, err := ioutil.TempFile("", "ca.crt")
	require.NoError(t, err)
	defer os.Remove(caFile.Name())
	caData, err := ioutil.ReadFile(caCert)
	require.NoError(t, err)
	_, err = caFile.Write(caData)
	require.NoError(t, err)
	caFile.Close()

	clientCaFile, err := ioutil.TempFile("", "clientCa.crt")
	require.NoError(t, err)
	defer os.Remove(clientCaFile.Name())
	clientCaData, err := ioutil.ReadFile(caCert)
	require.NoError(t, err)
	_, err = clientCaFile.Write(clientCaData)
	require.NoError(t, err)
	clientCaFile.Close()

	zcore, logObserver := observer.New(zapcore.InfoLevel)
	logger := zap.New(zcore)
	opts := Options{
		CAPath:       caFile.Name(),
		ClientCAPath: clientCaFile.Name(),
	}
	watcher, err := newCertWatcher(opts, logger)
	require.NoError(t, err)
	defer watcher.Close()

	certPool := x509.NewCertPool()
	require.NoError(t, err)
	go watcher.watchChangesLoop(certPool, certPool)

	require.NoError(t, os.Remove(caFile.Name()))
	require.NoError(t, os.Remove(clientCaFile.Name()))
	waitUntil(func() bool {
		return logObserver.FilterMessage("Certificate has been removed, using the last known version").Len() >= 2
	}, 100, time.Millisecond*100)
	assert.True(t, logObserver.FilterMessage("Certificate has been removed, using the last known version").FilterField(zap.String("certificate", caFile.Name())).Len() > 0)
	assert.True(t, logObserver.FilterMessage("Certificate has been removed, using the last known version").FilterField(zap.String("certificate", clientCaFile.Name())).Len() > 0)
}

func waitUntil(f func() bool, iterations int, sleepInterval time.Duration) {
	for i := 0; i < iterations; i++ {
		if f() {
			return
		}
		time.Sleep(sleepInterval)
	}
}

// syncWrite ensures data is written to the given filename and flushed to disk.
// This ensures that any watchers looking for file system changes can be reliably alerted.
func syncWrite(filename string, data []byte, perm os.FileMode) error {
	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE|os.O_TRUNC|os.O_SYNC, perm)
	if err != nil {
		return err
	}
	defer f.Close()
	if _, err = f.Write(data); err != nil {
		return err
	}
	return f.Sync()
}
<|endoftext|>


<commit_before>package action

import (
	"sync"
)

// Plan is a plan of actions
type Plan struct {
	// NodeMap is a map from key to a graph of actions, which must to be executed in order to get from actual state to
	// desired state. Key in the map corresponds to the key of the GraphNode
	NodeMap map[string]*GraphNode
}

// NewPlan creates a new Plan
func NewPlan() *Plan {
	return &Plan{
		NodeMap: make(map[string]*GraphNode),
	}
}

// GetActionGraphNode returns an action graph node for a given component instance key
func (plan *Plan) GetActionGraphNode(key string) *GraphNode {
	result, ok := plan.NodeMap[key]
	if !ok {
		result = NewGraphNode(key)
		plan.NodeMap[key] = result
	}
	return result
}

// Apply applies the action plan. It may call fn in multiple go routines, executing the plan in parallel
func (plan *Plan) Apply(fn ApplyFunction, resultUpdater ApplyResultUpdater) *ApplyResult {
	// update total number of actions and start the revision
	resultUpdater.SetTotal(plan.NumberOfActions())

	// apply the plan and calculate result (success/failed/skipped actions)
	plan.applyInternal(fn, resultUpdater)

	// tell results updater that we are done and return the results
	return resultUpdater.Done()
}

// Apply applies the action plan. It may call fn in multiple go routines, executing the plan in parallel
func (plan *Plan) applyInternal(fn ApplyFunction, resultUpdater ApplyResultUpdater) {
	deg := make(map[string]int)
	wasError := make(map[string]error)
	queue := make(chan string, len(plan.NodeMap))
	mutex := &sync.RWMutex{}

	// Initialize all degrees, put 0-degree leaf nodes into the queue
	var wg sync.WaitGroup
	for key := range plan.NodeMap {
		deg[key] = len(plan.NodeMap[key].Before)
		if deg[key] <= 0 {
			queue <- key
		}
		wg.Add(1)
	}

	// Start execution
	var done sync.WaitGroup
	done.Add(1)
	go func() {
		// This will keep running until the queue if not closed
		for key := range queue {
			// Take element off the queue, apply the block of actions and put into queue 0-degree nodes which are waiting on us
			go func(key string) {
				defer wg.Done()
				plan.applyActions(key, fn, queue, deg, wasError, mutex, resultUpdater)
			}(key)
		}
		done.Done()
	}()

	// Wait for all actions to finish
	wg.Wait()

	// Close the channel to ensure that the go routine launched above will exit
	close(queue)

	// Wait for the go routine to finish
	done.Wait()
}

// This function applies a block of actions and updates nodes which are waiting on this node
func (plan *Plan) applyActions(key string, fn ApplyFunction, queue chan string, deg map[string]int, wasError map[string]error, mutex *sync.RWMutex, resultUpdater ApplyResultUpdater) {
	// locate the node
	node := plan.NodeMap[key]

	// run all actions. if one of them fails, the rest won't be executed
	// only run them if all dependent nodes succeeded
	mutex.RLock()
	foundErr := wasError[key]
	mutex.RUnlock()
	for _, action := range node.Actions {
		// if an error happened before, all subsequent actions are getting marked as skipped
		if foundErr != nil {
			// fmt.Println("skipped ", action.GetName())
			resultUpdater.AddSkipped()
		} else {
			// Otherwise, let's run the action and see if it failed or not
			err := fn(action)
			if err != nil {
				// fmt.Println("failed ", action.GetName())
				resultUpdater.AddFailed()
				foundErr = err
			} else {
				// fmt.Println("success ", action.GetName())
				resultUpdater.AddSuccess()
			}
		}
	}

	// mark our node as failed, if we encountered an error
	if foundErr != nil {
		mutex.Lock()
		wasError[key] = foundErr
		mutex.Unlock()
	}

	// decrement degrees of nodes which are waiting on us
	for _, prevNode := range plan.NodeMap[node.Key].BeforeRev {
		mutex.Lock()
		deg[prevNode.Key]--
		if deg[prevNode.Key] < 0 {
			panic("negative node degree while applying actions in parallel")
		}
		if deg[prevNode.Key] == 0 {
			queue <- prevNode.Key
		}
		mutex.Unlock()
		if foundErr != nil {
			// Mark prev nodes failed too
			mutex.Lock()
			wasError[prevNode.Key] = foundErr
			mutex.Unlock()
		}
	}

}

// NumberOfActions returns the total number of actions that is expected to be executed in the whole action graph
func (plan *Plan) NumberOfActions() uint32 {
	resultUpdater := NewApplyResultUpdaterImpl()

	// apply the plan and calculate result (success/failed/skipped actions)
	plan.applyInternal(Noop(), resultUpdater)

	// return the number of success actions (all of them will be success due to Noop() action)
	return resultUpdater.Result.Success
}

// AsText returns the action plan as array of actions, each represented as text via NestedParameterMap
func (plan *Plan) AsText() *PlanAsText {
	result := NewPlanAsText()

	// apply the plan and capture actions as text
	plan.applyInternal(func(act Base) error {
		result.Actions = append(result.Actions, act.DescribeChanges())
		return nil
	}, NewApplyResultUpdaterImpl())

	return result
}
<commit_msg>Fixed #328 - rare smoke test failures - the problem is that append to actionPlan.Actions was not synchronized,   so sometimes it resulted in nil entries being added to the slice<commit_after>package action

import (
	"sync"
)

// Plan is a plan of actions
type Plan struct {
	// NodeMap is a map from key to a graph of actions, which must to be executed in order to get from actual state to
	// desired state. Key in the map corresponds to the key of the GraphNode
	NodeMap map[string]*GraphNode
}

// NewPlan creates a new Plan
func NewPlan() *Plan {
	return &Plan{
		NodeMap: make(map[string]*GraphNode),
	}
}

// GetActionGraphNode returns an action graph node for a given component instance key
func (plan *Plan) GetActionGraphNode(key string) *GraphNode {
	result, ok := plan.NodeMap[key]
	if !ok {
		result = NewGraphNode(key)
		plan.NodeMap[key] = result
	}
	return result
}

// Apply applies the action plan. It may call fn in multiple go routines, executing the plan in parallel
func (plan *Plan) Apply(fn ApplyFunction, resultUpdater ApplyResultUpdater) *ApplyResult {
	// update total number of actions and start the revision
	resultUpdater.SetTotal(plan.NumberOfActions())

	// apply the plan and calculate result (success/failed/skipped actions)
	plan.applyInternal(fn, resultUpdater)

	// tell results updater that we are done and return the results
	return resultUpdater.Done()
}

// Apply applies the action plan. It may call fn in multiple go routines, executing the plan in parallel
func (plan *Plan) applyInternal(fn ApplyFunction, resultUpdater ApplyResultUpdater) {
	deg := make(map[string]int)
	wasError := make(map[string]error)
	queue := make(chan string, len(plan.NodeMap))
	mutex := &sync.RWMutex{}

	// Initialize all degrees, put 0-degree leaf nodes into the queue
	var wg sync.WaitGroup
	for key := range plan.NodeMap {
		deg[key] = len(plan.NodeMap[key].Before)
		if deg[key] <= 0 {
			queue <- key
		}
		wg.Add(1)
	}

	// Start execution
	var done sync.WaitGroup
	done.Add(1)
	go func() {
		// This will keep running until the queue if not closed
		for key := range queue {
			// Take element off the queue, apply the block of actions and put into queue 0-degree nodes which are waiting on us
			go func(key string) {
				defer wg.Done()
				plan.applyActions(key, fn, queue, deg, wasError, mutex, resultUpdater)
			}(key)
		}
		done.Done()
	}()

	// Wait for all actions to finish
	wg.Wait()

	// Close the channel to ensure that the go routine launched above will exit
	close(queue)

	// Wait for the go routine to finish
	done.Wait()
}

// This function applies a block of actions and updates nodes which are waiting on this node
func (plan *Plan) applyActions(key string, fn ApplyFunction, queue chan string, deg map[string]int, wasError map[string]error, mutex *sync.RWMutex, resultUpdater ApplyResultUpdater) {
	// locate the node
	node := plan.NodeMap[key]

	// run all actions. if one of them fails, the rest won't be executed
	// only run them if all dependent nodes succeeded
	mutex.RLock()
	foundErr := wasError[key]
	mutex.RUnlock()
	for _, action := range node.Actions {
		// if an error happened before, all subsequent actions are getting marked as skipped
		if foundErr != nil {
			// fmt.Println("skipped ", action.GetName())
			resultUpdater.AddSkipped()
		} else {
			// Otherwise, let's run the action and see if it failed or not
			err := fn(action)
			if err != nil {
				// fmt.Println("failed ", action.GetName())
				resultUpdater.AddFailed()
				foundErr = err
			} else {
				// fmt.Println("success ", action.GetName())
				resultUpdater.AddSuccess()
			}
		}
	}

	// mark our node as failed, if we encountered an error
	if foundErr != nil {
		mutex.Lock()
		wasError[key] = foundErr
		mutex.Unlock()
	}

	// decrement degrees of nodes which are waiting on us
	for _, prevNode := range plan.NodeMap[node.Key].BeforeRev {
		mutex.Lock()
		deg[prevNode.Key]--
		if deg[prevNode.Key] < 0 {
			panic("negative node degree while applying actions in parallel")
		}
		if deg[prevNode.Key] == 0 {
			queue <- prevNode.Key
		}
		mutex.Unlock()
		if foundErr != nil {
			// Mark prev nodes failed too
			mutex.Lock()
			wasError[prevNode.Key] = foundErr
			mutex.Unlock()
		}
	}

}

// NumberOfActions returns the total number of actions that is expected to be executed in the whole action graph
func (plan *Plan) NumberOfActions() uint32 {
	resultUpdater := NewApplyResultUpdaterImpl()

	// apply the plan and calculate result (success/failed/skipped actions)
	plan.applyInternal(Noop(), resultUpdater)

	// return the number of success actions (all of them will be success due to Noop() action)
	return resultUpdater.Result.Success
}

// AsText returns the action plan as array of actions, each represented as text via NestedParameterMap
func (plan *Plan) AsText() *PlanAsText {
	result := NewPlanAsText()

	// apply the plan and capture actions as text
	plan.applyInternal(WrapSequential(func(act Base) error {
		result.Actions = append(result.Actions, act.DescribeChanges())
		return nil
	}), NewApplyResultUpdaterImpl())

	return result
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package streaming

import (
	"crypto/tls"
	"errors"
	"io"
	"net/http"
	"net/url"
	"path"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"

	restful "github.com/emicklei/go-restful"

	"k8s.io/apimachinery/pkg/types"
	remotecommandconsts "k8s.io/apimachinery/pkg/util/remotecommand"
	"k8s.io/client-go/tools/remotecommand"
	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
	"k8s.io/kubernetes/pkg/kubelet/server/portforward"
	remotecommandserver "k8s.io/kubernetes/pkg/kubelet/server/remotecommand"
)

// The library interface to serve the stream requests.
type Server interface {
	http.Handler

	// Get the serving URL for the requests.
	// Requests must not be nil. Responses may be nil iff an error is returned.
	GetExec(*runtimeapi.ExecRequest) (*runtimeapi.ExecResponse, error)
	GetAttach(req *runtimeapi.AttachRequest) (*runtimeapi.AttachResponse, error)
	GetPortForward(*runtimeapi.PortForwardRequest) (*runtimeapi.PortForwardResponse, error)

	// Start the server.
	// addr is the address to serve on (address:port) stayUp indicates whether the server should
	// listen until Stop() is called, or automatically stop after all expected connections are
	// closed. Calling Get{Exec,Attach,PortForward} increments the expected connection count.
	// Function does not return until the server is stopped.
	Start(stayUp bool) error
	// Stop the server, and terminate any open connections.
	Stop() error
}

// The interface to execute the commands and provide the streams.
type Runtime interface {
	Exec(containerID string, cmd []string, in io.Reader, out, err io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error
	Attach(containerID string, in io.Reader, out, err io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error
	PortForward(podSandboxID string, port int32, stream io.ReadWriteCloser) error
}

// Config defines the options used for running the stream server.
type Config struct {
	// The host:port address the server will listen on.
	Addr string
	// The optional base URL for constructing streaming URLs. If empty, the baseURL will be
	// constructed from the serve address.
	BaseURL *url.URL

	// How long to leave idle connections open for.
	StreamIdleTimeout time.Duration
	// How long to wait for clients to create streams. Only used for SPDY streaming.
	StreamCreationTimeout time.Duration

	// The streaming protocols the server supports (understands and permits).  See
	// k8s.io/kubernetes/pkg/kubelet/server/remotecommand/constants.go for available protocols.
	// Only used for SPDY streaming.
	SupportedRemoteCommandProtocols []string

	// The streaming protocols the server supports (understands and permits).  See
	// k8s.io/kubernetes/pkg/kubelet/server/portforward/constants.go for available protocols.
	// Only used for SPDY streaming.
	SupportedPortForwardProtocols []string

	// The config for serving over TLS. If nil, TLS will not be used.
	TLSConfig *tls.Config
}

// DefaultConfig provides default values for server Config. The DefaultConfig is partial, so
// some fields like Addr must still be provided.
var DefaultConfig = Config{
	StreamIdleTimeout:               4 * time.Hour,
	StreamCreationTimeout:           remotecommandconsts.DefaultStreamCreationTimeout,
	SupportedRemoteCommandProtocols: remotecommandconsts.SupportedStreamingProtocols,
	SupportedPortForwardProtocols:   portforward.SupportedProtocols,
}

// TODO(tallclair): Add auth(n/z) interface & handling.
func NewServer(config Config, runtime Runtime) (Server, error) {
	s := &server{
		config:  config,
		runtime: &criAdapter{runtime},
		cache:   newRequestCache(),
	}

	if s.config.BaseURL == nil {
		s.config.BaseURL = &url.URL{
			Scheme: "http",
			Host:   s.config.Addr,
		}
		if s.config.TLSConfig != nil {
			s.config.BaseURL.Scheme = "https"
		}
	}

	ws := &restful.WebService{}
	endpoints := []struct {
		path    string
		handler restful.RouteFunction
	}{
		{"/exec/{token}", s.serveExec},
		{"/attach/{token}", s.serveAttach},
		{"/portforward/{token}", s.servePortForward},
	}
	// If serving relative to a base path, set that here.
	pathPrefix := path.Dir(s.config.BaseURL.Path)
	for _, e := range endpoints {
		for _, method := range []string{"GET", "POST"} {
			ws.Route(ws.
				Method(method).
				Path(path.Join(pathPrefix, e.path)).
				To(e.handler))
		}
	}
	handler := restful.NewContainer()
	handler.Add(ws)
	s.handler = handler

	return s, nil
}

type server struct {
	config  Config
	runtime *criAdapter
	handler http.Handler
	cache   *requestCache
}

func (s *server) GetExec(req *runtimeapi.ExecRequest) (*runtimeapi.ExecResponse, error) {
	if req.ContainerId == "" {
		return nil, grpc.Errorf(codes.InvalidArgument, "missing required container_id")
	}
	token, err := s.cache.Insert(req)
	if err != nil {
		return nil, err
	}
	return &runtimeapi.ExecResponse{
		Url: s.buildURL("exec", token),
	}, nil
}

func (s *server) GetAttach(req *runtimeapi.AttachRequest) (*runtimeapi.AttachResponse, error) {
	if req.ContainerId == "" {
		return nil, grpc.Errorf(codes.InvalidArgument, "missing required container_id")
	}
	token, err := s.cache.Insert(req)
	if err != nil {
		return nil, err
	}
	return &runtimeapi.AttachResponse{
		Url: s.buildURL("attach", token),
	}, nil
}

func (s *server) GetPortForward(req *runtimeapi.PortForwardRequest) (*runtimeapi.PortForwardResponse, error) {
	if req.PodSandboxId == "" {
		return nil, grpc.Errorf(codes.InvalidArgument, "missing required pod_sandbox_id")
	}
	token, err := s.cache.Insert(req)
	if err != nil {
		return nil, err
	}
	return &runtimeapi.PortForwardResponse{
		Url: s.buildURL("portforward", token),
	}, nil
}

func (s *server) Start(stayUp bool) error {
	if !stayUp {
		// TODO(tallclair): Implement this.
		return errors.New("stayUp=false is not yet implemented")
	}

	server := &http.Server{
		Addr:      s.config.Addr,
		Handler:   s.handler,
		TLSConfig: s.config.TLSConfig,
	}
	if s.config.TLSConfig != nil {
		return server.ListenAndServeTLS("", "") // Use certs from TLSConfig.
	} else {
		return server.ListenAndServe()
	}
}

func (s *server) Stop() error {
	// TODO(tallclair): Implement this.
	return errors.New("not yet implemented")
}

func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	s.handler.ServeHTTP(w, r)
}

func (s *server) buildURL(method, token string) string {
	return s.config.BaseURL.ResolveReference(&url.URL{
		Path: path.Join(method, token),
	}).String()
}

func (s *server) serveExec(req *restful.Request, resp *restful.Response) {
	token := req.PathParameter("token")
	cachedRequest, ok := s.cache.Consume(token)
	if !ok {
		http.NotFound(resp.ResponseWriter, req.Request)
		return
	}
	exec, ok := cachedRequest.(*runtimeapi.ExecRequest)
	if !ok {
		http.NotFound(resp.ResponseWriter, req.Request)
		return
	}

	streamOpts := &remotecommandserver.Options{
		Stdin:  exec.Stdin,
		Stdout: true,
		Stderr: !exec.Tty,
		TTY:    exec.Tty,
	}

	remotecommandserver.ServeExec(
		resp.ResponseWriter,
		req.Request,
		s.runtime,
		"", // unused: podName
		"", // unusued: podUID
		exec.ContainerId,
		exec.Cmd,
		streamOpts,
		s.config.StreamIdleTimeout,
		s.config.StreamCreationTimeout,
		s.config.SupportedRemoteCommandProtocols)
}

func (s *server) serveAttach(req *restful.Request, resp *restful.Response) {
	token := req.PathParameter("token")
	cachedRequest, ok := s.cache.Consume(token)
	if !ok {
		http.NotFound(resp.ResponseWriter, req.Request)
		return
	}
	attach, ok := cachedRequest.(*runtimeapi.AttachRequest)
	if !ok {
		http.NotFound(resp.ResponseWriter, req.Request)
		return
	}

	streamOpts := &remotecommandserver.Options{
		Stdin:  attach.Stdin,
		Stdout: true,
		Stderr: !attach.Tty,
		TTY:    attach.Tty,
	}
	remotecommandserver.ServeAttach(
		resp.ResponseWriter,
		req.Request,
		s.runtime,
		"", // unused: podName
		"", // unusued: podUID
		attach.ContainerId,
		streamOpts,
		s.config.StreamIdleTimeout,
		s.config.StreamCreationTimeout,
		s.config.SupportedRemoteCommandProtocols)
}

func (s *server) servePortForward(req *restful.Request, resp *restful.Response) {
	token := req.PathParameter("token")
	cachedRequest, ok := s.cache.Consume(token)
	if !ok {
		http.NotFound(resp.ResponseWriter, req.Request)
		return
	}
	pf, ok := cachedRequest.(*runtimeapi.PortForwardRequest)
	if !ok {
		http.NotFound(resp.ResponseWriter, req.Request)
		return
	}

	portForwardOptions, err := portforward.BuildV4Options(pf.Port)
	if err != nil {
		resp.WriteError(http.StatusBadRequest, err)
		return
	}

	portforward.ServePortForward(
		resp.ResponseWriter,
		req.Request,
		s.runtime,
		pf.PodSandboxId,
		"", // unused: podUID
		portForwardOptions,
		s.config.StreamIdleTimeout,
		s.config.StreamCreationTimeout,
		s.config.SupportedPortForwardProtocols)
}

// criAdapter wraps the Runtime functions to conform to the remotecommand interfaces.
// The adapter binds the container ID to the container name argument, and the pod sandbox ID to the pod name.
type criAdapter struct {
	Runtime
}

var _ remotecommandserver.Executor = &criAdapter{}
var _ remotecommandserver.Attacher = &criAdapter{}
var _ portforward.PortForwarder = &criAdapter{}

func (a *criAdapter) ExecInContainer(podName string, podUID types.UID, container string, cmd []string, in io.Reader, out, err io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error {
	return a.Exec(container, cmd, in, out, err, tty, resize)
}

func (a *criAdapter) AttachContainer(podName string, podUID types.UID, container string, in io.Reader, out, err io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error {
	return a.Attach(container, in, out, err, tty, resize)
}

func (a *criAdapter) PortForward(podName string, podUID types.UID, port int32, stream io.ReadWriteCloser) error {
	return a.Runtime.PortForward(podName, port, stream)
}
<commit_msg>Implement stop function in streaming server.<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package streaming

import (
	"crypto/tls"
	"errors"
	"io"
	"net/http"
	"net/url"
	"path"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"

	restful "github.com/emicklei/go-restful"

	"k8s.io/apimachinery/pkg/types"
	remotecommandconsts "k8s.io/apimachinery/pkg/util/remotecommand"
	"k8s.io/client-go/tools/remotecommand"
	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/v1alpha1/runtime"
	"k8s.io/kubernetes/pkg/kubelet/server/portforward"
	remotecommandserver "k8s.io/kubernetes/pkg/kubelet/server/remotecommand"
)

// The library interface to serve the stream requests.
type Server interface {
	http.Handler

	// Get the serving URL for the requests.
	// Requests must not be nil. Responses may be nil iff an error is returned.
	GetExec(*runtimeapi.ExecRequest) (*runtimeapi.ExecResponse, error)
	GetAttach(req *runtimeapi.AttachRequest) (*runtimeapi.AttachResponse, error)
	GetPortForward(*runtimeapi.PortForwardRequest) (*runtimeapi.PortForwardResponse, error)

	// Start the server.
	// addr is the address to serve on (address:port) stayUp indicates whether the server should
	// listen until Stop() is called, or automatically stop after all expected connections are
	// closed. Calling Get{Exec,Attach,PortForward} increments the expected connection count.
	// Function does not return until the server is stopped.
	Start(stayUp bool) error
	// Stop the server, and terminate any open connections.
	Stop() error
}

// The interface to execute the commands and provide the streams.
type Runtime interface {
	Exec(containerID string, cmd []string, in io.Reader, out, err io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error
	Attach(containerID string, in io.Reader, out, err io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error
	PortForward(podSandboxID string, port int32, stream io.ReadWriteCloser) error
}

// Config defines the options used for running the stream server.
type Config struct {
	// The host:port address the server will listen on.
	Addr string
	// The optional base URL for constructing streaming URLs. If empty, the baseURL will be
	// constructed from the serve address.
	BaseURL *url.URL

	// How long to leave idle connections open for.
	StreamIdleTimeout time.Duration
	// How long to wait for clients to create streams. Only used for SPDY streaming.
	StreamCreationTimeout time.Duration

	// The streaming protocols the server supports (understands and permits).  See
	// k8s.io/kubernetes/pkg/kubelet/server/remotecommand/constants.go for available protocols.
	// Only used for SPDY streaming.
	SupportedRemoteCommandProtocols []string

	// The streaming protocols the server supports (understands and permits).  See
	// k8s.io/kubernetes/pkg/kubelet/server/portforward/constants.go for available protocols.
	// Only used for SPDY streaming.
	SupportedPortForwardProtocols []string

	// The config for serving over TLS. If nil, TLS will not be used.
	TLSConfig *tls.Config
}

// DefaultConfig provides default values for server Config. The DefaultConfig is partial, so
// some fields like Addr must still be provided.
var DefaultConfig = Config{
	StreamIdleTimeout:               4 * time.Hour,
	StreamCreationTimeout:           remotecommandconsts.DefaultStreamCreationTimeout,
	SupportedRemoteCommandProtocols: remotecommandconsts.SupportedStreamingProtocols,
	SupportedPortForwardProtocols:   portforward.SupportedProtocols,
}

// TODO(tallclair): Add auth(n/z) interface & handling.
func NewServer(config Config, runtime Runtime) (Server, error) {
	s := &server{
		config:  config,
		runtime: &criAdapter{runtime},
		cache:   newRequestCache(),
	}

	if s.config.BaseURL == nil {
		s.config.BaseURL = &url.URL{
			Scheme: "http",
			Host:   s.config.Addr,
		}
		if s.config.TLSConfig != nil {
			s.config.BaseURL.Scheme = "https"
		}
	}

	ws := &restful.WebService{}
	endpoints := []struct {
		path    string
		handler restful.RouteFunction
	}{
		{"/exec/{token}", s.serveExec},
		{"/attach/{token}", s.serveAttach},
		{"/portforward/{token}", s.servePortForward},
	}
	// If serving relative to a base path, set that here.
	pathPrefix := path.Dir(s.config.BaseURL.Path)
	for _, e := range endpoints {
		for _, method := range []string{"GET", "POST"} {
			ws.Route(ws.
				Method(method).
				Path(path.Join(pathPrefix, e.path)).
				To(e.handler))
		}
	}
	handler := restful.NewContainer()
	handler.Add(ws)
	s.handler = handler
	s.server = &http.Server{
		Addr:      s.config.Addr,
		Handler:   s.handler,
		TLSConfig: s.config.TLSConfig,
	}

	return s, nil
}

type server struct {
	config  Config
	runtime *criAdapter
	handler http.Handler
	cache   *requestCache
	server  *http.Server
}

func (s *server) GetExec(req *runtimeapi.ExecRequest) (*runtimeapi.ExecResponse, error) {
	if req.ContainerId == "" {
		return nil, grpc.Errorf(codes.InvalidArgument, "missing required container_id")
	}
	token, err := s.cache.Insert(req)
	if err != nil {
		return nil, err
	}
	return &runtimeapi.ExecResponse{
		Url: s.buildURL("exec", token),
	}, nil
}

func (s *server) GetAttach(req *runtimeapi.AttachRequest) (*runtimeapi.AttachResponse, error) {
	if req.ContainerId == "" {
		return nil, grpc.Errorf(codes.InvalidArgument, "missing required container_id")
	}
	token, err := s.cache.Insert(req)
	if err != nil {
		return nil, err
	}
	return &runtimeapi.AttachResponse{
		Url: s.buildURL("attach", token),
	}, nil
}

func (s *server) GetPortForward(req *runtimeapi.PortForwardRequest) (*runtimeapi.PortForwardResponse, error) {
	if req.PodSandboxId == "" {
		return nil, grpc.Errorf(codes.InvalidArgument, "missing required pod_sandbox_id")
	}
	token, err := s.cache.Insert(req)
	if err != nil {
		return nil, err
	}
	return &runtimeapi.PortForwardResponse{
		Url: s.buildURL("portforward", token),
	}, nil
}

func (s *server) Start(stayUp bool) error {
	if !stayUp {
		// TODO(tallclair): Implement this.
		return errors.New("stayUp=false is not yet implemented")
	}

	if s.config.TLSConfig != nil {
		return s.server.ListenAndServeTLS("", "") // Use certs from TLSConfig.
	} else {
		return s.server.ListenAndServe()
	}
}

func (s *server) Stop() error {
	return s.server.Close()
}

func (s *server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	s.handler.ServeHTTP(w, r)
}

func (s *server) buildURL(method, token string) string {
	return s.config.BaseURL.ResolveReference(&url.URL{
		Path: path.Join(method, token),
	}).String()
}

func (s *server) serveExec(req *restful.Request, resp *restful.Response) {
	token := req.PathParameter("token")
	cachedRequest, ok := s.cache.Consume(token)
	if !ok {
		http.NotFound(resp.ResponseWriter, req.Request)
		return
	}
	exec, ok := cachedRequest.(*runtimeapi.ExecRequest)
	if !ok {
		http.NotFound(resp.ResponseWriter, req.Request)
		return
	}

	streamOpts := &remotecommandserver.Options{
		Stdin:  exec.Stdin,
		Stdout: true,
		Stderr: !exec.Tty,
		TTY:    exec.Tty,
	}

	remotecommandserver.ServeExec(
		resp.ResponseWriter,
		req.Request,
		s.runtime,
		"", // unused: podName
		"", // unusued: podUID
		exec.ContainerId,
		exec.Cmd,
		streamOpts,
		s.config.StreamIdleTimeout,
		s.config.StreamCreationTimeout,
		s.config.SupportedRemoteCommandProtocols)
}

func (s *server) serveAttach(req *restful.Request, resp *restful.Response) {
	token := req.PathParameter("token")
	cachedRequest, ok := s.cache.Consume(token)
	if !ok {
		http.NotFound(resp.ResponseWriter, req.Request)
		return
	}
	attach, ok := cachedRequest.(*runtimeapi.AttachRequest)
	if !ok {
		http.NotFound(resp.ResponseWriter, req.Request)
		return
	}

	streamOpts := &remotecommandserver.Options{
		Stdin:  attach.Stdin,
		Stdout: true,
		Stderr: !attach.Tty,
		TTY:    attach.Tty,
	}
	remotecommandserver.ServeAttach(
		resp.ResponseWriter,
		req.Request,
		s.runtime,
		"", // unused: podName
		"", // unusued: podUID
		attach.ContainerId,
		streamOpts,
		s.config.StreamIdleTimeout,
		s.config.StreamCreationTimeout,
		s.config.SupportedRemoteCommandProtocols)
}

func (s *server) servePortForward(req *restful.Request, resp *restful.Response) {
	token := req.PathParameter("token")
	cachedRequest, ok := s.cache.Consume(token)
	if !ok {
		http.NotFound(resp.ResponseWriter, req.Request)
		return
	}
	pf, ok := cachedRequest.(*runtimeapi.PortForwardRequest)
	if !ok {
		http.NotFound(resp.ResponseWriter, req.Request)
		return
	}

	portForwardOptions, err := portforward.BuildV4Options(pf.Port)
	if err != nil {
		resp.WriteError(http.StatusBadRequest, err)
		return
	}

	portforward.ServePortForward(
		resp.ResponseWriter,
		req.Request,
		s.runtime,
		pf.PodSandboxId,
		"", // unused: podUID
		portForwardOptions,
		s.config.StreamIdleTimeout,
		s.config.StreamCreationTimeout,
		s.config.SupportedPortForwardProtocols)
}

// criAdapter wraps the Runtime functions to conform to the remotecommand interfaces.
// The adapter binds the container ID to the container name argument, and the pod sandbox ID to the pod name.
type criAdapter struct {
	Runtime
}

var _ remotecommandserver.Executor = &criAdapter{}
var _ remotecommandserver.Attacher = &criAdapter{}
var _ portforward.PortForwarder = &criAdapter{}

func (a *criAdapter) ExecInContainer(podName string, podUID types.UID, container string, cmd []string, in io.Reader, out, err io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize, timeout time.Duration) error {
	return a.Exec(container, cmd, in, out, err, tty, resize)
}

func (a *criAdapter) AttachContainer(podName string, podUID types.UID, container string, in io.Reader, out, err io.WriteCloser, tty bool, resize <-chan remotecommand.TerminalSize) error {
	return a.Attach(container, in, out, err, tty, resize)
}

func (a *criAdapter) PortForward(podName string, podUID types.UID, port int32, stream io.ReadWriteCloser) error {
	return a.Runtime.PortForward(podName, port, stream)
}
<|endoftext|>


<commit_before>// Package consumer implements command line crawl consumer from nsq.
package consumer

import (
	"errors"
	"fmt"
	"os"
	"os/signal"
	"time"

	"github.com/codegangsta/cli"
	"github.com/golang/glog"

	"github.com/crackcomm/crawl"
	"github.com/crackcomm/crawl/nsq/nsqcrawl"
)

// Spider - Spider registrator.
type Spider func(crawl.Crawler)

// App - Consumer command line application structure.
type App struct {
	// Ctx - Cli context, set on action.
	Ctx *cli.Context

	// Queue - NSQ queue. Constructed on first Action() call.
	*nsqcrawl.Queue

	// before - Flag requirements checking.
	before func(c *App) error

	// crawler - Accessed using Crawler() which constructs it on first call
	// using parameters from command line.
	crawler crawl.Crawler

	// crawlerConstructor - Constructs a crawler. Called only once in Crawler().
	// It can be changed using WithCrawlerConstructor()
	crawlerConstructor func(*App) crawl.Crawler

	// opts - Options which are applied on Action() call with all required
	// parameters from the command line context.
	opts []Option

	// spiderConstructors - List of functions which use flags to construct spider.
	spiderConstructors []func(*App) Spider
}

// Flags - Consumer app flags.
var Flags = []cli.Flag{
	cli.StringFlag{
		Name:   "topic",
		EnvVar: "TOPIC",
		Usage:  "crawl requests nsq topic (required)",
		Value:  "crawl_requests",
	},
	cli.StringFlag{
		Name:   "channel",
		EnvVar: "CHANNEL",
		Usage:  "crawl requests nsq channel (required)",
		Value:  "default",
	},
	cli.StringSliceFlag{
		Name:   "nsq-addr",
		EnvVar: "NSQ_ADDR",
	},
	cli.StringSliceFlag{
		Name:   "nsqlookup-addr",
		EnvVar: "NSQLOOKUP_ADDR",
	},
	cli.IntFlag{
		Name:   "concurrency",
		Value:  100,
		EnvVar: "CONCURRENCY",
	},
	cli.IntFlag{
		Name:   "timeout",
		Usage:  "default timeout in seconds",
		Value:  30,
		EnvVar: "TIMEOUT",
	},
}

// New - Creates nsq consumer app.
func New(opts ...Option) *cli.App {
	app := &App{opts: opts}
	cliapp := cli.NewApp()
	cliapp.Name = "crawler"
	cliapp.HelpName = cliapp.Name
	cliapp.Version = "0.0.1"
	cliapp.Usage = "nsq crawl consumer"
	cliapp.Flags = Flags
	cliapp.Action = app.Action
	return cliapp
}

// Action - Command line action.
func (app *App) Action(c *cli.Context) {
	app.Ctx = c
	app.Queue = nsqcrawl.NewQueue(c.String("topic"), c.String("channel"), c.Int("concurrency"))

	for _, opt := range app.opts {
		opt(app)
	}

	if err := app.Before(c); err != nil {
		glog.Fatal(err)
	}

	crawler := app.Crawler()

	if err := app.connectNSQ(c); err != nil {
		glog.Fatal(err)
	}

	for _, spiderConstructor := range app.spiderConstructors {
		if spider := spiderConstructor(app); spider != nil {
			spider(crawler)
		}
	}

	go func() {
		for err := range crawler.Errors() {
			glog.Warningf("crawl %v", err)
		}
	}()

	done := make(chan bool, 1)
	go func() {
		crawler.Start()
		done <- true
	}()

	glog.Infof("Started crawler (topic=%q)", c.String("topic"))

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt)

	for {
		select {
		case <-done:
			glog.Info("Crawler closed")
			return
		case s := <-sig:
			glog.Infof("Received signal %v, closing crawler", s)
			if err := app.Queue.Close(); err != nil {
				glog.Fatalf("Error closing queue: %v", err)
			}
			return
		}
	}
}

func (app *App) connectNSQ(c *cli.Context) (err error) {
	nsqAddr := c.StringSlice("nsq-addr")[0]
	if err := app.Queue.Producer.Connect(nsqAddr); err != nil {
		return fmt.Errorf("Error connecting producer to %q: %v", nsqAddr, err)
	}

	if addrs := c.StringSlice("nsq-addr"); len(addrs) != 0 {
		for _, addr := range addrs {
			glog.V(3).Infof("Connecting to nsq %s", addr)
			if err := app.Queue.Consumer.Connect(addr); err != nil {
				return fmt.Errorf("Error connecting to nsq %q: %v", addr, err)
			}
			glog.V(3).Infof("Connected to nsq %s", addr)
		}
	}

	if addrs := c.StringSlice("nsqlookup-addr"); len(addrs) != 0 {
		for _, addr := range addrs {
			glog.V(3).Infof("Connecting to nsq lookup %s", addr)
			if err := app.Queue.Consumer.ConnectLookupd(addr); err != nil {
				return fmt.Errorf("Error connecting to nsq lookup %q: %v", addr, err)
			}
			glog.V(3).Infof("Connected to nsq lookup %s", addr)
		}
	}

	return
}

// Before - Executed before action.
func (app *App) Before(c *cli.Context) (err error) {
	if app.before != nil {
		return app.before(app)
	}
	return beforeApp(c)
}

func beforeApp(c *cli.Context) error {
	if c.String("topic") == "" {
		return errors.New("flag --topic cannot be empty")
	}
	if len(c.StringSlice("nsq-addr")) == 0 && len(c.StringSlice("nsqlookup-addr")) == 0 {
		return errors.New("At least one --nsq-addr or --nsqlookup-addr is required")
	}
	return nil
}

// Crawler - Returns app crawler. Constructs if empty.
func (app *App) Crawler() crawl.Crawler {
	if app.crawler == nil {
		if app.crawlerConstructor != nil {
			app.crawler = app.crawlerConstructor(app)
		} else {
			app.crawler = crawlerConstructor(app)
		}
	}
	return app.crawler
}

func crawlerConstructor(app *App) crawl.Crawler {
	return crawl.New(
		crawl.WithQueue(app.Queue),
		crawl.WithConcurrency(app.Ctx.Int("concurrency")),
		crawl.WithDefaultTimeout(time.Duration(app.Ctx.Int("timeout"))*time.Second),
	)
}
<commit_msg>nsq consumer: before is now not over-writable<commit_after>// Package consumer implements command line crawl consumer from nsq.
package consumer

import (
	"errors"
	"fmt"
	"os"
	"os/signal"
	"time"

	"github.com/codegangsta/cli"
	"github.com/golang/glog"

	"github.com/crackcomm/crawl"
	"github.com/crackcomm/crawl/nsq/nsqcrawl"
)

// Spider - Spider registrator.
type Spider func(crawl.Crawler)

// App - Consumer command line application structure.
type App struct {
	// Ctx - Cli context, set on action.
	Ctx *cli.Context

	// Queue - NSQ queue. Constructed on first Action() call.
	*nsqcrawl.Queue

	// before - Flag requirements checking.
	before func(c *App) error

	// crawler - Accessed using Crawler() which constructs it on first call
	// using parameters from command line.
	crawler crawl.Crawler

	// crawlerConstructor - Constructs a crawler. Called only once in Crawler().
	// It can be changed using WithCrawlerConstructor()
	crawlerConstructor func(*App) crawl.Crawler

	// opts - Options which are applied on Action() call with all required
	// parameters from the command line context.
	opts []Option

	// spiderConstructors - List of functions which use flags to construct spider.
	spiderConstructors []func(*App) Spider
}

// Flags - Consumer app flags.
var Flags = []cli.Flag{
	cli.StringFlag{
		Name:   "topic",
		EnvVar: "TOPIC",
		Usage:  "crawl requests nsq topic (required)",
		Value:  "crawl_requests",
	},
	cli.StringFlag{
		Name:   "channel",
		EnvVar: "CHANNEL",
		Usage:  "crawl requests nsq channel (required)",
		Value:  "default",
	},
	cli.StringSliceFlag{
		Name:   "nsq-addr",
		EnvVar: "NSQ_ADDR",
	},
	cli.StringSliceFlag{
		Name:   "nsqlookup-addr",
		EnvVar: "NSQLOOKUP_ADDR",
	},
	cli.IntFlag{
		Name:   "concurrency",
		Value:  100,
		EnvVar: "CONCURRENCY",
	},
	cli.IntFlag{
		Name:   "timeout",
		Usage:  "default timeout in seconds",
		Value:  30,
		EnvVar: "TIMEOUT",
	},
}

// New - Creates nsq consumer app.
func New(opts ...Option) *cli.App {
	app := &App{opts: opts}
	cliapp := cli.NewApp()
	cliapp.Name = "crawler"
	cliapp.HelpName = cliapp.Name
	cliapp.Version = "0.0.1"
	cliapp.Usage = "nsq crawl consumer"
	cliapp.Flags = Flags
	cliapp.Action = app.Action
	return cliapp
}

// Action - Command line action.
func (app *App) Action(c *cli.Context) {
	app.Ctx = c
	app.Queue = nsqcrawl.NewQueue(c.String("topic"), c.String("channel"), c.Int("concurrency"))

	for _, opt := range app.opts {
		opt(app)
	}

	if err := app.Before(c); err != nil {
		glog.Fatal(err)
	}

	crawler := app.Crawler()

	if err := app.connectNSQ(c); err != nil {
		glog.Fatal(err)
	}

	for _, spiderConstructor := range app.spiderConstructors {
		if spider := spiderConstructor(app); spider != nil {
			spider(crawler)
		}
	}

	go func() {
		for err := range crawler.Errors() {
			glog.Warningf("crawl %v", err)
		}
	}()

	done := make(chan bool, 1)
	go func() {
		crawler.Start()
		done <- true
	}()

	glog.Infof("Started crawler (topic=%q)", c.String("topic"))

	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt)

	for {
		select {
		case <-done:
			glog.Info("Crawler closed")
			return
		case s := <-sig:
			glog.Infof("Received signal %v, closing crawler", s)
			if err := app.Queue.Close(); err != nil {
				glog.Fatalf("Error closing queue: %v", err)
			}
			return
		}
	}
}

func (app *App) connectNSQ(c *cli.Context) (err error) {
	nsqAddr := c.StringSlice("nsq-addr")[0]
	if err := app.Queue.Producer.Connect(nsqAddr); err != nil {
		return fmt.Errorf("Error connecting producer to %q: %v", nsqAddr, err)
	}

	if addrs := c.StringSlice("nsq-addr"); len(addrs) != 0 {
		for _, addr := range addrs {
			glog.V(3).Infof("Connecting to nsq %s", addr)
			if err := app.Queue.Consumer.Connect(addr); err != nil {
				return fmt.Errorf("Error connecting to nsq %q: %v", addr, err)
			}
			glog.V(3).Infof("Connected to nsq %s", addr)
		}
	}

	if addrs := c.StringSlice("nsqlookup-addr"); len(addrs) != 0 {
		for _, addr := range addrs {
			glog.V(3).Infof("Connecting to nsq lookup %s", addr)
			if err := app.Queue.Consumer.ConnectLookupd(addr); err != nil {
				return fmt.Errorf("Error connecting to nsq lookup %q: %v", addr, err)
			}
			glog.V(3).Infof("Connected to nsq lookup %s", addr)
		}
	}

	return
}

// Before - Executed before action.
func (app *App) Before(c *cli.Context) (err error) {
	if app.before != nil {
		err = app.before(app)
		if err != nil {
			return
		}
	}
	return beforeApp(c)
}

func beforeApp(c *cli.Context) error {
	if c.String("topic") == "" {
		return errors.New("flag --topic cannot be empty")
	}
	if len(c.StringSlice("nsq-addr")) == 0 && len(c.StringSlice("nsqlookup-addr")) == 0 {
		return errors.New("At least one --nsq-addr or --nsqlookup-addr is required")
	}
	return nil
}

// Crawler - Returns app crawler. Constructs if empty.
func (app *App) Crawler() crawl.Crawler {
	if app.crawler == nil {
		if app.crawlerConstructor != nil {
			app.crawler = app.crawlerConstructor(app)
		} else {
			app.crawler = crawlerConstructor(app)
		}
	}
	return app.crawler
}

func crawlerConstructor(app *App) crawl.Crawler {
	return crawl.New(
		crawl.WithQueue(app.Queue),
		crawl.WithConcurrency(app.Ctx.Int("concurrency")),
		crawl.WithDefaultTimeout(time.Duration(app.Ctx.Int("timeout"))*time.Second),
	)
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import "testing"

func TestGetVersion(t *testing.T) {
	testCases := []struct {
		groupVersion string
		output       string
	}{
		{
			"v1",
			"v1",
		},
		{
			"extensions/v1beta1",
			"v1beta1",
		},
	}
	for _, test := range testCases {
		actual := GetVersion(test.groupVersion)
		if test.output != actual {
			t.Errorf("expect version: %s, got: %s\n", test.output, actual)
		}
	}
}

func TestGetGroup(t *testing.T) {
	testCases := []struct {
		groupVersion string
		output       string
	}{
		{
			"v1",
			"",
		},
		{
			"extensions/v1beta1",
			"extensions",
		},
	}
	for _, test := range testCases {
		actual := GetGroup(test.groupVersion)
		if test.output != actual {
			t.Errorf("expect version: %s, got: %s\n", test.output, actual)
		}
	}
}
<commit_msg>add unit test for groupversion<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package util

import "testing"

func TestGetVersion(t *testing.T) {
	testCases := []struct {
		groupVersion string
		output       string
	}{
		{
			"v1",
			"v1",
		},
		{
			"extensions/v1beta1",
			"v1beta1",
		},
	}
	for _, test := range testCases {
		actual := GetVersion(test.groupVersion)
		if test.output != actual {
			t.Errorf("expect version: %s, got: %s\n", test.output, actual)
		}
	}
}

func TestGetGroup(t *testing.T) {
	testCases := []struct {
		groupVersion string
		output       string
	}{
		{
			"v1",
			"",
		},
		{
			"extensions/v1beta1",
			"extensions",
		},
	}
	for _, test := range testCases {
		actual := GetGroup(test.groupVersion)
		if test.output != actual {
			t.Errorf("expect version: %s, got: %s\n", test.output, actual)
		}
	}
}

func TestGetGroupVersion(t *testing.T) {
	testCases := []struct {
		group   string
		version string
		output  string
	}{
		{
			"",
			"v1",
			"v1",
		},
		{
			"extensions",
			"",
			"extensions/",
		},
		{
			"extensions",
			"v1beta1",
			"extensions/v1beta1",
		},
	}
	for _, test := range testCases {
		actual := GetGroupVersion(test.group, test.version)
		if test.output != actual {
			t.Errorf("expect version: %s, got: %s\n", test.output, actual)
		}
	}
}
<|endoftext|>