<commit_before>package master

import (
	"fmt"
	"time"

	"github.com/golang/glog"

	kapierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	ktypes "k8s.io/apimachinery/pkg/types"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/tools/cache"
	kapi "k8s.io/kubernetes/pkg/apis/core"
	kclientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	kinternalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
	kcoreinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion/core/internalversion"

	osconfigapi "github.com/openshift/origin/pkg/cmd/server/apis/config"
	"github.com/openshift/origin/pkg/network"
	networkapi "github.com/openshift/origin/pkg/network/apis/network"
	osapivalidation "github.com/openshift/origin/pkg/network/apis/network/validation"
	"github.com/openshift/origin/pkg/network/common"
	networkinternalinformers "github.com/openshift/origin/pkg/network/generated/informers/internalversion"
	networkinformers "github.com/openshift/origin/pkg/network/generated/informers/internalversion/network/internalversion"
	networkclient "github.com/openshift/origin/pkg/network/generated/internalclientset"
	"github.com/openshift/origin/pkg/util/netutils"
)

const (
	tun0 = "tun0"
)

type OsdnMaster struct {
	kClient       kclientset.Interface
	networkClient networkclient.Interface
	networkInfo   *common.NetworkInfo
	vnids         *masterVNIDMap

	nodeInformer         kcoreinformers.NodeInformer
	namespaceInformer    kcoreinformers.NamespaceInformer
	hostSubnetInformer   networkinformers.HostSubnetInformer
	netNamespaceInformer networkinformers.NetNamespaceInformer

	// Used for allocating subnets in order
	subnetAllocatorList []*SubnetAllocator
	// Used for clusterNetwork --> subnetAllocator lookup
	subnetAllocatorMap map[common.ClusterNetwork]*SubnetAllocator

	// Holds Node IP used in creating host subnet for a node
	hostSubnetNodeIPs map[ktypes.UID]string
}

func Start(networkConfig osconfigapi.MasterNetworkConfig, networkClient networkclient.Interface,
	kClient kclientset.Interface, kubeInformers kinternalinformers.SharedInformerFactory,
	networkInformers networkinternalinformers.SharedInformerFactory) error {
	glog.Infof("Initializing SDN master of type %q", networkConfig.NetworkPluginName)

	master := &OsdnMaster{
		kClient:       kClient,
		networkClient: networkClient,

		nodeInformer:         kubeInformers.Core().InternalVersion().Nodes(),
		namespaceInformer:    kubeInformers.Core().InternalVersion().Namespaces(),
		hostSubnetInformer:   networkInformers.Network().InternalVersion().HostSubnets(),
		netNamespaceInformer: networkInformers.Network().InternalVersion().NetNamespaces(),

		subnetAllocatorMap: map[common.ClusterNetwork]*SubnetAllocator{},
		hostSubnetNodeIPs:  map[ktypes.UID]string{},
	}

	var err error
	var clusterNetworkEntries []networkapi.ClusterNetworkEntry
	for _, entry := range networkConfig.ClusterNetworks {
		clusterNetworkEntries = append(clusterNetworkEntries, networkapi.ClusterNetworkEntry{CIDR: entry.CIDR, HostSubnetLength: entry.HostSubnetLength})
	}
	master.networkInfo, err = common.ParseNetworkInfo(clusterNetworkEntries, networkConfig.ServiceNetworkCIDR)
	if err != nil {
		return err
	}
	if len(clusterNetworkEntries) == 0 {
		panic("No ClusterNetworks set in networkConfig; should have been defaulted in if not configured")
	}

	var parsedClusterNetworkEntries []networkapi.ClusterNetworkEntry
	for _, entry := range master.networkInfo.ClusterNetworks {
		parsedClusterNetworkEntries = append(parsedClusterNetworkEntries, networkapi.ClusterNetworkEntry{CIDR: entry.ClusterCIDR.String(), HostSubnetLength: entry.HostSubnetLength})
	}

	configCN := &networkapi.ClusterNetwork{
		TypeMeta:   metav1.TypeMeta{Kind: "ClusterNetwork"},
		ObjectMeta: metav1.ObjectMeta{Name: networkapi.ClusterNetworkDefault},

		ClusterNetworks: parsedClusterNetworkEntries,
		ServiceNetwork:  master.networkInfo.ServiceNetwork.String(),
		PluginName:      networkConfig.NetworkPluginName,

		// Need to set these for backward compat
		Network:          parsedClusterNetworkEntries[0].CIDR,
		HostSubnetLength: parsedClusterNetworkEntries[0].HostSubnetLength,
	}
	osapivalidation.SetDefaultClusterNetwork(*configCN)

	// try this for a while before just dying
	var getError error
	err = wait.PollImmediate(1*time.Second, time.Minute, func() (bool, error) {
		// reset this so that failures come through correctly.
		getError = nil
		existingCN, err := master.networkClient.Network().ClusterNetworks().Get(networkapi.ClusterNetworkDefault, metav1.GetOptions{})
		if err != nil {
			if !kapierrors.IsNotFound(err) {
				// the first request can fail on permissions
				getError = err
				return false, nil
			}
			if err = master.checkClusterNetworkAgainstLocalNetworks(); err != nil {
				return false, err
			}

			if _, err = master.networkClient.Network().ClusterNetworks().Create(configCN); err != nil {
				return false, err
			}
			glog.Infof("Created ClusterNetwork %s", common.ClusterNetworkToString(configCN))

			if err = master.checkClusterNetworkAgainstClusterObjects(); err != nil {
				utilruntime.HandleError(fmt.Errorf("Cluster contains objects incompatible with new ClusterNetwork: %v", err))
			}
		} else {
			configChanged, err := clusterNetworkChanged(configCN, existingCN)
			if err != nil {
				return false, err
			}
			if configChanged {
				configCN.TypeMeta = existingCN.TypeMeta
				configCN.ObjectMeta = existingCN.ObjectMeta
				if err = master.checkClusterNetworkAgainstClusterObjects(); err != nil {
					utilruntime.HandleError(fmt.Errorf("Attempting to modify cluster to exclude existing objects: %v", err))
					return false, err
				}
				if _, err = master.networkClient.Network().ClusterNetworks().Update(configCN); err != nil {
					return false, err
				}
				glog.Infof("Updated ClusterNetwork %s", common.ClusterNetworkToString(configCN))
			} else {
				glog.V(5).Infof("No change to ClusterNetwork %s", common.ClusterNetworkToString(configCN))
			}
		}

		return true, nil
	})
	if err != nil {
		if getError != nil {
			return getError
		}
		return err
	}

	go master.startSubSystems(networkConfig.NetworkPluginName)

	return nil
}

func (master *OsdnMaster) startSubSystems(pluginName string) {
	// Wait for informer sync
	if !cache.WaitForCacheSync(wait.NeverStop,
		master.nodeInformer.Informer().GetController().HasSynced,
		master.namespaceInformer.Informer().GetController().HasSynced,
		master.hostSubnetInformer.Informer().GetController().HasSynced,
		master.netNamespaceInformer.Informer().GetController().HasSynced) {
		glog.Fatalf("failed to sync SDN master informers")
	}

	if err := master.startSubnetMaster(); err != nil {
		glog.Fatalf("failed to start subnet master: %v", err)
	}

	switch pluginName {
	case network.MultiTenantPluginName:
		master.vnids = newMasterVNIDMap(true)
	case network.NetworkPolicyPluginName:
		master.vnids = newMasterVNIDMap(false)
	}
	if master.vnids != nil {
		if err := master.startVNIDMaster(); err != nil {
			glog.Fatalf("failed to start VNID master: %v", err)
		}
	}
}

func (master *OsdnMaster) checkClusterNetworkAgainstLocalNetworks() error {
	hostIPNets, _, err := netutils.GetHostIPNetworks([]string{tun0})
	if err != nil {
		return err
	}
	return master.networkInfo.CheckHostNetworks(hostIPNets)
}

func (master *OsdnMaster) checkClusterNetworkAgainstClusterObjects() error {
	var subnets []networkapi.HostSubnet
	var pods []kapi.Pod
	var services []kapi.Service
	if subnetList, err := master.networkClient.Network().HostSubnets().List(metav1.ListOptions{}); err == nil {
		subnets = subnetList.Items
	}
	if podList, err := master.kClient.Core().Pods(metav1.NamespaceAll).List(metav1.ListOptions{}); err == nil {
		pods = podList.Items
	}
	if serviceList, err := master.kClient.Core().Services(metav1.NamespaceAll).List(metav1.ListOptions{}); err == nil {
		services = serviceList.Items
	}

	return master.networkInfo.CheckClusterObjects(subnets, pods, services)
}

func clusterNetworkChanged(obj *networkapi.ClusterNetwork, old *networkapi.ClusterNetwork) (bool, error) {

	if old.ServiceNetwork != obj.ServiceNetwork {
		return true, fmt.Errorf("cannot change the serviceNetworkCIDR of an already-deployed cluster")
	} else if old.PluginName != obj.PluginName {
		return true, nil
	} else if len(old.ClusterNetworks) != len(obj.ClusterNetworks) {
		return true, nil
	} else {
		changed := false
		for _, oldCIDR := range old.ClusterNetworks {
			found := false
			for _, newCIDR := range obj.ClusterNetworks {
				if newCIDR.CIDR == oldCIDR.CIDR && newCIDR.HostSubnetLength == oldCIDR.HostSubnetLength {
					found = true
					break
				}
			}
			if !found {
				changed = true
				break
			}
		}
		return changed, nil

	}
}
<commit_msg>Hoist sdn informer into Start()<commit_after>package master

import (
	"fmt"
	"time"

	"github.com/golang/glog"

	kapierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	ktypes "k8s.io/apimachinery/pkg/types"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/client-go/tools/cache"
	kapi "k8s.io/kubernetes/pkg/apis/core"
	kclientset "k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
	kinternalinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion"
	kcoreinformers "k8s.io/kubernetes/pkg/client/informers/informers_generated/internalversion/core/internalversion"

	osconfigapi "github.com/openshift/origin/pkg/cmd/server/apis/config"
	"github.com/openshift/origin/pkg/network"
	networkapi "github.com/openshift/origin/pkg/network/apis/network"
	osapivalidation "github.com/openshift/origin/pkg/network/apis/network/validation"
	"github.com/openshift/origin/pkg/network/common"
	networkinternalinformers "github.com/openshift/origin/pkg/network/generated/informers/internalversion"
	networkinformers "github.com/openshift/origin/pkg/network/generated/informers/internalversion/network/internalversion"
	networkclient "github.com/openshift/origin/pkg/network/generated/internalclientset"
	"github.com/openshift/origin/pkg/util/netutils"
)

const (
	tun0 = "tun0"
)

type OsdnMaster struct {
	kClient       kclientset.Interface
	networkClient networkclient.Interface
	networkInfo   *common.NetworkInfo
	vnids         *masterVNIDMap

	nodeInformer         kcoreinformers.NodeInformer
	namespaceInformer    kcoreinformers.NamespaceInformer
	hostSubnetInformer   networkinformers.HostSubnetInformer
	netNamespaceInformer networkinformers.NetNamespaceInformer

	// Used for allocating subnets in order
	subnetAllocatorList []*SubnetAllocator
	// Used for clusterNetwork --> subnetAllocator lookup
	subnetAllocatorMap map[common.ClusterNetwork]*SubnetAllocator

	// Holds Node IP used in creating host subnet for a node
	hostSubnetNodeIPs map[ktypes.UID]string
}

func Start(networkConfig osconfigapi.MasterNetworkConfig, networkClient networkclient.Interface,
	kClient kclientset.Interface, kubeInformers kinternalinformers.SharedInformerFactory,
	networkInformers networkinternalinformers.SharedInformerFactory) error {
	glog.Infof("Initializing SDN master of type %q", networkConfig.NetworkPluginName)

	master := &OsdnMaster{
		kClient:       kClient,
		networkClient: networkClient,

		nodeInformer:         kubeInformers.Core().InternalVersion().Nodes(),
		namespaceInformer:    kubeInformers.Core().InternalVersion().Namespaces(),
		hostSubnetInformer:   networkInformers.Network().InternalVersion().HostSubnets(),
		netNamespaceInformer: networkInformers.Network().InternalVersion().NetNamespaces(),

		subnetAllocatorMap: map[common.ClusterNetwork]*SubnetAllocator{},
		hostSubnetNodeIPs:  map[ktypes.UID]string{},
	}

	var err error
	var clusterNetworkEntries []networkapi.ClusterNetworkEntry
	for _, entry := range networkConfig.ClusterNetworks {
		clusterNetworkEntries = append(clusterNetworkEntries, networkapi.ClusterNetworkEntry{CIDR: entry.CIDR, HostSubnetLength: entry.HostSubnetLength})
	}
	master.networkInfo, err = common.ParseNetworkInfo(clusterNetworkEntries, networkConfig.ServiceNetworkCIDR)
	if err != nil {
		return err
	}
	if len(clusterNetworkEntries) == 0 {
		panic("No ClusterNetworks set in networkConfig; should have been defaulted in if not configured")
	}

	var parsedClusterNetworkEntries []networkapi.ClusterNetworkEntry
	for _, entry := range master.networkInfo.ClusterNetworks {
		parsedClusterNetworkEntries = append(parsedClusterNetworkEntries, networkapi.ClusterNetworkEntry{CIDR: entry.ClusterCIDR.String(), HostSubnetLength: entry.HostSubnetLength})
	}

	configCN := &networkapi.ClusterNetwork{
		TypeMeta:   metav1.TypeMeta{Kind: "ClusterNetwork"},
		ObjectMeta: metav1.ObjectMeta{Name: networkapi.ClusterNetworkDefault},

		ClusterNetworks: parsedClusterNetworkEntries,
		ServiceNetwork:  master.networkInfo.ServiceNetwork.String(),
		PluginName:      networkConfig.NetworkPluginName,

		// Need to set these for backward compat
		Network:          parsedClusterNetworkEntries[0].CIDR,
		HostSubnetLength: parsedClusterNetworkEntries[0].HostSubnetLength,
	}
	osapivalidation.SetDefaultClusterNetwork(*configCN)

	// try this for a while before just dying
	var getError error
	err = wait.PollImmediate(1*time.Second, time.Minute, func() (bool, error) {
		// reset this so that failures come through correctly.
		getError = nil
		existingCN, err := master.networkClient.Network().ClusterNetworks().Get(networkapi.ClusterNetworkDefault, metav1.GetOptions{})
		if err != nil {
			if !kapierrors.IsNotFound(err) {
				// the first request can fail on permissions
				getError = err
				return false, nil
			}
			if err = master.checkClusterNetworkAgainstLocalNetworks(); err != nil {
				return false, err
			}

			if _, err = master.networkClient.Network().ClusterNetworks().Create(configCN); err != nil {
				return false, err
			}
			glog.Infof("Created ClusterNetwork %s", common.ClusterNetworkToString(configCN))

			if err = master.checkClusterNetworkAgainstClusterObjects(); err != nil {
				utilruntime.HandleError(fmt.Errorf("Cluster contains objects incompatible with new ClusterNetwork: %v", err))
			}
		} else {
			configChanged, err := clusterNetworkChanged(configCN, existingCN)
			if err != nil {
				return false, err
			}
			if configChanged {
				configCN.TypeMeta = existingCN.TypeMeta
				configCN.ObjectMeta = existingCN.ObjectMeta
				if err = master.checkClusterNetworkAgainstClusterObjects(); err != nil {
					utilruntime.HandleError(fmt.Errorf("Attempting to modify cluster to exclude existing objects: %v", err))
					return false, err
				}
				if _, err = master.networkClient.Network().ClusterNetworks().Update(configCN); err != nil {
					return false, err
				}
				glog.Infof("Updated ClusterNetwork %s", common.ClusterNetworkToString(configCN))
			} else {
				glog.V(5).Infof("No change to ClusterNetwork %s", common.ClusterNetworkToString(configCN))
			}
		}

		return true, nil
	})
	if err != nil {
		if getError != nil {
			return getError
		}
		return err
	}

	// FIXME: this is required to register informers for the types we care about to ensure the informers are started.
	// FIXME: restructure this controller to add event handlers in Start() before returning, instead of inside startSubSystems.
	master.nodeInformer.Informer().GetController()
	master.namespaceInformer.Informer().GetController()
	master.hostSubnetInformer.Informer().GetController()
	master.netNamespaceInformer.Informer().GetController()

	go master.startSubSystems(networkConfig.NetworkPluginName)

	return nil
}

func (master *OsdnMaster) startSubSystems(pluginName string) {
	// Wait for informer sync
	if !cache.WaitForCacheSync(wait.NeverStop,
		master.nodeInformer.Informer().GetController().HasSynced,
		master.namespaceInformer.Informer().GetController().HasSynced,
		master.hostSubnetInformer.Informer().GetController().HasSynced,
		master.netNamespaceInformer.Informer().GetController().HasSynced) {
		glog.Fatalf("failed to sync SDN master informers")
	}

	if err := master.startSubnetMaster(); err != nil {
		glog.Fatalf("failed to start subnet master: %v", err)
	}

	switch pluginName {
	case network.MultiTenantPluginName:
		master.vnids = newMasterVNIDMap(true)
	case network.NetworkPolicyPluginName:
		master.vnids = newMasterVNIDMap(false)
	}
	if master.vnids != nil {
		if err := master.startVNIDMaster(); err != nil {
			glog.Fatalf("failed to start VNID master: %v", err)
		}
	}
}

func (master *OsdnMaster) checkClusterNetworkAgainstLocalNetworks() error {
	hostIPNets, _, err := netutils.GetHostIPNetworks([]string{tun0})
	if err != nil {
		return err
	}
	return master.networkInfo.CheckHostNetworks(hostIPNets)
}

func (master *OsdnMaster) checkClusterNetworkAgainstClusterObjects() error {
	var subnets []networkapi.HostSubnet
	var pods []kapi.Pod
	var services []kapi.Service
	if subnetList, err := master.networkClient.Network().HostSubnets().List(metav1.ListOptions{}); err == nil {
		subnets = subnetList.Items
	}
	if podList, err := master.kClient.Core().Pods(metav1.NamespaceAll).List(metav1.ListOptions{}); err == nil {
		pods = podList.Items
	}
	if serviceList, err := master.kClient.Core().Services(metav1.NamespaceAll).List(metav1.ListOptions{}); err == nil {
		services = serviceList.Items
	}

	return master.networkInfo.CheckClusterObjects(subnets, pods, services)
}

func clusterNetworkChanged(obj *networkapi.ClusterNetwork, old *networkapi.ClusterNetwork) (bool, error) {

	if old.ServiceNetwork != obj.ServiceNetwork {
		return true, fmt.Errorf("cannot change the serviceNetworkCIDR of an already-deployed cluster")
	} else if old.PluginName != obj.PluginName {
		return true, nil
	} else if len(old.ClusterNetworks) != len(obj.ClusterNetworks) {
		return true, nil
	} else {
		changed := false
		for _, oldCIDR := range old.ClusterNetworks {
			found := false
			for _, newCIDR := range obj.ClusterNetworks {
				if newCIDR.CIDR == oldCIDR.CIDR && newCIDR.HostSubnetLength == oldCIDR.HostSubnetLength {
					found = true
					break
				}
			}
			if !found {
				changed = true
				break
			}
		}
		return changed, nil

	}
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package stream_test

import (
	"context"
	"errors"
	"testing"
	"time"

	. "github.com/cilium/cilium/pkg/stream"
)

func TestMap(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	double := func(x int) int { return x * 2 }

	// 1. mapping a non-empty source
	{
		src := Range(0, 5)
		src = Map(src, double)
		result, err := ToSlice(ctx, src)
		assertNil(t, "case 1", err)
		assertSlice(t, "case 1", []int{0, 2, 4, 6, 8}, result)
	}

	// 2. mapping an empty source
	{
		src := Map(Empty[int](), double)
		result, err := ToSlice(ctx, src)
		assertNil(t, "case 2", err)
		assertSlice(t, "case 2", []int{}, result)
	}

	// 3. cancelled context
	checkCancelled(t, "case 3", Map(Range(0, 100), double))
}

func TestFilter(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	isOdd := func(x int) bool { return x%2 != 0 }

	// 1. filtering a non-empty source
	{
		src := Range(0, 5)
		src = Filter(src, isOdd)
		result, err := ToSlice(ctx, src)
		assertNil(t, "case 1", err)
		assertSlice(t, "case 1", []int{1, 3}, result)
	}

	// 2. filtering an empty source
	{
		src := Filter(Empty[int](), isOdd)
		result, err := ToSlice(ctx, src)
		assertNil(t, "case 2", err)
		assertSlice(t, "case 2", []int{}, result)
	}

	// 3. cancelled context
	checkCancelled(t, "case 3", Filter(Range(0, 100), isOdd))
}

func TestReduce(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	sum := func(result, x int) int { return result + x }

	// 1. Reducing a non-empty source
	{
		src := Range(0, 5)
		src = Reduce(src, 0, sum)
		result, err := ToSlice(ctx, src)
		assertNil(t, "case 1", err)
		assertSlice(t, "case 1", []int{0 + 0 + 1 + 2 + 3 + 4}, result)
	}

	// 2. Reducing an empty source
	{
		src := Reduce(Empty[int](), 0, sum)
		result, err := ToSlice(ctx, src)
		assertNil(t, "case 2", err)
		assertSlice(t, "case 2", []int{0}, result)
	}

	// 3. cancelled context
	checkCancelled(t, "case 3", Reduce(Range(0, 100), 0, sum))
}

func TestThrottle(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	waitMillis := 20
	go func() {
		time.Sleep(time.Duration(waitMillis) * time.Millisecond)
		cancel()
	}()

	ratePerSecond := 2000.0
	values, err := ToSlice(ctx, Throttle(Range(0, 100000), ratePerSecond, 1))
	if !errors.Is(err, context.Canceled) {
		t.Fatalf("expected Canceled error, got %s", err)
	}

	expectedLen := int(float64(waitMillis) / 1000.0 * ratePerSecond)

	lenDiff := len(values) - expectedLen
	if lenDiff < 0 {
		lenDiff *= -1
	}
	// Check that we're within 20%
	if lenDiff > expectedLen/5 {
		t.Fatalf("expected ~%d values, got %d, diff: %d", expectedLen, len(values), lenDiff)
	}
}

func TestRetry(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	var (
		err1 = errors.New("err1")
		err2 = errors.New("err2")
	)

	emit, complete, obs := fromCallback[int](1)
	shouldRetry := func(err error) bool {
		return errors.Is(err, err1)
	}
	// Retry if error is 'err1', otherwise stop.
	obs = Retry(obs, shouldRetry)

	errs := make(chan error)
	items := ToChannel(ctx, errs, obs)

	emit(1)
	if item := <-items; item != 1 {
		t.Fatalf("expected 1, got %d", item)
	}

	emit(2)
	if item := <-items; item != 2 {
		t.Fatalf("expected 2, got %d", item)
	}

	complete(err1) // this should be retried
	emit(3)
	if item := <-items; item != 3 {
		t.Fatalf("expected 3, got %d", item)
	}

	complete(err2) // this should stop the observing

	if item, ok := <-items; ok {
		t.Fatalf("expected items channel to be closed, got item %d", item)
	}

	if err := <-errs; err != err2 {
		t.Fatalf("expected error %s, got %s", err2, err)
	}
}

func TestRetryFuncs(t *testing.T) {
	err := errors.New("err")

	// Retry 10 times with exponential backoff up to 10ms.
	var retry RetryFunc
	retry = AlwaysRetry
	retry = BackoffRetry(retry, time.Millisecond, 10*time.Millisecond)
	retry = LimitRetries(retry, 6)

	t0 := time.Now()
	for i := 0; i < 10; i++ {
		if i < 6 {
			if !retry(err) {
				t.Fatalf("expected retry to succeed at attempt %d", i)
			}
		} else {
			if retry(err) {
				t.Fatalf("expected retry to fail at attempt %d", i)
			}
		}
	}
	tdiff := time.Now().Sub(t0)
	expectedDiff := time.Duration(1+2+4+8+10+10) * time.Millisecond

	if tdiff < expectedDiff || tdiff > 2*expectedDiff {
		t.Fatalf("expected backoff duration to be ~%s, it was %s", expectedDiff, tdiff)
	}
}

func TestDistict(t *testing.T) {
	items := FromSlice([]int{1, 2, 3, 3, 2, 1})
	result, err := ToSlice(context.TODO(), Distinct(items))
	assertNil(t, "ToSlice+Distict", err)
	assertSlice(t, "Distinct", []int{1, 2, 3, 2, 1}, result)
}

func TestDebounce(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	in := make(chan int, 16)
	src := FromChannel(in)
	src = Debounce(src, 5*time.Millisecond)
	errs := make(chan error)
	out := ToChannel(ctx, errs, src)

	in <- -1
	x := <-out // first item not delayed
	if x != -1 {
		t.Fatalf("expected -1, got %d", x)
	}
	// Emit 10 batches of 3 items. We should only
	// observe the last item of each batch.
	for i := 0; i < 10*3; i += 3 {
		in <- i
		in <- i + 1
		in <- i + 2
		time.Sleep(10 * time.Millisecond)
		x := <-out
		if x != i+2 {
			t.Fatalf("expected %d, got %d", i+2, x)
		}
	}
	cancel()
	err := <-errs
	if !errors.Is(err, context.Canceled) {
		t.Fatalf("expected Canceled error, got %s", err)
	}
}
<commit_msg>stream: operators_test: remove TestThrottle OS timer dependency<commit_after>// SPDX-License-Identifier: Apache-2.0
// Copyright Authors of Cilium

package stream_test

import (
	"context"
	"errors"
	"testing"
	"time"

	. "github.com/cilium/cilium/pkg/stream"
)

func TestMap(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	double := func(x int) int { return x * 2 }

	// 1. mapping a non-empty source
	{
		src := Range(0, 5)
		src = Map(src, double)
		result, err := ToSlice(ctx, src)
		assertNil(t, "case 1", err)
		assertSlice(t, "case 1", []int{0, 2, 4, 6, 8}, result)
	}

	// 2. mapping an empty source
	{
		src := Map(Empty[int](), double)
		result, err := ToSlice(ctx, src)
		assertNil(t, "case 2", err)
		assertSlice(t, "case 2", []int{}, result)
	}

	// 3. cancelled context
	checkCancelled(t, "case 3", Map(Range(0, 100), double))
}

func TestFilter(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	isOdd := func(x int) bool { return x%2 != 0 }

	// 1. filtering a non-empty source
	{
		src := Range(0, 5)
		src = Filter(src, isOdd)
		result, err := ToSlice(ctx, src)
		assertNil(t, "case 1", err)
		assertSlice(t, "case 1", []int{1, 3}, result)
	}

	// 2. filtering an empty source
	{
		src := Filter(Empty[int](), isOdd)
		result, err := ToSlice(ctx, src)
		assertNil(t, "case 2", err)
		assertSlice(t, "case 2", []int{}, result)
	}

	// 3. cancelled context
	checkCancelled(t, "case 3", Filter(Range(0, 100), isOdd))
}

func TestReduce(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	sum := func(result, x int) int { return result + x }

	// 1. Reducing a non-empty source
	{
		src := Range(0, 5)
		src = Reduce(src, 0, sum)
		result, err := ToSlice(ctx, src)
		assertNil(t, "case 1", err)
		assertSlice(t, "case 1", []int{0 + 0 + 1 + 2 + 3 + 4}, result)
	}

	// 2. Reducing an empty source
	{
		src := Reduce(Empty[int](), 0, sum)
		result, err := ToSlice(ctx, src)
		assertNil(t, "case 2", err)
		assertSlice(t, "case 2", []int{0}, result)
	}

	// 3. cancelled context
	checkCancelled(t, "case 3", Reduce(Range(0, 100), 0, sum))
}

func TestThrottle(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	item, err := First(ctx, Throttle(Range(42, 1000), 100.0, 1))
	assertNil(t, "First", err)

	if item != 42 {
		t.Fatalf("expected 42, got %d", item)
	}
}

func TestRetry(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	var (
		err1 = errors.New("err1")
		err2 = errors.New("err2")
	)

	emit, complete, obs := fromCallback[int](1)
	shouldRetry := func(err error) bool {
		return errors.Is(err, err1)
	}
	// Retry if error is 'err1', otherwise stop.
	obs = Retry(obs, shouldRetry)

	errs := make(chan error)
	items := ToChannel(ctx, errs, obs)

	emit(1)
	if item := <-items; item != 1 {
		t.Fatalf("expected 1, got %d", item)
	}

	emit(2)
	if item := <-items; item != 2 {
		t.Fatalf("expected 2, got %d", item)
	}

	complete(err1) // this should be retried
	emit(3)
	if item := <-items; item != 3 {
		t.Fatalf("expected 3, got %d", item)
	}

	complete(err2) // this should stop the observing

	if item, ok := <-items; ok {
		t.Fatalf("expected items channel to be closed, got item %d", item)
	}

	if err := <-errs; err != err2 {
		t.Fatalf("expected error %s, got %s", err2, err)
	}
}

func TestRetryFuncs(t *testing.T) {
	err := errors.New("err")

	// Retry 10 times with exponential backoff up to 10ms.
	var retry RetryFunc
	retry = AlwaysRetry
	retry = BackoffRetry(retry, time.Millisecond, 10*time.Millisecond)
	retry = LimitRetries(retry, 6)

	t0 := time.Now()
	for i := 0; i < 10; i++ {
		if i < 6 {
			if !retry(err) {
				t.Fatalf("expected retry to succeed at attempt %d", i)
			}
		} else {
			if retry(err) {
				t.Fatalf("expected retry to fail at attempt %d", i)
			}
		}
	}
	tdiff := time.Now().Sub(t0)
	expectedDiff := time.Duration(1+2+4+8+10+10) * time.Millisecond

	if tdiff < expectedDiff || tdiff > 2*expectedDiff {
		t.Fatalf("expected backoff duration to be ~%s, it was %s", expectedDiff, tdiff)
	}
}

func TestDistict(t *testing.T) {
	items := FromSlice([]int{1, 2, 3, 3, 2, 1})
	result, err := ToSlice(context.TODO(), Distinct(items))
	assertNil(t, "ToSlice+Distict", err)
	assertSlice(t, "Distinct", []int{1, 2, 3, 2, 1}, result)
}

func TestDebounce(t *testing.T) {
	ctx, cancel := context.WithCancel(context.Background())
	defer cancel()

	in := make(chan int, 16)
	src := FromChannel(in)
	src = Debounce(src, 5*time.Millisecond)
	errs := make(chan error)
	out := ToChannel(ctx, errs, src)

	in <- -1
	x := <-out // first item not delayed
	if x != -1 {
		t.Fatalf("expected -1, got %d", x)
	}
	// Emit 10 batches of 3 items. We should only
	// observe the last item of each batch.
	for i := 0; i < 10*3; i += 3 {
		in <- i
		in <- i + 1
		in <- i + 2
		time.Sleep(10 * time.Millisecond)
		x := <-out
		if x != i+2 {
			t.Fatalf("expected %d, got %d", i+2, x)
		}
	}
	cancel()
	err := <-errs
	if !errors.Is(err, context.Canceled) {
		t.Fatalf("expected Canceled error, got %s", err)
	}
}
<|endoftext|>


<commit_before>package backends

import (
	"encoding/json"
	"time"

	"github.com/RichardKnop/machinery/Godeps/_workspace/src/github.com/garyburd/redigo/redis"
	"github.com/RichardKnop/machinery/v1/config"
	"github.com/RichardKnop/machinery/v1/signatures"
)

// RedisBackend represents a Memcache result backend
type RedisBackend struct {
	config *config.Config
	host   string
	conn   redis.Conn
}

// NewRedisBackend creates RedisBackend instance
func NewRedisBackend(cnf *config.Config, host string) Backend {
	return Backend(&RedisBackend{
		config: cnf,
		host:   host,
	})
}

// InitGroup - saves UUIDs of all tasks in a group
func (redisBackend *RedisBackend) InitGroup(groupUUID string, taskUUIDs []string) error {
	groupMeta := &GroupMeta{
		GroupUUID: groupUUID,
		TaskUUIDs: taskUUIDs,
	}

	encoded, err := json.Marshal(&groupMeta)
	if err != nil {
		return err
	}

	conn, err := redisBackend.open()
	if err != nil {
		return err
	}
	defer conn.Close()

	_, err = conn.Do("SET", groupUUID, encoded)
	if err != nil {
		return err
	}

	return redisBackend.setExpirationTime(groupUUID)
}

// GroupCompleted - returns true if all tasks in a group finished
func (redisBackend *RedisBackend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) {
	groupMeta := GroupMeta{}

	conn, err := redisBackend.open()
	if err != nil {
		return false, err
	}
	defer conn.Close()

	item, err := redis.Bytes(conn.Do("GET", groupUUID))
	if err != nil {
		return false, err
	}

	if err := json.Unmarshal(item, &groupMeta); err != nil {
		return false, err
	}

	for _, taskUUID := range groupMeta.TaskUUIDs {
		taskState, err := redisBackend.GetState(taskUUID)
		if err != nil {
			return false, err
		}

		if !taskState.IsCompleted() {
			return false, nil
		}
	}

	return true, nil
}

// GroupTaskStates - returns states of all tasks in the group
func (redisBackend *RedisBackend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*TaskState, error) {
	taskStates := make([]*TaskState, groupTaskCount)

	groupMeta := GroupMeta{}

	conn, err := redisBackend.open()
	if err != nil {
		return taskStates, err
	}
	defer conn.Close()

	item, err := redis.Bytes(conn.Do("GET", groupUUID))
	if err != nil {
		return taskStates, err
	}

	if err := json.Unmarshal(item, &groupMeta); err != nil {
		return taskStates, err
	}

	for i, taskUUID := range groupMeta.TaskUUIDs {
		taskState, err := redisBackend.GetState(taskUUID)
		if err != nil {
			return taskStates, err
		}

		taskStates[i] = taskState
	}

	return taskStates, nil
}

// SetStatePending - sets task state to PENDING
func (redisBackend *RedisBackend) SetStatePending(signature *signatures.TaskSignature) error {
	taskState := NewPendingTaskState(signature)
	return redisBackend.updateState(taskState)
}

// SetStateReceived - sets task state to RECEIVED
func (redisBackend *RedisBackend) SetStateReceived(signature *signatures.TaskSignature) error {
	taskState := NewReceivedTaskState(signature)
	return redisBackend.updateState(taskState)
}

// SetStateStarted - sets task state to STARTED
func (redisBackend *RedisBackend) SetStateStarted(signature *signatures.TaskSignature) error {
	taskState := NewStartedTaskState(signature)
	return redisBackend.updateState(taskState)
}

// SetStateSuccess - sets task state to SUCCESS
func (redisBackend *RedisBackend) SetStateSuccess(signature *signatures.TaskSignature, result *TaskResult) error {
	taskState := NewSuccessTaskState(signature, result)
	return redisBackend.updateState(taskState)
}

// SetStateFailure - sets task state to FAILURE
func (redisBackend *RedisBackend) SetStateFailure(signature *signatures.TaskSignature, err string) error {
	taskState := NewFailureTaskState(signature, err)
	return redisBackend.updateState(taskState)
}

// GetState - returns the latest task state
func (redisBackend *RedisBackend) GetState(taskUUID string) (*TaskState, error) {
	taskState := TaskState{}

	conn, err := redisBackend.open()
	if err != nil {
		return nil, err
	}
	defer conn.Close()

	item, err := redis.Bytes(conn.Do("GET", taskUUID))
	if err != nil {
		return nil, err
	}

	if err := json.Unmarshal(item, &taskState); err != nil {
		return nil, err
	}

	return &taskState, nil
}

// PurgeState - deletes stored task state
func (redisBackend *RedisBackend) PurgeState(taskUUID string) error {
	conn, err := redisBackend.open()
	if err != nil {
		return err
	}
	defer conn.Close()

	_, err = conn.Do("DEL", taskUUID)
	if err != nil {
		return err
	}

	return nil
}

// PurgeGroupMeta - deletes stored group meta data
func (redisBackend *RedisBackend) PurgeGroupMeta(groupUUID string) error {
	conn, err := redisBackend.open()
	if err != nil {
		return err
	}
	defer conn.Close()

	_, err = conn.Do("DEL", groupUUID)
	if err != nil {
		return err
	}

	return nil
}

// Updates a task state
func (redisBackend *RedisBackend) updateState(taskState *TaskState) error {
	encoded, err := json.Marshal(&taskState)
	if err != nil {
		return err
	}

	conn, err := redisBackend.open()
	if err != nil {
		return err
	}
	defer conn.Close()

	_, err = conn.Do("SET", taskState.TaskUUID, encoded)
	if err != nil {
		return err
	}

	return redisBackend.setExpirationTime(taskState.TaskUUID)
}

// Sets expiration timestamp on a stored state
func (redisBackend *RedisBackend) setExpirationTime(key string) error {
	expiresIn := redisBackend.config.ResultsExpireIn
	if expiresIn == 0 {
		// // expire results after 1 hour by default
		expiresIn = 3600
	}
	expirationTimestamp := int32(time.Now().Unix() + int64(expiresIn))

	conn, err := redisBackend.open()
	if err != nil {
		return err
	}
	defer conn.Close()

	_, err = conn.Do("EXPIREAT", key, expirationTimestamp)
	if err != nil {
		return err
	}

	return nil
}

// Returns / creates instance of Redis connection
func (redisBackend *RedisBackend) open() (redis.Conn, error) {
	return redis.Dial("tcp", redisBackend.host)
}
<commit_msg>Using MGET to retrieve multiple values.<commit_after>package backends

import (
	"encoding/json"
	"fmt"
	"log"
	"time"

	"github.com/RichardKnop/machinery/Godeps/_workspace/src/github.com/garyburd/redigo/redis"
	"github.com/RichardKnop/machinery/v1/config"
	"github.com/RichardKnop/machinery/v1/signatures"
)

// RedisBackend represents a Memcache result backend
type RedisBackend struct {
	config *config.Config
	host   string
	conn   redis.Conn
}

// NewRedisBackend creates RedisBackend instance
func NewRedisBackend(cnf *config.Config, host string) Backend {
	return Backend(&RedisBackend{
		config: cnf,
		host:   host,
	})
}

// InitGroup - saves UUIDs of all tasks in a group
func (redisBackend *RedisBackend) InitGroup(groupUUID string, taskUUIDs []string) error {
	groupMeta := &GroupMeta{
		GroupUUID: groupUUID,
		TaskUUIDs: taskUUIDs,
	}

	encoded, err := json.Marshal(&groupMeta)
	if err != nil {
		return err
	}

	conn, err := redisBackend.open()
	if err != nil {
		return err
	}
	defer conn.Close()

	_, err = conn.Do("SET", groupUUID, encoded)
	if err != nil {
		return err
	}

	return redisBackend.setExpirationTime(groupUUID)
}

// GroupCompleted - returns true if all tasks in a group finished
func (redisBackend *RedisBackend) GroupCompleted(groupUUID string, groupTaskCount int) (bool, error) {
	groupMeta := GroupMeta{}

	conn, err := redisBackend.open()
	if err != nil {
		return false, err
	}
	defer conn.Close()

	item, err := redis.Bytes(conn.Do("GET", groupUUID))
	if err != nil {
		return false, err
	}

	if err := json.Unmarshal(item, &groupMeta); err != nil {
		return false, err
	}

	taskStates, err := redisBackend.getStates(groupMeta.TaskUUIDs...)
	if err != nil {
		return false, err
	}

	for _, taskState := range taskStates {
		if !taskState.IsCompleted() {
			return false, nil
		}
	}

	return true, nil
}

// GroupTaskStates - returns states of all tasks in the group
func (redisBackend *RedisBackend) GroupTaskStates(groupUUID string, groupTaskCount int) ([]*TaskState, error) {
	taskStates := make([]*TaskState, groupTaskCount)

	groupMeta := GroupMeta{}

	conn, err := redisBackend.open()
	if err != nil {
		return taskStates, err
	}
	defer conn.Close()

	item, err := redis.Bytes(conn.Do("GET", groupUUID))
	if err != nil {
		return taskStates, err
	}

	if err := json.Unmarshal(item, &groupMeta); err != nil {
		return taskStates, err
	}

	return redisBackend.getStates(groupMeta.TaskUUIDs...)
}

// SetStatePending - sets task state to PENDING
func (redisBackend *RedisBackend) SetStatePending(signature *signatures.TaskSignature) error {
	taskState := NewPendingTaskState(signature)
	return redisBackend.updateState(taskState)
}

// SetStateReceived - sets task state to RECEIVED
func (redisBackend *RedisBackend) SetStateReceived(signature *signatures.TaskSignature) error {
	taskState := NewReceivedTaskState(signature)
	return redisBackend.updateState(taskState)
}

// SetStateStarted - sets task state to STARTED
func (redisBackend *RedisBackend) SetStateStarted(signature *signatures.TaskSignature) error {
	taskState := NewStartedTaskState(signature)
	return redisBackend.updateState(taskState)
}

// SetStateSuccess - sets task state to SUCCESS
func (redisBackend *RedisBackend) SetStateSuccess(signature *signatures.TaskSignature, result *TaskResult) error {
	taskState := NewSuccessTaskState(signature, result)
	return redisBackend.updateState(taskState)
}

// SetStateFailure - sets task state to FAILURE
func (redisBackend *RedisBackend) SetStateFailure(signature *signatures.TaskSignature, err string) error {
	taskState := NewFailureTaskState(signature, err)
	return redisBackend.updateState(taskState)
}

// GetState - returns the latest task state
func (redisBackend *RedisBackend) GetState(taskUUID string) (*TaskState, error) {
	taskState := TaskState{}

	conn, err := redisBackend.open()
	if err != nil {
		return nil, err
	}
	defer conn.Close()

	item, err := redis.Bytes(conn.Do("GET", taskUUID))
	if err != nil {
		return nil, err
	}

	if err := json.Unmarshal(item, &taskState); err != nil {
		return nil, err
	}

	return &taskState, nil
}

// PurgeState - deletes stored task state
func (redisBackend *RedisBackend) PurgeState(taskUUID string) error {
	conn, err := redisBackend.open()
	if err != nil {
		return err
	}
	defer conn.Close()

	_, err = conn.Do("DEL", taskUUID)
	if err != nil {
		return err
	}

	return nil
}

// PurgeGroupMeta - deletes stored group meta data
func (redisBackend *RedisBackend) PurgeGroupMeta(groupUUID string) error {
	conn, err := redisBackend.open()
	if err != nil {
		return err
	}
	defer conn.Close()

	_, err = conn.Do("DEL", groupUUID)
	if err != nil {
		return err
	}

	return nil
}

// getStates Returns multiple task states with MGET
func (redisBackend *RedisBackend) getStates(taskUUIDs ...string) ([]*TaskState, error) {
	taskStates := make([]*TaskState, len(taskUUIDs))

	log.Print("Getting states")
	log.Print(taskUUIDs)

	conn, err := redisBackend.open()
	if err != nil {
		return taskStates, err
	}
	defer conn.Close()

	// conn.Do requires []interface{}... can't pass []string unfortunately
	taskUUIDInterfaces := make([]interface{}, len(taskUUIDs))
	for i, taskUUID := range taskUUIDs {
		taskUUIDInterfaces[i] = interface{}(taskUUID)
	}

	reply, err := redis.Values(conn.Do("MGET", taskUUIDInterfaces...))
	if err != nil {
		return taskStates, err
	}

	for i, value := range reply {
		bytes, ok := value.([]byte)
		if !ok {
			return taskStates, fmt.Errorf("Expected byte array, instead got: %v", value)
		}

		taskState := TaskState{}
		if err := json.Unmarshal(bytes, &taskState); err != nil {
			log.Print(err)
			return taskStates, err
		}

		taskStates[i] = &taskState
	}

	return taskStates, nil
}

// Updates a task state
func (redisBackend *RedisBackend) updateState(taskState *TaskState) error {
	encoded, err := json.Marshal(&taskState)
	if err != nil {
		return err
	}

	conn, err := redisBackend.open()
	if err != nil {
		return err
	}
	defer conn.Close()

	_, err = conn.Do("SET", taskState.TaskUUID, encoded)
	if err != nil {
		return err
	}

	return redisBackend.setExpirationTime(taskState.TaskUUID)
}

// Sets expiration timestamp on a stored state
func (redisBackend *RedisBackend) setExpirationTime(key string) error {
	expiresIn := redisBackend.config.ResultsExpireIn
	if expiresIn == 0 {
		// // expire results after 1 hour by default
		expiresIn = 3600
	}
	expirationTimestamp := int32(time.Now().Unix() + int64(expiresIn))

	conn, err := redisBackend.open()
	if err != nil {
		return err
	}
	defer conn.Close()

	_, err = conn.Do("EXPIREAT", key, expirationTimestamp)
	if err != nil {
		return err
	}

	return nil
}

// Returns / creates instance of Redis connection
func (redisBackend *RedisBackend) open() (redis.Conn, error) {
	return redis.Dial("tcp", redisBackend.host)
}
<|endoftext|>


<commit_before>package brightbox

import (
	"testing"

	"gotest.tools/assert"
)

func TestCloudIPs(t *testing.T) {
	instance := testAll[CloudIP](
		t,
		"CloudIP",
		"cloud_ips",
		"config map",
	)
	assert.Equal(t, instance.ID, "cip-k4a25")
}

func TestCloudIP(t *testing.T) {
	instance := testInstance[CloudIP](
		t,
		"CloudIP",
		"cloud_ips",
		"cloud_ip",
		"cip-k4a25",
	)
	assert.Equal(t, instance.Name, "product website ip")
}

func TestCreateCloudIP(t *testing.T) {
	newAC := CloudIPOptions{}
	_ = testCreate[CloudIP](
		t,
		"CloudIP",
		"cloud_ips",
		"cloud_ip",
		"cip-k4a25",
		&newAC,
		"{}",
	)
}

func TestUpdateCloudIP(t *testing.T) {
	name := "dev client"
	uac := CloudIPOptions{ID: "cip-k4a25", Name: &name}
	_ = testUpdate[CloudIP](
		t,
		"CloudIP",
		"cloud_ips",
		"cloud_ip",
		"cip-k4a25",
		&uac,
		`{"name":"dev client"}`,
	)
}

func TestDestroyCloudIP(t *testing.T) {
	testDestroy[CloudIP](
		t,
		"CloudIP",
		"cloud_ips",
		"cip-k4a25",
	)
}
<commit_msg>Test Cloud IP links<commit_after>package brightbox

import (
	"testing"

	"gotest.tools/assert"
)

func TestCloudIPs(t *testing.T) {
	instance := testAll[CloudIP](
		t,
		"CloudIP",
		"cloud_ips",
		"config map",
	)
	assert.Equal(t, instance.ID, "cip-k4a25")
}

func TestCloudIP(t *testing.T) {
	instance := testInstance[CloudIP](
		t,
		"CloudIP",
		"cloud_ips",
		"cloud_ip",
		"cip-k4a25",
	)
	assert.Equal(t, instance.Name, "product website ip")
	assert.Equal(t, instance.Account.ID, "acc-43ks4")
	assert.Equal(t, instance.Interface.ID, "int-ds42k")
	assert.Equal(t, instance.Server.ID, "srv-lv426")
}

func TestCreateCloudIP(t *testing.T) {
	newAC := CloudIPOptions{}
	_ = testCreate[CloudIP](
		t,
		"CloudIP",
		"cloud_ips",
		"cloud_ip",
		"cip-k4a25",
		&newAC,
		"{}",
	)
}

func TestUpdateCloudIP(t *testing.T) {
	name := "dev client"
	uac := CloudIPOptions{ID: "cip-k4a25", Name: &name}
	_ = testUpdate[CloudIP](
		t,
		"CloudIP",
		"cloud_ips",
		"cloud_ip",
		"cip-k4a25",
		&uac,
		`{"name":"dev client"}`,
	)
}

func TestDestroyCloudIP(t *testing.T) {
	testDestroy[CloudIP](
		t,
		"CloudIP",
		"cloud_ips",
		"cip-k4a25",
	)
}
<|endoftext|>


<commit_before>package ebs

import (
	"testing"

	"github.com/mitchellh/packer/packer"
)

func testConfig() map[string]interface{} {
	return map[string]interface{}{
		"access_key":    "foo",
		"secret_key":    "bar",
		"source_ami":    "foo",
		"instance_type": "foo",
		"region":        "us-east-1",
		"ssh_username":  "root",
		"ami_name":      "foo",
	}
}

func TestBuilder_ImplementsBuilder(t *testing.T) {
	var raw interface{}
	raw = &Builder{}
	if _, ok := raw.(packer.Builder); !ok {
		t.Fatalf("Builder should be a builder")
	}
}

func TestBuilder_Prepare_BadType(t *testing.T) {
	b := &Builder{}
	c := map[string]interface{}{
		"access_key": []string{},
	}

	warnings, err := b.Prepare(c)
	if len(warnings) > 0 {
		t.Fatalf("bad: %#v", warnings)
	}
	if err == nil {
		t.Fatalf("prepare should fail")
	}
}

func TestBuilderPrepare_AMIName(t *testing.T) {
	var b Builder
	config := testConfig()

	// Test good
	config["ami_name"] = "foo"
	warnings, err := b.Prepare(config)
	if len(warnings) > 0 {
		t.Fatalf("bad: %#v", warnings)
	}
	if err != nil {
		t.Fatalf("should not have error: %s", err)
	}

	// Test bad
	config["ami_name"] = "foo {{"
	b = Builder{}
	warnings, err = b.Prepare(config)
	if len(warnings) > 0 {
		t.Fatalf("bad: %#v", warnings)
	}
	if err == nil {
		t.Fatal("should have error")
	}

	// Test bad
	delete(config, "ami_name")
	b = Builder{}
	warnings, err = b.Prepare(config)
	if len(warnings) > 0 {
		t.Fatalf("bad: %#v", warnings)
	}
	if err == nil {
		t.Fatal("should have error")
	}
}

func TestBuilderPrepare_InvalidKey(t *testing.T) {
	var b Builder
	config := testConfig()

	// Add a random key
	config["i_should_not_be_valid"] = true
	warnings, err := b.Prepare(config)
	if len(warnings) > 0 {
		t.Fatalf("bad: %#v", warnings)
	}
	if err == nil {
		t.Fatal("should have error")
	}
}
<commit_msg>Add unit tests for shutdown behaviour<commit_after>package ebs

import (
	"testing"

	"github.com/mitchellh/packer/packer"
)

func testConfig() map[string]interface{} {
	return map[string]interface{}{
		"access_key":    "foo",
		"secret_key":    "bar",
		"source_ami":    "foo",
		"instance_type": "foo",
		"region":        "us-east-1",
		"ssh_username":  "root",
		"ami_name":      "foo",
	}
}

func TestBuilder_ImplementsBuilder(t *testing.T) {
	var raw interface{}
	raw = &Builder{}
	if _, ok := raw.(packer.Builder); !ok {
		t.Fatalf("Builder should be a builder")
	}
}

func TestBuilder_Prepare_BadType(t *testing.T) {
	b := &Builder{}
	c := map[string]interface{}{
		"access_key": []string{},
	}

	warnings, err := b.Prepare(c)
	if len(warnings) > 0 {
		t.Fatalf("bad: %#v", warnings)
	}
	if err == nil {
		t.Fatalf("prepare should fail")
	}
}

func TestBuilderPrepare_AMIName(t *testing.T) {
	var b Builder
	config := testConfig()

	// Test good
	config["ami_name"] = "foo"
	warnings, err := b.Prepare(config)
	if len(warnings) > 0 {
		t.Fatalf("bad: %#v", warnings)
	}
	if err != nil {
		t.Fatalf("should not have error: %s", err)
	}

	// Test bad
	config["ami_name"] = "foo {{"
	b = Builder{}
	warnings, err = b.Prepare(config)
	if len(warnings) > 0 {
		t.Fatalf("bad: %#v", warnings)
	}
	if err == nil {
		t.Fatal("should have error")
	}

	// Test bad
	delete(config, "ami_name")
	b = Builder{}
	warnings, err = b.Prepare(config)
	if len(warnings) > 0 {
		t.Fatalf("bad: %#v", warnings)
	}
	if err == nil {
		t.Fatal("should have error")
	}
}

func TestBuilderPrepare_InvalidKey(t *testing.T) {
	var b Builder
	config := testConfig()

	// Add a random key
	config["i_should_not_be_valid"] = true
	warnings, err := b.Prepare(config)
	if len(warnings) > 0 {
		t.Fatalf("bad: %#v", warnings)
	}
	if err == nil {
		t.Fatal("should have error")
	}
}

func TestBuilderPrepare_InvalidShutdownBehaviour(t *testing.T) {
	var b Builder
	config := testConfig()

	// Test good
	config["shutdown_behaviour"] = "terminate"
	warnings, err := b.Prepare(config)
	if len(warnings) > 0 {
		t.Fatalf("bad: %#v", warnings)
	}
	if err != nil {
		t.Fatalf("should not have error: %s", err)
	}

	// Test good
	config["shutdown_behaviour"] = "stop"
	warnings, err = b.Prepare(config)
	if len(warnings) > 0 {
		t.Fatalf("bad: %#v", warnings)
	}
	if err != nil {
		t.Fatalf("should not have error: %s", err)
	}

	// Test bad
	config["shutdown_behaviour"] = "foobar"
	warnings, err = b.Prepare(config)
	if len(warnings) > 0 {
		t.Fatalf("bad: %#v", warnings)
	}
	if err == nil {
		t.Fatal("should have error")
	}
}
<|endoftext|>


<commit_before>package pki

import (
	"context"
	"fmt"
	"strings"

	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/vault/sdk/framework"
	"github.com/hashicorp/vault/sdk/helper/errutil"
	"github.com/hashicorp/vault/sdk/logical"
)

func pathRevoke(b *backend) *framework.Path {
	return &framework.Path{
		Pattern: `revoke`,
		Fields: map[string]*framework.FieldSchema{
			"serial_number": &framework.FieldSchema{
				Type: framework.TypeString,
				Description: `Certificate serial number, in colon- or
hyphen-separated octal`,
			},
		},

		Callbacks: map[logical.Operation]framework.OperationFunc{
			logical.UpdateOperation: b.pathRevokeWrite,
		},

		HelpSynopsis:    pathRevokeHelpSyn,
		HelpDescription: pathRevokeHelpDesc,
	}
}

func pathRotateCRL(b *backend) *framework.Path {
	return &framework.Path{
		Pattern: `crl/rotate`,

		Callbacks: map[logical.Operation]framework.OperationFunc{
			logical.ReadOperation: b.pathRotateCRLRead,
		},

		HelpSynopsis:    pathRotateCRLHelpSyn,
		HelpDescription: pathRotateCRLHelpDesc,
	}
}

func (b *backend) pathRevokeWrite(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	serial := data.Get("serial_number").(string)
	if len(serial) == 0 {
		return logical.ErrorResponse("The serial number must be provided"), nil
	}

	// We store and identify by lowercase colon-separated hex, but other
	// utilities use dashes and/or uppercase, so normalize
	serial = strings.Replace(strings.ToLower(serial), "-", ":", -1)

	b.revokeStorageLock.Lock()
	defer b.revokeStorageLock.Unlock()

	return revokeCert(ctx, b, req, serial, false)
}

func (b *backend) pathRotateCRLRead(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	b.revokeStorageLock.RLock()
	defer b.revokeStorageLock.RUnlock()

	crlErr := buildCRL(ctx, b, req, false)
	switch crlErr.(type) {
	case errutil.UserError:
		return logical.ErrorResponse(fmt.Sprintf("Error during CRL building: %s", crlErr)), nil
	case errutil.InternalError:
		return nil, errwrap.Wrapf("error encountered during CRL building: {{err}}", crlErr)
	default:
		return &logical.Response{
			Data: map[string]interface{}{
				"success": true,
			},
		}, nil
	}
}

const pathRevokeHelpSyn = `
Revoke a certificate by serial number.
`

const pathRevokeHelpDesc = `
This allows certificates to be revoked using its serial number. A root token is required.
`

const pathRotateCRLHelpSyn = `
Force a rebuild of the CRL.
`

const pathRotateCRLHelpDesc = `
Force a rebuild of the CRL. This can be used to remove expired certificates from it if no certificates have been revoked. A root token is required.
`
<commit_msg>Don't try to revoke certs from PKI on perf standby (#7173)<commit_after>package pki

import (
	"context"
	"fmt"
	"strings"

	"github.com/hashicorp/errwrap"
	"github.com/hashicorp/vault/sdk/framework"
	"github.com/hashicorp/vault/sdk/helper/consts"
	"github.com/hashicorp/vault/sdk/helper/errutil"
	"github.com/hashicorp/vault/sdk/logical"
)

func pathRevoke(b *backend) *framework.Path {
	return &framework.Path{
		Pattern: `revoke`,
		Fields: map[string]*framework.FieldSchema{
			"serial_number": &framework.FieldSchema{
				Type: framework.TypeString,
				Description: `Certificate serial number, in colon- or
hyphen-separated octal`,
			},
		},

		Callbacks: map[logical.Operation]framework.OperationFunc{
			logical.UpdateOperation: b.pathRevokeWrite,
		},

		HelpSynopsis:    pathRevokeHelpSyn,
		HelpDescription: pathRevokeHelpDesc,
	}
}

func pathRotateCRL(b *backend) *framework.Path {
	return &framework.Path{
		Pattern: `crl/rotate`,

		Callbacks: map[logical.Operation]framework.OperationFunc{
			logical.ReadOperation: b.pathRotateCRLRead,
		},

		HelpSynopsis:    pathRotateCRLHelpSyn,
		HelpDescription: pathRotateCRLHelpDesc,
	}
}

func (b *backend) pathRevokeWrite(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	serial := data.Get("serial_number").(string)
	if len(serial) == 0 {
		return logical.ErrorResponse("The serial number must be provided"), nil
	}

	if b.System().ReplicationState().HasState(consts.ReplicationPerformanceStandby) {
		return nil, logical.ErrReadOnly
	}

	// We store and identify by lowercase colon-separated hex, but other
	// utilities use dashes and/or uppercase, so normalize
	serial = strings.Replace(strings.ToLower(serial), "-", ":", -1)

	b.revokeStorageLock.Lock()
	defer b.revokeStorageLock.Unlock()

	return revokeCert(ctx, b, req, serial, false)
}

func (b *backend) pathRotateCRLRead(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
	b.revokeStorageLock.RLock()
	defer b.revokeStorageLock.RUnlock()

	crlErr := buildCRL(ctx, b, req, false)
	switch crlErr.(type) {
	case errutil.UserError:
		return logical.ErrorResponse(fmt.Sprintf("Error during CRL building: %s", crlErr)), nil
	case errutil.InternalError:
		return nil, errwrap.Wrapf("error encountered during CRL building: {{err}}", crlErr)
	default:
		return &logical.Response{
			Data: map[string]interface{}{
				"success": true,
			},
		}, nil
	}
}

const pathRevokeHelpSyn = `
Revoke a certificate by serial number.
`

const pathRevokeHelpDesc = `
This allows certificates to be revoked using its serial number. A root token is required.
`

const pathRotateCRLHelpSyn = `
Force a rebuild of the CRL.
`

const pathRotateCRLHelpDesc = `
Force a rebuild of the CRL. This can be used to remove expired certificates from it if no certificates have been revoked. A root token is required.
`
<|endoftext|>


<commit_before>// Package power provides a library to read SysFS info on battery state.
//
// Icons are taken from the current GTK theme, typically stored at
// /usr/share/icons/gnome/32x32/status or similar.
package power

import (
	"errors"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strconv"
	"strings"
)

var (
	BasePath    = "/sys/class/power_supply/" // base SysFS path, under which battery info is assumed to exist.
	ErrNoFile   = errors.New("no such file or directory")
	ErrNoDevice = errors.New("no such device")
)

// Charge is the current charge of a battery.
type Charge float32

// Battery represents information about a battery.
type Battery struct {
	Charge Charge // current charge, [0.0, 1.0]
	State  State
}

func (c Charge) String() string {
	if c < 0.1 {
		return "empty"
	} else if c < 0.3 {
		return "caution"
	} else if c < 0.5 {
		return "low"
	} else if c < 0.9 {
		return "good"
	} else {
		return "full"
	}
}

// sysFile is a SysFS file.
type sysFile string

// sysInfo holds the contents read from SysFS.
type sysInfo map[sysFile]string

// sysFiles are the SysFS files to read.
var sysFiles = []sysFile{
	"energy_now",
	"energy_full",
	"status",
}

// State is the state of a battery.
type State int

const (
	Empty State = iota
	Full
	Charging
	Discharging
	Unknown
)

var states = [...]string{
	"Empty",
	"Full",
	"Charging",
	"Discharging",
	"Unknown",
}

// String returns the name of the state.
func (s State) String() string {
	return states[s]
}

// parse parses the state from the raw string.
func (s *State) parse(in string) error {
	for i, name := range states {
		if strings.EqualFold(name, in) {
			*s = State(i)
			return nil
		}
	}
	return fmt.Errorf("no state named %v", s)
}

// String returns a descriptive representation of the battery.
func (b Battery) String() string {
	return fmt.Sprintf("%-12s (%.2f%%)", b.State, 100*b.Charge)
}

// Desc describes the battery, in a way compatible with GTK icon names.
func (b Battery) Desc() string {
	charge := b.Charge.String()
	if b.State == Charging {
		if charge == "empty" {
			charge = "low" // there is no battery-empty-charging.
		}
		return fmt.Sprintf("battery-%s-%s", charge, strings.ToLower(b.State.String()))
	} else {
		return fmt.Sprintf("battery-%s", charge)
	}
}

// read reads the SysFS file.
func (s *sysFile) read(path string) (string, error) {
	path = filepath.Join(path, string(*s))
	bytes, err := ioutil.ReadFile(path)
	if err != nil {
		if strings.HasSuffix(err.Error(), ErrNoFile.Error()) {
			return "", ErrNoFile
		}
		if strings.HasSuffix(err.Error(), ErrNoDevice.Error()) {
			return "", ErrNoDevice
		}
		return "", fmt.Errorf("couldn't read from SysFS: %v\n", err)
	}
	return strings.TrimRight(string(bytes), "\n"), nil
}

// read reads a single SysFS file holding state of a battery.
func (s sysInfo) read(path string) error {
	for _, f := range sysFiles {
		c, err := f.read(path)
		if err != nil {
			return err
		}
		s[f] = c
	}
	return nil
}

// parseCharge parses the charge percentage from string values.
func parseCharge(p, f string) (Charge, error) {
	pi, err := strconv.ParseInt(p, 10, 0)
	if err != nil {
		return Charge(0.0), err
	}
	fi, err := strconv.ParseInt(f, 10, 0)
	if err != nil {
		return Charge(0.0), err
	}
	return Charge(float32(pi) / float32(fi)), nil
}

// GetNumber reads SysFS to find state of the battery with given index.
func GetNumber(i int) (Battery, error) {
	si := make(sysInfo)
	err := si.read(filepath.Join(BasePath, fmt.Sprintf("BAT%d", i)))
	if err != nil {
		return Battery{}, err
	}
	c, err := parseCharge(si["energy_now"], si["energy_full"])
	if err != nil {
		return Battery{}, fmt.Errorf("couldn't parse battery charge: %v\n", err)
	}
	state := Unknown
	err = state.parse(si["status"])
	if err != nil {
		return Battery{}, err
	}
	if state == Unknown && c.String() == "full" {
		// Full batteries sometimes are reported as "unknown", for some
		// reason.
		state = Full
	}
	return Battery{
		State:  state,
		Charge: c,
	}, nil
}

// Get reads SysFS to find state of all batteries.
func Get() ([]Battery, error) {
	path := filepath.Join(BasePath, "BAT*")
	glob, err := filepath.Glob(path)
	if err != nil {
		return []Battery{}, fmt.Errorf("failed to read battery info from SysFS: %v\n", err)
	}
	if len(glob) == 0 {
		return []Battery{}, fmt.Errorf("no battery info found in SysFS path %s\n", path)
	}
	result := make([]Battery, len(glob))
	for i := 0; i < len(glob); i++ {
		b, err := GetNumber(i)
		if err != nil {
			return []Battery{}, err
		}
		result[i] = b
	}
	return result, nil
}
<commit_msg>moves power to own repo<commit_after><|endoftext|>


<commit_before>package prj

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strings"
)

var project *Project
var Debug bool

type Project struct {
	rootFolder string
	// Global  GOPATH
	ggopath string
}

func GetProject() (*Project, error) {
	if project == nil {
		project = &Project{}
		gdir, gerr := Git("rev-parse --git-dir")
		gdir = strings.TrimSpace(gdir)
		// fmt.Printf("ko '%s' '%s'", gdir, gerr)
		if gerr != nil {
			return nil, gerr
		}
		if gdir != ".git" {
			project.rootFolder = gdir[:len(gdir)-5]
		} else {
			// fmt.Printf("ok")
			project.rootFolder = wd
		}
		project.ggopath = os.Getenv("GOPATH")
	}
	// fmt.Printf("prf '%s'", project.rootFolder)
	// fmt.Printf("prf '%s'", project.ggopath)
	return project, nil
}

func (p *Project) RootFolder() string {
	return p.rootFolder
}

// Inspired by https://github.com/ghthor/journal/blob/0bd4968a4f9841befdd0dde96b2096e6c930e74c/git/git.go

var gitPath string
var goPath string
var wd string

func init() {
	gitPath = getPathForExe("git")
	goPath = getPathForExe("go")
	var err error
	wd, err = os.Getwd()
	if err != nil {
		log.Fatal("Working directory not accessible")
	}
}

func getPathForExe(exe string) string {
	var err error
	var path = ""
	if path, err = exec.LookPath(exe); err != nil {
		aliases := ""
		if runtime.GOOS == "windows" {
			aliases, err = execcmd("doskey", "/macros")
		} else {
			aliases, err = execcmd("alias", "")
		}
		r := regexp.MustCompile(`(?m)^` + exe + `=(.*)\s+[\$%@\*].*$`)
		sm := r.FindAllStringSubmatch(aliases, 1)
		if len(sm) != 1 || len(sm[0]) != 2 {
			log.Fatalf("Unable to find '%s' path in aliases '%s'", exe)
		}
		return sm[0][1]
	}
	if runtime.GOOS == "windows" {
		if strings.HasSuffix(path, ".bat") {
			bat, err := ioutil.ReadFile(path)
			if err != nil {
				log.Fatalf("Unable to read '%s' for '%s'", path, exe)
			}
			bats := string(bat)
			r := regexp.MustCompile(`(?m)^\s*?(.*)\s+[\$%@\*].*$`)
			sm := r.FindAllStringSubmatch(bats, 1)
			if len(sm) != 1 || len(sm[0]) != 2 {
				log.Fatalf("Unable to find '%s' path in file '%s'", exe, path)
			}
			return sm[0][1]
		}
	}
	if path == "" {
		log.Fatalf("Unable to get path for '%s'", exe)
	}
	return path
}

// Construct an *exec.Cmd for `git {args}` with a workingDirectory
func Git(cmd string) (string, error) {
	return execcmd(gitPath, cmd)
}
func Golang(cmd string) (string, error) {
	os.Setenv("GOPATH", project.rootFolder+`/deps`)
	os.Setenv("GOBIN", project.rootFolder+`/bin`)
	return execcmd(goPath, cmd)
}

func execcmd(exe, cmd string) (string, error) {
	if Debug {
		fmt.Printf("%s %s\n", exe, cmd)
	}
	args := strings.Split(cmd, " ")
	c := exec.Command(exe, args...)
	c.Dir = project.rootFolder
	var bout bytes.Buffer
	c.Stdout = &bout
	var berr bytes.Buffer
	c.Stderr = &berr
	err := c.Run()
	if err != nil {
		return bout.String(), fmt.Errorf("Unable to run '%s %s' in '%s': err '%s'\n'%s'", exe, cmd, wd, err.Error(), berr.String())
	} else if berr.String() != "" {
		return bout.String(), fmt.Errorf("Warning on run '%s %s' in '%s': '%s'", exe, cmd, wd, berr.String())
	}
	return bout.String(), nil
}
<commit_msg>project.go: execcmd uses cmd /C for commands<commit_after>package prj

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"regexp"
	"runtime"
	"strings"
)

var project *Project
var Debug bool

type Project struct {
	rootFolder string
	// Global  GOPATH
	ggopath string
}

func GetProject() (*Project, error) {
	if project == nil {
		project = &Project{}
		gdir, gerr := Git("rev-parse --git-dir")
		gdir = strings.TrimSpace(gdir)
		// fmt.Printf("ko '%s' '%s'", gdir, gerr)
		if gerr != nil {
			return nil, gerr
		}
		if gdir != ".git" {
			project.rootFolder = gdir[:len(gdir)-5]
		} else {
			// fmt.Printf("ok")
			project.rootFolder = wd
		}
		project.ggopath = os.Getenv("GOPATH")
	}
	// fmt.Printf("prf '%s'", project.rootFolder)
	// fmt.Printf("prf '%s'", project.ggopath)
	return project, nil
}

func (p *Project) RootFolder() string {
	return p.rootFolder
}

// Inspired by https://github.com/ghthor/journal/blob/0bd4968a4f9841befdd0dde96b2096e6c930e74c/git/git.go

var gitPath string
var goPath string
var wd string

func init() {
	gitPath = getPathForExe("git")
	goPath = getPathForExe("go")
	var err error
	wd, err = os.Getwd()
	if err != nil {
		log.Fatal("Working directory not accessible")
	}
}

func getPathForExe(exe string) string {
	var err error
	var path = ""
	if path, err = exec.LookPath(exe); err != nil {
		aliases := ""
		if runtime.GOOS == "windows" {
			aliases, err = execcmd("doskey", "/macros")
		} else {
			aliases, err = execcmd("alias", "")
		}
		r := regexp.MustCompile(`(?m)^` + exe + `=(.*)\s+[\$%@\*].*$`)
		sm := r.FindAllStringSubmatch(aliases, 1)
		if len(sm) != 1 || len(sm[0]) != 2 {
			log.Fatalf("Unable to find '%s' path in aliases '%s'", exe)
		}
		return sm[0][1]
	}
	if runtime.GOOS == "windows" {
		if strings.HasSuffix(path, ".bat") {
			bat, err := ioutil.ReadFile(path)
			if err != nil {
				log.Fatalf("Unable to read '%s' for '%s'", path, exe)
			}
			bats := string(bat)
			r := regexp.MustCompile(`(?m)^\s*?(.*)\s+[\$%@\*].*$`)
			sm := r.FindAllStringSubmatch(bats, 1)
			if len(sm) != 1 || len(sm[0]) != 2 {
				log.Fatalf("Unable to find '%s' path in file '%s'", exe, path)
			}
			return sm[0][1]
		}
	}
	if path == "" {
		log.Fatalf("Unable to get path for '%s'", exe)
	}
	return path
}

// Construct an *exec.Cmd for `git {args}` with a workingDirectory
func Git(cmd string) (string, error) {
	return execcmd(gitPath, cmd)
}
func Golang(cmd string) (string, error) {
	os.Setenv("GOPATH", project.rootFolder+`/deps`)
	os.Setenv("GOBIN", project.rootFolder+`/bin`)
	return execcmd(goPath, cmd)
}

func execcmd(exe, cmd string) (string, error) {
	if Debug {
		fmt.Printf("%s %s\n", exe, cmd)
	}
	args := strings.Split(cmd, " ")
	args = append([]string{"/c", exe}, args...)
	c := exec.Command("cmd", args...)
	c.Dir = project.rootFolder
	var bout bytes.Buffer
	c.Stdout = &bout
	var berr bytes.Buffer
	c.Stderr = &berr
	err := c.Run()
	if err != nil {
		return bout.String(), fmt.Errorf("Unable to run '%s %s' in '%s': err '%s'\n'%s'", exe, cmd, wd, err.Error(), berr.String())
	} else if berr.String() != "" {
		return bout.String(), fmt.Errorf("Warning on run '%s %s' in '%s': '%s'", exe, cmd, wd, berr.String())
	}
	return bout.String(), nil
}
<|endoftext|>


<commit_before>package plugin

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/cloudfoundry/cli/plugin"
	"github.com/xchapter7x/deploycloud/remoteconfig"
)

const (
	//TokenEnvName - env var name to store your github oauth token
	TokenEnvName = "GH_TOKEN"
	//CFUserEnvName - env var name to store your cf user
	CFUserEnvName = "CF_USER"
	//CFPassEnvName - env var name to store your cf user's password
	CFPassEnvName = "CF_PASS"
)

var (
	//ErrInvalidArgs - invalid argument error
	ErrInvalidArgs = errors.New("invalid arguments")
	//Logger - default logger object
	Logger logger = log.New(os.Stdout, "[DeployCloudPlugin]", log.Lshortfile)
	//MakeConfigFetcher - function to create the default config fetcher (github oauth)
	MakeConfigFetcher = func(token, org, repo, branch, url string) (configFetcher *remoteconfig.ConfigFetcher) {
		configFetcher = &remoteconfig.ConfigFetcher{
			GithubOauthToken: token,
			GithubOrg:        org,
			Repo:             repo,
			Branch:           branch,
			GithubURL:        url,
		}
		configFetcher.UseOauthClient()
		return
	}
)

type (
	logger interface {
		Fatalln(...interface{})
		Println(...interface{})
	}
	//DeployCloudPlugin - deploy cloud plugin object
	DeployCloudPlugin struct {
		conn       plugin.CliConnection
		list       *bool
		run        *string
		show       *string
		org        *string
		repo       *string
		branch     *string
		url        *string
		token      *string
		cfuser     *string
		cfpass     *string
		configFile *string
		Errors     []error
	}
)

//GetMetadata - cf cli plugin metadata definition
func (s *DeployCloudPlugin) GetMetadata() plugin.PluginMetadata {
	return plugin.PluginMetadata{
		Name: "cloud-deploy",
		Commands: []plugin.Command{
			{
				Name:     "cloud-deploy",
				HelpText: "Deploy a cloud foundry app using a manifest and deployment details defined in a remote github repo",
			},
		},
	}
}

//Run - cf cli required Run method
func (s *DeployCloudPlugin) Run(cliConnection plugin.CliConnection, args []string) {
	args = args[1:]
	fs := new(flag.FlagSet)
	s.conn = cliConnection
	s.list = fs.Bool("list", false, "list apps available in config file")
	s.run = fs.String("run", "", "run the selected deployment")
	s.show = fs.String("show", "", "show the selected deployment's details")
	s.configFile = fs.String("config", remoteconfig.DefaultConfigPath, "path to remote config file")
	s.org = fs.String("org", "", "the target org")
	s.repo = fs.String("repo", "", "the target repo")
	s.branch = fs.String("branch", "", "the target branch")
	s.url = fs.String("url", remoteconfig.DefaultGithubURL, "the target github base url")
	s.token = fs.String("token", os.Getenv(TokenEnvName), "the oauth token for your github account (uses GH_TOKEN Env var by default)")
	s.cfuser = fs.String("cfuser", os.Getenv(CFUserEnvName), "the cf username to use when logging in to the deployment target")
	s.cfpass = fs.String("cfpass", os.Getenv(CFPassEnvName), "the cf user's password to use when logging in to the deployment target")
	fs.Parse(args)
	s.execute()
	s.errorOutput()
}

func (s *DeployCloudPlugin) errorOutput() {
	if len(s.Errors) > 0 {
		Logger.Fatalln("Errors: ", s.Errors)
	}
}

func (s *DeployCloudPlugin) execute() {

	if s.validArgs() {

		if config, err := s.fetchConfig(); err == nil {
			s.performActionOnConfig(config)
		}

	} else {
		s.appendError(ErrInvalidArgs)
	}
}

func (s *DeployCloudPlugin) performActionOnConfig(config *remoteconfig.ApplicationDeployments) {

	if *s.list {
		s.listApps(config)

	} else if *s.show != "" {
		s.showDeployment(config, *s.show)

	} else if *s.run != "" {
		s.runDeployment(config, *s.run)
	}
}

func (s *DeployCloudPlugin) runDeployment(config *remoteconfig.ApplicationDeployments, deployment string) {
	appName, deploymentName := s.parseAppDeployment(deployment)

	for _, deploymentInfo := range config.Applications[appName].Deployments {

		if deploymentInfo.Name == deploymentName {
			s.fetchManifest(deploymentInfo)
			s.cfLogin(deploymentInfo)
			s.cfDeploy(deploymentInfo)
		}
	}
}

func (s *DeployCloudPlugin) cfLogin(deploymentInfo remoteconfig.Deployment) {
	cmd := fmt.Sprintf("login -a %s -u %s -p %s -o %s -s %s",
		deploymentInfo.URL,
		*s.cfuser,
		*s.cfpass,
		deploymentInfo.Org,
		deploymentInfo.Space,
	)
	args := strings.Split(cmd, " ")

	if _, err := s.conn.CliCommand(args...); err != nil {
		s.appendError(err)
	}
}

func (s *DeployCloudPlugin) cfDeploy(deploymentInfo remoteconfig.Deployment) {
	args := strings.Split(deploymentInfo.PushCmd, " ")
	args = append(args, "-f", deploymentInfo.Name)

	if _, err := s.conn.CliCommand(args...); err != nil {
		s.appendError(err)
	}
}

func (s *DeployCloudPlugin) fetchManifest(deploymentInfo remoteconfig.Deployment) {
	if buf, err := s.fetch(deploymentInfo.Path); err == nil {
		ioutil.WriteFile(deploymentInfo.Name, buf.Bytes(), 0644)
	}
}

func (s *DeployCloudPlugin) parseAppDeployment(deployment string) (appname, deploymentname string) {
	d := strings.Split(deployment, ".")
	appname = d[0]
	deploymentname = d[1]
	return
}

func (s *DeployCloudPlugin) showDeployment(config *remoteconfig.ApplicationDeployments, deployment string) {
	appName, deploymentName := s.parseAppDeployment(deployment)

	for _, v := range config.Applications[appName].Deployments {

		if v.Name == deploymentName {
			b, _ := yaml.Marshal(v)
			Logger.Println(string(b[:]))
		}
	}
}

func (s *DeployCloudPlugin) listApps(config *remoteconfig.ApplicationDeployments) {
	for appName, app := range config.Applications {

		for _, deployment := range app.Deployments {
			Logger.Println(appName, ".", deployment.Name)
		}
	}
}

func (s *DeployCloudPlugin) appendError(e error) {
	s.Errors = append(s.Errors, e)
}

func (s *DeployCloudPlugin) validArgs() bool {
	return (s.hasFlag(*s.configFile, "configFile") &&
		s.hasFlag(*s.org, "org") &&
		s.hasFlag(*s.repo, "repo") &&
		s.hasFlag(*s.branch, "branch") &&
		s.hasFlag(*s.cfuser, "cfuser") &&
		s.hasFlag(*s.cfpass, "cfpass") &&
		s.hasFlag(*s.url, "url") &&
		s.hasFlag(*s.token, "token") &&
		(*s.list || *s.run != "" || *s.show != ""))
}

func (s *DeployCloudPlugin) hasFlag(f string, name string) (r bool) {
	if r = (f != ""); r == false {
		s.appendError(errors.New(fmt.Sprint("Invalid flag: ", name)))
	}
	return
}

func (s *DeployCloudPlugin) fetch(filePath string) (buf *bytes.Buffer, err error) {
	configFetcher := MakeConfigFetcher(
		*s.token,
		*s.org,
		*s.repo,
		*s.branch,
		*s.url,
	)

	if buf, err = configFetcher.Fetch(filePath); err != nil {
		s.appendError(err)
	}
	return
}

func (s *DeployCloudPlugin) fetchConfig() (config *remoteconfig.ApplicationDeployments, err error) {
	remoteconfig.DefaultConfigPath = *s.configFile
	configFetcher := MakeConfigFetcher(
		*s.token,
		*s.org,
		*s.repo,
		*s.branch,
		*s.url,
	)

	if config, err = configFetcher.FetchConfig(); err != nil {
		s.appendError(err)
	}
	return
}
<commit_msg>fixing bug to support calling plugins<commit_after>package plugin

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/cloudfoundry/cli/plugin"
	"github.com/xchapter7x/deploycloud/remoteconfig"
)

const (
	//TokenEnvName - env var name to store your github oauth token
	TokenEnvName = "GH_TOKEN"
	//CFUserEnvName - env var name to store your cf user
	CFUserEnvName = "CF_USER"
	//CFPassEnvName - env var name to store your cf user's password
	CFPassEnvName = "CF_PASS"
)

var (
	//ErrInvalidArgs - invalid argument error
	ErrInvalidArgs = errors.New("invalid arguments")
	//Logger - default logger object
	Logger logger = log.New(os.Stdout, "[DeployCloudPlugin]", log.Lshortfile)
	//MakeConfigFetcher - function to create the default config fetcher (github oauth)
	MakeConfigFetcher = func(token, org, repo, branch, url string) (configFetcher *remoteconfig.ConfigFetcher) {
		configFetcher = &remoteconfig.ConfigFetcher{
			GithubOauthToken: token,
			GithubOrg:        org,
			Repo:             repo,
			Branch:           branch,
			GithubURL:        url,
		}
		configFetcher.UseOauthClient()
		return
	}
)

type (
	logger interface {
		Fatalln(...interface{})
		Println(...interface{})
	}
	//DeployCloudPlugin - deploy cloud plugin object
	DeployCloudPlugin struct {
		conn       plugin.CliConnection
		list       *bool
		run        *string
		show       *string
		org        *string
		repo       *string
		branch     *string
		url        *string
		token      *string
		cfuser     *string
		cfpass     *string
		configFile *string
		Errors     []error
	}
)

//GetMetadata - cf cli plugin metadata definition
func (s *DeployCloudPlugin) GetMetadata() plugin.PluginMetadata {
	return plugin.PluginMetadata{
		Name: "cloud-deploy",
		Commands: []plugin.Command{
			{
				Name:     "cloud-deploy",
				HelpText: "Deploy a cloud foundry app using a manifest and deployment details defined in a remote github repo",
			},
		},
	}
}

//Run - cf cli required Run method
func (s *DeployCloudPlugin) Run(cliConnection plugin.CliConnection, args []string) {
	fs := new(flag.FlagSet)
	s.conn = cliConnection
	s.list = fs.Bool("list", false, "list apps available in config file")
	s.run = fs.String("run", "", "run the selected deployment")
	s.show = fs.String("show", "", "show the selected deployment's details")
	s.configFile = fs.String("config", remoteconfig.DefaultConfigPath, "path to remote config file")
	s.org = fs.String("org", "", "the target org")
	s.repo = fs.String("repo", "", "the target repo")
	s.branch = fs.String("branch", "", "the target branch")
	s.url = fs.String("url", remoteconfig.DefaultGithubURL, "the target github base url")
	s.token = fs.String("token", os.Getenv(TokenEnvName), "the oauth token for your github account (uses GH_TOKEN Env var by default)")
	s.cfuser = fs.String("cfuser", os.Getenv(CFUserEnvName), "the cf username to use when logging in to the deployment target")
	s.cfpass = fs.String("cfpass", os.Getenv(CFPassEnvName), "the cf user's password to use when logging in to the deployment target")
	fs.Parse(args[1:])
	s.execute()
	s.errorOutput()
}

func (s *DeployCloudPlugin) errorOutput() {
	if len(s.Errors) > 0 {
		Logger.Fatalln("Errors: ", s.Errors)
	}
}

func (s *DeployCloudPlugin) execute() {

	if s.validArgs() {

		if config, err := s.fetchConfig(); err == nil {
			s.performActionOnConfig(config)
		}

	} else {
		s.appendError(ErrInvalidArgs)
	}
}

func (s *DeployCloudPlugin) performActionOnConfig(config *remoteconfig.ApplicationDeployments) {

	if *s.list {
		s.listApps(config)

	} else if *s.show != "" {
		s.showDeployment(config, *s.show)

	} else if *s.run != "" {
		s.runDeployment(config, *s.run)
	}
}

func (s *DeployCloudPlugin) runDeployment(config *remoteconfig.ApplicationDeployments, deployment string) {
	appName, deploymentName := s.parseAppDeployment(deployment)

	for _, deploymentInfo := range config.Applications[appName].Deployments {

		if deploymentInfo.Name == deploymentName {
			s.fetchManifest(deploymentInfo)
			s.cfLogin(deploymentInfo)
			s.cfDeploy(deploymentInfo)
		}
	}
}

func (s *DeployCloudPlugin) cfLogin(deploymentInfo remoteconfig.Deployment) {
	cmd := fmt.Sprintf("login -a %s -u %s -p %s -o %s -s %s",
		deploymentInfo.URL,
		*s.cfuser,
		*s.cfpass,
		deploymentInfo.Org,
		deploymentInfo.Space,
	)
	args := strings.Split(cmd, " ")

	if _, err := s.conn.CliCommand(args...); err != nil {
		s.appendError(err)
	}
}

func (s *DeployCloudPlugin) cfDeploy(deploymentInfo remoteconfig.Deployment) {
	args := strings.Split(deploymentInfo.PushCmd, " ")
	args = append(args, "-f", deploymentInfo.Name)
	origArgs := os.Args
	os.Args = args
	defer func() { os.Args = origArgs }()

	if _, err := s.conn.CliCommand(args...); err != nil {
		s.appendError(err)
	}
}

func (s *DeployCloudPlugin) fetchManifest(deploymentInfo remoteconfig.Deployment) {
	if buf, err := s.fetch(deploymentInfo.Path); err == nil {
		ioutil.WriteFile(deploymentInfo.Name, buf.Bytes(), 0644)
	}
}

func (s *DeployCloudPlugin) parseAppDeployment(deployment string) (appname, deploymentname string) {
	d := strings.Split(deployment, ".")
	appname = d[0]
	deploymentname = d[1]
	return
}

func (s *DeployCloudPlugin) showDeployment(config *remoteconfig.ApplicationDeployments, deployment string) {
	appName, deploymentName := s.parseAppDeployment(deployment)

	for _, v := range config.Applications[appName].Deployments {

		if v.Name == deploymentName {
			b, _ := yaml.Marshal(v)
			Logger.Println(string(b[:]))
		}
	}
}

func (s *DeployCloudPlugin) listApps(config *remoteconfig.ApplicationDeployments) {
	for appName, app := range config.Applications {

		for _, deployment := range app.Deployments {
			Logger.Println(appName, ".", deployment.Name)
		}
	}
}

func (s *DeployCloudPlugin) appendError(e error) {
	s.Errors = append(s.Errors, e)
}

func (s *DeployCloudPlugin) validArgs() bool {
	return (s.hasFlag(*s.configFile, "configFile") &&
		s.hasFlag(*s.org, "org") &&
		s.hasFlag(*s.repo, "repo") &&
		s.hasFlag(*s.branch, "branch") &&
		s.hasFlag(*s.cfuser, "cfuser") &&
		s.hasFlag(*s.cfpass, "cfpass") &&
		s.hasFlag(*s.url, "url") &&
		s.hasFlag(*s.token, "token") &&
		(*s.list || *s.run != "" || *s.show != ""))
}

func (s *DeployCloudPlugin) hasFlag(f string, name string) (r bool) {
	if r = (f != ""); r == false {
		s.appendError(errors.New(fmt.Sprint("Invalid flag: ", name)))
	}
	return
}

func (s *DeployCloudPlugin) fetch(filePath string) (buf *bytes.Buffer, err error) {
	configFetcher := MakeConfigFetcher(
		*s.token,
		*s.org,
		*s.repo,
		*s.branch,
		*s.url,
	)

	if buf, err = configFetcher.Fetch(filePath); err != nil {
		s.appendError(err)
	}
	return
}

func (s *DeployCloudPlugin) fetchConfig() (config *remoteconfig.ApplicationDeployments, err error) {
	remoteconfig.DefaultConfigPath = *s.configFile
	configFetcher := MakeConfigFetcher(
		*s.token,
		*s.org,
		*s.repo,
		*s.branch,
		*s.url,
	)

	if config, err = configFetcher.FetchConfig(); err != nil {
		s.appendError(err)
	}
	return
}
<|endoftext|>


<commit_before>package proxy

import (
	"bytes"
	"fmt"
	"github.com/robryk/zproxy/cache"
	"github.com/robryk/zproxy/hasher"
	"io"
	"io/ioutil"
	"log"
	"net/http"
)

const SizeCutoff = 10 // for testing

type Proxy struct {
	Cr    hasher.ChunkedRetriever
	Cache cache.Cache
}

func sanitizeRequest(req *http.Request) *http.Request {
	r := &http.Request{}
	// FIXME: What if old proto?

	r.Method = req.Method
	r.URL = req.URL // FIXME: Should this be deeper?
	// Proto* are ignored on outgoing requests
	r.Header = make(http.Header)
	for k, vv := range req.Header {
		if k == "Connection" {
			continue
		}
		for _, v := range vv {
			r.Header.Add(k, v)
		}
		// FIXME: Remove other headers
	}
	r.Body = req.Body // We might wish to warn about this
	r.ContentLength = req.ContentLength
	if r.ContentLength == -1 {
		r.ContentLength = 0 // necessary?
	}
	// FIXME: Deal better with Transfer-Encoding
	r.TransferEncoding = nil
	r.Close = false
	r.Host = req.Host // necessary?
	// *Form is ignored on outgoing requests
	// We set nil Trailer, because there is no way for us to set it early enough (or so I think)
	// RemoteAddr, RequestURI and TLS make no sense on an outgoing request
	return r
}

type byteRange struct {
	begin int64
	end   int64
}

func getContents(url string, r byteRange) ([]byte, http.Header, int, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, nil, 0, err
	}
	if r.begin != 0 || r.end != 0 {
		req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", r.begin, r.end))
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, nil, 0, err
	}
	defer resp.Body.Close()

	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, nil, 0, err
	}

	return contents, resp.Header, resp.StatusCode, nil
}

func copyHeaders(dst http.Header, src http.Header) {
	for k := range dst {
		delete(dst, k)
	}
	for k, v := range src {
		dst[k] = v
	}
}

func directProxy(rw http.ResponseWriter, req *http.Request) {
	r := sanitizeRequest(req)
	resp, err := http.DefaultClient.Do(r)
	if err != nil {
		http.Error(rw, fmt.Sprintf("Proxy error: %v", err), http.StatusInternalServerError)
		return
	}
	defer resp.Body.Close()

	// FIXME: Are there any headers we should omit?
	copyHeaders(rw.Header(), resp.Header)
	rw.WriteHeader(resp.StatusCode)
	io.Copy(rw, resp.Body)
	// FIXME: Maybe deal with Trailer? We should send the Trailer if our request was http/1.1
}

func isDirect(req *http.Request) bool {
	if req.Method != "GET" {
		return true
	}

	headReq := sanitizeRequest(req)
	headReq.Method = "HEAD"
	headReq.Body = nil
	headReq.ContentLength = 0
	headResp, err := http.DefaultClient.Do(headReq)
	if err != nil {
		log.Printf("Head request to %s failed: %v", req.URL, err)
		return true
	}
	headResp.Body.Close()
	if headResp.ContentLength < SizeCutoff || headResp.ContentLength == -1 {
		// If the server can't return a Content-Length, chances are high the page changes often
		// even if the server claims otherwise.
		return true
	}
	// FIXME: StatusMethodNotAvailable?
	// FIXME: Cache-Control
	return false
}

func (p *Proxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	// FIXME: Deal with recursion
	url := req.URL.String()
	log.Printf("Serving %s", url)

	if isDirect(req) {
		log.Printf("Serving directly")
		directProxy(rw, req)
		return
	}
	chunked, err := p.Cr.GetChunked(url)
	if err != nil {
		directProxy(rw, req)
		return
	}

	// TODO: deal with status and headers
	// TODO: emit content-length!

	for _, chunk := range chunked.Chunks {
		r, err := p.Cache.Read(string(chunk.Digest))
		if err == nil {
			io.Copy(rw, r)
		} else {
			// TODO: handle cache write failures
			contents, _, _, err := getContents(string(url), byteRange{int64(chunk.Offset), int64(chunk.Offset + chunk.Length)})
			if err != nil {
				log.Printf("Error retrieving missing chunk: %v", err)
				return
			}
			// errors fixme
			go p.Cache.Write(string(chunk.Digest), bytes.NewBuffer(contents))
			rw.Write(contents)
		}
	}
}
<commit_msg>Added support for marshalling HTTP requests.<commit_after>package proxy

import (
	"bytes"
	"fmt"
	"github.com/robryk/zproxy/cache"
	"github.com/robryk/zproxy/hasher"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
)

const SizeCutoff = 10 // for testing

type Proxy struct {
	Cr    hasher.ChunkedRetriever
	Cache cache.Cache
}

type Request struct {
	Method string
	URL    *url.URL
	Host   string //???
	Header http.Header
}

func sanitizeRequest(req *http.Request) *http.Request {
	r := &http.Request{}
	// FIXME: What if old proto?

	r.Method = req.Method
	r.URL = req.URL // FIXME: Should this be deeper?
	// Proto* are ignored on outgoing requests
	r.Header = make(http.Header)
	for k, vv := range req.Header {
		if k == "Connection" {
			continue
		}
		for _, v := range vv {
			r.Header.Add(k, v)
		}
		// FIXME: Remove other headers
	}
	r.Body = req.Body // We might wish to warn about this
	r.ContentLength = req.ContentLength
	if r.ContentLength == -1 {
		r.ContentLength = 0 // necessary?
	}
	// FIXME: Deal better with Transfer-Encoding
	r.TransferEncoding = nil
	r.Close = false
	r.Host = req.Host // necessary?
	// *Form is ignored on outgoing requests
	// We set nil Trailer, because there is no way for us to set it early enough (or so I think)
	// RemoteAddr, RequestURI and TLS make no sense on an outgoing request
	return r
}

func MarshallRequest(req *http.Request) (*Request, error) {
	if req.ContentLength != 0 {
		return nil, fmt.Errorf("Cannot marshall a request with nonzero length body")
	}
	sanitizedReq := sanitizeRequest(req)
	r := &Request{
		Method: sanitizedReq.Method,
		URL:    sanitizedReq.URL,
		Header: sanitizedReq.Header,
		Host:   sanitizedReq.Host,
	}
	return r, nil
}

func UnmarshallRequest(req *Request) *http.Request {
	return &http.Request{
		Method: req.Method,
		URL:    req.URL,
		Header: req.Header,
		Host:   req.Host,
	}
}

type byteRange struct {
	begin int64
	end   int64
}

func getContents(url string, r byteRange) ([]byte, http.Header, int, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, nil, 0, err
	}
	if r.begin != 0 || r.end != 0 {
		req.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", r.begin, r.end))
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, nil, 0, err
	}
	defer resp.Body.Close()

	contents, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, nil, 0, err
	}

	return contents, resp.Header, resp.StatusCode, nil
}

func copyHeaders(dst http.Header, src http.Header) {
	for k := range dst {
		delete(dst, k)
	}
	for k, v := range src {
		dst[k] = v
	}
}

func directProxy(rw http.ResponseWriter, req *http.Request) {
	r := sanitizeRequest(req)
	resp, err := http.DefaultClient.Do(r)
	if err != nil {
		http.Error(rw, fmt.Sprintf("Proxy error: %v", err), http.StatusInternalServerError)
		return
	}
	defer resp.Body.Close()

	// FIXME: Are there any headers we should omit?
	copyHeaders(rw.Header(), resp.Header)
	rw.WriteHeader(resp.StatusCode)
	io.Copy(rw, resp.Body)
	// FIXME: Maybe deal with Trailer? We should send the Trailer if our request was http/1.1
}

func isDirect(req *http.Request) bool {
	if req.Method != "GET" {
		return true
	}

	headReq := sanitizeRequest(req)
	headReq.Method = "HEAD"
	headReq.Body = nil
	headReq.ContentLength = 0
	headResp, err := http.DefaultClient.Do(headReq)
	if err != nil {
		log.Printf("Head request to %s failed: %v", req.URL, err)
		return true
	}
	headResp.Body.Close()
	if headResp.ContentLength < SizeCutoff || headResp.ContentLength == -1 {
		// If the server can't return a Content-Length, chances are high the page changes often
		// even if the server claims otherwise.
		return true
	}
	// FIXME: StatusMethodNotAvailable?
	// FIXME: Cache-Control
	return false
}

func (p *Proxy) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	// FIXME: Deal with recursion
	url := req.URL.String()
	log.Printf("Serving %s", url)

	if isDirect(req) {
		log.Printf("Serving directly")
		directProxy(rw, req)
		return
	}
	chunked, err := p.Cr.GetChunked(url)
	if err != nil {
		directProxy(rw, req)
		return
	}

	// TODO: deal with status and headers
	// TODO: emit content-length!

	for _, chunk := range chunked.Chunks {
		r, err := p.Cache.Read(string(chunk.Digest))
		if err == nil {
			io.Copy(rw, r)
		} else {
			// TODO: handle cache write failures
			contents, _, _, err := getContents(string(url), byteRange{int64(chunk.Offset), int64(chunk.Offset + chunk.Length)})
			if err != nil {
				log.Printf("Error retrieving missing chunk: %v", err)
				return
			}
			// errors fixme
			go p.Cache.Write(string(chunk.Digest), bytes.NewBuffer(contents))
			rw.Write(contents)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package proxy provides support for a variety of protocols to proxy network
// data.
package proxy

import (
	"errors"
	"net"
	"net/url"
	"os"
)

// A Dialer is a means to establish a connection.
type Dialer interface {
	// Dial connects to the given address via the proxy.
	Dial(network, addr string) (c net.Conn, err error)
}

// Auth contains authentication parameters that specific Dialers may require.
type Auth struct {
	User, Password string
}

// DefaultDialer returns the dialer specified by the proxy related variables in
// the environment.
func FromEnvironment() Dialer {
	allProxy := os.Getenv("all_proxy")
	if len(allProxy) == 0 {
		return Direct
	}

	proxyURL, err := url.Parse(allProxy)
	if err != nil {
		return Direct
	}
	proxy, err := FromURL(proxyURL, Direct)
	if err != nil {
		return Direct
	}

	noProxy := os.Getenv("no_proxy")
	if len(noProxy) == 0 {
		return proxy
	}

	perHost := NewPerHost(proxy, Direct)
	perHost.AddFromString(noProxy)
	return perHost
}

// proxySchemes is a map from URL schemes to a function that creates a Dialer
// from a URL with such a scheme.
var proxySchemes map[string]func(*url.URL, Dialer) (Dialer, error)

// RegisterDialerType takes a URL scheme and a function to generate Dialers from
// a URL with that scheme and a forwarding Dialer. Registered schemes are used
// by FromURL.
func RegisterDialerType(scheme string, f func(*url.URL, Dialer) (Dialer, error)) {
	if proxySchemes == nil {
		proxySchemes = make(map[string]func(*url.URL, Dialer) (Dialer, error))
	}
	proxySchemes[scheme] = f
}

// FromURL returns a Dialer given a URL specification and an underlying
// Dialer for it to make network requests.
func FromURL(u *url.URL, forward Dialer) (Dialer, error) {
	var auth *Auth
	if u.User != nil {
		auth = new(Auth)
		auth.User = u.User.Username()
		if p, ok := u.User.Password(); ok {
			auth.Password = p
		}
	}

	switch u.Scheme {
	case "socks5":
		return SOCKS5("tcp", u.Host, auth, forward)
	}

	// If the scheme doesn't match any of the built-in schemes, see if it
	// was registered by another package.
	if proxySchemes != nil {
		if f, ok := proxySchemes[u.Scheme]; ok {
			return f(u, forward)
		}
	}

	return nil, errors.New("proxy: unknown scheme: " + u.Scheme)
}
<commit_msg>go.net/proxy: fix typo in doc<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package proxy provides support for a variety of protocols to proxy network
// data.
package proxy

import (
	"errors"
	"net"
	"net/url"
	"os"
)

// A Dialer is a means to establish a connection.
type Dialer interface {
	// Dial connects to the given address via the proxy.
	Dial(network, addr string) (c net.Conn, err error)
}

// Auth contains authentication parameters that specific Dialers may require.
type Auth struct {
	User, Password string
}

// FromEnvironment returns the dialer specified by the proxy related variables in
// the environment.
func FromEnvironment() Dialer {
	allProxy := os.Getenv("all_proxy")
	if len(allProxy) == 0 {
		return Direct
	}

	proxyURL, err := url.Parse(allProxy)
	if err != nil {
		return Direct
	}
	proxy, err := FromURL(proxyURL, Direct)
	if err != nil {
		return Direct
	}

	noProxy := os.Getenv("no_proxy")
	if len(noProxy) == 0 {
		return proxy
	}

	perHost := NewPerHost(proxy, Direct)
	perHost.AddFromString(noProxy)
	return perHost
}

// proxySchemes is a map from URL schemes to a function that creates a Dialer
// from a URL with such a scheme.
var proxySchemes map[string]func(*url.URL, Dialer) (Dialer, error)

// RegisterDialerType takes a URL scheme and a function to generate Dialers from
// a URL with that scheme and a forwarding Dialer. Registered schemes are used
// by FromURL.
func RegisterDialerType(scheme string, f func(*url.URL, Dialer) (Dialer, error)) {
	if proxySchemes == nil {
		proxySchemes = make(map[string]func(*url.URL, Dialer) (Dialer, error))
	}
	proxySchemes[scheme] = f
}

// FromURL returns a Dialer given a URL specification and an underlying
// Dialer for it to make network requests.
func FromURL(u *url.URL, forward Dialer) (Dialer, error) {
	var auth *Auth
	if u.User != nil {
		auth = new(Auth)
		auth.User = u.User.Username()
		if p, ok := u.User.Password(); ok {
			auth.Password = p
		}
	}

	switch u.Scheme {
	case "socks5":
		return SOCKS5("tcp", u.Host, auth, forward)
	}

	// If the scheme doesn't match any of the built-in schemes, see if it
	// was registered by another package.
	if proxySchemes != nil {
		if f, ok := proxySchemes[u.Scheme]; ok {
			return f(u, forward)
		}
	}

	return nil, errors.New("proxy: unknown scheme: " + u.Scheme)
}
<|endoftext|>