<commit_before>package libstorage

import (
	"io"

	"github.com/emccode/libstorage/api/context"
	"github.com/emccode/libstorage/api/registry"
	"github.com/emccode/libstorage/api/types"
	"github.com/emccode/libstorage/api/utils"
)

func (c *client) Instances(
	ctx types.Context) (map[string]*types.Instance, error) {

	if c.isController() {
		return nil, utils.NewUnsupportedForClientTypeError(
			c.clientType, "Instances")
	}

	ctx = c.withAllInstanceIDs(c.requireCtx(ctx))
	return c.APIClient.Instances(ctx)
}

func (c *client) InstanceInspect(
	ctx types.Context, service string) (*types.Instance, error) {

	if c.isController() {
		return nil, utils.NewUnsupportedForClientTypeError(
			c.clientType, "InstanceInspect")
	}

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	i, err := c.APIClient.InstanceInspect(ctx, service)
	if err != nil {
		return nil, err
	}
	return i, nil
}

func (c *client) Root(
	ctx types.Context) ([]string, error) {

	return c.APIClient.Root(c.requireCtx(ctx))
}

func (c *client) Services(
	ctx types.Context) (map[string]*types.ServiceInfo, error) {

	ctx = c.withAllInstanceIDs(c.requireCtx(ctx))

	svcInfo, err := c.APIClient.Services(ctx)
	if err != nil {
		return nil, err
	}
	for k, v := range svcInfo {
		c.serviceCache.Set(k, v)
	}
	return svcInfo, err
}

func (c *client) ServiceInspect(
	ctx types.Context, service string) (*types.ServiceInfo, error) {

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	return c.APIClient.ServiceInspect(ctx, service)
}

func (c *client) Volumes(
	ctx types.Context,
	attachments bool) (types.ServiceVolumeMap, error) {

	ctx = c.requireCtx(ctx)

	ctxA, err := c.withAllLocalDevices(ctx)
	if err != nil {
		return nil, err
	}
	ctx = c.withAllInstanceIDs(ctxA)

	return c.APIClient.Volumes(ctx, attachments)
}

func (c *client) VolumesByService(
	ctx types.Context,
	service string,
	attachments bool) (types.VolumeMap, error) {

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	ctxA, err := c.withAllLocalDevices(ctx)
	if err != nil {
		return nil, err
	}
	ctx = ctxA

	return c.APIClient.VolumesByService(ctx, service, attachments)
}

func (c *client) VolumeInspect(
	ctx types.Context,
	service, volumeID string,
	attachments bool) (*types.Volume, error) {

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	ctxA, err := c.withAllLocalDevices(ctx)
	if err != nil {
		return nil, err
	}
	ctx = ctxA

	return c.APIClient.VolumeInspect(ctx, service, volumeID, attachments)
}

func (c *client) VolumeCreate(
	ctx types.Context,
	service string,
	request *types.VolumeCreateRequest) (*types.Volume, error) {

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	ctxA, err := c.withAllLocalDevices(ctx)
	if err != nil {
		return nil, err
	}
	ctx = ctxA

	lsd, _ := registry.NewClientDriver(service)
	if lsd != nil {
		if err := lsd.Init(ctx, c.config); err != nil {
			return nil, err
		}

		if err := lsd.VolumeCreateBefore(
			&ctx, service, request); err != nil {
			return nil, err
		}
	}

	vol, err := c.APIClient.VolumeCreate(ctx, service, request)
	if err != nil {
		return nil, err
	}

	if lsd != nil {
		lsd.VolumeCreateAfter(ctx, vol)
	}

	return vol, nil
}

func (c *client) VolumeCreateFromSnapshot(
	ctx types.Context,
	service, snapshotID string,
	request *types.VolumeCreateRequest) (*types.Volume, error) {

	ctx = c.requireCtx(ctx).WithValue(context.ServiceKey, service)

	lsd, _ := registry.NewClientDriver(service)
	if lsd != nil {
		if err := lsd.Init(ctx, c.config); err != nil {
			return nil, err
		}

		if err := lsd.VolumeCreateFromSnapshotBefore(
			&ctx, service, snapshotID, request); err != nil {
			return nil, err
		}
	}

	vol, err := c.APIClient.VolumeCreateFromSnapshot(
		ctx, service, snapshotID, request)
	if err != nil {
		return nil, err
	}

	if lsd != nil {
		lsd.VolumeCreateFromSnapshotAfter(ctx, vol)
	}

	return vol, nil
}

func (c *client) VolumeCopy(
	ctx types.Context,
	service, volumeID string,
	request *types.VolumeCopyRequest) (*types.Volume, error) {

	ctx = c.requireCtx(ctx).WithValue(context.ServiceKey, service)

	lsd, _ := registry.NewClientDriver(service)
	if lsd != nil {
		if err := lsd.Init(ctx, c.config); err != nil {
			return nil, err
		}

		if err := lsd.VolumeCopyBefore(
			&ctx, service, volumeID, request); err != nil {
			return nil, err
		}
	}

	vol, err := c.APIClient.VolumeCopy(ctx, service, volumeID, request)
	if err != nil {
		return nil, err
	}

	if lsd != nil {
		lsd.VolumeCopyAfter(ctx, vol)
	}

	return vol, nil
}

func (c *client) VolumeRemove(
	ctx types.Context,
	service, volumeID string) error {

	ctx = c.requireCtx(ctx).WithValue(context.ServiceKey, service)

	lsd, _ := registry.NewClientDriver(service)
	if lsd != nil {
		if err := lsd.Init(ctx, c.config); err != nil {
			return err
		}

		if err := lsd.VolumeRemoveBefore(
			&ctx, service, volumeID); err != nil {
			return err
		}
	}

	err := c.APIClient.VolumeRemove(ctx, service, volumeID)
	if err != nil {
		return err
	}

	if lsd != nil {
		lsd.VolumeRemoveAfter(ctx, service, volumeID)
	}

	return nil
}

func (c *client) VolumeAttach(
	ctx types.Context,
	service string,
	volumeID string,
	request *types.VolumeAttachRequest) (*types.Volume, string, error) {

	if c.isController() {
		return nil, "", utils.NewUnsupportedForClientTypeError(
			c.clientType, "VolumeAttach")
	}

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	ctxA, err := c.withAllLocalDevices(ctx)
	if err != nil {
		return nil, "", err
	}
	ctx = ctxA

	return c.APIClient.VolumeAttach(ctx, service, volumeID, request)
}

func (c *client) VolumeDetach(
	ctx types.Context,
	service string,
	volumeID string,
	request *types.VolumeDetachRequest) (*types.Volume, error) {

	if c.isController() {
		return nil, utils.NewUnsupportedForClientTypeError(
			c.clientType, "VolumeDetach")
	}

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	ctxA, err := c.withAllLocalDevices(ctx)
	if err != nil {
		return nil, err
	}
	ctx = ctxA

	return c.APIClient.VolumeDetach(ctx, service, volumeID, request)
}

func (c *client) VolumeDetachAll(
	ctx types.Context,
	request *types.VolumeDetachRequest) (types.ServiceVolumeMap, error) {

	if c.isController() {
		return nil, utils.NewUnsupportedForClientTypeError(
			c.clientType, "VolumeDetachAll")
	}

	ctx = c.withAllInstanceIDs(c.requireCtx(ctx))
	ctxA, err := c.withAllLocalDevices(ctx)
	if err != nil {
		return nil, err
	}
	ctx = ctxA

	return c.APIClient.VolumeDetachAll(ctx, request)
}

func (c *client) VolumeDetachAllForService(
	ctx types.Context,
	service string,
	request *types.VolumeDetachRequest) (types.VolumeMap, error) {

	if c.isController() {
		return nil, utils.NewUnsupportedForClientTypeError(
			c.clientType, "VolumeDetachAllForService")
	}

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	ctxA, err := c.withAllLocalDevices(ctx)
	if err != nil {
		return nil, err
	}
	ctx = ctxA

	return c.APIClient.VolumeDetachAllForService(ctx, service, request)
}

func (c *client) VolumeSnapshot(
	ctx types.Context,
	service string,
	volumeID string,
	request *types.VolumeSnapshotRequest) (*types.Snapshot, error) {

	ctx = c.requireCtx(ctx).WithValue(context.ServiceKey, service)
	return c.APIClient.VolumeSnapshot(ctx, service, volumeID, request)
}

func (c *client) Snapshots(
	ctx types.Context) (types.ServiceSnapshotMap, error) {

	ctx = c.requireCtx(ctx)
	return c.APIClient.Snapshots(ctx)
}

func (c *client) SnapshotsByService(
	ctx types.Context, service string) (types.SnapshotMap, error) {

	ctx = c.requireCtx(ctx).WithValue(context.ServiceKey, service)
	return c.APIClient.SnapshotsByService(ctx, service)
}

func (c *client) SnapshotInspect(
	ctx types.Context,
	service, snapshotID string) (*types.Snapshot, error) {

	ctx = c.requireCtx(ctx).WithValue(context.ServiceKey, service)
	return c.APIClient.SnapshotInspect(ctx, service, snapshotID)
}

func (c *client) SnapshotRemove(
	ctx types.Context,
	service, snapshotID string) error {

	ctx = c.requireCtx(ctx).WithValue(context.ServiceKey, service)
	return c.APIClient.SnapshotRemove(ctx, service, snapshotID)
}

func (c *client) SnapshotCopy(
	ctx types.Context,
	service, snapshotID string,
	request *types.SnapshotCopyRequest) (*types.Snapshot, error) {

	ctx = c.requireCtx(ctx).WithValue(context.ServiceKey, service)
	return c.APIClient.SnapshotCopy(ctx, service, snapshotID, request)
}

func (c *client) Executors(
	ctx types.Context) (map[string]*types.ExecutorInfo, error) {

	if c.isController() {
		return nil, utils.NewUnsupportedForClientTypeError(
			c.clientType, "Executors")
	}

	ctx = c.requireCtx(ctx)
	lsxInfo, err := c.APIClient.Executors(ctx)
	if err != nil {
		return nil, err
	}
	for k, v := range lsxInfo {
		c.lsxCache.Set(k, v)
	}
	return lsxInfo, nil
}

func (c *client) ExecutorHead(
	ctx types.Context,
	name string) (*types.ExecutorInfo, error) {

	if c.isController() {
		return nil, utils.NewUnsupportedForClientTypeError(
			c.clientType, "ExecutorHead")
	}

	ctx = c.requireCtx(ctx)
	return c.APIClient.ExecutorHead(ctx, name)
}

func (c *client) ExecutorGet(
	ctx types.Context, name string) (io.ReadCloser, error) {

	if c.isController() {
		return nil, utils.NewUnsupportedForClientTypeError(
			c.clientType, "ExecutorGet")
	}

	ctx = c.requireCtx(ctx)
	return c.APIClient.ExecutorGet(ctx, name)
}
<commit_msg>Some Ops Did Not Include Instance ID<commit_after>package libstorage

import (
	"io"

	"github.com/emccode/libstorage/api/registry"
	"github.com/emccode/libstorage/api/types"
	"github.com/emccode/libstorage/api/utils"
)

func (c *client) Instances(
	ctx types.Context) (map[string]*types.Instance, error) {

	if c.isController() {
		return nil, utils.NewUnsupportedForClientTypeError(
			c.clientType, "Instances")
	}

	ctx = c.withAllInstanceIDs(c.requireCtx(ctx))
	return c.APIClient.Instances(ctx)
}

func (c *client) InstanceInspect(
	ctx types.Context, service string) (*types.Instance, error) {

	if c.isController() {
		return nil, utils.NewUnsupportedForClientTypeError(
			c.clientType, "InstanceInspect")
	}

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	i, err := c.APIClient.InstanceInspect(ctx, service)
	if err != nil {
		return nil, err
	}
	return i, nil
}

func (c *client) Root(
	ctx types.Context) ([]string, error) {

	return c.APIClient.Root(c.requireCtx(ctx))
}

func (c *client) Services(
	ctx types.Context) (map[string]*types.ServiceInfo, error) {

	ctx = c.withAllInstanceIDs(c.requireCtx(ctx))

	svcInfo, err := c.APIClient.Services(ctx)
	if err != nil {
		return nil, err
	}
	for k, v := range svcInfo {
		c.serviceCache.Set(k, v)
	}
	return svcInfo, err
}

func (c *client) ServiceInspect(
	ctx types.Context, service string) (*types.ServiceInfo, error) {

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	return c.APIClient.ServiceInspect(ctx, service)
}

func (c *client) Volumes(
	ctx types.Context,
	attachments bool) (types.ServiceVolumeMap, error) {

	ctx = c.requireCtx(ctx)

	ctxA, err := c.withAllLocalDevices(ctx)
	if err != nil {
		return nil, err
	}
	ctx = c.withAllInstanceIDs(ctxA)

	return c.APIClient.Volumes(ctx, attachments)
}

func (c *client) VolumesByService(
	ctx types.Context,
	service string,
	attachments bool) (types.VolumeMap, error) {

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	ctxA, err := c.withAllLocalDevices(ctx)
	if err != nil {
		return nil, err
	}
	ctx = ctxA

	return c.APIClient.VolumesByService(ctx, service, attachments)
}

func (c *client) VolumeInspect(
	ctx types.Context,
	service, volumeID string,
	attachments bool) (*types.Volume, error) {

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	ctxA, err := c.withAllLocalDevices(ctx)
	if err != nil {
		return nil, err
	}
	ctx = ctxA

	return c.APIClient.VolumeInspect(ctx, service, volumeID, attachments)
}

func (c *client) VolumeCreate(
	ctx types.Context,
	service string,
	request *types.VolumeCreateRequest) (*types.Volume, error) {

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	ctxA, err := c.withAllLocalDevices(ctx)
	if err != nil {
		return nil, err
	}
	ctx = ctxA

	lsd, _ := registry.NewClientDriver(service)
	if lsd != nil {
		if err := lsd.Init(ctx, c.config); err != nil {
			return nil, err
		}

		if err := lsd.VolumeCreateBefore(
			&ctx, service, request); err != nil {
			return nil, err
		}
	}

	vol, err := c.APIClient.VolumeCreate(ctx, service, request)
	if err != nil {
		return nil, err
	}

	if lsd != nil {
		lsd.VolumeCreateAfter(ctx, vol)
	}

	return vol, nil
}

func (c *client) VolumeCreateFromSnapshot(
	ctx types.Context,
	service, snapshotID string,
	request *types.VolumeCreateRequest) (*types.Volume, error) {

	ctx = c.withInstanceID(c.requireCtx(ctx), service)

	lsd, _ := registry.NewClientDriver(service)
	if lsd != nil {
		if err := lsd.Init(ctx, c.config); err != nil {
			return nil, err
		}

		if err := lsd.VolumeCreateFromSnapshotBefore(
			&ctx, service, snapshotID, request); err != nil {
			return nil, err
		}
	}

	vol, err := c.APIClient.VolumeCreateFromSnapshot(
		ctx, service, snapshotID, request)
	if err != nil {
		return nil, err
	}

	if lsd != nil {
		lsd.VolumeCreateFromSnapshotAfter(ctx, vol)
	}

	return vol, nil
}

func (c *client) VolumeCopy(
	ctx types.Context,
	service, volumeID string,
	request *types.VolumeCopyRequest) (*types.Volume, error) {

	ctx = c.withInstanceID(c.requireCtx(ctx), service)

	lsd, _ := registry.NewClientDriver(service)
	if lsd != nil {
		if err := lsd.Init(ctx, c.config); err != nil {
			return nil, err
		}

		if err := lsd.VolumeCopyBefore(
			&ctx, service, volumeID, request); err != nil {
			return nil, err
		}
	}

	vol, err := c.APIClient.VolumeCopy(ctx, service, volumeID, request)
	if err != nil {
		return nil, err
	}

	if lsd != nil {
		lsd.VolumeCopyAfter(ctx, vol)
	}

	return vol, nil
}

func (c *client) VolumeRemove(
	ctx types.Context,
	service, volumeID string) error {

	ctx = c.withInstanceID(c.requireCtx(ctx), service)

	lsd, _ := registry.NewClientDriver(service)
	if lsd != nil {
		if err := lsd.Init(ctx, c.config); err != nil {
			return err
		}

		if err := lsd.VolumeRemoveBefore(
			&ctx, service, volumeID); err != nil {
			return err
		}
	}

	err := c.APIClient.VolumeRemove(ctx, service, volumeID)
	if err != nil {
		return err
	}

	if lsd != nil {
		lsd.VolumeRemoveAfter(ctx, service, volumeID)
	}

	return nil
}

func (c *client) VolumeAttach(
	ctx types.Context,
	service string,
	volumeID string,
	request *types.VolumeAttachRequest) (*types.Volume, string, error) {

	if c.isController() {
		return nil, "", utils.NewUnsupportedForClientTypeError(
			c.clientType, "VolumeAttach")
	}

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	ctxA, err := c.withAllLocalDevices(ctx)
	if err != nil {
		return nil, "", err
	}
	ctx = ctxA

	return c.APIClient.VolumeAttach(ctx, service, volumeID, request)
}

func (c *client) VolumeDetach(
	ctx types.Context,
	service string,
	volumeID string,
	request *types.VolumeDetachRequest) (*types.Volume, error) {

	if c.isController() {
		return nil, utils.NewUnsupportedForClientTypeError(
			c.clientType, "VolumeDetach")
	}

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	ctxA, err := c.withAllLocalDevices(ctx)
	if err != nil {
		return nil, err
	}
	ctx = ctxA

	return c.APIClient.VolumeDetach(ctx, service, volumeID, request)
}

func (c *client) VolumeDetachAll(
	ctx types.Context,
	request *types.VolumeDetachRequest) (types.ServiceVolumeMap, error) {

	if c.isController() {
		return nil, utils.NewUnsupportedForClientTypeError(
			c.clientType, "VolumeDetachAll")
	}

	ctx = c.withAllInstanceIDs(c.requireCtx(ctx))
	ctxA, err := c.withAllLocalDevices(ctx)
	if err != nil {
		return nil, err
	}
	ctx = ctxA

	return c.APIClient.VolumeDetachAll(ctx, request)
}

func (c *client) VolumeDetachAllForService(
	ctx types.Context,
	service string,
	request *types.VolumeDetachRequest) (types.VolumeMap, error) {

	if c.isController() {
		return nil, utils.NewUnsupportedForClientTypeError(
			c.clientType, "VolumeDetachAllForService")
	}

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	ctxA, err := c.withAllLocalDevices(ctx)
	if err != nil {
		return nil, err
	}
	ctx = ctxA

	return c.APIClient.VolumeDetachAllForService(ctx, service, request)
}

func (c *client) VolumeSnapshot(
	ctx types.Context,
	service string,
	volumeID string,
	request *types.VolumeSnapshotRequest) (*types.Snapshot, error) {

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	return c.APIClient.VolumeSnapshot(ctx, service, volumeID, request)
}

func (c *client) Snapshots(
	ctx types.Context) (types.ServiceSnapshotMap, error) {

	ctx = c.withAllInstanceIDs(c.requireCtx(ctx))
	return c.APIClient.Snapshots(ctx)
}

func (c *client) SnapshotsByService(
	ctx types.Context, service string) (types.SnapshotMap, error) {

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	return c.APIClient.SnapshotsByService(ctx, service)
}

func (c *client) SnapshotInspect(
	ctx types.Context,
	service, snapshotID string) (*types.Snapshot, error) {

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	return c.APIClient.SnapshotInspect(ctx, service, snapshotID)
}

func (c *client) SnapshotRemove(
	ctx types.Context,
	service, snapshotID string) error {

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	return c.APIClient.SnapshotRemove(ctx, service, snapshotID)
}

func (c *client) SnapshotCopy(
	ctx types.Context,
	service, snapshotID string,
	request *types.SnapshotCopyRequest) (*types.Snapshot, error) {

	ctx = c.withInstanceID(c.requireCtx(ctx), service)
	return c.APIClient.SnapshotCopy(ctx, service, snapshotID, request)
}

func (c *client) Executors(
	ctx types.Context) (map[string]*types.ExecutorInfo, error) {

	if c.isController() {
		return nil, utils.NewUnsupportedForClientTypeError(
			c.clientType, "Executors")
	}

	ctx = c.requireCtx(ctx)
	lsxInfo, err := c.APIClient.Executors(ctx)
	if err != nil {
		return nil, err
	}
	for k, v := range lsxInfo {
		c.lsxCache.Set(k, v)
	}
	return lsxInfo, nil
}

func (c *client) ExecutorHead(
	ctx types.Context,
	name string) (*types.ExecutorInfo, error) {

	if c.isController() {
		return nil, utils.NewUnsupportedForClientTypeError(
			c.clientType, "ExecutorHead")
	}

	ctx = c.requireCtx(ctx)
	return c.APIClient.ExecutorHead(ctx, name)
}

func (c *client) ExecutorGet(
	ctx types.Context, name string) (io.ReadCloser, error) {

	if c.isController() {
		return nil, utils.NewUnsupportedForClientTypeError(
			c.clientType, "ExecutorGet")
	}

	ctx = c.requireCtx(ctx)
	return c.APIClient.ExecutorGet(ctx, name)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"syscall"

	"github.com/Shopify/sarama"
)

var (
	fConfig = flag.String("config", "", "xml file configuring processing streams")

	logger = log.New(os.Stderr, "processor", log.LstdFlags)
)

func flagbad(f string, i ...interface{}) {
	fmt.Fprintf(os.Stderr, f, i...)
	flag.PrintDefaults()
	os.Exit(1)
}

func main() {
	flag.Parse()
	if *fConfig == "" {
		flagbad("-config is empty\n")
	}

	cfg, err := ParseConfig(*fConfig)
	if err != nil {
		logger.Panicf("Failed to parse config: %v", err)
	}

	config := sarama.NewConfig()
	config.Producer.RequiredAcks = sarama.WaitForAll
	config.ClientID = "kafkaproc.processor"

	client, err := sarama.NewClient(cfg.Kafka.Peer, config)
	if err != nil {
		logger.Panicf("Creating sarama client: %v", err)
	}
	defer client.Close()

	consumer, err := sarama.NewConsumerFromClient(client)
	if err != nil {
		logger.Panicf("Creating sarama consumer: %v", err)
	}
	defer consumer.Close()

	producer, err := sarama.NewSyncProducerFromClient(client)
	if err != nil {
		logger.Panicf("Creating sarama syncproducer: %v", err)
	}
	defer producer.Close()

	cl := NewConsumerList()

	for _, stream := range cfg.Stream {
		for _, partition := range stream.Partition {
			partitionconsumer, err := consumer.ConsumePartition(stream.TopicSrc, partition, sarama.OffsetNewest)
			if err != nil {
				logger.Panicf("Creating partition consumer: %v", err)
			}
			//defer partitionconsumer.Close()
			pc := &ProcConsumer{
				consumer:  partitionconsumer,
				producer:  producer,
				src:       stream.TopicSrc,
				dst:       stream.TopicDst,
				partition: partition,
				function:  stream.Function,
				killchan:  make(chan struct{}),
			}
			cl.Add(pc)
		}
	}

	sigchan := make(chan os.Signal)
	signal.Notify(sigchan, syscall.SIGINT)
	go func() {
		<-sigchan
		fmt.Printf("SIGINT\n")
		cl.Close()
	}()
	fmt.Printf("wait\n")
	<-cl.killchan
	fmt.Printf("waited\n")
}
<commit_msg>Remove another panic<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"os/signal"
	"syscall"

	"github.com/Shopify/sarama"
)

var (
	fConfig = flag.String("config", "", "xml file configuring processing streams")

	logger = log.New(os.Stderr, "processor", log.LstdFlags)
)

func flagbad(f string, i ...interface{}) {
	fmt.Fprintf(os.Stderr, f, i...)
	flag.PrintDefaults()
	os.Exit(1)
}

func main() {
	flag.Parse()
	if *fConfig == "" {
		flagbad("-config is empty\n")
	}

	cfg, err := ParseConfig(*fConfig)
	if err != nil {
		logger.Panicf("Failed to parse config: %v", err)
	}

	config := sarama.NewConfig()
	config.Producer.RequiredAcks = sarama.WaitForAll
	config.ClientID = "kafkaproc.processor"

	client, err := sarama.NewClient(cfg.Kafka.Peer, config)
	if err != nil {
		logger.Panicf("Creating sarama client: %v", err)
	}
	defer client.Close()

	consumer, err := sarama.NewConsumerFromClient(client)
	if err != nil {
		logger.Panicf("Creating sarama consumer: %v", err)
	}
	defer consumer.Close()

	producer, err := sarama.NewSyncProducerFromClient(client)
	if err != nil {
		logger.Panicf("Creating sarama syncproducer: %v", err)
	}
	defer producer.Close()

	cl := NewConsumerList()

	for _, stream := range cfg.Stream {
		for _, partition := range stream.Partition {
			partitionconsumer, err := consumer.ConsumePartition(stream.TopicSrc, partition, sarama.OffsetNewest)
			if err != nil {
				logger.Printf("Failed to create partition consumer %s:%d: %v", stream.TopicSrc, partition, err)
				continue
			}
			pc := &ProcConsumer{
				consumer:  partitionconsumer,
				producer:  producer,
				src:       stream.TopicSrc,
				dst:       stream.TopicDst,
				partition: partition,
				function:  stream.Function,
				killchan:  make(chan struct{}),
			}
			cl.Add(pc)
		}
	}

	sigchan := make(chan os.Signal)
	signal.Notify(sigchan, syscall.SIGINT)
	go func() {
		<-sigchan
		fmt.Printf("SIGINT\n")
		cl.Close()
	}()
	fmt.Printf("wait\n")
	<-cl.killchan
	fmt.Printf("waited\n")
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"sort"
	"time"
    "strings"
	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/controller/framework"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/util"
	utildbus "k8s.io/kubernetes/pkg/util/dbus"
	"k8s.io/kubernetes/pkg/util/exec"
	"k8s.io/kubernetes/pkg/util/intstr"
	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
)

const (
	reloadQPS    = 10.0
	resyncPeriod = 10 * time.Second
)

type service struct {
	IP   string
	Port int
}

type serviceByIPPort []service

func (c serviceByIPPort) Len() int      { return len(c) }
func (c serviceByIPPort) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
func (c serviceByIPPort) Less(i, j int) bool {
	iIP := c[i].IP
	jIP := c[j].IP
	if iIP != jIP {
		return iIP < jIP
	}

	iPort := c[i].Port
	jPort := c[j].Port
	return iPort < jPort
}

type vip struct {
	Name      string
	IP        string
	Port      int
	Protocol  string
	LVSMethod string
	Backends  []service
}

type vipByNameIPPort []vip

func (c vipByNameIPPort) Len() int      { return len(c) }
func (c vipByNameIPPort) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
func (c vipByNameIPPort) Less(i, j int) bool {
	iName := c[i].Name
	jName := c[j].Name
	if iName != jName {
		return iName < jName
	}

	iIP := c[i].IP
	jIP := c[j].IP
	if iIP != jIP {
		return iIP < jIP
	}

	iPort := c[i].Port
	jPort := c[j].Port
	return iPort < jPort
}

// ipvsControllerController watches the kubernetes api and adds/removes
// services from LVS throgh ipvsadmin.
type ipvsControllerController struct {
	client            *unversioned.Client
	epController      *framework.Controller
	svcController     *framework.Controller
	svcLister         cache.StoreToServiceLister
	epLister          cache.StoreToEndpointsLister
	reloadRateLimiter util.RateLimiter
	keepalived        *keepalived
	configMapName     string
	ruCfg             []vip
	ruMD5             string
}

// getEndpoints returns a list of <endpoint ip>:<port> for a given service/target port combination.
func (ipvsc *ipvsControllerController) getEndpoints(
	s *api.Service, servicePort *api.ServicePort) []service {
	ep, err := ipvsc.epLister.GetServiceEndpoints(s)
	if err != nil {
		glog.Warningf("unexpected error getting service endpoints: %v", err)
		return []service{}
	}

	var endpoints []service

	// The intent here is to create a union of all subsets that match a targetPort.
	// We know the endpoint already matches the service, so all pod ips that have
	// the target port are capable of service traffic for it.
	for _, ss := range ep.Subsets {
		for _, epPort := range ss.Ports {
			var targetPort int
			switch servicePort.TargetPort.Type {
			case intstr.Int:
				if epPort.Port == servicePort.TargetPort.IntValue() {
					targetPort = epPort.Port
				}
			case intstr.String:
				if epPort.Name == servicePort.TargetPort.StrVal {
					targetPort = epPort.Port
				}
			}
			if targetPort == 0 {
				continue
			}
			for _, epAddress := range ss.Addresses {
				endpoints = append(endpoints, service{IP: epAddress.IP, Port: targetPort})
			}
		}
	}

	return endpoints
}

// getServices returns a list of services and their endpoints.
func (ipvsc *ipvsControllerController) getServices() []vip {
	svcs := []vip{}

	ns, name, err := parseNsName(ipvsc.configMapName)
	if err != nil {
		glog.Warningf("%v", err)
		return []vip{}
	}
	cfgMap, err := ipvsc.getConfigMap(ns, name)
	if err != nil {
		glog.Warningf("%v", err)
		return []vip{}
	}

	// k -> IP to use
	// v -> <namespace>/<service name>:<lvs method>
	for externalIPIndex, nsSvcLvs := range cfgMap.Data {
		var externalIP string
		if colonIndex := strings.Index(externalIPIndex, "-"); colonIndex < 0 {
			externalIP = externalIPIndex
		} else {
			externalIP = externalIPIndex[:colonIndex]
		}
		ns, svc, lvsm, err := parseNsSvcLVS(nsSvcLvs)
		if err != nil {
			glog.Warningf("%v", err)
			continue
		}

		nsSvc := fmt.Sprintf("%v/%v", ns, svc)
		svcObj, svcExists, err := ipvsc.svcLister.Store.GetByKey(nsSvc)
		if err != nil {
			glog.Warningf("error getting service %v: %v", nsSvc, err)
			continue
		}

		if !svcExists {
			glog.Warningf("service %v not found", nsSvc)
			continue
		}

		s := svcObj.(*api.Service)
		for _, servicePort := range s.Spec.Ports {
            		np := servicePort.NodePort
           		if np == 0 {
           		    	glog.Infof("No nodePort found for service %v, port %+v", s.Name, servicePort)
                		continue
            		}

			ep := ipvsc.getEndpoints(s, &servicePort)
			if len(ep) == 0 {
				glog.Warningf("no endpoints found for service %v, port %+v", s.Name, servicePort)
				continue
			}

			sort.Sort(serviceByIPPort(ep))

			svcs = append(svcs, vip{
				Name:      fmt.Sprintf("%v/%v", s.Namespace, s.Name),
				IP:        externalIP,
				Port:      servicePort.NodePort,
				LVSMethod: lvsm,
				Backends:  ep,
				Protocol:  fmt.Sprintf("%v", servicePort.Protocol),
			})
			glog.V(2).Infof("Found service: %v %v %v:%v->%v:%v", s.Name, servicePort.Protocol, externalIP, servicePort.NodePort, s.Spec.ClusterIP, servicePort.Port)
		}
	}

	sort.Sort(vipByNameIPPort(svcs))

	return svcs
}

func (ipvsc *ipvsControllerController) getConfigMap(ns, name string) (*api.ConfigMap, error) {
	return ipvsc.client.ConfigMaps(ns).Get(name)
}

// sync all services with the
func (ipvsc *ipvsControllerController) sync() {
	ipvsc.reloadRateLimiter.Accept()

	if !ipvsc.epController.HasSynced() || !ipvsc.svcController.HasSynced() {
		time.Sleep(100 * time.Millisecond)
		return
	}

	svc := ipvsc.getServices()
	ipvsc.ruCfg = svc

	err := ipvsc.keepalived.WriteCfg(svc)
	if err != nil {
		return
	}
	glog.V(2).Infof("services: %v", svc)

	md5, err := checksum(keepalivedCfg)
	if err == nil && md5 == ipvsc.ruMD5 {
		return
	}

	ipvsc.ruMD5 = md5
	err = ipvsc.keepalived.Reload()
	if err != nil {
		glog.Errorf("error reloading keepalived: %v", err)
	}
}

// newIPVSController creates a new controller from the given config.
func newIPVSController(kubeClient *unversioned.Client, namespace string, useUnicast bool, configMapName string) *ipvsControllerController {
	ipvsc := ipvsControllerController{
		client:            kubeClient,
		reloadRateLimiter: util.NewTokenBucketRateLimiter(reloadQPS, int(reloadQPS)),
		ruCfg:             []vip{},
		configMapName:     configMapName,
	}

	podInfo, err := getPodDetails(kubeClient)
	if err != nil {
		glog.Fatalf("Error getting POD information: %v", err)
	}

	pod, err := kubeClient.Pods(podInfo.PodNamespace).Get(podInfo.PodName)
	if err != nil {
		glog.Fatalf("Error getting %v: %v", podInfo.PodName, err)
	}

	selector := parseNodeSelector(pod.Spec.NodeSelector)
	clusterNodes := getClusterNodesIP(kubeClient, selector)

	nodeInfo, err := getNetworkInfo(podInfo.NodeIP)
	if err != nil {
		glog.Fatalf("Error getting local IP from nodes in the cluster: %v", err)
	}

	neighbors := getNodeNeighbors(nodeInfo, clusterNodes)

	execer := exec.New()
	dbus := utildbus.New()
	iptInterface := utiliptables.New(execer, dbus, utiliptables.ProtocolIpv4)

	ipvsc.keepalived = &keepalived{
		iface:      nodeInfo.iface,
		ip:         nodeInfo.ip,
		netmask:    nodeInfo.netmask,
		nodes:      clusterNodes,
		neighbors:  neighbors,
		priority:   getNodePriority(nodeInfo.ip, clusterNodes),
		useUnicast: useUnicast,
		ipt:        iptInterface,
	}

	err = ipvsc.keepalived.loadTemplate()
	if err != nil {
		glog.Fatalf("Error loading keepalived template: %v", err)
	}

	eventHandlers := framework.ResourceEventHandlerFuncs{}

	ipvsc.svcLister.Store, ipvsc.svcController = framework.NewInformer(
		cache.NewListWatchFromClient(
			ipvsc.client, "services", namespace, fields.Everything()),
		&api.Service{}, resyncPeriod, eventHandlers)

	ipvsc.epLister.Store, ipvsc.epController = framework.NewInformer(
		cache.NewListWatchFromClient(
			ipvsc.client, "endpoints", namespace, fields.Everything()),
		&api.Endpoints{}, resyncPeriod, eventHandlers)

	return &ipvsc
}

func checksum(filename string) (string, error) {
	var result []byte
	file, err := os.Open(filename)
	if err != nil {
		return "", err
	}
	defer file.Close()

	hash := md5.New()
	_, err = io.Copy(hash, file)
	if err != nil {
		return "", err
	}
	return hex.EncodeToString(hash.Sum(result)), nil
}
<commit_msg>manage some format problem<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"sort"
	"time"
    "strings"
	"github.com/golang/glog"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/controller/framework"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/util"
	utildbus "k8s.io/kubernetes/pkg/util/dbus"
	"k8s.io/kubernetes/pkg/util/exec"
	"k8s.io/kubernetes/pkg/util/intstr"
	utiliptables "k8s.io/kubernetes/pkg/util/iptables"
)

const (
	reloadQPS    = 10.0
	resyncPeriod = 10 * time.Second
)

type service struct {
	IP   string
	Port int
}

type serviceByIPPort []service

func (c serviceByIPPort) Len() int      { return len(c) }
func (c serviceByIPPort) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
func (c serviceByIPPort) Less(i, j int) bool {
	iIP := c[i].IP
	jIP := c[j].IP
	if iIP != jIP {
		return iIP < jIP
	}

	iPort := c[i].Port
	jPort := c[j].Port
	return iPort < jPort
}

type vip struct {
	Name      string
	IP        string
	Port      int
	Protocol  string
	LVSMethod string
	Backends  []service
}

type vipByNameIPPort []vip

func (c vipByNameIPPort) Len() int      { return len(c) }
func (c vipByNameIPPort) Swap(i, j int) { c[i], c[j] = c[j], c[i] }
func (c vipByNameIPPort) Less(i, j int) bool {
	iName := c[i].Name
	jName := c[j].Name
	if iName != jName {
		return iName < jName
	}

	iIP := c[i].IP
	jIP := c[j].IP
	if iIP != jIP {
		return iIP < jIP
	}

	iPort := c[i].Port
	jPort := c[j].Port
	return iPort < jPort
}

// ipvsControllerController watches the kubernetes api and adds/removes
// services from LVS throgh ipvsadmin.
type ipvsControllerController struct {
	client            *unversioned.Client
	epController      *framework.Controller
	svcController     *framework.Controller
	svcLister         cache.StoreToServiceLister
	epLister          cache.StoreToEndpointsLister
	reloadRateLimiter util.RateLimiter
	keepalived        *keepalived
	configMapName     string
	ruCfg             []vip
	ruMD5             string
}

// getEndpoints returns a list of <endpoint ip>:<port> for a given service/target port combination.
func (ipvsc *ipvsControllerController) getEndpoints(
	s *api.Service, servicePort *api.ServicePort) []service {
	ep, err := ipvsc.epLister.GetServiceEndpoints(s)
	if err != nil {
		glog.Warningf("unexpected error getting service endpoints: %v", err)
		return []service{}
	}

	var endpoints []service

	// The intent here is to create a union of all subsets that match a targetPort.
	// We know the endpoint already matches the service, so all pod ips that have
	// the target port are capable of service traffic for it.
	for _, ss := range ep.Subsets {
		for _, epPort := range ss.Ports {
			var targetPort int
			switch servicePort.TargetPort.Type {
			case intstr.Int:
				if epPort.Port == servicePort.TargetPort.IntValue() {
					targetPort = epPort.Port
				}
			case intstr.String:
				if epPort.Name == servicePort.TargetPort.StrVal {
					targetPort = epPort.Port
				}
			}
			if targetPort == 0 {
				continue
			}
			for _, epAddress := range ss.Addresses {
				endpoints = append(endpoints, service{IP: epAddress.IP, Port: targetPort})
			}
		}
	}

	return endpoints
}

// getServices returns a list of services and their endpoints.
func (ipvsc *ipvsControllerController) getServices() []vip {
	svcs := []vip{}

	ns, name, err := parseNsName(ipvsc.configMapName)
	if err != nil {
		glog.Warningf("%v", err)
		return []vip{}
	}
	cfgMap, err := ipvsc.getConfigMap(ns, name)
	if err != nil {
		glog.Warningf("%v", err)
		return []vip{}
	}

	// k -> IP to use
	// v -> <namespace>/<service name>:<lvs method>
	for externalIPIndex, nsSvcLvs := range cfgMap.Data {
		var externalIP string
		if colonIndex := strings.Index(externalIPIndex, "-"); colonIndex < 0 {
			externalIP = externalIPIndex
		} else {
			externalIP = externalIPIndex[:colonIndex]
		}
		ns, svc, lvsm, err := parseNsSvcLVS(nsSvcLvs)
		if err != nil {
			glog.Warningf("%v", err)
			continue
		}

		nsSvc := fmt.Sprintf("%v/%v", ns, svc)
		svcObj, svcExists, err := ipvsc.svcLister.Store.GetByKey(nsSvc)
		if err != nil {
			glog.Warningf("error getting service %v: %v", nsSvc, err)
			continue
		}

		if !svcExists {
			glog.Warningf("service %v not found", nsSvc)
			continue
		}

		s := svcObj.(*api.Service)
		for _, servicePort := range s.Spec.Ports {
			np := servicePort.NodePort
			if np == 0 {
				glog.Infof("No nodePort found for service %v, port %+v", s.Name, servicePort)
				continue
			}

			ep := ipvsc.getEndpoints(s, &servicePort)
			if len(ep) == 0 {
				glog.Warningf("no endpoints found for service %v, port %+v", s.Name, servicePort)
				continue
			}

			sort.Sort(serviceByIPPort(ep))

			svcs = append(svcs, vip{
				Name:      fmt.Sprintf("%v/%v", s.Namespace, s.Name),
				IP:        externalIP,
				Port:      servicePort.NodePort,
				LVSMethod: lvsm,
				Backends:  ep,
				Protocol:  fmt.Sprintf("%v", servicePort.Protocol),
			})
			glog.V(2).Infof("Found service: %v %v %v:%v->%v:%v", s.Name, servicePort.Protocol, externalIP, servicePort.NodePort, s.Spec.ClusterIP, servicePort.Port)
		}
	}

	sort.Sort(vipByNameIPPort(svcs))

	return svcs
}

func (ipvsc *ipvsControllerController) getConfigMap(ns, name string) (*api.ConfigMap, error) {
	return ipvsc.client.ConfigMaps(ns).Get(name)
}

// sync all services with the
func (ipvsc *ipvsControllerController) sync() {
	ipvsc.reloadRateLimiter.Accept()

	if !ipvsc.epController.HasSynced() || !ipvsc.svcController.HasSynced() {
		time.Sleep(100 * time.Millisecond)
		return
	}

	svc := ipvsc.getServices()
	ipvsc.ruCfg = svc

	err := ipvsc.keepalived.WriteCfg(svc)
	if err != nil {
		return
	}
	glog.V(2).Infof("services: %v", svc)

	md5, err := checksum(keepalivedCfg)
	if err == nil && md5 == ipvsc.ruMD5 {
		return
	}

	ipvsc.ruMD5 = md5
	err = ipvsc.keepalived.Reload()
	if err != nil {
		glog.Errorf("error reloading keepalived: %v", err)
	}
}

// newIPVSController creates a new controller from the given config.
func newIPVSController(kubeClient *unversioned.Client, namespace string, useUnicast bool, configMapName string) *ipvsControllerController {
	ipvsc := ipvsControllerController{
		client:            kubeClient,
		reloadRateLimiter: util.NewTokenBucketRateLimiter(reloadQPS, int(reloadQPS)),
		ruCfg:             []vip{},
		configMapName:     configMapName,
	}

	podInfo, err := getPodDetails(kubeClient)
	if err != nil {
		glog.Fatalf("Error getting POD information: %v", err)
	}

	pod, err := kubeClient.Pods(podInfo.PodNamespace).Get(podInfo.PodName)
	if err != nil {
		glog.Fatalf("Error getting %v: %v", podInfo.PodName, err)
	}

	selector := parseNodeSelector(pod.Spec.NodeSelector)
	clusterNodes := getClusterNodesIP(kubeClient, selector)

	nodeInfo, err := getNetworkInfo(podInfo.NodeIP)
	if err != nil {
		glog.Fatalf("Error getting local IP from nodes in the cluster: %v", err)
	}

	neighbors := getNodeNeighbors(nodeInfo, clusterNodes)

	execer := exec.New()
	dbus := utildbus.New()
	iptInterface := utiliptables.New(execer, dbus, utiliptables.ProtocolIpv4)

	ipvsc.keepalived = &keepalived{
		iface:      nodeInfo.iface,
		ip:         nodeInfo.ip,
		netmask:    nodeInfo.netmask,
		nodes:      clusterNodes,
		neighbors:  neighbors,
		priority:   getNodePriority(nodeInfo.ip, clusterNodes),
		useUnicast: useUnicast,
		ipt:        iptInterface,
	}

	err = ipvsc.keepalived.loadTemplate()
	if err != nil {
		glog.Fatalf("Error loading keepalived template: %v", err)
	}

	eventHandlers := framework.ResourceEventHandlerFuncs{}

	ipvsc.svcLister.Store, ipvsc.svcController = framework.NewInformer(
		cache.NewListWatchFromClient(
			ipvsc.client, "services", namespace, fields.Everything()),
		&api.Service{}, resyncPeriod, eventHandlers)

	ipvsc.epLister.Store, ipvsc.epController = framework.NewInformer(
		cache.NewListWatchFromClient(
			ipvsc.client, "endpoints", namespace, fields.Everything()),
		&api.Endpoints{}, resyncPeriod, eventHandlers)

	return &ipvsc
}

func checksum(filename string) (string, error) {
	var result []byte
	file, err := os.Open(filename)
	if err != nil {
		return "", err
	}
	defer file.Close()

	hash := md5.New()
	_, err = io.Copy(hash, file)
	if err != nil {
		return "", err
	}
	return hex.EncodeToString(hash.Sum(result)), nil
}
<|endoftext|>


<commit_before><commit_msg>fix (engine): check nb worker spawned (#2935)<commit_after><|endoftext|>


<commit_before>package ql

import (
	"bytes"
	"database/sql/driver"
	"reflect"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

// Need to turn \x00, \n, \r, \, ', " and \x1a.
// Returns an escaped, quoted string. eg, "hello 'world'" -> "'hello \'world\''".
func escapeAndQuoteString(val string) string {
	buf := bytes.Buffer{}

	buf.WriteRune('\'')
	for _, char := range val {
		switch char {
		case '\'':
			buf.WriteString(`\'`)
		case '"':
			buf.WriteString(`\"`)
		case '\\':
			buf.WriteString(`\\`)
		case '\n':
			buf.WriteString(`\n`)
		case '\r':
			buf.WriteString(`\r`)
		case 0:
			buf.WriteString(`\x00`)
		case 0x1a:
			buf.WriteString(`\x1a`)
		default:
			buf.WriteRune(char)
		}
	}
	buf.WriteRune('\'')

	return buf.String()
}

func isUint(k reflect.Kind) bool {
	return k == reflect.Uint || k == reflect.Uint8 || k == reflect.Uint16 || k == reflect.Uint32 ||
		k == reflect.Uint64
}

func isInt(k reflect.Kind) bool {
	return k == reflect.Int || k == reflect.Int8 || k == reflect.Int16 || k == reflect.Int32 ||
		k == reflect.Int64
}

func isFloat(k reflect.Kind) bool {
	return k == reflect.Float32 || k == reflect.Float64
}

// sql is like "id = ? OR username = ?"
// vals is like []interface{}{4, "bob"}
// NOTE that vals can only have values of certain types:
//   - Integers (signed and unsigned)
//   - floats
//   - strings (that are valid utf-8)
//   - booleans
//   - times
var typeOfTime = reflect.TypeOf(time.Time{})

// Preprocess takes an SQL string with placeholders and a list of arguments to
// replace them with. It returns a blank string and error if the number of placeholders
// does not match the number of arguments.
func Preprocess(sql string, vals []interface{}) (string, error) {
	// Get the number of arguments to add to this query
	maxVals := len(vals)

	// If our query is blank and has no args return early
	// Args with a blank query is an error
	if sql == "" {
		if maxVals != 0 {
			return "", ErrArgumentMismatch
		}
		return "", nil
	}

	// Iterate over each rune in the sql string and replace with the next arg if it's a place holder
	curVal := 0
	buf := new(bytes.Buffer)

	pos := 0
	for pos < len(sql) {
		r, w := utf8.DecodeRuneInString(sql[pos:])
		pos += w

		switch {
		case r == '?' && curVal < maxVals:
			if err := interpolate(buf, vals[curVal]); err != nil {
				return "", err
			}
			curVal++
		case r == '`', r == '\'', r == '"':
			p := strings.IndexRune(sql[pos:], r)
			if p == -1 {
				return "", ErrInvalidSyntax
			}
			if r == '"' {
				r = '\''
			}
			buf.WriteRune(r)
			buf.WriteString(sql[pos : pos+p])
			buf.WriteRune(r)
			pos += p + 1
		case r == '[':
			w := strings.IndexRune(sql[pos:], ']')
			col := sql[pos : pos+w]
			Quoter.writeQuotedColumn(col, buf)
			pos += w + 1 // size of ']'
		case r != '?':
			buf.WriteRune(r)
		default:
			return "", ErrArgumentMismatch
		}
	}

	if curVal != maxVals {
		return "", ErrArgumentMismatch
	}

	return buf.String(), nil
}

func interpolate(buf *bytes.Buffer, v interface{}) error {
	valuer, ok := v.(driver.Valuer)
	if ok {
		val, err := valuer.Value()
		if err != nil {
			return err
		}
		v = val
	}

	valueOfV := reflect.ValueOf(v)
	kindOfV := valueOfV.Kind()

	if v == nil {
		buf.WriteString("NULL")
	} else if isInt(kindOfV) {
		var ival = valueOfV.Int()

		buf.WriteString(strconv.FormatInt(ival, 10))
	} else if isUint(kindOfV) {
		var uival = valueOfV.Uint()

		buf.WriteString(strconv.FormatUint(uival, 10))
	} else if kindOfV == reflect.String {
		var str = valueOfV.String()

		if !utf8.ValidString(str) {
			return ErrNotUTF8
		}

		buf.WriteString(escapeAndQuoteString(str))
	} else if isFloat(kindOfV) {
		var fval = valueOfV.Float()

		buf.WriteString(strconv.FormatFloat(fval, 'f', -1, 64))
	} else if kindOfV == reflect.Bool {
		var bval = valueOfV.Bool()

		if bval {
			buf.WriteRune('1')
		} else {
			buf.WriteRune('0')
		}
	} else if kindOfV == reflect.Struct {
		if typeOfV := valueOfV.Type(); typeOfV == typeOfTime {
			t := valueOfV.Interface().(time.Time)
			buf.WriteString(escapeAndQuoteString(t.UTC().Format(timeFormat)))
		} else {
			return ErrInvalidValue
		}
	} else if kindOfV == reflect.Slice {
		typeOfV := reflect.TypeOf(v)
		subtype := typeOfV.Elem()
		kindOfSubtype := subtype.Kind()

		sliceLen := valueOfV.Len()
		stringSlice := make([]string, 0, sliceLen)

		if sliceLen == 0 {
			return ErrInvalidSliceLength
		} else if isInt(kindOfSubtype) {
			for i := 0; i < sliceLen; i++ {
				var ival = valueOfV.Index(i).Int()
				stringSlice = append(stringSlice, strconv.FormatInt(ival, 10))
			}
		} else if isUint(kindOfSubtype) {
			for i := 0; i < sliceLen; i++ {
				var uival = valueOfV.Index(i).Uint()
				stringSlice = append(stringSlice, strconv.FormatUint(uival, 10))
			}
		} else if kindOfSubtype == reflect.String {
			for i := 0; i < sliceLen; i++ {
				var str = valueOfV.Index(i).String()
				if !utf8.ValidString(str) {
					return ErrNotUTF8
				}
				stringSlice = append(stringSlice, escapeAndQuoteString(str))
			}
		} else {
			return ErrInvalidSliceValue
		}
		buf.WriteRune('(')
		buf.WriteString(strings.Join(stringSlice, ","))
		buf.WriteRune(')')
	} else {
		return ErrInvalidValue
	}
	return nil
}
<commit_msg>change if/elses to switch (no semantic change)<commit_after>package ql

import (
	"bytes"
	"database/sql/driver"
	"reflect"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

// Need to turn \x00, \n, \r, \, ', " and \x1a.
// Returns an escaped, quoted string. eg, "hello 'world'" -> "'hello \'world\''".
func escapeAndQuoteString(val string) string {
	buf := bytes.Buffer{}

	buf.WriteRune('\'')
	for _, char := range val {
		switch char {
		case '\'':
			buf.WriteString(`\'`)
		case '"':
			buf.WriteString(`\"`)
		case '\\':
			buf.WriteString(`\\`)
		case '\n':
			buf.WriteString(`\n`)
		case '\r':
			buf.WriteString(`\r`)
		case 0:
			buf.WriteString(`\x00`)
		case 0x1a:
			buf.WriteString(`\x1a`)
		default:
			buf.WriteRune(char)
		}
	}
	buf.WriteRune('\'')

	return buf.String()
}

func isUint(k reflect.Kind) bool {
	return k == reflect.Uint || k == reflect.Uint8 || k == reflect.Uint16 || k == reflect.Uint32 ||
		k == reflect.Uint64
}

func isInt(k reflect.Kind) bool {
	return k == reflect.Int || k == reflect.Int8 || k == reflect.Int16 || k == reflect.Int32 ||
		k == reflect.Int64
}

func isFloat(k reflect.Kind) bool {
	return k == reflect.Float32 || k == reflect.Float64
}

// sql is like "id = ? OR username = ?"
// vals is like []interface{}{4, "bob"}
// NOTE that vals can only have values of certain types:
//   - Integers (signed and unsigned)
//   - floats
//   - strings (that are valid utf-8)
//   - booleans
//   - times
var typeOfTime = reflect.TypeOf(time.Time{})

// Preprocess takes an SQL string with placeholders and a list of arguments to
// replace them with. It returns a blank string and error if the number of placeholders
// does not match the number of arguments.
func Preprocess(sql string, vals []interface{}) (string, error) {
	// Get the number of arguments to add to this query
	maxVals := len(vals)

	// If our query is blank and has no args return early
	// Args with a blank query is an error
	if sql == "" {
		if maxVals != 0 {
			return "", ErrArgumentMismatch
		}
		return "", nil
	}

	// Iterate over each rune in the sql string and replace with the next arg if it's a place holder
	curVal := 0
	buf := new(bytes.Buffer)

	pos := 0
	for pos < len(sql) {
		r, w := utf8.DecodeRuneInString(sql[pos:])
		pos += w

		switch {
		case r == '?' && curVal < maxVals:
			if err := interpolate(buf, vals[curVal]); err != nil {
				return "", err
			}
			curVal++
		case r == '`', r == '\'', r == '"':
			p := strings.IndexRune(sql[pos:], r)
			if p == -1 {
				return "", ErrInvalidSyntax
			}
			if r == '"' {
				r = '\''
			}
			buf.WriteRune(r)
			buf.WriteString(sql[pos : pos+p])
			buf.WriteRune(r)
			pos += p + 1
		case r == '[':
			w := strings.IndexRune(sql[pos:], ']')
			col := sql[pos : pos+w]
			Quoter.writeQuotedColumn(col, buf)
			pos += w + 1 // size of ']'
		case r != '?':
			buf.WriteRune(r)
		default:
			return "", ErrArgumentMismatch
		}
	}

	if curVal != maxVals {
		return "", ErrArgumentMismatch
	}

	return buf.String(), nil
}

func interpolate(buf *bytes.Buffer, v interface{}) error {
	valuer, ok := v.(driver.Valuer)
	if ok {
		val, err := valuer.Value()
		if err != nil {
			return err
		}
		v = val
	}

	valueOfV := reflect.ValueOf(v)
	kindOfV := valueOfV.Kind()

	switch {
	case v == nil:
		buf.WriteString("NULL")
	case isInt(kindOfV):
		var ival = valueOfV.Int()

		buf.WriteString(strconv.FormatInt(ival, 10))
	case isUint(kindOfV):
		var uival = valueOfV.Uint()

		buf.WriteString(strconv.FormatUint(uival, 10))
	case kindOfV == reflect.String:
		var str = valueOfV.String()

		if !utf8.ValidString(str) {
			return ErrNotUTF8
		}

		buf.WriteString(escapeAndQuoteString(str))
	case isFloat(kindOfV):
		var fval = valueOfV.Float()

		buf.WriteString(strconv.FormatFloat(fval, 'f', -1, 64))
	case kindOfV == reflect.Bool:
		var bval = valueOfV.Bool()

		if bval {
			buf.WriteRune('1')
		} else {
			buf.WriteRune('0')
		}
	case kindOfV == reflect.Struct:
		if typeOfV := valueOfV.Type(); typeOfV == typeOfTime {
			t := valueOfV.Interface().(time.Time)
			buf.WriteString(escapeAndQuoteString(t.UTC().Format(timeFormat)))
		} else {
			return ErrInvalidValue
		}
	case kindOfV == reflect.Slice:
		typeOfV := reflect.TypeOf(v)
		subtype := typeOfV.Elem()
		kindOfSubtype := subtype.Kind()

		sliceLen := valueOfV.Len()
		stringSlice := make([]string, 0, sliceLen)

		switch {
		case sliceLen == 0:
			return ErrInvalidSliceLength
		case isInt(kindOfSubtype):
			for i := 0; i < sliceLen; i++ {
				var ival = valueOfV.Index(i).Int()
				stringSlice = append(stringSlice, strconv.FormatInt(ival, 10))
			}
		case isUint(kindOfSubtype):
			for i := 0; i < sliceLen; i++ {
				var uival = valueOfV.Index(i).Uint()
				stringSlice = append(stringSlice, strconv.FormatUint(uival, 10))
			}
		case kindOfSubtype == reflect.String:
			for i := 0; i < sliceLen; i++ {
				var str = valueOfV.Index(i).String()
				if !utf8.ValidString(str) {
					return ErrNotUTF8
				}
				stringSlice = append(stringSlice, escapeAndQuoteString(str))
			}
		default:
			return ErrInvalidSliceValue
		}
		buf.WriteRune('(')
		buf.WriteString(strings.Join(stringSlice, ","))
		buf.WriteRune(')')
	default:
		return ErrInvalidValue
	}
	return nil
}
<|endoftext|>


<commit_before>package jmespath

import (
	"errors"
	"reflect"
	"unicode"
	"unicode/utf8"
)

/* This is a tree based interpreter.  It walks the AST and directly
   interprets the AST to search through a JSON document.
*/

type treeInterpreter struct {
	fCall *functionCaller
}

func newInterpreter() *treeInterpreter {
	interpreter := treeInterpreter{}
	interpreter.fCall = newFunctionCaller()
	return &interpreter
}

type expRef struct {
	ref ASTNode
}

// Execute takes an ASTNode and input data and interprets the AST directly.
// It will produce the result of applying the JMESPath expression associated
// with the ASTNode to the input data "value".
func (intr *treeInterpreter) Execute(node ASTNode, value interface{}) (interface{}, error) {
	switch node.nodeType {
	case ASTComparator:
		left, err := intr.Execute(node.children[0], value)
		if err != nil {
			return nil, err
		}
		right, err := intr.Execute(node.children[1], value)
		if err != nil {
			return nil, err
		}
		switch node.value {
		case tEQ:
			return objsEqual(left, right), nil
		case tNE:
			return !objsEqual(left, right), nil
		}
		leftNum, ok := left.(float64)
		if !ok {
			return nil, nil
		}
		rightNum, ok := right.(float64)
		if !ok {
			return nil, nil
		}
		switch node.value {
		case tGT:
			return leftNum > rightNum, nil
		case tGTE:
			return leftNum >= rightNum, nil
		case tLT:
			return leftNum < rightNum, nil
		case tLTE:
			return leftNum <= rightNum, nil
		}
	case ASTExpRef:
		return expRef{ref: node.children[0]}, nil
	case ASTFunctionExpression:
		resolvedArgs := make([]interface{}, 0, 0)
		for _, arg := range node.children {
			current, err := intr.Execute(arg, value)
			if err != nil {
				return nil, err
			}
			resolvedArgs = append(resolvedArgs, current)
		}
		return intr.fCall.CallFunction(node.value.(string), resolvedArgs, intr)
	case ASTField:
		if m, ok := value.(map[string]interface{}); ok {
			key := node.value.(string)
			return m[key], nil
		}
		return intr.fieldFromStruct(node.value.(string), value)
	case ASTFilterProjection:
		left, err := intr.Execute(node.children[0], value)
		if err != nil {
			return nil, nil
		}
		sliceType, ok := left.([]interface{})
		if !ok {
			return nil, nil
		}
		compareNode := node.children[2]
		collected := make([]interface{}, 0, 0)
		for _, element := range sliceType {
			result, err := intr.Execute(compareNode, element)
			if err != nil {
				return nil, err
			}
			if !isFalse(result) {
				current, err := intr.Execute(node.children[1], element)
				if err != nil {
					return nil, err
				}
				if current != nil {
					collected = append(collected, current)
				}
			}
		}
		return collected, nil
	case ASTFlatten:
		left, err := intr.Execute(node.children[0], value)
		if err != nil {
			return nil, err
		}
		if left == nil {
			return nil, nil
		}
		if reflect.TypeOf(left).Kind() != reflect.Slice {
			// Can't flatten a non slice object.
			return nil, nil
		}
		v := reflect.ValueOf(left)
		flattened := make([]interface{}, 0, 0)
		for i := 0; i < v.Len(); i++ {
			element := v.Index(i).Interface()
			if reflect.TypeOf(element).Kind() == reflect.Slice {
				// Then insert the contents of the element
				// slice into the flattened slice,
				// i.e flattened = append(flattened, mySlice...)
				elementV := reflect.ValueOf(element)
				for j := 0; j < elementV.Len(); j++ {
					flattened = append(
						flattened, elementV.Index(j).Interface())
				}
			} else {
				flattened = append(flattened, element)
			}
		}
		return flattened, nil
	case ASTIdentity, ASTCurrentNode:
		return value, nil
	case ASTIndex:
		if sliceType, ok := value.([]interface{}); ok {
			index := node.value.(int)
			if index < 0 {
				index += len(sliceType)
			}
			if index < len(sliceType) && index >= 0 {
				return sliceType[index], nil
			}
			return nil, nil
		}
		// Otherwise try via reflection.
		rv := reflect.ValueOf(value)
		if rv.Kind() == reflect.Slice {
			index := node.value.(int)
			if index < 0 {
				index += rv.Len()
			}
			if index < rv.Len() && index >= 0 {
				v := rv.Index(index)
				return v.Interface(), nil
			}
		}
		return nil, nil
	case ASTKeyValPair:
		return intr.Execute(node.children[0], value)
	case ASTLiteral:
		return node.value, nil
	case ASTMultiSelectHash:
		if value == nil {
			return nil, nil
		}
		collected := make(map[string]interface{})
		for _, child := range node.children {
			current, err := intr.Execute(child, value)
			if err != nil {
				return nil, err
			}
			key := child.value.(string)
			collected[key] = current
		}
		return collected, nil
	case ASTMultiSelectList:
		if value == nil {
			return nil, nil
		}
		collected := make([]interface{}, 0, 0)
		for _, child := range node.children {
			current, err := intr.Execute(child, value)
			if err != nil {
				return nil, err
			}
			collected = append(collected, current)
		}
		return collected, nil
	case ASTOrExpression:
		matched, err := intr.Execute(node.children[0], value)
		if err != nil {
			return nil, err
		}
		if isFalse(matched) {
			matched, err = intr.Execute(node.children[1], value)
			if err != nil {
				return nil, err
			}
		}
		return matched, nil
	case ASTAndExpression:
		matched, err := intr.Execute(node.children[0], value)
		if err != nil {
			return nil, err
		}
		if isFalse(matched) {
			return matched, nil
		}
		return intr.Execute(node.children[1], value)
	case ASTNotExpression:
		matched, err := intr.Execute(node.children[0], value)
		if err != nil {
			return nil, err
		}
		if isFalse(matched) {
			return true, nil
		}
		return false, nil
	case ASTPipe:
		result := value
		var err error
		for _, child := range node.children {
			result, err = intr.Execute(child, result)
			if err != nil {
				return nil, err
			}
		}
		return result, nil
	case ASTProjection:
		left, err := intr.Execute(node.children[0], value)
		if err != nil {
			return nil, err
		}
		if sliceType, ok := left.([]interface{}); ok {
			collected := make([]interface{}, 0, 0)
			var current interface{}
			var err error
			for _, element := range sliceType {
				current, err = intr.Execute(node.children[1], element)
				if err != nil {
					return nil, err
				}
				if current != nil {
					collected = append(collected, current)
				}
			}
			return collected, nil
		}
		return nil, nil
	case ASTSubexpression, ASTIndexExpression:
		left, err := intr.Execute(node.children[0], value)
		if err != nil {
			return nil, err
		}
		return intr.Execute(node.children[1], left)
	case ASTSlice:
		sliceType, ok := value.([]interface{})
		if !ok {
			return nil, nil
		}
		parts := node.value.([]*int)
		sliceParams := make([]sliceParam, 3)
		for i, part := range parts {
			if part != nil {
				sliceParams[i].Specified = true
				sliceParams[i].N = *part
			}
		}
		return slice(sliceType, sliceParams)
	case ASTValueProjection:
		left, err := intr.Execute(node.children[0], value)
		if err != nil {
			return nil, nil
		}
		mapType, ok := left.(map[string]interface{})
		if !ok {
			return nil, nil
		}
		values := make([]interface{}, len(mapType))
		for _, value := range mapType {
			values = append(values, value)
		}
		collected := make([]interface{}, 0, 0)
		for _, element := range values {
			current, err := intr.Execute(node.children[1], element)
			if err != nil {
				return nil, err
			}
			if current != nil {
				collected = append(collected, current)
			}
		}
		return collected, nil
	}
	return nil, errors.New("Unknown AST node: " + node.nodeType.String())
}

func (intr *treeInterpreter) fieldFromStruct(key string, value interface{}) (interface{}, error) {
	rv := reflect.ValueOf(value)
	first, n := utf8.DecodeRuneInString(key)
	fieldName := string(unicode.ToUpper(first)) + key[n:]
	if rv.Kind() == reflect.Struct {
		v := rv.FieldByName(fieldName)
		if !v.IsValid() {
			return nil, nil
		}
		return v.Interface(), nil
	} else if rv.Kind() == reflect.Ptr {
		// Handle multiple levels of indirection?
		rv = rv.Elem()
		v := rv.FieldByName(fieldName)
		if !v.IsValid() {
			return nil, nil
		}
		return v.Interface(), nil
	}
	return nil, nil
}
<commit_msg>Keep reflection code path separate<commit_after>package jmespath

import (
	"errors"
	"reflect"
	"unicode"
	"unicode/utf8"
)

/* This is a tree based interpreter.  It walks the AST and directly
   interprets the AST to search through a JSON document.
*/

type treeInterpreter struct {
	fCall *functionCaller
}

func newInterpreter() *treeInterpreter {
	interpreter := treeInterpreter{}
	interpreter.fCall = newFunctionCaller()
	return &interpreter
}

type expRef struct {
	ref ASTNode
}

// Execute takes an ASTNode and input data and interprets the AST directly.
// It will produce the result of applying the JMESPath expression associated
// with the ASTNode to the input data "value".
func (intr *treeInterpreter) Execute(node ASTNode, value interface{}) (interface{}, error) {
	switch node.nodeType {
	case ASTComparator:
		left, err := intr.Execute(node.children[0], value)
		if err != nil {
			return nil, err
		}
		right, err := intr.Execute(node.children[1], value)
		if err != nil {
			return nil, err
		}
		switch node.value {
		case tEQ:
			return objsEqual(left, right), nil
		case tNE:
			return !objsEqual(left, right), nil
		}
		leftNum, ok := left.(float64)
		if !ok {
			return nil, nil
		}
		rightNum, ok := right.(float64)
		if !ok {
			return nil, nil
		}
		switch node.value {
		case tGT:
			return leftNum > rightNum, nil
		case tGTE:
			return leftNum >= rightNum, nil
		case tLT:
			return leftNum < rightNum, nil
		case tLTE:
			return leftNum <= rightNum, nil
		}
	case ASTExpRef:
		return expRef{ref: node.children[0]}, nil
	case ASTFunctionExpression:
		resolvedArgs := make([]interface{}, 0, 0)
		for _, arg := range node.children {
			current, err := intr.Execute(arg, value)
			if err != nil {
				return nil, err
			}
			resolvedArgs = append(resolvedArgs, current)
		}
		return intr.fCall.CallFunction(node.value.(string), resolvedArgs, intr)
	case ASTField:
		if m, ok := value.(map[string]interface{}); ok {
			key := node.value.(string)
			return m[key], nil
		}
		return intr.fieldFromStruct(node.value.(string), value)
	case ASTFilterProjection:
		left, err := intr.Execute(node.children[0], value)
		if err != nil {
			return nil, nil
		}
		sliceType, ok := left.([]interface{})
		if !ok {
			return nil, nil
		}
		compareNode := node.children[2]
		collected := make([]interface{}, 0, 0)
		for _, element := range sliceType {
			result, err := intr.Execute(compareNode, element)
			if err != nil {
				return nil, err
			}
			if !isFalse(result) {
				current, err := intr.Execute(node.children[1], element)
				if err != nil {
					return nil, err
				}
				if current != nil {
					collected = append(collected, current)
				}
			}
		}
		return collected, nil
	case ASTFlatten:
		left, err := intr.Execute(node.children[0], value)
		if err != nil {
			return nil, nil
		}
		sliceType, ok := left.([]interface{})
		if !ok {
			// If we can't type convert to []interface{}, there's
			// a chance this could still work via reflection if we're
			// dealing with user provided types.
			if isSliceType(left) {
				return intr.flattenWithReflection(left)
			}
			return nil, nil
		}
		flattened := make([]interface{}, 0, 0)
		for _, element := range sliceType {
			if elementSlice, ok := element.([]interface{}); ok {
				flattened = append(flattened, elementSlice...)
			} else {
				flattened = append(flattened, element)
			}
		}
		return flattened, nil
	case ASTIdentity, ASTCurrentNode:
		return value, nil
	case ASTIndex:
		if sliceType, ok := value.([]interface{}); ok {
			index := node.value.(int)
			if index < 0 {
				index += len(sliceType)
			}
			if index < len(sliceType) && index >= 0 {
				return sliceType[index], nil
			}
			return nil, nil
		}
		// Otherwise try via reflection.
		rv := reflect.ValueOf(value)
		if rv.Kind() == reflect.Slice {
			index := node.value.(int)
			if index < 0 {
				index += rv.Len()
			}
			if index < rv.Len() && index >= 0 {
				v := rv.Index(index)
				return v.Interface(), nil
			}
		}
		return nil, nil
	case ASTKeyValPair:
		return intr.Execute(node.children[0], value)
	case ASTLiteral:
		return node.value, nil
	case ASTMultiSelectHash:
		if value == nil {
			return nil, nil
		}
		collected := make(map[string]interface{})
		for _, child := range node.children {
			current, err := intr.Execute(child, value)
			if err != nil {
				return nil, err
			}
			key := child.value.(string)
			collected[key] = current
		}
		return collected, nil
	case ASTMultiSelectList:
		if value == nil {
			return nil, nil
		}
		collected := make([]interface{}, 0, 0)
		for _, child := range node.children {
			current, err := intr.Execute(child, value)
			if err != nil {
				return nil, err
			}
			collected = append(collected, current)
		}
		return collected, nil
	case ASTOrExpression:
		matched, err := intr.Execute(node.children[0], value)
		if err != nil {
			return nil, err
		}
		if isFalse(matched) {
			matched, err = intr.Execute(node.children[1], value)
			if err != nil {
				return nil, err
			}
		}
		return matched, nil
	case ASTAndExpression:
		matched, err := intr.Execute(node.children[0], value)
		if err != nil {
			return nil, err
		}
		if isFalse(matched) {
			return matched, nil
		}
		return intr.Execute(node.children[1], value)
	case ASTNotExpression:
		matched, err := intr.Execute(node.children[0], value)
		if err != nil {
			return nil, err
		}
		if isFalse(matched) {
			return true, nil
		}
		return false, nil
	case ASTPipe:
		result := value
		var err error
		for _, child := range node.children {
			result, err = intr.Execute(child, result)
			if err != nil {
				return nil, err
			}
		}
		return result, nil
	case ASTProjection:
		left, err := intr.Execute(node.children[0], value)
		if err != nil {
			return nil, err
		}
		if sliceType, ok := left.([]interface{}); ok {
			collected := make([]interface{}, 0, 0)
			var current interface{}
			var err error
			for _, element := range sliceType {
				current, err = intr.Execute(node.children[1], element)
				if err != nil {
					return nil, err
				}
				if current != nil {
					collected = append(collected, current)
				}
			}
			return collected, nil
		}
		return nil, nil
	case ASTSubexpression, ASTIndexExpression:
		left, err := intr.Execute(node.children[0], value)
		if err != nil {
			return nil, err
		}
		return intr.Execute(node.children[1], left)
	case ASTSlice:
		sliceType, ok := value.([]interface{})
		if !ok {
			return nil, nil
		}
		parts := node.value.([]*int)
		sliceParams := make([]sliceParam, 3)
		for i, part := range parts {
			if part != nil {
				sliceParams[i].Specified = true
				sliceParams[i].N = *part
			}
		}
		return slice(sliceType, sliceParams)
	case ASTValueProjection:
		left, err := intr.Execute(node.children[0], value)
		if err != nil {
			return nil, nil
		}
		mapType, ok := left.(map[string]interface{})
		if !ok {
			return nil, nil
		}
		values := make([]interface{}, len(mapType))
		for _, value := range mapType {
			values = append(values, value)
		}
		collected := make([]interface{}, 0, 0)
		for _, element := range values {
			current, err := intr.Execute(node.children[1], element)
			if err != nil {
				return nil, err
			}
			if current != nil {
				collected = append(collected, current)
			}
		}
		return collected, nil
	}
	return nil, errors.New("Unknown AST node: " + node.nodeType.String())
}

func (intr *treeInterpreter) fieldFromStruct(key string, value interface{}) (interface{}, error) {
	rv := reflect.ValueOf(value)
	first, n := utf8.DecodeRuneInString(key)
	fieldName := string(unicode.ToUpper(first)) + key[n:]
	if rv.Kind() == reflect.Struct {
		v := rv.FieldByName(fieldName)
		if !v.IsValid() {
			return nil, nil
		}
		return v.Interface(), nil
	} else if rv.Kind() == reflect.Ptr {
		// Handle multiple levels of indirection?
		rv = rv.Elem()
		v := rv.FieldByName(fieldName)
		if !v.IsValid() {
			return nil, nil
		}
		return v.Interface(), nil
	}
	return nil, nil
}

func (intr *treeInterpreter) flattenWithReflection(value interface{}) (interface{}, error) {
	v := reflect.ValueOf(value)
	flattened := make([]interface{}, 0, 0)
	for i := 0; i < v.Len(); i++ {
		element := v.Index(i).Interface()
		if reflect.TypeOf(element).Kind() == reflect.Slice {
			// Then insert the contents of the element
			// slice into the flattened slice,
			// i.e flattened = append(flattened, mySlice...)
			elementV := reflect.ValueOf(element)
			for j := 0; j < elementV.Len(); j++ {
				flattened = append(
					flattened, elementV.Index(j).Interface())
			}
		} else {
			flattened = append(flattened, element)
		}
	}
	return flattened, nil
}
<|endoftext|>


<commit_before><commit_msg>added short random sleep between queries in batch<commit_after><|endoftext|>


<commit_before>package medtronic

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"reflect"
	"testing"
	"time"
)

func TestEncodeBasalRate(t *testing.T) {
	cases := []struct {
		family Family
		rate   Insulin
		actual Insulin
	}{
		{22, 1000, 1000},
		{22, 2550, 2550},
		{23, 575, 575},
		{23, 2575, 2550},
		{23, 11250, 11200},
	}
	log.SetOutput(ioutil.Discard)
	for _, c := range cases {
		name := fmt.Sprintf("%d_%d", c.family, c.rate)
		t.Run(name, func(t *testing.T) {
			r, err := encodeBasalRate("basal", c.rate, c.family)
			if err != nil {
				t.Errorf("encodeBasalRate(%d, %d) raised error (%v)", c.rate, c.family, err)
			}
			a := Insulin(r) * milliUnitsPerStroke(23)
			if a != c.actual {
				t.Errorf("encodeBasalRate(%v, %d) == %d, want %d", c.rate, c.family, a, c.actual)
			}
		})
	}
}

func TestBasalRates(t *testing.T) {
	cases := []struct {
		family Family
		data   []byte
		sched  BasalRateSchedule
	}{
		{
			22,
			parseBytes("28 00 00 30 00 12"),
			BasalRateSchedule{
				{parseTD("00:00"), 1000},
				{parseTD("09:00"), 1200},
			},
		},
		{
			23,
			parseBytes("20 00 00 26 00 0D 2C 00 13 26 00 1C"),
			BasalRateSchedule{
				{parseTD("00:00"), 800},
				{parseTD("06:30"), 950},
				{parseTD("09:30"), 1100},
				{parseTD("14:00"), 950},
			},
		},
		{
			22,
			parseBytes("28 00 00 28 00 06 2C 00 0C 30 00 14 30 00 2C"),
			BasalRateSchedule{
				{parseTD("00:00"), 1000},
				{parseTD("03:00"), 1000},
				{parseTD("06:00"), 1100},
				{parseTD("10:00"), 1200},
				{parseTD("22:00"), 1200},
			},
		},
		{
			22,
			parseBytes("00 00 00 04 00 02 08 00 04 0C 00 06 10 00 08 14 00 0A 18 00 0C 1C 00 0E 20 00 10 24 00 12 28 00 14 2C 00 16 30 00 18 34 00 1A 38 00 1C 3C 00 1E 40 00 20 44 00 22 48 00 24 4C 00 26 50 00 28 54 00 2A 58 00 2C 5C 00 2E"),
			BasalRateSchedule{
				{parseTD("00:00"), 0},
				{parseTD("01:00"), 100},
				{parseTD("02:00"), 200},
				{parseTD("03:00"), 300},
				{parseTD("04:00"), 400},
				{parseTD("05:00"), 500},
				{parseTD("06:00"), 600},
				{parseTD("07:00"), 700},
				{parseTD("08:00"), 800},
				{parseTD("09:00"), 900},
				{parseTD("10:00"), 1000},
				{parseTD("11:00"), 1100},
				{parseTD("12:00"), 1200},
				{parseTD("13:00"), 1300},
				{parseTD("14:00"), 1400},
				{parseTD("15:00"), 1500},
				{parseTD("16:00"), 1600},
				{parseTD("17:00"), 1700},
				{parseTD("18:00"), 1800},
				{parseTD("19:00"), 1900},
				{parseTD("20:00"), 2000},
				{parseTD("21:00"), 2100},
				{parseTD("22:00"), 2200},
				{parseTD("23:00"), 2300},
			},
		},
		{
			22,
			parseBytes("28 00 00 40 01 08 28 00 2C"),
			BasalRateSchedule{
				{parseTD("00:00"), 1000},
				{parseTD("04:00"), 8000},
				{parseTD("22:00"), 1000},
			},
		},
	}
	for _, c := range cases {
		name := fmt.Sprintf("%d_%d", c.family, len(c.sched))
		t.Run("decode_"+name, func(t *testing.T) {
			s := decodeBasalRateSchedule(c.data)
			if !reflect.DeepEqual(s, c.sched) {
				t.Errorf("decodeBasalRateSchedule(% X) == %+v, want %+v", c.data, s, c.sched)
			}
		})
		t.Run("encode_"+name, func(t *testing.T) {
			data, err := encodeBasalRateSchedule(c.sched, c.family)
			if err != nil {
				t.Errorf("encodeBasalRateSchedule(%+v) raised error (%v)", c.sched, err)
			}
			if !bytes.Equal(data, c.data) {
				t.Errorf("encodeBasalRateSchedule(%+v) == % X, want % X", c.sched, data, c.data)
			}
		})
	}
}

func TestBasalRateAt(t *testing.T) {
	cases := []struct {
		sched  BasalRateSchedule
		at     time.Time
		target BasalRate
	}{
		{
			BasalRateSchedule{
				{parseTD("00:00"), 1000},
			},
			parseTime("2016-11-06T23:00:00"),
			BasalRate{parseTD("00:00"), 1000},
		},
	}
	for _, c := range cases {
		t.Run("", func(t *testing.T) {
			target := c.sched.BasalRateAt(c.at)
			if !reflect.DeepEqual(target, c.target) {
				t.Errorf("%v.BasalRateAt(%v) == %+v, want %+v", c.sched, c.at, target, c.target)
			}
		})
	}
}
<commit_msg>Add 24-entry test data from @andyrozman<commit_after>package medtronic

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"reflect"
	"testing"
	"time"
)

func TestEncodeBasalRate(t *testing.T) {
	cases := []struct {
		family Family
		rate   Insulin
		actual Insulin
	}{
		{22, 1000, 1000},
		{22, 2550, 2550},
		{23, 575, 575},
		{23, 2575, 2550},
		{23, 11250, 11200},
	}
	log.SetOutput(ioutil.Discard)
	for _, c := range cases {
		name := fmt.Sprintf("%d_%d", c.family, c.rate)
		t.Run(name, func(t *testing.T) {
			r, err := encodeBasalRate("basal", c.rate, c.family)
			if err != nil {
				t.Errorf("encodeBasalRate(%d, %d) raised error (%v)", c.rate, c.family, err)
			}
			a := Insulin(r) * milliUnitsPerStroke(23)
			if a != c.actual {
				t.Errorf("encodeBasalRate(%v, %d) == %d, want %d", c.rate, c.family, a, c.actual)
			}
		})
	}
}

func TestBasalRates(t *testing.T) {
	cases := []struct {
		family Family
		data   []byte
		sched  BasalRateSchedule
	}{
		{
			22,
			parseBytes("28 00 00 30 00 12"),
			BasalRateSchedule{
				{parseTD("00:00"), 1000},
				{parseTD("09:00"), 1200},
			},
		},
		{
			22,
			parseBytes("28 00 00 40 01 08 28 00 2C"),
			BasalRateSchedule{
				{parseTD("00:00"), 1000},
				{parseTD("04:00"), 8000},
				{parseTD("22:00"), 1000},
			},
		},
		{
			23,
			parseBytes("20 00 00 26 00 0D 2C 00 13 26 00 1C"),
			BasalRateSchedule{
				{parseTD("00:00"), 800},
				{parseTD("06:30"), 950},
				{parseTD("09:30"), 1100},
				{parseTD("14:00"), 950},
			},
		},
		{
			22,
			parseBytes("28 00 00 28 00 06 2C 00 0C 30 00 14 30 00 2C"),
			BasalRateSchedule{
				{parseTD("00:00"), 1000},
				{parseTD("03:00"), 1000},
				{parseTD("06:00"), 1100},
				{parseTD("10:00"), 1200},
				{parseTD("22:00"), 1200},
			},
		},
		{
			22,
			parseBytes("00 00 00 04 00 02 08 00 04 0C 00 06 10 00 08 14 00 0A 18 00 0C 1C 00 0E 20 00 10 24 00 12 28 00 14 2C 00 16 30 00 18 34 00 1A 38 00 1C 3C 00 1E 40 00 20 44 00 22 48 00 24 4C 00 26 50 00 28 54 00 2A 58 00 2C 5C 00 2E"),
			BasalRateSchedule{
				{parseTD("00:00"), 0},
				{parseTD("01:00"), 100},
				{parseTD("02:00"), 200},
				{parseTD("03:00"), 300},
				{parseTD("04:00"), 400},
				{parseTD("05:00"), 500},
				{parseTD("06:00"), 600},
				{parseTD("07:00"), 700},
				{parseTD("08:00"), 800},
				{parseTD("09:00"), 900},
				{parseTD("10:00"), 1000},
				{parseTD("11:00"), 1100},
				{parseTD("12:00"), 1200},
				{parseTD("13:00"), 1300},
				{parseTD("14:00"), 1400},
				{parseTD("15:00"), 1500},
				{parseTD("16:00"), 1600},
				{parseTD("17:00"), 1700},
				{parseTD("18:00"), 1800},
				{parseTD("19:00"), 1900},
				{parseTD("20:00"), 2000},
				{parseTD("21:00"), 2100},
				{parseTD("22:00"), 2200},
				{parseTD("23:00"), 2300},
			},
		},
		{
			23,
			parseBytes("42 00 00 40 00 02 3A 00 04 3E 00 06 36 00 08 46 00 0A 4A 00 0C 4C 00 0E 4E 00 10 4C 00 12 4E 00 14 50 00 16 50 00 18 4E 00 1A 4A 00 1C 4A 00 1E 4A 00 20 4C 00 22 4A 00 24 4A 00 26 48 00 28 46 00 2A 4A 00 2C 4A 00 2E"),
			BasalRateSchedule{
				{parseTD("00:00"), 1650},
				{parseTD("01:00"), 1600},
				{parseTD("02:00"), 1450},
				{parseTD("03:00"), 1550},
				{parseTD("04:00"), 1350},
				{parseTD("05:00"), 1750},
				{parseTD("06:00"), 1850},
				{parseTD("07:00"), 1900},
				{parseTD("08:00"), 1950},
				{parseTD("09:00"), 1900},
				{parseTD("10:00"), 1950},
				{parseTD("11:00"), 2000},
				{parseTD("12:00"), 2000},
				{parseTD("13:00"), 1950},
				{parseTD("14:00"), 1850},
				{parseTD("15:00"), 1850},
				{parseTD("16:00"), 1850},
				{parseTD("17:00"), 1900},
				{parseTD("18:00"), 1850},
				{parseTD("19:00"), 1850},
				{parseTD("20:00"), 1800},
				{parseTD("21:00"), 1750},
				{parseTD("22:00"), 1850},
				{parseTD("23:00"), 1850},
			},
		},
	}
	for _, c := range cases {
		name := fmt.Sprintf("%d_%d", c.family, len(c.sched))
		t.Run("decode_"+name, func(t *testing.T) {
			s := decodeBasalRateSchedule(c.data)
			if !reflect.DeepEqual(s, c.sched) {
				t.Errorf("decodeBasalRateSchedule(% X) == %+v, want %+v", c.data, s, c.sched)
			}
		})
		t.Run("encode_"+name, func(t *testing.T) {
			data, err := encodeBasalRateSchedule(c.sched, c.family)
			if err != nil {
				t.Errorf("encodeBasalRateSchedule(%+v) raised error (%v)", c.sched, err)
			}
			if !bytes.Equal(data, c.data) {
				t.Errorf("encodeBasalRateSchedule(%+v) == % X, want % X", c.sched, data, c.data)
			}
		})
	}
}

func TestBasalRateAt(t *testing.T) {
	cases := []struct {
		sched  BasalRateSchedule
		at     time.Time
		target BasalRate
	}{
		{
			BasalRateSchedule{
				{parseTD("00:00"), 1000},
			},
			parseTime("2016-11-06T23:00:00"),
			BasalRate{parseTD("00:00"), 1000},
		},
	}
	for _, c := range cases {
		t.Run("", func(t *testing.T) {
			target := c.sched.BasalRateAt(c.at)
			if !reflect.DeepEqual(target, c.target) {
				t.Errorf("%v.BasalRateAt(%v) == %+v, want %+v", c.sched, c.at, target, c.target)
			}
		})
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package networking

import (
	"fmt"
	"log"
	"net"
	"os"
	"path/filepath"
	"syscall"

	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/appc/spec/schema/types"
	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/vishvananda/netlink"

	"github.com/coreos/rocket/networking/util"
)

const (
	ifnamePattern = "eth%d"
	selfNetNS     = "/proc/self/ns/net"
)

type activeNet struct {
	Net
	ifName string
	ip     net.IP
}

// "base" struct that's populated from the beginning
// describing the environment in which the container
// is running in
type containerEnv struct {
	rktRoot string
	contID  types.UUID
}

// Networking describes the networking details of a container.
type Networking struct {
	containerEnv

	MetadataIP net.IP

	contID     types.UUID
	hostNS     *os.File
	contNS     *os.File
	contNSPath string
	nets       []activeNet
}

// Setup produces a Networking object for a given container ID.
func Setup(rktRoot string, contID types.UUID) (*Networking, error) {
	var err error
	n := Networking{
		containerEnv: containerEnv{
			rktRoot: rktRoot,
			contID:  contID,
		},
	}

	defer func() {
		// cleanup on error
		if err != nil {
			n.Teardown()
		}
	}()

	if n.hostNS, n.contNS, err = basicNetNS(); err != nil {
		return nil, err
	}
	// we're in contNS!

	contNSPath := filepath.Join("/var/lib/rkt/containers", contID.String(), "ns")
	if err = bindMountFile(selfNetNS, contNSPath, "net"); err != nil {
		return nil, err
	}
	n.contNSPath = filepath.Join(contNSPath, "net")

	if err != nil {
		return nil, fmt.Errorf("error loading plugin definitions: %v", err)
	}

	nets, err := n.loadNets()
	if err != nil {
		return nil, fmt.Errorf("error loading network definitions: %v", err)
	}

	err = withNetNS(n.contNS, n.hostNS, func() error {
		n.nets, err = n.setupNets(n.contNSPath, nets)
		return err
	})
	if err != nil {
		return nil, err
	}

	if len(n.nets) == 0 {
		return nil, fmt.Errorf("no nets successfully setup")
	}

	// last net is the default
	n.MetadataIP = n.nets[len(n.nets)-1].ip

	return &n, nil
}

// Teardown cleans up a produced Networking object.
func (n *Networking) Teardown() {
	// Teardown everything in reverse order of setup.
	// This is called during error cases as well, so
	// not everything may be setup.
	// N.B. better to keep going in case of errors
	// to get as much cleaned up as possible.

	if n.contNS == nil || n.hostNS == nil {
		return
	}

	if err := n.EnterHostNS(); err != nil {
		log.Print(err)
		return
	}

	n.teardownNets(n.contNSPath, n.nets)

	if n.contNSPath == "" {
		return
	}

	if err := syscall.Unmount(n.contNSPath, 0); err != nil {
		log.Printf("Error unmounting %q: %v", n.contNSPath, err)
	}
}

// sets up new netns with just lo
func basicNetNS() (hostNS, contNS *os.File, err error) {
	hostNS, contNS, err = newNetNS()
	if err != nil {
		err = fmt.Errorf("failed to create new netns: %v", err)
		return
	}
	// we're in contNS!!

	if err = loUp(); err != nil {
		hostNS.Close()
		contNS.Close()
		return nil, nil, err
	}

	return
}

// EnterHostNS moves into the host's network namespace.
func (n *Networking) EnterHostNS() error {
	return util.SetNS(n.hostNS, syscall.CLONE_NEWNET)
}

// EnterContNS moves into the container's network namespace.
func (n *Networking) EnterContNS() error {
	return util.SetNS(n.contNS, syscall.CLONE_NEWNET)
}

func (e *containerEnv) setupNets(netns string, nets []Net) ([]activeNet, error) {
	var err error

	active := []activeNet{}

	for i, nt := range nets {
		an := activeNet{
			Net:    nt,
			ifName: fmt.Sprintf(ifnamePattern, i),
		}

		log.Printf("Setup: executing net-plugin %v", nt.Type)

		an.ip, err = e.netPluginAdd(&nt, netns, nt.args, an.ifName)
		if err != nil {
			err = fmt.Errorf("error adding network %q: %v", nt.Name, err)
			break
		}

		active = append(active, an)
	}

	if err != nil {
		e.teardownNets(netns, active)
		return nil, err
	}

	return active, nil
}

func (e *containerEnv) teardownNets(netns string, nets []activeNet) {
	for i := len(nets) - 1; i >= 0; i-- {
		nt := nets[i]

		log.Printf("Teardown: executing net-plugin %v", nt.Type)
		err := e.netPluginDel(&nt.Net, netns, nt.args, nt.ifName)
		if err != nil {
			log.Printf("Error deleting %q: %v", nt.Name, err)
		}
	}
}

func newNetNS() (hostNS, childNS *os.File, err error) {
	defer func() {
		if err != nil {
			if hostNS != nil {
				hostNS.Close()
			}
			if childNS != nil {
				childNS.Close()
			}
		}
	}()

	hostNS, err = os.Open(selfNetNS)
	if err != nil {
		return
	}

	if err = syscall.Unshare(syscall.CLONE_NEWNET); err != nil {
		return
	}

	childNS, err = os.Open(selfNetNS)
	if err != nil {
		util.SetNS(hostNS, syscall.CLONE_NEWNET)
		return
	}

	return
}

// execute f() in tgtNS
func withNetNS(curNS, tgtNS *os.File, f func() error) error {
	if err := util.SetNS(tgtNS, syscall.CLONE_NEWNET); err != nil {
		return err
	}

	if err := f(); err != nil {
		return err
	}

	return util.SetNS(curNS, syscall.CLONE_NEWNET)
}

func loUp() error {
	lo, err := netlink.LinkByName("lo")
	if err != nil {
		return fmt.Errorf("failed to lookup lo: %v", err)
	}

	if err := netlink.LinkSetUp(lo); err != nil {
		return fmt.Errorf("failed to set lo up: %v", err)
	}

	return nil
}

func bindMountFile(src, dstDir, dstFile string) error {
	if err := os.MkdirAll(dstDir, 0755); err != nil {
		return err
	}

	dst := filepath.Join(dstDir, dstFile)

	// mount point has to be an existing file
	f, err := os.Create(dst)
	if err != nil {
		return err
	}
	f.Close()

	return syscall.Mount(src, dst, "none", syscall.MS_BIND, "")
}
<commit_msg>net: copy netconf files into container dir<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package networking

import (
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"path/filepath"
	"syscall"

	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/appc/spec/schema/types"
	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/vishvananda/netlink"

	"github.com/coreos/rocket/networking/util"
)

const (
	ifnamePattern = "eth%d"
	selfNetNS     = "/proc/self/ns/net"
)

type activeNet struct {
	Net
	ifName string
	ip     net.IP
}

// "base" struct that's populated from the beginning
// describing the environment in which the container
// is running in
type containerEnv struct {
	rktRoot string
	contID  types.UUID
}

// Networking describes the networking details of a container.
type Networking struct {
	containerEnv

	MetadataIP net.IP

	contID     types.UUID
	hostNS     *os.File
	contNS     *os.File
	contNSPath string
	nets       []activeNet
}

// Setup produces a Networking object for a given container ID.
func Setup(rktRoot string, contID types.UUID) (*Networking, error) {
	var err error
	n := Networking{
		containerEnv: containerEnv{
			rktRoot: rktRoot,
			contID:  contID,
		},
	}

	defer func() {
		// cleanup on error
		if err != nil {
			n.Teardown()
		}
	}()

	if n.hostNS, n.contNS, err = basicNetNS(); err != nil {
		return nil, err
	}
	// we're in contNS!

	contNSPath := filepath.Join("/var/lib/rkt/containers", contID.String(), "ns")
	if err = bindMountFile(selfNetNS, contNSPath, "net"); err != nil {
		return nil, err
	}
	n.contNSPath = filepath.Join(contNSPath, "net")

	if err != nil {
		return nil, fmt.Errorf("error loading plugin definitions: %v", err)
	}

	nets, err := n.loadNets()
	if err != nil {
		return nil, fmt.Errorf("error loading network definitions: %v", err)
	}

	err = withNetNS(n.contNS, n.hostNS, func() error {
		n.nets, err = n.setupNets(n.contNSPath, nets)
		return err
	})
	if err != nil {
		return nil, err
	}

	if len(n.nets) == 0 {
		return nil, fmt.Errorf("no nets successfully setup")
	}

	// last net is the default
	n.MetadataIP = n.nets[len(n.nets)-1].ip

	return &n, nil
}

// Teardown cleans up a produced Networking object.
func (n *Networking) Teardown() {
	// Teardown everything in reverse order of setup.
	// This is called during error cases as well, so
	// not everything may be setup.
	// N.B. better to keep going in case of errors
	// to get as much cleaned up as possible.

	if n.contNS == nil || n.hostNS == nil {
		return
	}

	if err := n.EnterHostNS(); err != nil {
		log.Print(err)
		return
	}

	n.teardownNets(n.contNSPath, n.nets)

	if n.contNSPath == "" {
		return
	}

	if err := syscall.Unmount(n.contNSPath, 0); err != nil {
		log.Printf("Error unmounting %q: %v", n.contNSPath, err)
	}
}

// sets up new netns with just lo
func basicNetNS() (hostNS, contNS *os.File, err error) {
	hostNS, contNS, err = newNetNS()
	if err != nil {
		err = fmt.Errorf("failed to create new netns: %v", err)
		return
	}
	// we're in contNS!!

	if err = loUp(); err != nil {
		hostNS.Close()
		contNS.Close()
		return nil, nil, err
	}

	return
}

// EnterHostNS moves into the host's network namespace.
func (n *Networking) EnterHostNS() error {
	return util.SetNS(n.hostNS, syscall.CLONE_NEWNET)
}

// EnterContNS moves into the container's network namespace.
func (n *Networking) EnterContNS() error {
	return util.SetNS(n.contNS, syscall.CLONE_NEWNET)
}

func (e *containerEnv) netDir() string {
	return filepath.Join(e.rktRoot, "net")
}

func (e *containerEnv) setupNets(netns string, nets []Net) ([]activeNet, error) {
	err := os.MkdirAll(e.netDir(), 0755)
	if err != nil {
		return nil, err
	}

	active := []activeNet{}

	for i, nt := range nets {
		log.Printf("Setup: executing net-plugin %v", nt.Type)

		an := activeNet{
			Net:    nt,
			ifName: fmt.Sprintf(ifnamePattern, i),
		}

		if an.Filename, err = copyFileToDir(nt.Filename, e.netDir()); err != nil {
			err = fmt.Errorf("error copying %q to %q: %v", nt.Filename, e.netDir(), err)
			break
		}

		an.ip, err = e.netPluginAdd(&nt, netns, nt.args, an.ifName)
		if err != nil {
			err = fmt.Errorf("error adding network %q: %v", nt.Name, err)
			break
		}

		active = append(active, an)
	}

	if err != nil {
		e.teardownNets(netns, active)
		return nil, err
	}

	return active, nil
}

func (e *containerEnv) teardownNets(netns string, nets []activeNet) {
	for i := len(nets) - 1; i >= 0; i-- {
		nt := nets[i]

		log.Printf("Teardown: executing net-plugin %v", nt.Type)

		err := e.netPluginDel(&nt.Net, netns, nt.args, nt.ifName)
		if err != nil {
			log.Printf("Error deleting %q: %v", nt.Name, err)
		}

		// Delete the conf file to signal that the network was
		// torn down (or at least attempted to)
		if err = os.Remove(nt.Filename); err != nil {
			log.Printf("Error deleting %q: %v", nt.Filename, err)
		}
	}
}

func newNetNS() (hostNS, childNS *os.File, err error) {
	defer func() {
		if err != nil {
			if hostNS != nil {
				hostNS.Close()
			}
			if childNS != nil {
				childNS.Close()
			}
		}
	}()

	hostNS, err = os.Open(selfNetNS)
	if err != nil {
		return
	}

	if err = syscall.Unshare(syscall.CLONE_NEWNET); err != nil {
		return
	}

	childNS, err = os.Open(selfNetNS)
	if err != nil {
		util.SetNS(hostNS, syscall.CLONE_NEWNET)
		return
	}

	return
}

// execute f() in tgtNS
func withNetNS(curNS, tgtNS *os.File, f func() error) error {
	if err := util.SetNS(tgtNS, syscall.CLONE_NEWNET); err != nil {
		return err
	}

	if err := f(); err != nil {
		return err
	}

	return util.SetNS(curNS, syscall.CLONE_NEWNET)
}

func loUp() error {
	lo, err := netlink.LinkByName("lo")
	if err != nil {
		return fmt.Errorf("failed to lookup lo: %v", err)
	}

	if err := netlink.LinkSetUp(lo); err != nil {
		return fmt.Errorf("failed to set lo up: %v", err)
	}

	return nil
}

func bindMountFile(src, dstDir, dstFile string) error {
	if err := os.MkdirAll(dstDir, 0755); err != nil {
		return err
	}

	dst := filepath.Join(dstDir, dstFile)

	// mount point has to be an existing file
	f, err := os.Create(dst)
	if err != nil {
		return err
	}
	f.Close()

	return syscall.Mount(src, dst, "none", syscall.MS_BIND, "")
}

func copyFileToDir(src, dstdir string) (string, error) {
	dst := filepath.Join(dstdir, filepath.Base(src))

	s, err := os.Open(src)
	if err != nil {
		return "", err
	}
	defer s.Close()

	d, err := os.Create(dst)
	if err != nil {
		return "", err
	}
	defer d.Close()

	_, err = io.Copy(d, s)
	return dst, err
}
<|endoftext|>


<commit_before>/*
------------------------------------------------------------------------------
isaac.go: an implementation of Bob Jenkins' random number generator ISAAC based on 'readable.c'
* 18 Aug 2014 -- direct port of readable.c to Go
------------------------------------------------------------------------------
*/

package isaac

/* external results */
var randrsl [256]uint32
var randcnt uint32

/* internal state */
var mm [256]uint32
var aa, bb, cc uint32 // zero by default

func Isaac() {
	var x, y uint32

	cc = cc + 1  /* cc just gets incremented once per 256 results */
	bb = bb + cc /* then combined with bb */

	for i := 0; i < 256; i++ {
		x = mm[i]

		switch i % 4 {
		case 0:
			aa = aa ^ (aa << 13)
			break
		case 1:
			aa = aa ^ (aa >> 6)
			break
		case 2:
			aa = aa ^ (aa << 2)
			break
		case 3:
			aa = aa ^ (aa >> 16)
			break
		}

		aa = mm[(i+128)%256] + aa
		y = mm[(x>>2)%256] + aa + bb
		bb = mm[(y>>10)%256] + x

		mm[i] = y
		randrsl[i] = bb
	}
}

/* if (flag!=0), then use the contents of randrsl[] to initialize mm[]. */
func mix(a, b, c, d, e, f, g, h uint32) (uint32, uint32, uint32, uint32, uint32, uint32, uint32, uint32) {
	a ^= b << 11
	d += a
	b += c
	b ^= c >> 2
	e += b
	c += d
	c ^= d << 8
	f += c
	d += e
	d ^= e >> 16
	g += d
	e += f
	e ^= f << 10
	h += e
	f += g
	f ^= g >> 4
	a += f
	g += h
	g ^= h << 8
	b += g
	h += a
	h ^= a >> 9
	c += h
	a += b
	return a, b, c, d, e, f, g, h
}

func Randinit(flag bool) {
	var a, b, c, d, e, f, g, h uint32
	aa, bb, cc = 0, 0, 0

	a, b, c, d, e, f, g, h = 0x9e3779b9, 0x9e3779b9, 0x9e3779b9, 0x9e3779b9, 0x9e3779b9, 0x9e3779b9, 0x9e3779b9, 0x9e3779b9

	for i := 0; i < 4; i++ {
		a, b, c, d, e, f, g, h = mix(a, b, c, d, e, f, g, h)
	}

	for i := 0; i < 256; i += 8 { /* fill mm[] with messy stuff */
		if flag { /* use all the information in the seed */
			a += randrsl[i]
			b += randrsl[i+1]
			c += randrsl[i+2]
			d += randrsl[i+3]
			e += randrsl[i+4]
			f += randrsl[i+5]
			g += randrsl[i+6]
			h += randrsl[i+7]
		}
		a, b, c, d, e, f, g, h = mix(a, b, c, d, e, f, g, h)
		mm[i] = a
		mm[i+1] = b
		mm[i+2] = c
		mm[i+3] = d
		mm[i+4] = e
		mm[i+5] = f
		mm[i+6] = g
		mm[i+7] = h
	}

	if flag { /* do a second pass to make all of the seed affect all of mm */
		for i := 0; i < 256; i += 8 {
			a += mm[i]
			b += mm[i+1]
			c += mm[i+2]
			d += mm[i+3]
			e += mm[i+4]
			f += mm[i+5]
			g += mm[i+6]
			h += mm[i+7]
			a, b, c, d, e, f, g, h = mix(a, b, c, d, e, f, g, h)
			mm[i] = a
			mm[i+1] = b
			mm[i+2] = c
			mm[i+3] = d
			mm[i+4] = e
			mm[i+5] = f
			mm[i+6] = g
			mm[i+7] = h
		}
	}

	Isaac()
	randcnt = 256
}

func Randcnt() uint32 {
	return randcnt
}

func Randrsl() [256]uint32 {
	return randrsl
}
<commit_msg>update isaac to be a little more idiomatic<commit_after>/*
------------------------------------------------------------------------------
isaac.go: an implementation of Bob Jenkins' random number generator ISAAC based on 'readable.c'
* 18 Aug 2014 -- direct port of readable.c to Go
* 10 Sep 2014 -- updated to be more idiomatic Go
------------------------------------------------------------------------------
*/

package isaac

type isaac struct {
	/* external results */
	randrsl [256]uint32
	randcnt uint32

	/* internal state */
	mm         [256]uint32
	aa, bb, cc uint32
}

func (r *isaac) isaac() {
	var x, y uint32

	r.cc++       /* cc just gets incremented once per 256 results */
	r.bb += r.cc /* then combined with bb                         */

	for i := 0; i < 256; i++ {
		x = r.mm[i]

		switch i % 4 {
		case 0:
			r.aa ^= (r.aa << 13)
		case 1:
			r.aa ^= (r.aa >> 6)
		case 2:
			r.aa ^= (r.aa << 2)
		case 3:
			r.aa ^= (r.aa >> 16)
		}

		r.aa = r.mm[(i+128)%256] + r.aa
		y = r.mm[(x>>2)%256] + r.aa + r.bb
		r.bb = r.mm[(y>>10)%256] + x

		r.mm[i] = y
		r.randrsl[i] = r.bb

		/* Note that bits 2..9 are chosen from x but 10..17 are chosen
		   from y.  The only important thing here is that 2..9 and 10..17
		   don't overlap.  2..9 and 10..17 were then chosen for speed in
		   the optimized version (rand.c) */
		/* See http://burtleburtle.net/bob/rand/isaac.html
		   for further explanations and analysis. */

	}
}

func mix(a, b, c, d, e, f, g, h uint32) (uint32, uint32, uint32, uint32, uint32, uint32, uint32, uint32) {
	a ^= b << 11
	d += a
	b += c
	b ^= c >> 2
	e += b
	c += d
	c ^= d << 8
	f += c
	d += e
	d ^= e >> 16
	g += d
	e += f
	e ^= f << 10
	h += e
	f += g
	f ^= g >> 4
	a += f
	g += h
	g ^= h << 8
	b += g
	h += a
	h ^= a >> 9
	c += h
	a += b
	return a, b, c, d, e, f, g, h
}

/* if (flag==true), then use the contents of randrsl[] to initialize mm[]. */
func (r *isaac) randInit(flag bool) {
	var a, b, c, d, e, f, g, h uint32
	a, b, c, d, e, f, g, h = 0x9e3779b9, 0x9e3779b9, 0x9e3779b9, 0x9e3779b9, 0x9e3779b9, 0x9e3779b9, 0x9e3779b9, 0x9e3779b9

	for i := 0; i < 4; i++ {
		a, b, c, d, e, f, g, h = mix(a, b, c, d, e, f, g, h)
	}

	for i := 0; i < 256; i += 8 { /* fill mm[] with messy stuff */
		if flag { /* use all the information in the seed */
			a += r.randrsl[i]
			b += r.randrsl[i+1]
			c += r.randrsl[i+2]
			d += r.randrsl[i+3]
			e += r.randrsl[i+4]
			f += r.randrsl[i+5]
			g += r.randrsl[i+6]
			h += r.randrsl[i+7]
		}
		a, b, c, d, e, f, g, h = mix(a, b, c, d, e, f, g, h)
		r.mm[i] = a
		r.mm[i+1] = b
		r.mm[i+2] = c
		r.mm[i+3] = d
		r.mm[i+4] = e
		r.mm[i+5] = f
		r.mm[i+6] = g
		r.mm[i+7] = h
	}

	if flag { /* do a second pass to make all of the seed affect all of mm */
		for i := 0; i < 256; i += 8 {
			a += r.mm[i]
			b += r.mm[i+1]
			c += r.mm[i+2]
			d += r.mm[i+3]
			e += r.mm[i+4]
			f += r.mm[i+5]
			g += r.mm[i+6]
			h += r.mm[i+7]
			a, b, c, d, e, f, g, h = mix(a, b, c, d, e, f, g, h)
			r.mm[i] = a
			r.mm[i+1] = b
			r.mm[i+2] = c
			r.mm[i+3] = d
			r.mm[i+4] = e
			r.mm[i+5] = f
			r.mm[i+6] = g
			r.mm[i+7] = h
		}
	}

	r.isaac()     /* fill in the first set of results */
	r.randcnt = 0 /* reset the counter */
}

/* there is no official method for doing this, but just writing the key to the
 * state array is how the demo code does it */
func (r *isaac) Seed(key string) {
	for idx, c := range key {
		if idx == len(r.randrsl) {
			break
		}
		r.randrsl[idx] = uint32(c)
		r.randInit(true)
	}
}

/* retrieve the next number in the sequence */
func (r *isaac) Rand() uint32 {
	rnd := r.randrsl[r.randcnt]
	r.randcnt++
	if r.randcnt == uint32(len(r.randrsl)) {
		// reset for another 256
		r.isaac()
		r.randcnt = 0
	}
	return rnd
}
<|endoftext|>