<commit_before>// Copyright (c) 2020 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build fvtests

package fv_test

import (
	"context"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/projectcalico/felix/fv/infrastructure"
	"github.com/projectcalico/libcalico-go/lib/apiconfig"
	api "github.com/projectcalico/libcalico-go/lib/apis/v3"
	client "github.com/projectcalico/libcalico-go/lib/clientv3"
	"github.com/projectcalico/libcalico-go/lib/errors"
	"github.com/projectcalico/libcalico-go/lib/options"
)

var _ = infrastructure.DatastoreDescribe("service loop prevention; with 2 nodes", []apiconfig.DatastoreType{apiconfig.EtcdV3, apiconfig.Kubernetes}, func(getInfra infrastructure.InfraFactory) {

	var (
		infra   infrastructure.DatastoreInfra
		felixes []*infrastructure.Felix
		client  client.Interface
	)

	BeforeEach(func() {
		infra = getInfra()

		options := infrastructure.DefaultTopologyOptions()
		options.IPIPEnabled = false
		felixes, client = infrastructure.StartNNodeTopology(2, options, infra)
	})

	AfterEach(func() {
		if CurrentGinkgoTestDescription().Failed {
			for _, felix := range felixes {
				felix.Exec("iptables-save", "-c")
				felix.Exec("ipset", "list")
				felix.Exec("ip", "r")
				felix.Exec("ip", "a")
			}
		}
		for _, felix := range felixes {
			felix.Stop()
		}
		if CurrentGinkgoTestDescription().Failed {
			infra.DumpErrorData()
		}
		infra.Stop()
	})

	updateFelixConfig := func(deltaFn func(*api.FelixConfiguration)) {
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()
		cfg, err := client.FelixConfigurations().Get(ctx, "default", options.GetOptions{})
		if _, doesNotExist := err.(errors.ErrorResourceDoesNotExist); doesNotExist {
			cfg = api.NewFelixConfiguration()
			cfg.Name = "default"
			deltaFn(cfg)
			_, err = client.FelixConfigurations().Create(ctx, cfg, options.SetOptions{})
			Expect(err).NotTo(HaveOccurred())
		} else {
			Expect(err).NotTo(HaveOccurred())
			deltaFn(cfg)
			_, err = client.FelixConfigurations().Update(ctx, cfg, options.SetOptions{})
			Expect(err).NotTo(HaveOccurred())
		}
	}

	updateBGPConfig := func(deltaFn func(*api.BGPConfiguration)) {
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()
		cfg, err := client.BGPConfigurations().Get(ctx, "default", options.GetOptions{})
		if _, doesNotExist := err.(errors.ErrorResourceDoesNotExist); doesNotExist {
			cfg = api.NewBGPConfiguration()
			cfg.Name = "default"
			deltaFn(cfg)
			_, err = client.BGPConfigurations().Create(ctx, cfg, options.SetOptions{})
			Expect(err).NotTo(HaveOccurred())
		} else {
			Expect(err).NotTo(HaveOccurred())
			deltaFn(cfg)
			_, err = client.BGPConfigurations().Update(ctx, cfg, options.SetOptions{})
			Expect(err).NotTo(HaveOccurred())
		}
	}

	getCIDRBlockRules := func(felix *infrastructure.Felix, saveCommand string) func() []string {
		return func() []string {
			out, err := felix.ExecOutput(saveCommand, "-t", "filter")
			Expect(err).NotTo(HaveOccurred())
			var cidrBlockLines []string
			for _, line := range strings.Split(out, "\n") {
				if strings.Contains(line, "-A cali-cidr-block") {
					cidrBlockLines = append(cidrBlockLines, line)
				}
			}
			return cidrBlockLines
		}
	}

	It("programs iptables as expected to block service routing loops", func() {

		By("configuring service cluster IPs")
		updateBGPConfig(func(cfg *api.BGPConfiguration) {
			cfg.Spec.ServiceClusterIPs = []api.ServiceClusterIPBlock{
				{
					CIDR: "1.2.0.0/16",
				},
				{
					CIDR: "fd5f::/119",
				},
			}
		})

		// Default ServiceLoopPrevention is Drop, so expect to see rules in cali-cidr-block
		// chains with DROP.  (Felix handles BGPConfiguration without restarting, so this
		// should be quick.)
		for _, felix := range felixes {
			Eventually(getCIDRBlockRules(felix, "iptables-save")).Should(ConsistOf(
				MatchRegexp("-A cali-cidr-block -d 1\\.2\\.0\\.0/16 .* -j DROP"),
			))
			Eventually(getCIDRBlockRules(felix, "ip6tables-save")).Should(ConsistOf(
				MatchRegexp("-A cali-cidr-block -d fd5f::/119 .* -j DROP"),
			))
		}

		By("configuring ServiceLoopPrevention=Reject")
		updateFelixConfig(func(cfg *api.FelixConfiguration) {
			cfg.Spec.ServiceLoopPrevention = "Reject"
		})

		// Expect to see rules in cali-cidr-block chains with REJECT.  (Allowing time for a
		// Felix restart.)
		for _, felix := range felixes {
			Eventually(getCIDRBlockRules(felix, "iptables-save"), "4s", "0.5s").Should(ConsistOf(
				MatchRegexp("-A cali-cidr-block -d 1\\.2\\.0\\.0/16 .* -j REJECT"),
			))
			Eventually(getCIDRBlockRules(felix, "ip6tables-save"), "4s", "0.5s").Should(ConsistOf(
				MatchRegexp("-A cali-cidr-block -d fd5f::/119 .* -j REJECT"),
			))
		}

		By("configuring ServiceLoopPrevention=Disabled")
		updateFelixConfig(func(cfg *api.FelixConfiguration) {
			cfg.Spec.ServiceLoopPrevention = "Disabled"
		})

		// Expect to see empty cali-cidr-block chains.  (Allowing time for a Felix restart.)
		for _, felix := range felixes {
			Eventually(getCIDRBlockRules(felix, "iptables-save"), "4s", "0.5s").Should(BeEmpty())
			Eventually(getCIDRBlockRules(felix, "ip6tables-save"), "4s", "0.5s").Should(BeEmpty())
		}

		By("configuring ServiceLoopPrevention=Drop")
		updateFelixConfig(func(cfg *api.FelixConfiguration) {
			cfg.Spec.ServiceLoopPrevention = "Drop"
		})

		// Expect to see rules in cali-cidr-block chains with DROP.  (Allowing time for a
		// Felix restart.)
		for _, felix := range felixes {
			Eventually(getCIDRBlockRules(felix, "iptables-save"), "4s", "0.5s").Should(ConsistOf(
				MatchRegexp("-A cali-cidr-block -d 1\\.2\\.0\\.0/16 .* -j DROP"),
			))
			Eventually(getCIDRBlockRules(felix, "ip6tables-save"), "4s", "0.5s").Should(ConsistOf(
				MatchRegexp("-A cali-cidr-block -d fd5f::/119 .* -j DROP"),
			))
		}

		By("updating the service CIDRs")
		updateBGPConfig(func(cfg *api.BGPConfiguration) {
			cfg.Spec.ServiceClusterIPs = []api.ServiceClusterIPBlock{
				{
					CIDR: "1.1.0.0/16",
				},
				{
					CIDR: "fd5e::/119",
				},
			}
		})

		// Expect to see rules in cali-cidr-block chains with DROP and the updated CIDRs.
		// (BGPConfiguration change is handled without needing a restart.)
		for _, felix := range felixes {
			Eventually(getCIDRBlockRules(felix, "iptables-save")).Should(ConsistOf(
				MatchRegexp("-A cali-cidr-block -d 1\\.1\\.0\\.0/16 .* -j DROP"),
			))
			Eventually(getCIDRBlockRules(felix, "ip6tables-save")).Should(ConsistOf(
				MatchRegexp("-A cali-cidr-block -d fd5e::/119 .* -j DROP"),
			))
		}
	})
})
<commit_msg>Use tcpdump to check that there is no routing loop<commit_after>// Copyright (c) 2020 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build fvtests

package fv_test

import (
	"context"
	"regexp"
	"strings"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/projectcalico/felix/fv/containers"
	"github.com/projectcalico/felix/fv/infrastructure"
	"github.com/projectcalico/felix/fv/utils"
	"github.com/projectcalico/libcalico-go/lib/apiconfig"
	api "github.com/projectcalico/libcalico-go/lib/apis/v3"
	client "github.com/projectcalico/libcalico-go/lib/clientv3"
	"github.com/projectcalico/libcalico-go/lib/errors"
	"github.com/projectcalico/libcalico-go/lib/options"
)

var _ = infrastructure.DatastoreDescribe("service loop prevention; with 2 nodes", []apiconfig.DatastoreType{apiconfig.EtcdV3, apiconfig.Kubernetes}, func(getInfra infrastructure.InfraFactory) {

	var (
		infra   infrastructure.DatastoreInfra
		felixes []*infrastructure.Felix
		client  client.Interface
	)

	BeforeEach(func() {
		infra = getInfra()

		options := infrastructure.DefaultTopologyOptions()
		options.IPIPEnabled = false
		felixes, client = infrastructure.StartNNodeTopology(2, options, infra)
	})

	AfterEach(func() {
		if CurrentGinkgoTestDescription().Failed {
			for _, felix := range felixes {
				felix.Exec("iptables-save", "-c")
				felix.Exec("ipset", "list")
				felix.Exec("ip", "r")
				felix.Exec("ip", "a")
			}
		}
		for _, felix := range felixes {
			felix.Stop()
		}
		if CurrentGinkgoTestDescription().Failed {
			infra.DumpErrorData()
		}
		infra.Stop()
	})

	updateFelixConfig := func(deltaFn func(*api.FelixConfiguration)) {
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()
		cfg, err := client.FelixConfigurations().Get(ctx, "default", options.GetOptions{})
		if _, doesNotExist := err.(errors.ErrorResourceDoesNotExist); doesNotExist {
			cfg = api.NewFelixConfiguration()
			cfg.Name = "default"
			deltaFn(cfg)
			_, err = client.FelixConfigurations().Create(ctx, cfg, options.SetOptions{})
			Expect(err).NotTo(HaveOccurred())
		} else {
			Expect(err).NotTo(HaveOccurred())
			deltaFn(cfg)
			_, err = client.FelixConfigurations().Update(ctx, cfg, options.SetOptions{})
			Expect(err).NotTo(HaveOccurred())
		}
	}

	updateBGPConfig := func(deltaFn func(*api.BGPConfiguration)) {
		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		defer cancel()
		cfg, err := client.BGPConfigurations().Get(ctx, "default", options.GetOptions{})
		if _, doesNotExist := err.(errors.ErrorResourceDoesNotExist); doesNotExist {
			cfg = api.NewBGPConfiguration()
			cfg.Name = "default"
			deltaFn(cfg)
			_, err = client.BGPConfigurations().Create(ctx, cfg, options.SetOptions{})
			Expect(err).NotTo(HaveOccurred())
		} else {
			Expect(err).NotTo(HaveOccurred())
			deltaFn(cfg)
			_, err = client.BGPConfigurations().Update(ctx, cfg, options.SetOptions{})
			Expect(err).NotTo(HaveOccurred())
		}
	}

	getCIDRBlockRules := func(felix *infrastructure.Felix, saveCommand string) func() []string {
		return func() []string {
			out, err := felix.ExecOutput(saveCommand, "-t", "filter")
			Expect(err).NotTo(HaveOccurred())
			var cidrBlockLines []string
			for _, line := range strings.Split(out, "\n") {
				if strings.Contains(line, "-A cali-cidr-block") {
					cidrBlockLines = append(cidrBlockLines, line)
				}
			}
			return cidrBlockLines
		}
	}

	tryRoutingLoop := func(expectLoop bool) {

		// Run containers to model a default gateway, and an external client connecting to
		// services within the cluster via that gateway.
		externalGW := containers.Run("external-gw",
			containers.RunOpts{AutoRemove: true},
			"--privileged", // So that we can add routes inside the container.
			utils.Config.BusyboxImage,
			"/bin/sh", "-c", "sleep 1000")
		defer externalGW.Stop()

		externalClient := containers.Run("external-client",
			containers.RunOpts{AutoRemove: true},
			"--privileged", // So that we can add routes inside the container.
			utils.Config.BusyboxImage,
			"/bin/sh", "-c", "sleep 1000")
		defer externalClient.Stop()

		// Add a service CIDR route in those containers, similar to the routes that they
		// would have via BGP per our service advertisement feature.  (This should really be
		// an ECMP route to both Felixes, but busybox's ip can't program ECMP routes, and a
		// non-ECMP route is sufficient to demonstrate the looping issue.)
		externalClient.Exec("ip", "r", "a", "10.96.0.0/17", "via", externalGW.IP)
		externalGW.Exec("ip", "r", "a", "10.96.0.0/17", "via", felixes[0].IP)

		// Configure the external gateway client to forward, in order to create the
		// conditions for looping.
		externalClient.Exec("sysctl", "-w", "net.ipv4.ip_forward=1")
		externalGW.Exec("sysctl", "-w", "net.ipv4.ip_forward=1")

		// Also tell Felix to route that CIDR to the external gateway.
		felixes[0].ExecMayFail("ip", "r", "d", "10.96.0.0/17")
		felixes[0].Exec("ip", "r", "a", "10.96.0.0/17", "via", externalGW.IP)
		felixes[0].Exec("iptables", "-P", "FORWARD", "ACCEPT")

		tryPing := func() int {
			// Start monitoring all packets, on the Felix, to or from a specific (but
			// unused) service IP.
			tcpdumpF := felixes[0].AttachTCPDump("eth0")
			tcpdumpF.AddMatcher("serviceIPPackets", regexp.MustCompile("10\\.96\\.0\\.19"))
			tcpdumpF.Start()
			defer tcpdumpF.Stop()

			// Send a single ping from the external client to the unused service IP.
			err := externalClient.ExecMayFail("ping", "-c", "1", "-W", "1", "10.96.0.19")
			Expect(err).To(HaveOccurred())

			// Return the number of packets observed to/from 10.96.0.19.
			return tcpdumpF.MatchCount("serviceIPPackets")
		}

		if expectLoop {
			// Tcpdump should see more than 2 packets, because of looping.  Note: 2
			// packets would be Felix receiving the ping and then forwarding it out
			// again.  I want to check here that it's also looped around again by the
			// gateway, resulting in MORE THAN 2 packets.
			Expect(tryPing()).To(BeNumerically(">", 2))
		} else {
			// Tcpdump should see just 1 packet, the request, with no response (because
			// we DROP) and no looping.
			Expect(tryPing()).To(BeNumerically("==", 1))
		}
	}

	It("programs iptables as expected to block service routing loops", func() {

		By("configuring service cluster IPs")
		updateBGPConfig(func(cfg *api.BGPConfiguration) {
			cfg.Spec.ServiceClusterIPs = []api.ServiceClusterIPBlock{
				{
					CIDR: "10.96.0.0/17",
				},
				{
					CIDR: "fd5f::/119",
				},
			}
		})

		// Default ServiceLoopPrevention is Drop, so expect to see rules in cali-cidr-block
		// chains with DROP.  (Felix handles BGPConfiguration without restarting, so this
		// should be quick.)
		for _, felix := range felixes {
			Eventually(getCIDRBlockRules(felix, "iptables-save")).Should(ConsistOf(
				MatchRegexp("-A cali-cidr-block -d 10\\.96\\.0\\.0/17 .* -j DROP"),
			))
			Eventually(getCIDRBlockRules(felix, "ip6tables-save")).Should(ConsistOf(
				MatchRegexp("-A cali-cidr-block -d fd5f::/119 .* -j DROP"),
			))
		}

		By("test that we don't get a routing loop")
		tryRoutingLoop(false)

		By("configuring ServiceLoopPrevention=Reject")
		updateFelixConfig(func(cfg *api.FelixConfiguration) {
			cfg.Spec.ServiceLoopPrevention = "Reject"
		})

		// Expect to see rules in cali-cidr-block chains with REJECT.  (Allowing time for a
		// Felix restart.)
		for _, felix := range felixes {
			Eventually(getCIDRBlockRules(felix, "iptables-save"), "8s", "0.5s").Should(ConsistOf(
				MatchRegexp("-A cali-cidr-block -d 10\\.96\\.0\\.0/17 .* -j REJECT"),
			))
			Eventually(getCIDRBlockRules(felix, "ip6tables-save"), "8s", "0.5s").Should(ConsistOf(
				MatchRegexp("-A cali-cidr-block -d fd5f::/119 .* -j REJECT"),
			))
		}

		By("configuring ServiceLoopPrevention=Disabled")
		updateFelixConfig(func(cfg *api.FelixConfiguration) {
			cfg.Spec.ServiceLoopPrevention = "Disabled"
		})

		// Expect to see empty cali-cidr-block chains.  (Allowing time for a Felix restart.)
		for _, felix := range felixes {
			Eventually(getCIDRBlockRules(felix, "iptables-save"), "8s", "0.5s").Should(BeEmpty())
			Eventually(getCIDRBlockRules(felix, "ip6tables-save"), "8s", "0.5s").Should(BeEmpty())
		}

		By("test that we DO get a routing loop")
		// (In order to test that the tryRoutingLoop setup is genuine.)
		tryRoutingLoop(true)

		By("configuring ServiceLoopPrevention=Drop")
		updateFelixConfig(func(cfg *api.FelixConfiguration) {
			cfg.Spec.ServiceLoopPrevention = "Drop"
		})

		// Expect to see rules in cali-cidr-block chains with DROP.  (Allowing time for a
		// Felix restart.)
		for _, felix := range felixes {
			Eventually(getCIDRBlockRules(felix, "iptables-save"), "8s", "0.5s").Should(ConsistOf(
				MatchRegexp("-A cali-cidr-block -d 10\\.96\\.0\\.0/17 .* -j DROP"),
			))
			Eventually(getCIDRBlockRules(felix, "ip6tables-save"), "8s", "0.5s").Should(ConsistOf(
				MatchRegexp("-A cali-cidr-block -d fd5f::/119 .* -j DROP"),
			))
		}

		By("updating the service CIDRs")
		updateBGPConfig(func(cfg *api.BGPConfiguration) {
			cfg.Spec.ServiceClusterIPs = []api.ServiceClusterIPBlock{
				{
					CIDR: "1.1.0.0/16",
				},
				{
					CIDR: "fd5e::/119",
				},
			}
		})

		// Expect to see rules in cali-cidr-block chains with DROP and the updated CIDRs.
		// (BGPConfiguration change is handled without needing a restart.)
		for _, felix := range felixes {
			Eventually(getCIDRBlockRules(felix, "iptables-save")).Should(ConsistOf(
				MatchRegexp("-A cali-cidr-block -d 1\\.1\\.0\\.0/16 .* -j DROP"),
			))
			Eventually(getCIDRBlockRules(felix, "ip6tables-save")).Should(ConsistOf(
				MatchRegexp("-A cali-cidr-block -d fd5e::/119 .* -j DROP"),
			))
		}
	})
})
<|endoftext|>


<commit_before>package survey

import (
	"errors"
	"fmt"
	"reflect"
)

// Required does not allow an empty value
func Required(val interface{}) error {
	// if the value passed in is the zero value of the appropriate type
	if isZero(reflect.ValueOf(val)) {
		return errors.New("Value is required")
	}
	return nil
}

// MaxLength requires that the string is no longer than the specified value
func MaxLength(length int) Validator {
	// return a validator that checks the length of the string
	return func(val interface{}) error {
		if str, ok := val.(string); ok {
			// if the string is longer than the given value
			if len(str) > length {
				// yell loudly
				return fmt.Errorf("value is too long. Max length is %v", length)
			}
		} else {
			// otherwise we cannot convert the value into a string and cannot enforce length
			return fmt.Errorf("cannot enforce length on response of type %v", reflect.TypeOf(val).Name())
		}

		// the input is fine
		return nil
	}
}

// MinLength requires that the string is longer or equal in length to the specified value
func MinLength(length int) Validator {
	// return a validator that checks the length of the string
	return func(val interface{}) error {
		if str, ok := val.(string); ok {
			// if the string is shorter than the given value
			if len(str) < length {
				// yell loudly
				return fmt.Errorf("value is too short. Min length is %v", length)
			}
		} else {
			// otherwise we cannot convert the value into a string and cannot enforce length
			return fmt.Errorf("cannot enforce length on response of type %v", reflect.TypeOf(val).Name())
		}

		// the input is fine
		return nil
	}
}

// ComposeValidators is a variadic function used to create one validator from many.
func ComposeValidators(validators ...Validator) Validator {
	// return a validator that calls each one sequentially
	return func(val interface{}) error {
		// execute each validator
		for _, validator := range validators {
			// if the string is not valid
			if err := validator(val); err != nil {
				// return the error
				return err
			}
		}
		// we passed all validators, the string is valid
		return nil
	}
}

// isZero returns true if the passed value is the zero object
func isZero(v reflect.Value) bool {
	switch v.Kind() {
	case reflect.Slice:
		return v.Len() == 0
	case reflect.Func, reflect.Map:
		return v.IsNil()
	case reflect.Array:
		z := true
		for i := 0; i < v.Len(); i++ {
			z = z && isZero(v.Index(i))
		}
		return z
	case reflect.Struct:
		z := true
		for i := 0; i < v.NumField(); i++ {
			if v.Field(i).CanSet() {
				z = z && isZero(v.Field(i))
			}
		}
		return z
	case reflect.Ptr:
		return isZero(reflect.Indirect(v))
	}
	// Compare other types directly:
	z := reflect.Zero(v.Type())
	result := v.Interface() == z.Interface()

	return result
}
<commit_msg>simplified required logic<commit_after>package survey

import (
	"errors"
	"fmt"
	"reflect"
)

// Required does not allow an empty value
func Required(val interface{}) error {
	// if the value passed in is the zero value of the appropriate type
	if isZero(reflect.ValueOf(val)) {
		return errors.New("Value is required")
	}
	return nil
}

// MaxLength requires that the string is no longer than the specified value
func MaxLength(length int) Validator {
	// return a validator that checks the length of the string
	return func(val interface{}) error {
		if str, ok := val.(string); ok {
			// if the string is longer than the given value
			if len(str) > length {
				// yell loudly
				return fmt.Errorf("value is too long. Max length is %v", length)
			}
		} else {
			// otherwise we cannot convert the value into a string and cannot enforce length
			return fmt.Errorf("cannot enforce length on response of type %v", reflect.TypeOf(val).Name())
		}

		// the input is fine
		return nil
	}
}

// MinLength requires that the string is longer or equal in length to the specified value
func MinLength(length int) Validator {
	// return a validator that checks the length of the string
	return func(val interface{}) error {
		if str, ok := val.(string); ok {
			// if the string is shorter than the given value
			if len(str) < length {
				// yell loudly
				return fmt.Errorf("value is too short. Min length is %v", length)
			}
		} else {
			// otherwise we cannot convert the value into a string and cannot enforce length
			return fmt.Errorf("cannot enforce length on response of type %v", reflect.TypeOf(val).Name())
		}

		// the input is fine
		return nil
	}
}

// ComposeValidators is a variadic function used to create one validator from many.
func ComposeValidators(validators ...Validator) Validator {
	// return a validator that calls each one sequentially
	return func(val interface{}) error {
		// execute each validator
		for _, validator := range validators {
			// if the string is not valid
			if err := validator(val); err != nil {
				// return the error
				return err
			}
		}
		// we passed all validators, the string is valid
		return nil
	}
}

// isZero returns true if the passed value is the zero object
func isZero(v reflect.Value) bool {
	switch v.Kind() {
	case reflect.Slice:
		return v.Len() == 0
	}

	// compare the types directly with more general coverage
	return reflect.DeepEqual(v.Interface(), reflect.Zero(v.Type()).Interface())
}
<|endoftext|>


<commit_before>package cache

import (
	"encoding/binary"
	"net"
	"reflect"
	"testing"
	"time"

	"github.com/lomik/go-carbon/points"
	"github.com/stretchr/testify/assert"
)

const sampleCacheQuery = "\x00\x00\x00Y\x80\x02}q\x01(U\x06metricq\x02U,carbon.agents.carbon_agent_server.cache.sizeq\x03U\x04typeq\x04U\x0bcache-queryq\x05u."
const sampleCacheQuery2 = "\x00\x00\x00Y\x80\x02}q\x01(U\x04typeq\x04U\x0bcache-queryq\x05U\x06metricq\x02U,carbon.agents.carbon_agent_server.param.sizeq\x03u."
const sampleCacheQuery3 = "\x00\x00\x00R\x80\x02}(U\x06metricX,\x00\x00\x00carbon.agents.carbon_agent_server.param.sizeU\x04typeU\x0bcache-queryu." // unicode metric

func TestCarbonlink(t *testing.T) {
	assert := assert.New(t)

	cache := New()

	msg1 := points.OnePoint(
		"carbon.agents.carbon_agent_server.cache.size",
		42.17,
		1422797285,
	)

	msg2 := points.OnePoint(
		"carbon.agents.carbon_agent_server.param.size",
		-42.14,
		1422797267,
	)

	msg3 := points.OnePoint(
		"carbon.agents.carbon_agent_server.param.size",
		15,
		1422795966,
	)

	cache.Add(msg1)
	cache.Add(msg2)
	cache.Add(msg3)

	defer cache.Stop()

	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
	assert.NoError(err)

	carbonlink := NewCarbonlinkListener(cache)
	defer carbonlink.Stop()

	assert.NoError(carbonlink.Listen(addr))

	NewClient := func() (conn net.Conn, cleanup func()) {
		conn, err := net.Dial("tcp", carbonlink.Addr().String())
		assert.NoError(err)

		conn.SetDeadline(time.Now().Add(time.Second))
		return conn, func() { conn.Close() }
	}

	var replyLength uint32
	var data []byte

	/* MESSAGE 1 */
	conn, cleanup := NewClient()
	_, err = conn.Write([]byte(sampleCacheQuery))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)
	assert.NoError(err)

	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	// {'datapoints': [(1422797285, 42.17)]}
	assert.Equal([]byte("\x80\x02}U\ndatapoints]J\xe5)\xceTG@E\x15\xc2\x8f\\(\xf6\x86as."), data)
	cleanup()

	/* MESSAGE 2 */
	conn, cleanup = NewClient()
	_, err = conn.Write([]byte(sampleCacheQuery2))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)
	assert.NoError(err)

	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	// {'datapoints': [(1422797267, -42.14), (1422795966, 15.0)]}
	assert.Equal("\x80\x02}U\ndatapoints](J\xd3)\xceTG\xc0E\x11\xeb\x85\x1e\xb8R\x86J\xbe$\xceTG@.\x00\x00\x00\x00\x00\x00\x86es.",
		string(data))
	cleanup()

	/* MESSAGE 2.5 - unicode */
	conn, cleanup = NewClient()
	_, err = conn.Write([]byte(sampleCacheQuery3))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)
	assert.NoError(err)

	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	// {'datapoints': [(1422797267, -42.14), (1422795966, 15.0)]}
	assert.Equal("\x80\x02}U\ndatapoints](J\xd3)\xceTG\xc0E\x11\xeb\x85\x1e\xb8R\x86J\xbe$\xceTG@.\x00\x00\x00\x00\x00\x00\x86es.",
		string(data))
	cleanup()

	/* MESSAGE 3 */
	/* Remove carbon.agents.carbon_agent_server.param.size from cache and request again */
	conn, cleanup = NewClient()
	cache.Pop("carbon.agents.carbon_agent_server.param.size")

	_, err = conn.Write([]byte(sampleCacheQuery2))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)
	assert.NoError(err)

	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	assert.Equal("\x80\x02}U\ndatapoints]s.", string(data))
	cleanup()

	/* WRONG MESSAGE TEST */
	conn, cleanup = NewClient()
	_, err = conn.Write([]byte("\x00\x00\x00\x05aaaaa"))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)

	assert.Error(err)
	cleanup()

	/* Invalid querytype */
	conn, cleanup = NewClient()
	_, err = conn.Write([]byte("\x00\x00\x00(\x80\x02}q\x00(U\x06metricq\x01U\x03barq\x02U\x04typeq\x03U\x03fooq\x04u."))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)

	assert.NoError(err)
	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	assert.Equal("\x80\x02}q\x00U\x05errorq\x01U\x1aInvalid request type \"foo\"q\x02s.", string(data))
	cleanup()
}

func TestCarbonlinkErrors(t *testing.T) {
	assert := assert.New(t)

	cache := New()

	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
	assert.NoError(err)

	carbonlink := NewCarbonlinkListener(cache)
	listenerTimeout := 10 * time.Millisecond
	carbonlink.SetReadTimeout(listenerTimeout)
	defer carbonlink.Stop()

	err = carbonlink.Listen(addr)
	if err != nil {
		t.Errorf("Received %q when attempting to start CarbonlinkListener", err)
	}

	table := []*struct {
		waitAfterWrite time.Duration
		msg            []byte
	}{
		{ // connect and disconnect
			time.Millisecond * 0,
			[]byte{},
		},
		{ // connect, send msg length and disconnect
			time.Millisecond * 0,
			[]byte(sampleCacheQuery2[:4]),
		},
		{ // connect and wait timeout
			listenerTimeout * 2,
			[]byte{},
		},
		{ // connect, send msg length and wait timeout
			listenerTimeout * 2,
			[]byte(sampleCacheQuery2[:4]),
		},
		{ // send broken pickle
			listenerTimeout * 2,
			[]byte(sampleCacheQuery2[:len(sampleCacheQuery2)-1] + "a"),
		},
	}

	for _, test := range table {
		remoteaddr, _ := net.ResolveTCPAddr("tcp", carbonlink.Addr().String())
		conn, err := net.DialTCP("tcp", addr, remoteaddr)
		assert.NoError(err)

		if len(test.msg) > 0 {
			_, err = conn.Write(test.msg)
			if err != nil {
				t.Errorf("Received %q when attempting to write to CarbonlinkListener", err)
			}
		}
		time.Sleep(test.waitAfterWrite)
		if test.waitAfterWrite > listenerTimeout {
			conn.SetWriteBuffer(0)
			n, err := conn.Write([]byte("x"))
			if err == nil || n > 0 {
				t.Errorf("CarbonlinkListener did not close connection as expected")
			}
		}
		conn.Close()

	}
}

func TestNewCarbonlinkRequest(t *testing.T) {
	want := &CarbonlinkRequest{}
	got := NewCarbonlinkRequest()
	if !reflect.DeepEqual(got, want) {
		t.Errorf("NewCarbonlinkRequest() = %v, want %v", got, want)
	}
}

func TestParseCarbonlinkRequest(t *testing.T) {
	tests := []struct {
		name    string
		data    []byte
		want    *CarbonlinkRequest
		wantErr bool
	}{
		{name: "Good query",
			data: []byte(sampleCacheQuery)[4:],
			want: &CarbonlinkRequest{
				Type:   "cache-query",
				Metric: "carbon.agents.carbon_agent_server.cache.size"},
		},
		{name: "Good query2",
			data: []byte(sampleCacheQuery2)[4:],
			want: &CarbonlinkRequest{
				Type:   "cache-query",
				Metric: "carbon.agents.carbon_agent_server.param.size"},
		},
		{name: "Invalid query type",
			data: []byte("\x80\x02}q\x00(U\x06metricq\x01U\x03barq\x02U\x04typeq\x03U\x03fooq\x04u."),
			want: &CarbonlinkRequest{
				Type:   "foo",
				Metric: "bar",
			},
		},
		{name: "Garbage",
			data:    []byte("garbage"),
			wantErr: true,
		},
	}
	for _, tt := range tests {
		got, err := ParseCarbonlinkRequest(tt.data)
		if (err != nil) != tt.wantErr {
			t.Errorf("%q. ParseCarbonlinkRequest() error = %v, wantErr %v", tt.name, err, tt.wantErr)
			continue
		}
		if !reflect.DeepEqual(got, tt.want) {
			t.Errorf("%q. ParseCarbonlinkRequest() = %v, want %v", tt.name, got, tt.want)
		}
	}
}

func BenchmarkCarbonLinkPickleParse(b *testing.B) {
	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			ParseCarbonlinkRequest([]byte(sampleCacheQuery))
		}
	})
}

func BenchmarkCarbonLinkPackReply(b *testing.B) {
	p := points.OnePoint("carbon.agents.carbon_agent_server.param.size", 15, 1422795966).Add(15, 9000000).Add(16, 9000000)

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			packReply(p.Data)
		}
	})
}
<commit_msg>fix unpack carbonlink benchmark<commit_after>package cache

import (
	"encoding/binary"
	"net"
	"reflect"
	"testing"
	"time"

	"github.com/lomik/go-carbon/points"
	"github.com/stretchr/testify/assert"
)

const sampleCacheQuery = "\x00\x00\x00Y\x80\x02}q\x01(U\x06metricq\x02U,carbon.agents.carbon_agent_server.cache.sizeq\x03U\x04typeq\x04U\x0bcache-queryq\x05u."
const sampleCacheQuery2 = "\x00\x00\x00Y\x80\x02}q\x01(U\x04typeq\x04U\x0bcache-queryq\x05U\x06metricq\x02U,carbon.agents.carbon_agent_server.param.sizeq\x03u."
const sampleCacheQuery3 = "\x00\x00\x00R\x80\x02}(U\x06metricX,\x00\x00\x00carbon.agents.carbon_agent_server.param.sizeU\x04typeU\x0bcache-queryu." // unicode metric

func TestCarbonlink(t *testing.T) {
	assert := assert.New(t)

	cache := New()

	msg1 := points.OnePoint(
		"carbon.agents.carbon_agent_server.cache.size",
		42.17,
		1422797285,
	)

	msg2 := points.OnePoint(
		"carbon.agents.carbon_agent_server.param.size",
		-42.14,
		1422797267,
	)

	msg3 := points.OnePoint(
		"carbon.agents.carbon_agent_server.param.size",
		15,
		1422795966,
	)

	cache.Add(msg1)
	cache.Add(msg2)
	cache.Add(msg3)

	defer cache.Stop()

	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
	assert.NoError(err)

	carbonlink := NewCarbonlinkListener(cache)
	defer carbonlink.Stop()

	assert.NoError(carbonlink.Listen(addr))

	NewClient := func() (conn net.Conn, cleanup func()) {
		conn, err := net.Dial("tcp", carbonlink.Addr().String())
		assert.NoError(err)

		conn.SetDeadline(time.Now().Add(time.Second))
		return conn, func() { conn.Close() }
	}

	var replyLength uint32
	var data []byte

	/* MESSAGE 1 */
	conn, cleanup := NewClient()
	_, err = conn.Write([]byte(sampleCacheQuery))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)
	assert.NoError(err)

	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	// {'datapoints': [(1422797285, 42.17)]}
	assert.Equal([]byte("\x80\x02}U\ndatapoints]J\xe5)\xceTG@E\x15\xc2\x8f\\(\xf6\x86as."), data)
	cleanup()

	/* MESSAGE 2 */
	conn, cleanup = NewClient()
	_, err = conn.Write([]byte(sampleCacheQuery2))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)
	assert.NoError(err)

	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	// {'datapoints': [(1422797267, -42.14), (1422795966, 15.0)]}
	assert.Equal("\x80\x02}U\ndatapoints](J\xd3)\xceTG\xc0E\x11\xeb\x85\x1e\xb8R\x86J\xbe$\xceTG@.\x00\x00\x00\x00\x00\x00\x86es.",
		string(data))
	cleanup()

	/* MESSAGE 2.5 - unicode */
	conn, cleanup = NewClient()
	_, err = conn.Write([]byte(sampleCacheQuery3))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)
	assert.NoError(err)

	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	// {'datapoints': [(1422797267, -42.14), (1422795966, 15.0)]}
	assert.Equal("\x80\x02}U\ndatapoints](J\xd3)\xceTG\xc0E\x11\xeb\x85\x1e\xb8R\x86J\xbe$\xceTG@.\x00\x00\x00\x00\x00\x00\x86es.",
		string(data))
	cleanup()

	/* MESSAGE 3 */
	/* Remove carbon.agents.carbon_agent_server.param.size from cache and request again */
	conn, cleanup = NewClient()
	cache.Pop("carbon.agents.carbon_agent_server.param.size")

	_, err = conn.Write([]byte(sampleCacheQuery2))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)
	assert.NoError(err)

	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	assert.Equal("\x80\x02}U\ndatapoints]s.", string(data))
	cleanup()

	/* WRONG MESSAGE TEST */
	conn, cleanup = NewClient()
	_, err = conn.Write([]byte("\x00\x00\x00\x05aaaaa"))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)

	assert.Error(err)
	cleanup()

	/* Invalid querytype */
	conn, cleanup = NewClient()
	_, err = conn.Write([]byte("\x00\x00\x00(\x80\x02}q\x00(U\x06metricq\x01U\x03barq\x02U\x04typeq\x03U\x03fooq\x04u."))
	assert.NoError(err)

	err = binary.Read(conn, binary.BigEndian, &replyLength)

	assert.NoError(err)
	data = make([]byte, replyLength)

	err = binary.Read(conn, binary.BigEndian, data)
	assert.NoError(err)

	assert.Equal("\x80\x02}q\x00U\x05errorq\x01U\x1aInvalid request type \"foo\"q\x02s.", string(data))
	cleanup()
}

func TestCarbonlinkErrors(t *testing.T) {
	assert := assert.New(t)

	cache := New()

	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
	assert.NoError(err)

	carbonlink := NewCarbonlinkListener(cache)
	listenerTimeout := 10 * time.Millisecond
	carbonlink.SetReadTimeout(listenerTimeout)
	defer carbonlink.Stop()

	err = carbonlink.Listen(addr)
	if err != nil {
		t.Errorf("Received %q when attempting to start CarbonlinkListener", err)
	}

	table := []*struct {
		waitAfterWrite time.Duration
		msg            []byte
	}{
		{ // connect and disconnect
			time.Millisecond * 0,
			[]byte{},
		},
		{ // connect, send msg length and disconnect
			time.Millisecond * 0,
			[]byte(sampleCacheQuery2[:4]),
		},
		{ // connect and wait timeout
			listenerTimeout * 2,
			[]byte{},
		},
		{ // connect, send msg length and wait timeout
			listenerTimeout * 2,
			[]byte(sampleCacheQuery2[:4]),
		},
		{ // send broken pickle
			listenerTimeout * 2,
			[]byte(sampleCacheQuery2[:len(sampleCacheQuery2)-1] + "a"),
		},
	}

	for _, test := range table {
		remoteaddr, _ := net.ResolveTCPAddr("tcp", carbonlink.Addr().String())
		conn, err := net.DialTCP("tcp", addr, remoteaddr)
		assert.NoError(err)

		if len(test.msg) > 0 {
			_, err = conn.Write(test.msg)
			if err != nil {
				t.Errorf("Received %q when attempting to write to CarbonlinkListener", err)
			}
		}
		time.Sleep(test.waitAfterWrite)
		if test.waitAfterWrite > listenerTimeout {
			conn.SetWriteBuffer(0)
			n, err := conn.Write([]byte("x"))
			if err == nil || n > 0 {
				t.Errorf("CarbonlinkListener did not close connection as expected")
			}
		}
		conn.Close()

	}
}

func TestNewCarbonlinkRequest(t *testing.T) {
	want := &CarbonlinkRequest{}
	got := NewCarbonlinkRequest()
	if !reflect.DeepEqual(got, want) {
		t.Errorf("NewCarbonlinkRequest() = %v, want %v", got, want)
	}
}

func TestParseCarbonlinkRequest(t *testing.T) {
	tests := []struct {
		name    string
		data    []byte
		want    *CarbonlinkRequest
		wantErr bool
	}{
		{name: "Good query",
			data: []byte(sampleCacheQuery)[4:],
			want: &CarbonlinkRequest{
				Type:   "cache-query",
				Metric: "carbon.agents.carbon_agent_server.cache.size"},
		},
		{name: "Good query2",
			data: []byte(sampleCacheQuery2)[4:],
			want: &CarbonlinkRequest{
				Type:   "cache-query",
				Metric: "carbon.agents.carbon_agent_server.param.size"},
		},
		{name: "Invalid query type",
			data: []byte("\x80\x02}q\x00(U\x06metricq\x01U\x03barq\x02U\x04typeq\x03U\x03fooq\x04u."),
			want: &CarbonlinkRequest{
				Type:   "foo",
				Metric: "bar",
			},
		},
		{name: "Garbage",
			data:    []byte("garbage"),
			wantErr: true,
		},
	}
	for _, tt := range tests {
		got, err := ParseCarbonlinkRequest(tt.data)
		if (err != nil) != tt.wantErr {
			t.Errorf("%q. ParseCarbonlinkRequest() error = %v, wantErr %v", tt.name, err, tt.wantErr)
			continue
		}
		if !reflect.DeepEqual(got, tt.want) {
			t.Errorf("%q. ParseCarbonlinkRequest() = %v, want %v", tt.name, got, tt.want)
		}
	}
}

func BenchmarkCarbonLinkPickleParse(b *testing.B) {
	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			ParseCarbonlinkRequest([]byte(sampleCacheQuery)[4:])
		}
	})
}

func BenchmarkCarbonLinkPackReply(b *testing.B) {
	p := points.OnePoint("carbon.agents.carbon_agent_server.param.size", 15, 1422795966).Add(15, 9000000).Add(16, 9000000)

	b.RunParallel(func(pb *testing.PB) {
		for pb.Next() {
			packReply(p.Data)
		}
	})
}
<|endoftext|>


<commit_before>package batchinsert

import (
	"database/sql"
	"strings"
) //import

type BatchInsert_t struct {
	bindParams []interface{}
	dbh        *sql.DB
	insertCtr  uint
	insertRate uint
	queryPart1 string
	queryPart2 string
	values     string
} //BatchInsert_t

func NewBatchInsert(dbh *sql.DB, insertRate uint) *BatchInsert_t {
	return &BatchInsert_t{
		dbh:        dbh,
		bindParams: make([]interface{}, 0),
		insertRate: insertRate,
		values:     " VALUES",
	} //return
} //NewBatchInsert

func (this *BatchInsert_t) Insert(query string, params ...interface{}) (err error) {
	// Only split out query the first time Insert is called
	if this.queryPart1 == "" {
		this.splitQuery(query)
	} //if

	this.insertCtr++

	// Build VALUES seciton of query and add to parameter slice
	this.values += this.queryPart2
	this.bindParams = append(this.bindParams, params...)

	// If the batch interval has been hit, execute a batch insert
	if this.insertCtr >= this.insertRate {
		var (
			stmt *sql.Stmt
		) //var

		// Prepare query
		if stmt, err = this.dbh.Prepare(this.queryPart1 + this.values[:len(this.values)-1]); err != nil {
			return (err)
		} //if
		defer stmt.Close()

		// Executate batch insert
		if _, err = stmt.Exec(this.bindParams...); err != nil {
			return (err)
		} //if

		// Reset vars
		_ = stmt.Close()
		this.values = " VALUES"
		this.bindParams = make([]interface{}, 0)
		this.insertCtr = 0
	} //if

	return err
} //Insert

func (this *BatchInsert_t) Flush() (err error) {
	var (
		stmt *sql.Stmt
	) //var

	// Prepare query
	if stmt, err = this.dbh.Prepare(this.queryPart1 + this.values[:len(this.values)-1]); err != nil {
		return (err)
	} //if
	defer stmt.Close()

	// Executate batch insert
	if _, err = stmt.Exec(this.bindParams...); err != nil {
		return (err)
	} //if

	// Reset vars
	_ = stmt.Close()
	this.values = " VALUES"
	this.bindParams = make([]interface{}, 0)
	this.insertCtr = 0

	return err
} //Flush

func (this *BatchInsert_t) splitQuery(query string) {
	var (
		ndxParens, ndxValues int
	) //var

	// Normalize and split query
	query = strings.ToLower(query)
	ndxValues = strings.LastIndex(query, "values")
	ndxParens = strings.LastIndex(query, ")")

	// Save the first and second parts of the query separately for easier building later
	this.queryPart1 = strings.TrimSpace(query[:ndxValues])
	this.queryPart2 = query[ndxValues+6:ndxParens+1] + ","
} //splitQuery
<commit_msg>Removed duplicate code, just calling Flush() instead<commit_after>package batchinsert

import (
	"database/sql"
	"strings"
) //import

type BatchInsert_t struct {
	bindParams []interface{}
	dbh        *sql.DB
	insertCtr  uint
	insertRate uint
	queryPart1 string
	queryPart2 string
	values     string
} //BatchInsert_t

func NewBatchInsert(dbh *sql.DB, insertRate uint) *BatchInsert_t {
	return &BatchInsert_t{
		dbh:        dbh,
		bindParams: make([]interface{}, 0),
		insertRate: insertRate,
		values:     " VALUES",
	} //return
} //NewBatchInsert

func (this *BatchInsert_t) Insert(query string, params ...interface{}) (err error) {
	// Only split out query the first time Insert is called
	if this.queryPart1 == "" {
		this.splitQuery(query)
	} //if

	this.insertCtr++

	// Build VALUES seciton of query and add to parameter slice
	this.values += this.queryPart2
	this.bindParams = append(this.bindParams, params...)

	// If the batch interval has been hit, execute a batch insert
	if this.insertCtr >= this.insertRate {
		err = this.Flush()
	} //if

	return err
} //Insert

func (this *BatchInsert_t) Flush() (err error) {
	var (
		stmt *sql.Stmt
	) //var

	// Prepare query
	if stmt, err = this.dbh.Prepare(this.queryPart1 + this.values[:len(this.values)-1]); err != nil {
		return (err)
	} //if
	defer stmt.Close()

	// Executate batch insert
	if _, err = stmt.Exec(this.bindParams...); err != nil {
		return (err)
	} //if

	// Reset vars
	_ = stmt.Close()
	this.values = " VALUES"
	this.bindParams = make([]interface{}, 0)
	this.insertCtr = 0

	return err
} //Flush

func (this *BatchInsert_t) splitQuery(query string) {
	var (
		ndxParens, ndxValues int
	) //var

	// Normalize and split query
	query = strings.ToLower(query)
	ndxValues = strings.LastIndex(query, "values")
	ndxParens = strings.LastIndex(query, ")")

	// Save the first and second parts of the query separately for easier building later
	this.queryPart1 = strings.TrimSpace(query[:ndxValues])
	this.queryPart2 = query[ndxValues+6:ndxParens+1] + ","
} //splitQuery
<|endoftext|>


<commit_before>// Copyright (c) 2017 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bgpsyncer

import (
	apiv3 "github.com/projectcalico/libcalico-go/lib/apis/v3"
	"github.com/projectcalico/libcalico-go/lib/backend/api"
	"github.com/projectcalico/libcalico-go/lib/backend/model"
	"github.com/projectcalico/libcalico-go/lib/backend/syncersv1/updateprocessors"
	"github.com/projectcalico/libcalico-go/lib/backend/watchersyncer"
)

// New creates a new BGP v1 Syncer.  Since only etcdv3 supports Watchers for all of
// the required resource types, the WatcherSyncer will go into a polling loop for
// KDD.  An optional node name may be supplied.  If set, the syncer only watches
// the specified node rather than all nodes.
func New(client api.Client, callbacks api.SyncerCallbacks, node string, watchAllNodes bool) api.Syncer {
	// Create the set of ResourceTypes required for Felix.  Since the update processors
	// also cache state, we need to create individual ones per syncer rather than create
	// a common global set.
	// For BGP we always only care about affinity blocks assigned to our own particuar node.
	// However, depending on whether we are in full-mesh mode or not changes whether we want
	// to watch all Node resources or just our own.
	nodeToWatch := node
	if watchAllNodes {
		nodeToWatch = ""
	}
	resourceTypes := []watchersyncer.ResourceType{
		{
			ListInterface:   model.ResourceListOptions{Kind: apiv3.KindIPPool},
			UpdateProcessor: updateprocessors.NewIPPoolUpdateProcessor(),
		},
		{
			ListInterface:   model.ResourceListOptions{Kind: apiv3.KindBGPConfiguration},
			UpdateProcessor: updateprocessors.NewBGPConfigUpdateProcessor(),
		},
		{
			ListInterface: model.ResourceListOptions{
				Kind: apiv3.KindNode,
				Name: nodeToWatch,
			},
			UpdateProcessor: updateprocessors.NewBGPNodeUpdateProcessor(),
		},
		{
			ListInterface:   model.ResourceListOptions{Kind: apiv3.KindBGPPeer},
			UpdateProcessor: updateprocessors.NewBGPPeerUpdateProcessor(),
		},
	}
	if node != "" {
		resourceTypes = append(resourceTypes, watchersyncer.ResourceType{
			ListInterface: model.BlockAffinityListOptions{Host: node},
		})
	}

	return watchersyncer.New(
		client,
		resourceTypes,
		callbacks,
	)
}
<commit_msg>Add explanatory comment<commit_after>// Copyright (c) 2017 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package bgpsyncer

import (
	apiv3 "github.com/projectcalico/libcalico-go/lib/apis/v3"
	"github.com/projectcalico/libcalico-go/lib/backend/api"
	"github.com/projectcalico/libcalico-go/lib/backend/model"
	"github.com/projectcalico/libcalico-go/lib/backend/syncersv1/updateprocessors"
	"github.com/projectcalico/libcalico-go/lib/backend/watchersyncer"
)

// New creates a new BGP v1 Syncer.  Since only etcdv3 supports Watchers for all of
// the required resource types, the WatcherSyncer will go into a polling loop for
// KDD.  An optional node name may be supplied.  If set, the syncer only watches
// the specified node rather than all nodes.
func New(client api.Client, callbacks api.SyncerCallbacks, node string, watchAllNodes bool) api.Syncer {
	// Create the set of ResourceTypes required for Felix.  Since the update processors
	// also cache state, we need to create individual ones per syncer rather than create
	// a common global set.
	// For BGP we always only care about affinity blocks assigned to our own particuar node.
	// However, depending on whether we are in full-mesh mode or not changes whether we want
	// to watch all Node resources or just our own.
	nodeToWatch := node
	if watchAllNodes {
		nodeToWatch = ""
	}
	resourceTypes := []watchersyncer.ResourceType{
		{
			ListInterface:   model.ResourceListOptions{Kind: apiv3.KindIPPool},
			UpdateProcessor: updateprocessors.NewIPPoolUpdateProcessor(),
		},
		{
			ListInterface:   model.ResourceListOptions{Kind: apiv3.KindBGPConfiguration},
			UpdateProcessor: updateprocessors.NewBGPConfigUpdateProcessor(),
		},
		{
			ListInterface: model.ResourceListOptions{
				Kind: apiv3.KindNode,
				Name: nodeToWatch,
			},
			UpdateProcessor: updateprocessors.NewBGPNodeUpdateProcessor(),
		},
		{
			ListInterface:   model.ResourceListOptions{Kind: apiv3.KindBGPPeer},
			UpdateProcessor: updateprocessors.NewBGPPeerUpdateProcessor(),
		},
	}
	// When this syncer is used (via confd) in calico/node, it needs to know the affinity blocks
	// for that node so that it can set up the blackhole routes; in that case,
	// node is non-empty.  When this syncer is used (also via confd) in
	// calico/routereflector, it has no need for affinity block information, so we skip that
	// here; in that case, node is empty.
	if node != "" {
		resourceTypes = append(resourceTypes, watchersyncer.ResourceType{
			ListInterface: model.BlockAffinityListOptions{Host: node},
		})
	}

	return watchersyncer.New(
		client,
		resourceTypes,
		callbacks,
	)
}
<|endoftext|>


<commit_before>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mesh

import (
	"istio.io/api/mesh/v1alpha1"
	"istio.io/istio/pkg/config/mesh"
)

// DefaultMeshConfig returns a default meshconfig.
func DefaultMeshConfig() *v1alpha1.MeshConfig {
	meshconfig := mesh.DefaultMeshConfig()
	meshconfig.IngressClass = "istio"
	meshconfig.IngressControllerMode = v1alpha1.MeshConfig_STRICT
	return &meshconfig
}

// DefaultMeshNetworks returns a default meshnetworks configuration.
// By default, it is empty.
func DefaultMeshNetworks() *v1alpha1.MeshNetworks {
	mn := mesh.EmptyMeshNetworks()
	return &mn
}
<commit_msg>Repeat set value (#33789)<commit_after>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package mesh

import (
	"istio.io/api/mesh/v1alpha1"
	"istio.io/istio/pkg/config/mesh"
)

// DefaultMeshConfig returns a default meshconfig.
func DefaultMeshConfig() *v1alpha1.MeshConfig {
	meshconfig := mesh.DefaultMeshConfig()
	return &meshconfig
}

// DefaultMeshNetworks returns a default meshnetworks configuration.
// By default, it is empty.
func DefaultMeshNetworks() *v1alpha1.MeshNetworks {
	mn := mesh.EmptyMeshNetworks()
	return &mn
}
<|endoftext|>


<commit_before>package kvs

import (
	"bytes"
	"compress/gzip"
	"encoding/gob"
	"errors"
	"fmt"
	"io/ioutil"
	"time"

	"golang.org/x/net/context"

	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/memcache"
)

const kind = "kvs"

var NotFound = fmt.Errorf("Key-value pair was not found")

// use App Engine's datastore as a simple key-value store

type KV struct {
	Key     string `datastore:",noindex"`
	Value   []byte `datastore:",noindex"`
	Expires time.Time

	Ttl time.Duration `datastore:"-"` // convenient alternative to Expires
}

// GC defines options for how to perform garbage collection on KV entities.
type GC struct {
	// Ttl describes how much time a single GC operation should be allowed to run.
	// This can be tuned based on how frequently GC jobs are executed.  For
	// example, if GC runs once per minute, you might set Ttl to 50 seconds.
	//
	// Ttl is only a guideline for the GC operation.  It might run longer or
	// shorter than this target.
	//
	// Defaults to 50 seconds.
	Ttl time.Duration
}

// Find looks for an existing key-value pair.  Returns
// NotFound if the key does not exist.
func Find(c context.Context, k string) (*KV, error) {
	// is the kv in memcache?
	kv := new(KV)
	memcacheKey := memKey(k)
	item, err := memcache.Get(c, memcacheKey)
	if err == nil {
		kv.Key = k
		kv.Value = item.Value
		return kv, nil
	}

	// nope, look in the datastore
	key := datastore.NewKey(c, kind, k, 0, nil)
	err = datastore.Get(c, key, kv)
	if err == datastore.ErrNoSuchEntity {
		return nil, NotFound
	}
	if err != nil {
		return nil, err
	}
	if !kv.Expires.IsZero() && kv.Expires.Before(time.Now()) {
		// key has expired. pretend it doesn't exist
		return nil, NotFound
	}

	// store result in memcache for later
	item = &memcache.Item{
		Key:   memcacheKey,
		Value: kv.Value,
	}
	if !kv.Expires.IsZero() {
		item.Expiration = kv.Expires.Sub(time.Now())
	}
	err = memcache.Set(c, item)
	_ = err // memcache is an optimization. ignore its errors.

	return kv, nil
}

func (kv *KV) datastoreKey(c context.Context) *datastore.Key {
	return datastore.NewKey(c, kind, kv.Key, 0, nil)
}

// Put stores a key-value pair until its expiration.
func (kv *KV) Put(c context.Context) error {
	// prepare a memcache item for later
	memcacheKey := memKey(kv.Key)
	item := &memcache.Item{
		Key:   memcacheKey,
		Value: kv.Value,
	}

	// calculate key-value expiration time
	if kv.Ttl > 0 {
		item.Expiration = kv.Ttl
		kv.Expires = time.Now().Add(kv.Ttl)
		kv.Ttl = 0
	} else if !kv.Expires.IsZero() {
		item.Expiration = kv.Expires.Sub(time.Now())
	}

	// store kv into datastore for permanent storage
	_, err := datastore.Put(c, kv.datastoreKey(c), kv)
	if err != nil {
		return err
	}

	// cache kv for faster access next time
	err = memcache.Set(c, item)
	_ = err // memcache is an optimization. ignore errors

	return nil
}

// Remove a rule in the datastore
func (kv *KV) Delete(c context.Context) error {
	// delete from datastore
	err := datastore.Delete(c, kv.datastoreKey(c))
	if err != nil {
		return err
	}

	// delete from memcache too
	err = memcache.Delete(c, memKey(kv.Key))
	_ = err // memcache is an optimization. ignore errors.
	return nil
}

// Compress rewrites the Value field by compressing it with gzip.
func (kv *KV) Compress() error {
	var buf bytes.Buffer
	w := gzip.NewWriter(&buf)
	_, err := w.Write(kv.Value)
	if err != nil {
		return err
	}
	err = w.Close()
	if err != nil {
		return err
	}

	kv.Value = buf.Bytes()
	return nil
}

// Decompress rewrites the Value field by decompressing it with gzip.
func (kv *KV) Decompress() error {
	buf := bytes.NewBuffer(kv.Value)
	r, err := gzip.NewReader(buf)
	if err != nil {
		return err
	}
	val, err := ioutil.ReadAll(r)
	if err != nil {
		return err
	}

	kv.Value = val
	return nil
}

// Encode sets the Value field by gob encoding a Go value.
func (kv *KV) Encode(x interface{}) error {
	var buf bytes.Buffer
	err := gob.NewEncoder(&buf).Encode(x)
	if err != nil {
		return err
	}

	kv.Value = buf.Bytes()
	return nil
}

// Decode extracts the Value field by gob decoding into a Go value.
func (kv *KV) Decode(x interface{}) error {
	buf := bytes.NewBuffer(kv.Value)
	return gob.NewDecoder(buf).Decode(x)
}

// returns a key for use with memcache
func memKey(key string) string {
	return fmt.Sprintf("%s: %s", kind, key)
}

var CollectGarbageTimeout = errors.New("CollectGarbage timed out")

// CollectGarbage deletes expired kv entities from the datastore. This function
// should be called regularly to prevent expired kvs from accumulating in the
// datastore.  Returns the number of entities that were removed from datastore.
//
// If GC.Ttl is reached, returns CollectGarbageTimeout regardless how many
// entities were expired before then.
func CollectGarbage(c context.Context, opts *GC) (int, error) {
	if opts == nil {
		opts = &GC{}
	}
	if opts.Ttl == 0 {
		opts.Ttl = 50 * time.Second
	}
	quittingTime := time.Now().Add(opts.Ttl)

	n := 0
	q := datastore.NewQuery(kind).
		Filter("Expires<", time.Now()).
		Order("Expires").
		Limit(400).
		KeysOnly()
	for {
		if time.Now().After(quittingTime) {
			return n, CollectGarbageTimeout
		}

		keys, err := q.GetAll(c, nil)
		if len(keys) > 0 {
			err = datastore.DeleteMulti(c, keys)
			// don't have to clear memcache. it expires on its own
			if err == nil {
				n += len(keys)
			}
		}
		if err != nil {
			return n, err
		}
		if len(keys) == 0 {
			break
		}
	}

	return n, nil
}
<commit_msg>Add GC.Leeway<commit_after>package kvs

import (
	"bytes"
	"compress/gzip"
	"encoding/gob"
	"errors"
	"fmt"
	"io/ioutil"
	"time"

	"golang.org/x/net/context"

	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/memcache"
)

const kind = "kvs"

var NotFound = fmt.Errorf("Key-value pair was not found")

// use App Engine's datastore as a simple key-value store

type KV struct {
	Key     string `datastore:",noindex"`
	Value   []byte `datastore:",noindex"`
	Expires time.Time

	Ttl time.Duration `datastore:"-"` // convenient alternative to Expires
}

// GC defines options for how to perform garbage collection on KV entities.
type GC struct {
	// Ttl describes how much time a single GC operation should be allowed to run.
	// This can be tuned based on how frequently GC jobs are executed.  For
	// example, if GC runs once per minute, you might set Ttl to 50 seconds.
	//
	// Ttl is only a guideline for the GC operation.  It might run longer or
	// shorter than this target.
	//
	// Defaults to 50 seconds.
	Ttl time.Duration

	// Leeway describes how far past its expiration a KV must be before it's
	// considered for garbage collection.  It should be set high enough that
	// the probability of a KV being used again is very low.
	//
	// Defaults to 24 hours.
	Leeway time.Duration
}

// Find looks for an existing key-value pair.  Returns
// NotFound if the key does not exist.
func Find(c context.Context, k string) (*KV, error) {
	// is the kv in memcache?
	kv := new(KV)
	memcacheKey := memKey(k)
	item, err := memcache.Get(c, memcacheKey)
	if err == nil {
		kv.Key = k
		kv.Value = item.Value
		return kv, nil
	}

	// nope, look in the datastore
	key := datastore.NewKey(c, kind, k, 0, nil)
	err = datastore.Get(c, key, kv)
	if err == datastore.ErrNoSuchEntity {
		return nil, NotFound
	}
	if err != nil {
		return nil, err
	}
	if !kv.Expires.IsZero() && kv.Expires.Before(time.Now()) {
		// key has expired. pretend it doesn't exist
		return nil, NotFound
	}

	// store result in memcache for later
	item = &memcache.Item{
		Key:   memcacheKey,
		Value: kv.Value,
	}
	if !kv.Expires.IsZero() {
		item.Expiration = kv.Expires.Sub(time.Now())
	}
	err = memcache.Set(c, item)
	_ = err // memcache is an optimization. ignore its errors.

	return kv, nil
}

func (kv *KV) datastoreKey(c context.Context) *datastore.Key {
	return datastore.NewKey(c, kind, kv.Key, 0, nil)
}

// Put stores a key-value pair until its expiration.
func (kv *KV) Put(c context.Context) error {
	// prepare a memcache item for later
	memcacheKey := memKey(kv.Key)
	item := &memcache.Item{
		Key:   memcacheKey,
		Value: kv.Value,
	}

	// calculate key-value expiration time
	if kv.Ttl > 0 {
		item.Expiration = kv.Ttl
		kv.Expires = time.Now().Add(kv.Ttl)
		kv.Ttl = 0
	} else if !kv.Expires.IsZero() {
		item.Expiration = kv.Expires.Sub(time.Now())
	}

	// store kv into datastore for permanent storage
	_, err := datastore.Put(c, kv.datastoreKey(c), kv)
	if err != nil {
		return err
	}

	// cache kv for faster access next time
	err = memcache.Set(c, item)
	_ = err // memcache is an optimization. ignore errors

	return nil
}

// Remove a rule in the datastore
func (kv *KV) Delete(c context.Context) error {
	// delete from datastore
	err := datastore.Delete(c, kv.datastoreKey(c))
	if err != nil {
		return err
	}

	// delete from memcache too
	err = memcache.Delete(c, memKey(kv.Key))
	_ = err // memcache is an optimization. ignore errors.
	return nil
}

// Compress rewrites the Value field by compressing it with gzip.
func (kv *KV) Compress() error {
	var buf bytes.Buffer
	w := gzip.NewWriter(&buf)
	_, err := w.Write(kv.Value)
	if err != nil {
		return err
	}
	err = w.Close()
	if err != nil {
		return err
	}

	kv.Value = buf.Bytes()
	return nil
}

// Decompress rewrites the Value field by decompressing it with gzip.
func (kv *KV) Decompress() error {
	buf := bytes.NewBuffer(kv.Value)
	r, err := gzip.NewReader(buf)
	if err != nil {
		return err
	}
	val, err := ioutil.ReadAll(r)
	if err != nil {
		return err
	}

	kv.Value = val
	return nil
}

// Encode sets the Value field by gob encoding a Go value.
func (kv *KV) Encode(x interface{}) error {
	var buf bytes.Buffer
	err := gob.NewEncoder(&buf).Encode(x)
	if err != nil {
		return err
	}

	kv.Value = buf.Bytes()
	return nil
}

// Decode extracts the Value field by gob decoding into a Go value.
func (kv *KV) Decode(x interface{}) error {
	buf := bytes.NewBuffer(kv.Value)
	return gob.NewDecoder(buf).Decode(x)
}

// returns a key for use with memcache
func memKey(key string) string {
	return fmt.Sprintf("%s: %s", kind, key)
}

var CollectGarbageTimeout = errors.New("CollectGarbage timed out")

// CollectGarbage deletes expired kv entities from the datastore. This function
// should be called regularly to prevent expired kvs from accumulating in the
// datastore.  Returns the number of entities that were removed from datastore.
//
// If GC.Ttl is reached, returns CollectGarbageTimeout regardless how many
// entities were expired before then.
func CollectGarbage(c context.Context, opts *GC) (int, error) {
	if opts == nil {
		opts = &GC{}
	}
	if opts.Ttl == 0 {
		opts.Ttl = 50 * time.Second
	}
	if opts.Leeway == 0 {
		opts.Leeway = 24 * time.Hour
	}
	quittingTime := time.Now().Add(opts.Ttl)
	cutOff := time.Now().Add(-opts.Leeway)

	n := 0
	q := datastore.NewQuery(kind).
		Filter("Expires<", cutOff).
		Order("Expires").
		Limit(400).
		KeysOnly()
	for {
		if time.Now().After(quittingTime) {
			return n, CollectGarbageTimeout
		}

		keys, err := q.GetAll(c, nil)
		if len(keys) > 0 {
			err = datastore.DeleteMulti(c, keys)
			// don't have to clear memcache. it expires on its own
			if err == nil {
				n += len(keys)
			}
		}
		if err != nil {
			return n, err
		}
		if len(keys) == 0 {
			break
		}
	}

	return n, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"sync"

	"github.com/maruel/subcommands"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/client/downloader"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/isolated"
	"go.chromium.org/luci/common/isolatedclient"
	"go.chromium.org/luci/common/system/signals"
)

func cmdDownload(authOpts auth.Options) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: "download <options>...",
		ShortDesc: "downloads a file or a .isolated tree from an isolate server.",
		LongDesc: `Downloads one or multiple files, or a isolated tree from the isolate server.

Files are referenced by their hash`,
		CommandRun: func() subcommands.CommandRun {
			c := downloadRun{}
			c.commonFlags.Init(authOpts)
			// TODO(mknyszek): Add support for downloading individual files.
			c.Flags.StringVar(&c.outputDir, "output-dir", ".", "The directory where files will be downloaded to.")
			c.Flags.StringVar(&c.outputFiles, "output-files", "", "File into which the full list of downloaded files is written to.")
			c.Flags.StringVar(&c.isolated, "isolated", "", "Hash of a .isolated tree to download.")
			return &c
		},
	}
}

type downloadRun struct {
	commonFlags
	outputDir   string
	outputFiles string
	isolated    string
}

func (c *downloadRun) Parse(a subcommands.Application, args []string) error {
	if err := c.commonFlags.Parse(); err != nil {
		return err
	}
	if len(args) != 0 {
		return errors.New("position arguments not expected")
	}
	if c.isolated == "" {
		return errors.New("isolated is required")
	}
	return nil
}

func (c *downloadRun) main(a subcommands.Application, args []string) error {
	// Prepare isolated client.
	ctx, cancel := context.WithCancel(c.defaultFlags.MakeLoggingContext(os.Stderr))
	signals.HandleInterrupt(cancel)
	authClient, err := c.createAuthClient(ctx)
	if err != nil {
		return err
	}
	client := isolatedclient.New(nil, authClient, c.isolatedFlags.ServerURL, c.isolatedFlags.Namespace, nil, nil)
	var filesMu sync.Mutex
	var files []string
	dl := downloader.New(ctx, client, isolated.HexDigest(c.isolated), c.outputDir, &downloader.Options{
		FileCallback: func(name string, _ *isolated.File) {
			filesMu.Lock()
			files = append(files, name)
			filesMu.Unlock()
		},
	})
	if err := dl.Wait(); err != nil {
		return errors.Annotate(err, "failed to call FetchIsolated()").Err()
	}
	if c.outputFiles != "" {
		filesData := strings.Join(files, "\n")
		if len(files) > 0 {
			filesData += "\n"
		}

		if err := ioutil.WriteFile(c.outputFiles, []byte(filesData), 0664); err != nil {
			return errors.Annotate(err, "failed to call WriteFile(%s, ...)", c.outputFiles).Err()
		}
	}
	return nil
}

func (c *downloadRun) Run(a subcommands.Application, args []string, _ subcommands.Env) int {
	if err := c.Parse(a, args); err != nil {
		fmt.Fprintf(a.GetErr(), "%s: failed to call Parse(%s): %v\n", a.GetName(), args, err)
		return 1
	}
	cl, err := c.defaultFlags.StartTracing()
	if err != nil {
		fmt.Fprintf(a.GetErr(), "%s: failed to call StartTracing(): %v\n", a.GetName(), err)
		return 1
	}
	defer cl.Close()
	if err := c.main(a, args); err != nil {
		fmt.Fprintf(a.GetErr(), "%s: failed to call main(%s): %v\n", a.GetName(), args, err)
		return 1
	}
	return 0
}
<commit_msg>isolated: add cache support in download sub command<commit_after>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"sync"

	"github.com/maruel/subcommands"

	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/client/downloader"
	"go.chromium.org/luci/common/data/caching/cache"
	"go.chromium.org/luci/common/data/text/units"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/isolated"
	"go.chromium.org/luci/common/isolatedclient"
	"go.chromium.org/luci/common/system/signals"
)

func cmdDownload(authOpts auth.Options) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: "download <options>...",
		ShortDesc: "downloads a file or a .isolated tree from an isolate server.",
		LongDesc: `Downloads one or multiple files, or a isolated tree from the isolate server.

Files are referenced by their hash`,
		CommandRun: func() subcommands.CommandRun {
			c := downloadRun{}
			c.commonFlags.Init(authOpts)
			// TODO(mknyszek): Add support for downloading individual files.
			c.Flags.StringVar(&c.outputDir, "output-dir", ".", "The directory where files will be downloaded to.")
			c.Flags.StringVar(&c.outputFiles, "output-files", "", "File into which the full list of downloaded files is written to.")
			c.Flags.StringVar(&c.isolated, "isolated", "", "Hash of a .isolated tree to download.")

			c.Flags.StringVar(&c.cacheDir, "cache-dir", "", "Cache directory to store downloaded files.")
			c.Flags.Int64Var(&c.maxSize, "cache-max-size", 0, "Cache is trimmed if the cache gets larger than this value.")
			c.Flags.IntVar(&c.maxItems, "cache-max-items", 0, "Maximum number of items to keep in the cache.")
			c.Flags.Int64Var(&c.minFreeSpace, "cache-min-free-space", 0, "Cache is trimmed if disk free space becomes lower than this value.")
			return &c
		},
	}
}

type downloadRun struct {
	commonFlags
	outputDir   string
	outputFiles string
	isolated    string

	cacheDir     string
	maxSize      int64
	maxItems     int
	minFreeSpace int64
}

func (c *downloadRun) Parse(a subcommands.Application, args []string) error {
	if err := c.commonFlags.Parse(); err != nil {
		return err
	}
	if len(args) != 0 {
		return errors.New("position arguments not expected")
	}
	if c.isolated == "" {
		return errors.New("isolated is required")
	}

	if c.cacheDir == "" && (c.maxSize != 0 || c.maxItems != 0 || c.minFreeSpace != 0) {
		return errors.New("cache-dir is necessary when cache-max-size, cache-max-items or cache-min-free-space are specified,")
	}
	return nil
}

func (c *downloadRun) main(a subcommands.Application, args []string) error {
	// Prepare isolated client.
	ctx, cancel := context.WithCancel(c.defaultFlags.MakeLoggingContext(os.Stderr))
	signals.HandleInterrupt(cancel)
	authClient, err := c.createAuthClient(ctx)
	if err != nil {
		return err
	}
	client := isolatedclient.New(nil, authClient, c.isolatedFlags.ServerURL, c.isolatedFlags.Namespace, nil, nil)
	var filesMu sync.Mutex
	var files []string

	var diskCache cache.Cache
	if c.cacheDir != "" {
		diskCache, err = cache.NewDisk(cache.Policies{
			MaxSize:      units.Size(c.maxSize),
			MaxItems:     c.maxItems,
			MinFreeSpace: units.Size(c.minFreeSpace),
		}, c.cacheDir, c.isolatedFlags.Namespace)
		if err != nil {
			return err
		}
		defer diskCache.Close()
	}

	dl := downloader.New(ctx, client, isolated.HexDigest(c.isolated), c.outputDir, &downloader.Options{
		FileCallback: func(name string, _ *isolated.File) {
			filesMu.Lock()
			files = append(files, name)
			filesMu.Unlock()
		},
		Cache: diskCache,
	})
	if err := dl.Wait(); err != nil {
		return errors.Annotate(err, "failed to call FetchIsolated()").Err()
	}
	if c.outputFiles != "" {
		filesData := strings.Join(files, "\n")
		if len(files) > 0 {
			filesData += "\n"
		}

		if err := ioutil.WriteFile(c.outputFiles, []byte(filesData), 0664); err != nil {
			return errors.Annotate(err, "failed to call WriteFile(%s, ...)", c.outputFiles).Err()
		}
	}
	return nil
}

func (c *downloadRun) Run(a subcommands.Application, args []string, _ subcommands.Env) int {
	if err := c.Parse(a, args); err != nil {
		fmt.Fprintf(a.GetErr(), "%s: failed to call Parse(%s): %v\n", a.GetName(), args, err)
		return 1
	}
	cl, err := c.defaultFlags.StartTracing()
	if err != nil {
		fmt.Fprintf(a.GetErr(), "%s: failed to call StartTracing(): %v\n", a.GetName(), err)
		return 1
	}
	defer cl.Close()
	if err := c.main(a, args); err != nil {
		fmt.Fprintf(a.GetErr(), "%s: failed to call main(%s): %v\n", a.GetName(), args, err)
		return 1
	}
	return 0
}
<|endoftext|>


<commit_before>package pretty

import (
	"bytes"
	"io"
	"strconv"
	"strings"
)

// An Options represents optional configuration parameters for formatting.
//
// Some options, notably ShortList, dramatically increase the overhead
// of pretty-printing a value.
type Options struct {
	Compact  bool // One-line output. Overrides Diffable.
	Diffable bool // Adds extra newlines for more easily diffable output.

	ShortList int // Maximum character length for short lists if nonzero.
}

var DefaultOptions = &Options{}

func (o *Options) dup() *Options {
	o2 := *o
	return &o2
}

type node interface {
	WriteTo(w io.Writer, indent string, opts *Options)
}

func compactString(n node) string {
	switch k := n.(type) {
	case stringVal:
		return string(k)
	case rawVal:
		return string(k)
	}

	buf := new(bytes.Buffer)
	n.WriteTo(buf, "", &Options{Compact: true})
	return buf.String()
}

type stringVal string

func (str stringVal) WriteTo(w io.Writer, indent string, opts *Options) {
	io.WriteString(w, strconv.Quote(string(str)))
}

type rawVal string

func (r rawVal) WriteTo(w io.Writer, indent string, opts *Options) {
	io.WriteString(w, string(r))
}

type keyval struct {
	key string
	val node
}

type keyvals []keyval

func (l keyvals) Len() int           { return len(l) }
func (l keyvals) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }
func (l keyvals) Less(i, j int) bool { return l[i].key < l[j].key }

func (l keyvals) WriteTo(w io.Writer, indent string, opts *Options) {
	keyWidth := 0

	for _, kv := range l {
		if kw := len(kv.key); kw > keyWidth {
			keyWidth = kw
		}
	}
	padding := strings.Repeat(" ", keyWidth+1)

	inner := indent + "  " + padding
	io.WriteString(w, "{")
	for i, kv := range l {
		if opts.Compact {
			io.WriteString(w, kv.key)
			io.WriteString(w, ":")
		} else {
			if i > 0 || opts.Diffable {
				io.WriteString(w, "\n ")
				io.WriteString(w, indent)
			}
			io.WriteString(w, kv.key)
			io.WriteString(w, ":")
			io.WriteString(w, padding[len(kv.key):])
		}
		kv.val.WriteTo(w, inner, opts)
		if i+1 < len(l) || opts.Diffable {
			io.WriteString(w, ",")
		}
	}
	if !opts.Compact && opts.Diffable && len(l) > 0 {
		io.WriteString(w, "\n")
		io.WriteString(w, indent)
	}
	io.WriteString(w, "}")
}

type list []node

func (l list) WriteTo(w io.Writer, indent string, opts *Options) {
	if max := opts.ShortList; max > 0 {
		short := compactString(l)
		if len(short) <= max {
			io.WriteString(w, short)
			return
		}
	}

	inner := indent + " "
	io.WriteString(w, "[")
	for i, v := range l {
		if !opts.Compact && (i > 0 || opts.Diffable) {
			io.WriteString(w, "\n")
			io.WriteString(w, inner)
		}
		v.WriteTo(w, inner, opts)
		if i+1 < len(l) || opts.Diffable {
			io.WriteString(w, ",")
		}
	}
	if !opts.Compact && opts.Diffable && len(l) > 0 {
		io.WriteString(w, "\n")
		io.WriteString(w, indent)
	}
	io.WriteString(w, "]")
}
<commit_msg>Call bytes.Buffer methods directly<commit_after>package pretty

import (
	"bytes"
	"strconv"
	"strings"
)

// An Options represents optional configuration parameters for formatting.
//
// Some options, notably ShortList, dramatically increase the overhead
// of pretty-printing a value.
type Options struct {
	Compact  bool // One-line output. Overrides Diffable.
	Diffable bool // Adds extra newlines for more easily diffable output.

	ShortList int // Maximum character length for short lists if nonzero.
}

var DefaultOptions = &Options{}

func (o *Options) dup() *Options {
	o2 := *o
	return &o2
}

type node interface {
	WriteTo(w *bytes.Buffer, indent string, opts *Options)
}

func compactString(n node) string {
	switch k := n.(type) {
	case stringVal:
		return string(k)
	case rawVal:
		return string(k)
	}

	buf := new(bytes.Buffer)
	n.WriteTo(buf, "", &Options{Compact: true})
	return buf.String()
}

type stringVal string

func (str stringVal) WriteTo(w *bytes.Buffer, indent string, opts *Options) {
	w.WriteString(strconv.Quote(string(str)))
}

type rawVal string

func (r rawVal) WriteTo(w *bytes.Buffer, indent string, opts *Options) {
	w.WriteString(string(r))
}

type keyval struct {
	key string
	val node
}

type keyvals []keyval

func (l keyvals) Len() int           { return len(l) }
func (l keyvals) Swap(i, j int)      { l[i], l[j] = l[j], l[i] }
func (l keyvals) Less(i, j int) bool { return l[i].key < l[j].key }

func (l keyvals) WriteTo(w *bytes.Buffer, indent string, opts *Options) {
	keyWidth := 0

	for _, kv := range l {
		if kw := len(kv.key); kw > keyWidth {
			keyWidth = kw
		}
	}
	padding := strings.Repeat(" ", keyWidth+1)

	inner := indent + "  " + padding
	w.WriteByte('{')
	for i, kv := range l {
		if opts.Compact {
			w.WriteString(kv.key)
			w.WriteByte(':')
		} else {
			if i > 0 || opts.Diffable {
				w.WriteString("\n ")
				w.WriteString(indent)
			}
			w.WriteString(kv.key)
			w.WriteByte(':')
			w.WriteString(padding[len(kv.key):])
		}
		kv.val.WriteTo(w, inner, opts)
		if i+1 < len(l) || opts.Diffable {
			w.WriteByte(',')
		}
	}
	if !opts.Compact && opts.Diffable && len(l) > 0 {
		w.WriteString("\n")
		w.WriteString(indent)
	}
	w.WriteByte('}')
}

type list []node

func (l list) WriteTo(w *bytes.Buffer, indent string, opts *Options) {
	if max := opts.ShortList; max > 0 {
		short := compactString(l)
		if len(short) <= max {
			w.WriteString(short)
			return
		}
	}

	inner := indent + " "
	w.WriteByte('[')
	for i, v := range l {
		if !opts.Compact && (i > 0 || opts.Diffable) {
			w.WriteByte('\n')
			w.WriteString(inner)
		}
		v.WriteTo(w, inner, opts)
		if i+1 < len(l) || opts.Diffable {
			w.WriteByte(',')
		}
	}
	if !opts.Compact && opts.Diffable && len(l) > 0 {
		w.WriteByte('\n')
		w.WriteString(indent)
	}
	w.WriteByte(']')
}
<|endoftext|>


<commit_before>package daemon

import (
	"errors"
	"fmt"
	"github.com/nytlabs/streamtools/blocks"
	"net/url"
	"strconv"
	"sync"
)

// so i don't forget:
// all blocks should clear a message's route
// connects should set messages route.

type BlockInfo struct {
	Id       string
	Type     string
	Rule     interface{}
	Position *Coords
	in       chan interface{}
}

type ConnectionInfo struct {
	Id      string
	FromId  string
	ToId    string
	ToRoute string
	in      chan interface{}
}

type Coords struct {
	X float64
	Y float64
}

type BlockManager struct {
	blockMap map[string]*BlockInfo
	connMap  map[string]*ConnectionInfo
	genId    chan string
	Mu       *sync.Mutex
}

func IDService(idChan chan string) {
	i := 1
	for {
		id := strconv.Itoa(i)
		idChan <- id
		i += 1
	}
}

func NewBlockManager() *BlockManager {
	idChan := make(chan string)
	go IDService(idChan)
	blocks.BuildLibrary()
	return &BlockManager{
		blockMap: make(map[string]*BlockInfo),
		connMap:  make(map[string]*ConnectionInfo),
		genId:    idChan,
		Mu:       &sync.Mutex{},
	}
}

func (b *BlockManager) GetId() string {
	id := <-b.genId
	ok := b.IdExists(id)
	for ok {
		id = <-b.genId
		ok = b.IdExists(id)
	}
	return id
}

func (b *BlockManager) IdExists(id string) bool {
	_, okB := b.blockMap[id]
	_, okC := b.connMap[id]
	return okB || okC
}

func (b *BlockManager) IdSafe(id string) bool {
	return url.QueryEscape(id) == id && id != "DAEMON"
}

func (b *BlockManager) Create(block *BlockInfo) (*BlockInfo, error) {
	if block == nil {
		return nil, errors.New(fmt.Sprintf("Cannot create block %s: no block data.", block.Id))
	}

	// check to see if the ID is OK
	if !b.IdSafe(block.Id) {
		return nil, errors.New(fmt.Sprintf("Cannot create block %s: invalid id", block.Id))
	}

	// create ID if there is none
	if block.Id == "" {
		block.Id = b.GetId()
	}

	// make sure ID doesn't already exist
	if b.IdExists(block.Id) {
		return nil, errors.New(fmt.Sprintf("Cannot create block %s: id already exists", block.Id))
	}

	// check block Type
	_, ok := blocks.Library[block.Type]
	if !ok {
		return nil, errors.New(fmt.Sprintf("Cannot create block %s: invalid block type %s", block.Id, block.Type))
	}

	// go blockroutine create block here
	b.blockMap[block.Id] = block

	// if rule != nil
	// do a send on the rule.

	return block, nil
}

func (b *BlockManager) UpdateBlock(id string, coord *Coords) (*BlockInfo, error) {
	block, ok := b.blockMap[id]
	if !ok {
		return nil, errors.New(fmt.Sprintf("Cannot update block %s: does not exist", id))
	}

	block.Position = coord

	return block, nil
}

func (b *BlockManager) Send(id string, route string, msg interface{}) error {
	_, ok := b.blockMap[id]
	if !ok {
		return errors.New(fmt.Sprintf("Cannot send to block %s: does not exist", id))
	}
	// send message to block here

	return nil
}

func (b *BlockManager) Query(id string, route string) (interface{}, error) {
	_, ok := b.blockMap[id]
	if !ok {
		return nil, errors.New(fmt.Sprintf("Cannot query block %s: does not exist", id))
	}
	// send qury to block here

	return nil, nil
}

func (b *BlockManager) Connect(conn *ConnectionInfo) (*ConnectionInfo, error) {
	if conn == nil {
		return nil, errors.New("Cannot create: no connection data.")
	}

	// check to see if the ID is OK
	if !b.IdSafe(conn.Id) {
		return nil, errors.New(fmt.Sprintf("Cannot create block %s: invalid id", conn.Id))
	}

	// create ID if there is none
	if conn.Id == "" {
		conn.Id = b.GetId()
	}

	// make sure ID doesn't already exist
	if b.IdExists(conn.Id) {
		return nil, errors.New(fmt.Sprintf("Cannot create connection %s: id already exists", conn.Id))
	}

	// check to see if the blocks that we are attaching to exist
	fromExists := b.IdExists(conn.FromId)
	if !fromExists {
		return nil, errors.New(fmt.Sprintf("Cannot create connection %s: FromId block does not exist", conn.Id))
	}

	toExists := b.IdExists(conn.ToId)
	if !toExists {
		return nil, errors.New(fmt.Sprintf("Cannot create connection %s: ToId ID does not exist", conn.Id))
	}

	// go blockroutine create block here
	b.connMap[conn.Id] = conn

	return conn, nil
}

func (b *BlockManager) GetBlock(id string) (*BlockInfo, error) {
	block, ok := b.blockMap[id]
	if !ok {
		return nil, errors.New(fmt.Sprintf("Cannot get block %s: does not exist", id))
	}

	// retrieve block's and set it here.
	block.Rule = "test" // "retrieve fresh rule from the block here..."

	return block, nil
}

func (b *BlockManager) GetConnection(id string) (*ConnectionInfo, error) {
	_, ok := b.connMap[id]
	if !ok {
		return nil, errors.New(fmt.Sprintf("Cannot get connection %s: does not exist", id))
	}
	return b.connMap[id], nil
}

func (b *BlockManager) DeleteBlock(id string) error {
	_, ok := b.blockMap[id]
	if !ok {
		return errors.New(fmt.Sprintf("Cannot delete block %s: does not exist", id))
	}

	// delete connections that reference this block
	for _, c := range b.connMap {
		if c.FromId == id {
			b.DeleteConnection(c.Id)
		}
		if c.ToId == id {
			b.DeleteConnection(c.Id)
		}
	}

	// turn off block here
	// close channels, whatever.

	delete(b.blockMap, id)

	return nil
}

func (b *BlockManager) DeleteConnection(id string) error {
	_, ok := b.connMap[id]
	if !ok {
		return errors.New(fmt.Sprintf("Cannot delete connection %s: does not exist", id))
	}

	// call disconnecting stuff here
	// remove channel from FromBlock, etc
	// turn off connection block
	delete(b.connMap, id)

	return nil
}

func (b *BlockManager) ListBlocks() []*BlockInfo {
	i := 0
	blocks := make([]*BlockInfo, len(b.blockMap), len(b.blockMap))
	for k, _ := range b.blockMap {
		v, err := b.GetBlock(k)
		if err != nil {
			continue
		}
		blocks[i] = v
		i++
	}

	return blocks
}

func (b *BlockManager) ListConnections() []*ConnectionInfo {
	i := 0
	conns := make([]*ConnectionInfo, len(b.connMap), len(b.connMap))
	for _, v := range b.connMap {
		conns[i] = v
		i++
	}
	return conns
}
<commit_msg>MUTEXES: HOW DO THEY WORK?<commit_after>package daemon

import (
	"errors"
	"fmt"
	"github.com/nytlabs/streamtools/blocks"
	"strconv"
	"net/url"
)

// so i don't forget:
// all blocks should clear a message's route
// connects should set messages route.

type BlockInfo struct {
	Id       string
	Type     string
	Rule     interface{}
	Position *Coords
	in       chan interface{}
}

type ConnectionInfo struct {
	Id      string
	FromId  string
	ToId    string
	ToRoute string
	in      chan interface{}
}

type Coords struct {
	X float64
	Y float64
}

type BlockManager struct {
	blockMap map[string]*BlockInfo
	connMap  map[string]*ConnectionInfo
	genId    chan string
}

func IDService(idChan chan string) {
	i := 1
	for {
		id := strconv.Itoa(i)
		idChan <- id
		i += 1
	}
}

func NewBlockManager() *BlockManager {
	idChan := make(chan string)
	go IDService(idChan)
	blocks.BuildLibrary()
	return &BlockManager{
		blockMap: make(map[string]*BlockInfo),
		connMap:  make(map[string]*ConnectionInfo),
		genId:    idChan,
	}
}

func (b *BlockManager) GetId() string {
	id := <-b.genId
	ok := b.IdExists(id)
	for ok {
		id = <-b.genId
		ok = b.IdExists(id)
	}
	return id
}

func (b *BlockManager) IdExists(id string) bool {
	_, okB := b.blockMap[id]
	_, okC := b.connMap[id]
	return okB || okC
}

func (b *BlockManager) IdSafe(id string) bool {
	return url.QueryEscape(id) == id && id != "DAEMON"
}

func (b *BlockManager) Create(block *BlockInfo) (*BlockInfo, error) {
	if block == nil {
		return nil, errors.New(fmt.Sprintf("Cannot create block %s: no block data.", block.Id))
	}

	// check to see if the ID is OK
	if !b.IdSafe(block.Id) {
		return nil, errors.New(fmt.Sprintf("Cannot create block %s: invalid id", block.Id))
	}

	// create ID if there is none
	if block.Id == "" {
		block.Id = b.GetId()
	}

	// make sure ID doesn't already exist
	if b.IdExists(block.Id) {
		return nil, errors.New(fmt.Sprintf("Cannot create block %s: id already exists", block.Id))
	}

	// check block Type
	_, ok := blocks.Library[block.Type]
	if !ok {
		return nil, errors.New(fmt.Sprintf("Cannot create block %s: invalid block type %s", block.Id, block.Type))
	}

	// go blockroutine create block here
	b.blockMap[block.Id] = block

	// if rule != nil
	// do a send on the rule.

	return block, nil
}

func (b *BlockManager) UpdateBlock(id string, coord *Coords) (*BlockInfo, error) {
	block, ok := b.blockMap[id]
	if !ok {
		return nil, errors.New(fmt.Sprintf("Cannot update block %s: does not exist", id))
	}

	block.Position = coord

	return block, nil
}

func (b *BlockManager) Send(id string, route string, msg interface{}) error {
	_, ok := b.blockMap[id]
	if !ok {
		return errors.New(fmt.Sprintf("Cannot send to block %s: does not exist", id))
	}
	// send message to block here

	return nil
}

func (b *BlockManager) Query(id string, route string) (interface{}, error) {
	_, ok := b.blockMap[id]
	if !ok {
		return nil, errors.New(fmt.Sprintf("Cannot query block %s: does not exist", id))
	}
	// send qury to block here

	return nil, nil
}

func (b *BlockManager) Connect(conn *ConnectionInfo) (*ConnectionInfo, error) {
	if conn == nil {
		return nil, errors.New("Cannot create: no connection data.")
	}

	// check to see if the ID is OK
	if !b.IdSafe(conn.Id) {
		return nil, errors.New(fmt.Sprintf("Cannot create block %s: invalid id",conn.Id))
	}

	// create ID if there is none
	if conn.Id == "" {
		conn.Id = b.GetId()
	}

	// make sure ID doesn't already exist
	if b.IdExists(conn.Id) {
		return nil, errors.New(fmt.Sprintf("Cannot create connection %s: id already exists", conn.Id))
	}

	// check to see if the blocks that we are attaching to exist
	fromExists := b.IdExists(conn.FromId)
	if !fromExists {
		return nil, errors.New(fmt.Sprintf("Cannot create connection %s: FromId block does not exist", conn.Id))
	}

	toExists := b.IdExists(conn.ToId)
	if !toExists {
		return nil, errors.New(fmt.Sprintf("Cannot create connection %s: ToId ID does not exist", conn.Id))
	}

	// go blockroutine create block here
	b.connMap[conn.Id] = conn

	return conn, nil
}

func (b *BlockManager) GetBlock(id string) (*BlockInfo, error) {
	block, ok := b.blockMap[id]
	if !ok {
		return nil, errors.New(fmt.Sprintf("Cannot get block %s: does not exist", id))
	}

	// retrieve block's and set it here.
	block.Rule = "test" // "retrieve fresh rule from the block here..."

	return block, nil
}

func (b *BlockManager) GetConnection(id string) (*ConnectionInfo, error) {
	_, ok := b.connMap[id]
	if !ok {
		return nil, errors.New(fmt.Sprintf("Cannot get connection %s: does not exist", id))
	}
	return b.connMap[id], nil
}

func (b *BlockManager) DeleteBlock(id string) error {
	_, ok := b.blockMap[id]
	if !ok {
		return errors.New(fmt.Sprintf("Cannot delete block %s: does not exist", id))
	}

	// delete connections that reference this block
	for _, c := range b.connMap {
		if c.FromId == id {
			b.DeleteConnection(c.Id)
		}
		if c.ToId == id {
			b.DeleteConnection(c.Id)
		}
	}

	// turn off block here
	// close channels, whatever.

	delete(b.blockMap, id)

	return nil
}

func (b *BlockManager) DeleteConnection(id string) error {
	_, ok := b.connMap[id]
	if !ok {
		return errors.New(fmt.Sprintf("Cannot delete connection %s: does not exist", id))
	}

	// call disconnecting stuff here
	// remove channel from FromBlock, etc
	// turn off connection block
	delete(b.connMap, id)

	return nil
}

func (b *BlockManager) ListBlocks() []*BlockInfo {
	i := 0
	blocks := make([]*BlockInfo, len(b.blockMap), len(b.blockMap))
	for k, _ := range b.blockMap {
		v, err := b.GetBlock(k)
		if err != nil {
			continue
		}
		blocks[i] = v
		i++
	}

	return blocks
}

func (b *BlockManager) ListConnections() []*ConnectionInfo {
	i := 0
	conns := make([]*ConnectionInfo, len(b.connMap), len(b.connMap))
	for _, v := range b.connMap {
		conns[i] = v
		i++
	}
	return conns
}
<|endoftext|>