<commit_before>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inmemory

import (
	"bufio"
	"bytes"
	"crypto/sha1"
	"fmt"
	"io"
	"strings"
	"sync"

	"k8s.io/apimachinery/pkg/util/yaml"

	"github.com/hashicorp/go-multierror"
	kubeJson "k8s.io/apimachinery/pkg/runtime/serializer/json"

	"istio.io/istio/galley/pkg/config/event"
	"istio.io/istio/galley/pkg/config/meta/schema"
	"istio.io/istio/galley/pkg/config/meta/schema/collection"
	"istio.io/istio/galley/pkg/config/resource"
	"istio.io/istio/galley/pkg/config/scope"
	"istio.io/istio/galley/pkg/config/source/inmemory"
	"istio.io/istio/galley/pkg/config/source/kube/rt"
)

var inMemoryKubeNameDiscriminator int64

// KubeSource is an in-memory source implementation that can handle K8s style resources.
type KubeSource struct {
	mu sync.Mutex

	name      string
	resources schema.KubeResources
	source    *inmemory.Source
	defaultNs string

	versionCtr int64
	shas       map[kubeResourceKey]resourceSha
	byFile     map[string]map[kubeResourceKey]collection.Name
}

type resourceSha [sha1.Size]byte

type kubeResource struct {
	entry *resource.Entry
	spec  schema.KubeResource
	sha   resourceSha
}

func (r *kubeResource) newKey() kubeResourceKey {
	return kubeResourceKey{
		kind:     r.spec.Kind,
		fullName: r.entry.Metadata.Name,
	}
}

type kubeResourceKey struct {
	fullName resource.Name
	kind     string
}

var _ event.Source = &KubeSource{}

// NewKubeSource returns a new in-memory Source that works with Kubernetes resources.
func NewKubeSource(resources schema.KubeResources) *KubeSource {
	name := fmt.Sprintf("kube-inmemory-%d", inMemoryKubeNameDiscriminator)
	inMemoryKubeNameDiscriminator++

	s := inmemory.New(resources.Collections())

	return &KubeSource{
		name:      name,
		resources: resources,
		source:    s,
		shas:      make(map[kubeResourceKey]resourceSha),
		byFile:    make(map[string]map[kubeResourceKey]collection.Name),
	}
}

// SetDefaultNamespace enables injecting a default namespace for resources where none is already specified
func (s *KubeSource) SetDefaultNamespace(defaultNs string) {
	s.defaultNs = defaultNs
}

// Start implements processor.Source
func (s *KubeSource) Start() {
	s.source.Start()
}

// Stop implements processor.Source
func (s *KubeSource) Stop() {
	s.source.Stop()
}

// Clear the contents of this source
func (s *KubeSource) Clear() {
	s.versionCtr = 0
	s.shas = make(map[kubeResourceKey]resourceSha)
	s.byFile = make(map[string]map[kubeResourceKey]collection.Name)
	s.source.Clear()
}

// Dispatch implements processor.Source
func (s *KubeSource) Dispatch(h event.Handler) {
	s.source.Dispatch(h)
}

// Get returns the named collection.
func (s *KubeSource) Get(collection collection.Name) *inmemory.Collection {
	return s.source.Get(collection)
}

// ContentNames returns the names known to this source.
func (s *KubeSource) ContentNames() map[string]struct{} {
	s.mu.Lock()
	defer s.mu.Unlock()

	result := make(map[string]struct{})
	for n := range s.byFile {
		result[n] = struct{}{}
	}

	return result
}

// ApplyContent applies the given yamltext to this source. The content is tracked with the given name. If ApplyContent
// gets called multiple times with the same name, the contents applied by the previous incarnation will be overwritten
// or removed, depending on the new content.
// Returns an error if any were encountered, but that still may represent a partial success
func (s *KubeSource) ApplyContent(name, yamlText string) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	// We hold off on dealing with parseErr until the end, since partial success is possible
	resources, parseErrs := s.parseContent(s.resources, name, yamlText)

	oldKeys := s.byFile[name]
	newKeys := make(map[kubeResourceKey]collection.Name)

	for _, r := range resources {
		key := r.newKey()

		oldSha, found := s.shas[key]
		if !found || oldSha != r.sha {
			s.versionCtr++
			r.entry.Metadata.Version = resource.Version(fmt.Sprintf("v%d", s.versionCtr))
			scope.Source.Debuga("KubeSource.ApplyContent: Set: ", r.spec.Collection.Name, r.entry.Metadata.Name)
			s.source.Get(r.spec.Collection.Name).Set(r.entry)
			s.shas[key] = r.sha
		}
		newKeys[key] = r.spec.Collection.Name
		if oldKeys != nil {
			scope.Source.Debuga("KubeSource.ApplyContent: Delete: ", r.spec.Collection.Name, key)
			delete(oldKeys, key)
		}
	}

	for k, col := range oldKeys {
		s.source.Get(col).Remove(k.fullName)
	}
	s.byFile[name] = newKeys

	if parseErrs != nil {
		return fmt.Errorf("errors parsing content %q: %v", name, parseErrs)
	}
	return nil
}

// RemoveContent removes the content for the given name
func (s *KubeSource) RemoveContent(name string) {
	s.mu.Lock()
	defer s.mu.Unlock()

	keys := s.byFile[name]
	if keys != nil {
		for key, col := range keys {
			s.source.Get(col).Remove(key.fullName)
			delete(s.shas, key)
		}

		delete(s.byFile, name)
	}
}

func (s *KubeSource) parseContent(r schema.KubeResources, name, yamlText string) ([]kubeResource, error) {
	var resources []kubeResource
	var errs error

	reader := bufio.NewReader(strings.NewReader(yamlText))
	decoder := yaml.NewYAMLReader(reader)

	for {
		doc, err := decoder.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			e := fmt.Errorf("error reading documents in %s: %v", name, err)
			scope.Source.Warnf("%v - skipping", e)
			scope.Source.Debugf("Failed to parse yamlText chunk: %v", yamlText)
			errs = multierror.Append(errs, e)
			break
		}

		chunk := bytes.TrimSpace(doc)
		r, err := s.parseChunk(r, chunk)
		if err != nil {
			e := fmt.Errorf("error processing %s: %v", name, err)
			scope.Source.Warnf("%v - skipping", e)
			scope.Source.Debugf("Failed to parse yaml chunk: %v", string(chunk))
			errs = multierror.Append(errs, e)
			continue
		}
		resources = append(resources, r)
	}

	return resources, errs
}

func (s *KubeSource) parseChunk(r schema.KubeResources, yamlChunk []byte) (kubeResource, error) {
	// Convert to JSON
	jsonChunk, err := yaml.ToJSON(yamlChunk)
	if err != nil {
		return kubeResource{}, fmt.Errorf("failed converting YAML to JSON: %v", err)
	}

	// Peek at the beginning of the JSON to
	groupVersionKind, err := kubeJson.DefaultMetaFactory.Interpret(jsonChunk)
	if err != nil {
		return kubeResource{}, fmt.Errorf("failed interpreting jsonChunk: %v", err)
	}

	resourceSpec, found := r.Find(groupVersionKind.Group, groupVersionKind.Kind)
	if !found {
		return kubeResource{}, fmt.Errorf("failed finding spec for group/kind: %s/%s", groupVersionKind.Group, groupVersionKind.Kind)
	}

	t := rt.DefaultProvider().GetAdapter(resourceSpec)
	obj, err := t.ParseJSON(jsonChunk)
	if err != nil {
		return kubeResource{}, fmt.Errorf("failed parsing JSON for built-in type: %v", err)
	}
	objMeta := t.ExtractObject(obj)

	// If namespace is blank and we have a default set, fill in the default
	// (This mirrors the behavior if you kubectl apply a resource without a namespace defined)
	// Don't do this for cluster scoped resources
	if !resourceSpec.ClusterScoped {
		if objMeta.GetNamespace() == "" && s.defaultNs != "" {
			scope.Source.Debugf("KubeSource.parseChunk: namespace not specified for %q, using %q", objMeta.GetName(), s.defaultNs)
			objMeta.SetNamespace(s.defaultNs)
		}
	} else {
		// Clear the namespace if there is any specified.
		objMeta.SetNamespace("")
	}

	item, err := t.ExtractResource(obj)
	if err != nil {
		return kubeResource{}, err
	}

	return kubeResource{
		spec:  resourceSpec,
		sha:   sha1.Sum(yamlChunk),
		entry: rt.ToResourceEntry(objMeta, &resourceSpec, item),
	}, nil
}
<commit_msg>Put chunk count back into error message in kubesource (#19358)<commit_after>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package inmemory

import (
	"bufio"
	"bytes"
	"crypto/sha1"
	"fmt"
	"io"
	"strings"
	"sync"

	"k8s.io/apimachinery/pkg/util/yaml"

	"github.com/hashicorp/go-multierror"
	kubeJson "k8s.io/apimachinery/pkg/runtime/serializer/json"

	"istio.io/istio/galley/pkg/config/event"
	"istio.io/istio/galley/pkg/config/meta/schema"
	"istio.io/istio/galley/pkg/config/meta/schema/collection"
	"istio.io/istio/galley/pkg/config/resource"
	"istio.io/istio/galley/pkg/config/scope"
	"istio.io/istio/galley/pkg/config/source/inmemory"
	"istio.io/istio/galley/pkg/config/source/kube/rt"
)

var inMemoryKubeNameDiscriminator int64

// KubeSource is an in-memory source implementation that can handle K8s style resources.
type KubeSource struct {
	mu sync.Mutex

	name      string
	resources schema.KubeResources
	source    *inmemory.Source
	defaultNs string

	versionCtr int64
	shas       map[kubeResourceKey]resourceSha
	byFile     map[string]map[kubeResourceKey]collection.Name
}

type resourceSha [sha1.Size]byte

type kubeResource struct {
	entry *resource.Entry
	spec  schema.KubeResource
	sha   resourceSha
}

func (r *kubeResource) newKey() kubeResourceKey {
	return kubeResourceKey{
		kind:     r.spec.Kind,
		fullName: r.entry.Metadata.Name,
	}
}

type kubeResourceKey struct {
	fullName resource.Name
	kind     string
}

var _ event.Source = &KubeSource{}

// NewKubeSource returns a new in-memory Source that works with Kubernetes resources.
func NewKubeSource(resources schema.KubeResources) *KubeSource {
	name := fmt.Sprintf("kube-inmemory-%d", inMemoryKubeNameDiscriminator)
	inMemoryKubeNameDiscriminator++

	s := inmemory.New(resources.Collections())

	return &KubeSource{
		name:      name,
		resources: resources,
		source:    s,
		shas:      make(map[kubeResourceKey]resourceSha),
		byFile:    make(map[string]map[kubeResourceKey]collection.Name),
	}
}

// SetDefaultNamespace enables injecting a default namespace for resources where none is already specified
func (s *KubeSource) SetDefaultNamespace(defaultNs string) {
	s.defaultNs = defaultNs
}

// Start implements processor.Source
func (s *KubeSource) Start() {
	s.source.Start()
}

// Stop implements processor.Source
func (s *KubeSource) Stop() {
	s.source.Stop()
}

// Clear the contents of this source
func (s *KubeSource) Clear() {
	s.versionCtr = 0
	s.shas = make(map[kubeResourceKey]resourceSha)
	s.byFile = make(map[string]map[kubeResourceKey]collection.Name)
	s.source.Clear()
}

// Dispatch implements processor.Source
func (s *KubeSource) Dispatch(h event.Handler) {
	s.source.Dispatch(h)
}

// Get returns the named collection.
func (s *KubeSource) Get(collection collection.Name) *inmemory.Collection {
	return s.source.Get(collection)
}

// ContentNames returns the names known to this source.
func (s *KubeSource) ContentNames() map[string]struct{} {
	s.mu.Lock()
	defer s.mu.Unlock()

	result := make(map[string]struct{})
	for n := range s.byFile {
		result[n] = struct{}{}
	}

	return result
}

// ApplyContent applies the given yamltext to this source. The content is tracked with the given name. If ApplyContent
// gets called multiple times with the same name, the contents applied by the previous incarnation will be overwritten
// or removed, depending on the new content.
// Returns an error if any were encountered, but that still may represent a partial success
func (s *KubeSource) ApplyContent(name, yamlText string) error {
	s.mu.Lock()
	defer s.mu.Unlock()

	// We hold off on dealing with parseErr until the end, since partial success is possible
	resources, parseErrs := s.parseContent(s.resources, name, yamlText)

	oldKeys := s.byFile[name]
	newKeys := make(map[kubeResourceKey]collection.Name)

	for _, r := range resources {
		key := r.newKey()

		oldSha, found := s.shas[key]
		if !found || oldSha != r.sha {
			s.versionCtr++
			r.entry.Metadata.Version = resource.Version(fmt.Sprintf("v%d", s.versionCtr))
			scope.Source.Debuga("KubeSource.ApplyContent: Set: ", r.spec.Collection.Name, r.entry.Metadata.Name)
			s.source.Get(r.spec.Collection.Name).Set(r.entry)
			s.shas[key] = r.sha
		}
		newKeys[key] = r.spec.Collection.Name
		if oldKeys != nil {
			scope.Source.Debuga("KubeSource.ApplyContent: Delete: ", r.spec.Collection.Name, key)
			delete(oldKeys, key)
		}
	}

	for k, col := range oldKeys {
		s.source.Get(col).Remove(k.fullName)
	}
	s.byFile[name] = newKeys

	if parseErrs != nil {
		return fmt.Errorf("errors parsing content %q: %v", name, parseErrs)
	}
	return nil
}

// RemoveContent removes the content for the given name
func (s *KubeSource) RemoveContent(name string) {
	s.mu.Lock()
	defer s.mu.Unlock()

	keys := s.byFile[name]
	if keys != nil {
		for key, col := range keys {
			s.source.Get(col).Remove(key.fullName)
			delete(s.shas, key)
		}

		delete(s.byFile, name)
	}
}

func (s *KubeSource) parseContent(r schema.KubeResources, name, yamlText string) ([]kubeResource, error) {
	var resources []kubeResource
	var errs error

	reader := bufio.NewReader(strings.NewReader(yamlText))
	decoder := yaml.NewYAMLReader(reader)
	chunkCount := -1

	for {
		chunkCount++
		doc, err := decoder.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			e := fmt.Errorf("error reading documents in %s[%d]: %v", name, chunkCount, err)
			scope.Source.Warnf("%v - skipping", e)
			scope.Source.Debugf("Failed to parse yamlText chunk: %v", yamlText)
			errs = multierror.Append(errs, e)
			break
		}

		chunk := bytes.TrimSpace(doc)
		r, err := s.parseChunk(r, chunk)
		if err != nil {
			e := fmt.Errorf("error processing %s[%d]: %v", name, chunkCount, err)
			scope.Source.Warnf("%v - skipping", e)
			scope.Source.Debugf("Failed to parse yaml chunk: %v", string(chunk))
			errs = multierror.Append(errs, e)
			continue
		}
		resources = append(resources, r)
	}

	return resources, errs
}

func (s *KubeSource) parseChunk(r schema.KubeResources, yamlChunk []byte) (kubeResource, error) {
	// Convert to JSON
	jsonChunk, err := yaml.ToJSON(yamlChunk)
	if err != nil {
		return kubeResource{}, fmt.Errorf("failed converting YAML to JSON: %v", err)
	}

	// Peek at the beginning of the JSON to
	groupVersionKind, err := kubeJson.DefaultMetaFactory.Interpret(jsonChunk)
	if err != nil {
		return kubeResource{}, fmt.Errorf("failed interpreting jsonChunk: %v", err)
	}

	resourceSpec, found := r.Find(groupVersionKind.Group, groupVersionKind.Kind)
	if !found {
		return kubeResource{}, fmt.Errorf("failed finding spec for group/kind: %s/%s", groupVersionKind.Group, groupVersionKind.Kind)
	}

	t := rt.DefaultProvider().GetAdapter(resourceSpec)
	obj, err := t.ParseJSON(jsonChunk)
	if err != nil {
		return kubeResource{}, fmt.Errorf("failed parsing JSON for built-in type: %v", err)
	}
	objMeta := t.ExtractObject(obj)

	// If namespace is blank and we have a default set, fill in the default
	// (This mirrors the behavior if you kubectl apply a resource without a namespace defined)
	// Don't do this for cluster scoped resources
	if !resourceSpec.ClusterScoped {
		if objMeta.GetNamespace() == "" && s.defaultNs != "" {
			scope.Source.Debugf("KubeSource.parseChunk: namespace not specified for %q, using %q", objMeta.GetName(), s.defaultNs)
			objMeta.SetNamespace(s.defaultNs)
		}
	} else {
		// Clear the namespace if there is any specified.
		objMeta.SetNamespace("")
	}

	item, err := t.ExtractResource(obj)
	if err != nil {
		return kubeResource{}, err
	}

	return kubeResource{
		spec:  resourceSpec,
		sha:   sha1.Sum(yamlChunk),
		entry: rt.ToResourceEntry(objMeta, &resourceSpec, item),
	}, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package integration

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"testing"
	"time"

	heapster_api "github.com/GoogleCloudPlatform/heapster/api/v1"
	heapster_sink_api "github.com/GoogleCloudPlatform/heapster/sinks/api"
	"github.com/GoogleCloudPlatform/heapster/sinks/cache"
	kube_api "github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/golang/glog"
	"github.com/stretchr/testify/require"
)

const (
	kTestZone        = "us-central1-b"
	targetTags       = "kubernetes-minion"
	heapsterBuildDir = "../deploy/docker"
	makefile         = "../Makefile"
)

var (
	kubeVersions           = flag.String("kube_versions", "", "Comma separated list of kube versions to test against. By default will run the test against an existing cluster")
	heapsterControllerFile = flag.String("heapster_controller", "../deploy/kube-config/standalone/heapster-controller.json", "Path to heapster replication controller file.")
	heapsterServiceFile    = flag.String("heapster_service", "../deploy/kube-config/standalone/heapster-service.json", "Path to heapster service file.")
	heapsterImage          = flag.String("heapster_image", "heapster:e2e_test", "heapster docker image that needs to be tested.")
	avoidBuild             = flag.Bool("nobuild", false, "When true, a new heapster docker image will not be created and pushed to test cluster nodes.")
	namespace              = flag.String("namespace", "default", "namespace to be used for testing")
	maxRetries             = flag.Int("retries", 10, "Number of attempts before failing this test.")
	runForever             = flag.Bool("run_forever", false, "If true, the tests are run in a loop forever.")
)

func deleteAll(fm kubeFramework, ns string, service *kube_api.Service, rc *kube_api.ReplicationController) {
	if err := fm.DeleteRC(ns, rc); err != nil {
		glog.Error(err)
	}

	if err := fm.DeleteService(ns, service); err != nil {
		glog.Error(err)
	}
}

func createAll(fm kubeFramework, ns string, service **kube_api.Service, rc **kube_api.ReplicationController) error {
	if newRc, err := fm.CreateRC(ns, *rc); err != nil {
		return err
	} else {
		*rc = newRc
	}

	if newSvc, err := fm.CreateService(ns, *service); err != nil {
		return err
	} else {
		*service = newSvc
	}
	return nil
}

func removeHeapsterImage(fm kubeFramework) error {
	if err := removeDockerImage(*heapsterImage); err != nil {
		glog.Error(err)
	}
	if nodes, err := fm.GetNodes(); err == nil {
		for _, node := range nodes {
			host := strings.Split(node, ".")[0]
			cleanupRemoteHost(host, kTestZone)
		}
	} else {
		glog.Errorf("failed to cleanup nodes - %v", err)
	}
	return nil
}

func buildAndPushHeapsterImage(hostnames []string) error {
	curwd, err := os.Getwd()
	if err != nil {
		return err
	}
	if err := os.Chdir(heapsterBuildDir); err != nil {
		return err
	}
	if err := make("build", path.Join(curwd, makefile)); err != nil {
		return err
	}
	if err := buildDockerImage(*heapsterImage); err != nil {
		return err
	}
	for _, host := range hostnames {
		if err := copyDockerImage(*heapsterImage, host, kTestZone); err != nil {
			return err
		}
	}
	glog.V(2).Info("built and pushed heapster image")
	return os.Chdir(curwd)
}

func getHeapsterRcAndSvc(fm kubeFramework) (*kube_api.Service, *kube_api.ReplicationController, error) {
	// Add test docker image
	rc, err := fm.ParseRC(*heapsterControllerFile)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to parse heapster controller - %v", err)
	}
	rc.Spec.Template.Spec.Containers[0].Image = *heapsterImage
	rc.Spec.Template.Spec.Containers[0].ImagePullPolicy = kube_api.PullNever
	// increase logging level
	rc.Spec.Template.Spec.Containers[0].Env = append(rc.Spec.Template.Spec.Containers[0].Env, kube_api.EnvVar{Name: "FLAGS", Value: "--vmodule=*=3"})
	svc, err := fm.ParseService(*heapsterServiceFile)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to parse heapster service - %v", err)
	}

	return svc, rc, nil
}

func buildAndPushDockerImages(fm kubeFramework) error {
	if *avoidBuild {
		return nil
	}
	nodes, err := fm.GetNodes()
	if err != nil {
		return err
	}
	hostnames := []string{}
	for _, node := range nodes {
		hostnames = append(hostnames, strings.Split(node, ".")[0])
	}

	return buildAndPushHeapsterImage(hostnames)
}

const metricsEndpoint = "/api/v1/metric-export"

func runHeapsterMetricsTest(fm kubeFramework, svc *kube_api.Service, expectedNodes, expectedPods []string) error {
	body, err := fm.Client().Get().
		Namespace(svc.Namespace).
		Prefix("proxy").
		Resource("services").
		Name(svc.Name).
		Suffix(metricsEndpoint).
		Do().Raw()
	if err != nil {
		return err
	}
	var timeseries []*heapster_api.Timeseries
	if err := json.Unmarshal(body, &timeseries); err != nil {
		glog.V(2).Infof("response body: %v", string(body))
		return err
	}

	if len(timeseries) == 0 {
		return fmt.Errorf("expected non zero timeseries")
	}
	actualPods := map[string]bool{}
	actualNodes := map[string]bool{}
	for _, point := range timeseries {
		if podName, ok := point.Labels[heapster_sink_api.LabelPodName]; ok {
			actualPods[podName] = true

		} else if cName, ok := point.Labels[heapster_sink_api.LabelContainerName]; ok && cName == cache.NodeContainerName {
			hostname, ok := point.Labels[heapster_sink_api.LabelHostname]
			if !ok {
				return fmt.Errorf("hostname label missing on node container %+v", point.Labels)
			}
			actualNodes[hostname] = true
		}
	}
	if !expectedItemsExist(expectedPods, actualPods) {
		return fmt.Errorf("expected pods don't exist.\nExpected: %v\nActual:%v", expectedPods, actualPods)
	}
	if !expectedItemsExist(expectedNodes, actualNodes) {
		return fmt.Errorf("expected nodes don't exist.\nExpected: %v\nActual:%v", expectedNodes, actualNodes)
	}

	return nil
}

func expectedItemsExist(expectedItems []string, actualItems map[string]bool) bool {
	if len(actualItems) < len(expectedItems) {
		return false
	}
	for _, item := range expectedItems {
		if _, found := actualItems[item]; !found {
			return false
		}
	}
	return true
}

func apiTest(kubeVersion string) error {
	fm, err := newKubeFramework(kubeVersion)
	if err != nil {
		return err
	}
	if err := buildAndPushDockerImages(fm); err != nil {
		return err
	}
	// Create heapster pod and service.
	svc, rc, err := getHeapsterRcAndSvc(fm)
	if err != nil {
		return err
	}
	ns := *namespace
	deleteAll(fm, ns, svc, rc)
	if err = createAll(fm, ns, &svc, &rc); err != nil {
		return err
	}
	defer func() {
		deleteAll(fm, ns, svc, rc)
		removeHeapsterImage(fm)
	}()
	if err := fm.WaitUntilPodRunning(ns, rc.Spec.Template.Labels, time.Minute); err != nil {
		return err
	}
	if err := fm.WaitUntilServiceActive(svc, time.Minute); err != nil {
		return err
	}
	expectedPods, err := fm.GetPods()
	if err != nil {
		return err
	}
	expectedNodes, err := fm.GetNodes()
	if err != nil {
		return err
	}
	attempts := *maxRetries
	for {
		err := runHeapsterMetricsTest(fm, svc, expectedNodes, expectedPods)
		if *runForever {
			continue
		}
		if err == nil {
			return nil
		}
		if attempts == 0 {
			return err
		}
		attempts--
	}
}

func runApiTest() error {
	tempDir, err := ioutil.TempDir("", "deploy")
	if err != nil {
		return nil
	}
	defer os.RemoveAll(tempDir)
	if *kubeVersions == "" {
		return apiTest("")
	}
	kubeVersionsList := strings.Split(*kubeVersions, ",")
	for _, kubeVersion := range kubeVersionsList {
		return apiTest(kubeVersion)
	}
	return nil
}

func TestHeapster(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping heapster kubernetes integration test.")
	}
	require.NoError(t, runApiTest())
}
<commit_msg>Add a sleep between test retries.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package integration

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"testing"
	"time"

	heapster_api "github.com/GoogleCloudPlatform/heapster/api/v1"
	heapster_sink_api "github.com/GoogleCloudPlatform/heapster/sinks/api"
	"github.com/GoogleCloudPlatform/heapster/sinks/cache"
	kube_api "github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/golang/glog"
	"github.com/stretchr/testify/require"
)

const (
	kTestZone        = "us-central1-b"
	targetTags       = "kubernetes-minion"
	heapsterBuildDir = "../deploy/docker"
	makefile         = "../Makefile"
)

var (
	kubeVersions           = flag.String("kube_versions", "", "Comma separated list of kube versions to test against. By default will run the test against an existing cluster")
	heapsterControllerFile = flag.String("heapster_controller", "../deploy/kube-config/standalone/heapster-controller.json", "Path to heapster replication controller file.")
	heapsterServiceFile    = flag.String("heapster_service", "../deploy/kube-config/standalone/heapster-service.json", "Path to heapster service file.")
	heapsterImage          = flag.String("heapster_image", "heapster:e2e_test", "heapster docker image that needs to be tested.")
	avoidBuild             = flag.Bool("nobuild", false, "When true, a new heapster docker image will not be created and pushed to test cluster nodes.")
	namespace              = flag.String("namespace", "default", "namespace to be used for testing")
	maxRetries             = flag.Int("retries", 100, "Number of attempts before failing this test.")
	runForever             = flag.Bool("run_forever", false, "If true, the tests are run in a loop forever.")
)

func deleteAll(fm kubeFramework, ns string, service *kube_api.Service, rc *kube_api.ReplicationController) {
	if err := fm.DeleteRC(ns, rc); err != nil {
		glog.Error(err)
	}

	if err := fm.DeleteService(ns, service); err != nil {
		glog.Error(err)
	}
}

func createAll(fm kubeFramework, ns string, service **kube_api.Service, rc **kube_api.ReplicationController) error {
	if newRc, err := fm.CreateRC(ns, *rc); err != nil {
		return err
	} else {
		*rc = newRc
	}

	if newSvc, err := fm.CreateService(ns, *service); err != nil {
		return err
	} else {
		*service = newSvc
	}
	return nil
}

func removeHeapsterImage(fm kubeFramework) error {
	if err := removeDockerImage(*heapsterImage); err != nil {
		glog.Error(err)
	}
	if nodes, err := fm.GetNodes(); err == nil {
		for _, node := range nodes {
			host := strings.Split(node, ".")[0]
			cleanupRemoteHost(host, kTestZone)
		}
	} else {
		glog.Errorf("failed to cleanup nodes - %v", err)
	}
	return nil
}

func buildAndPushHeapsterImage(hostnames []string) error {
	curwd, err := os.Getwd()
	if err != nil {
		return err
	}
	if err := os.Chdir(heapsterBuildDir); err != nil {
		return err
	}
	if err := make("build", path.Join(curwd, makefile)); err != nil {
		return err
	}
	if err := buildDockerImage(*heapsterImage); err != nil {
		return err
	}
	for _, host := range hostnames {
		if err := copyDockerImage(*heapsterImage, host, kTestZone); err != nil {
			return err
		}
	}
	glog.V(2).Info("built and pushed heapster image")
	return os.Chdir(curwd)
}

func getHeapsterRcAndSvc(fm kubeFramework) (*kube_api.Service, *kube_api.ReplicationController, error) {
	// Add test docker image
	rc, err := fm.ParseRC(*heapsterControllerFile)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to parse heapster controller - %v", err)
	}
	rc.Spec.Template.Spec.Containers[0].Image = *heapsterImage
	rc.Spec.Template.Spec.Containers[0].ImagePullPolicy = kube_api.PullNever
	// increase logging level
	rc.Spec.Template.Spec.Containers[0].Env = append(rc.Spec.Template.Spec.Containers[0].Env, kube_api.EnvVar{Name: "FLAGS", Value: "--vmodule=*=3"})
	svc, err := fm.ParseService(*heapsterServiceFile)
	if err != nil {
		return nil, nil, fmt.Errorf("failed to parse heapster service - %v", err)
	}

	return svc, rc, nil
}

func buildAndPushDockerImages(fm kubeFramework) error {
	if *avoidBuild {
		return nil
	}
	nodes, err := fm.GetNodes()
	if err != nil {
		return err
	}
	hostnames := []string{}
	for _, node := range nodes {
		hostnames = append(hostnames, strings.Split(node, ".")[0])
	}

	return buildAndPushHeapsterImage(hostnames)
}

const metricsEndpoint = "/api/v1/metric-export"

func runHeapsterMetricsTest(fm kubeFramework, svc *kube_api.Service, expectedNodes, expectedPods []string) error {
	body, err := fm.Client().Get().
		Namespace(svc.Namespace).
		Prefix("proxy").
		Resource("services").
		Name(svc.Name).
		Suffix(metricsEndpoint).
		Do().Raw()
	if err != nil {
		return err
	}
	var timeseries []*heapster_api.Timeseries
	if err := json.Unmarshal(body, &timeseries); err != nil {
		glog.V(2).Infof("response body: %v", string(body))
		return err
	}

	if len(timeseries) == 0 {
		return fmt.Errorf("expected non zero timeseries")
	}
	actualPods := map[string]bool{}
	actualNodes := map[string]bool{}
	for _, point := range timeseries {
		if podName, ok := point.Labels[heapster_sink_api.LabelPodName]; ok {
			actualPods[podName] = true

		} else if cName, ok := point.Labels[heapster_sink_api.LabelContainerName]; ok && cName == cache.NodeContainerName {
			hostname, ok := point.Labels[heapster_sink_api.LabelHostname]
			if !ok {
				return fmt.Errorf("hostname label missing on node container %+v", point.Labels)
			}
			actualNodes[hostname] = true
		}
	}
	if !expectedItemsExist(expectedPods, actualPods) {
		return fmt.Errorf("expected pods don't exist.\nExpected: %v\nActual:%v", expectedPods, actualPods)
	}
	if !expectedItemsExist(expectedNodes, actualNodes) {
		return fmt.Errorf("expected nodes don't exist.\nExpected: %v\nActual:%v", expectedNodes, actualNodes)
	}

	return nil
}

func expectedItemsExist(expectedItems []string, actualItems map[string]bool) bool {
	if len(actualItems) < len(expectedItems) {
		return false
	}
	for _, item := range expectedItems {
		if _, found := actualItems[item]; !found {
			return false
		}
	}
	return true
}

func apiTest(kubeVersion string) error {
	fm, err := newKubeFramework(kubeVersion)
	if err != nil {
		return err
	}
	if err := buildAndPushDockerImages(fm); err != nil {
		return err
	}
	// Create heapster pod and service.
	svc, rc, err := getHeapsterRcAndSvc(fm)
	if err != nil {
		return err
	}
	ns := *namespace
	deleteAll(fm, ns, svc, rc)
	if err = createAll(fm, ns, &svc, &rc); err != nil {
		return err
	}
	defer func() {
		deleteAll(fm, ns, svc, rc)
		removeHeapsterImage(fm)
	}()
	if err := fm.WaitUntilPodRunning(ns, rc.Spec.Template.Labels, time.Minute); err != nil {
		return err
	}
	if err := fm.WaitUntilServiceActive(svc, time.Minute); err != nil {
		return err
	}
	expectedPods, err := fm.GetPods()
	if err != nil {
		return err
	}
	expectedNodes, err := fm.GetNodes()
	if err != nil {
		return err
	}
	attempts := *maxRetries
	for {
		err := runHeapsterMetricsTest(fm, svc, expectedNodes, expectedPods)
		if *runForever {
			continue
		}
		if err == nil {
			return nil
		}
		if attempts == 0 {
			return err
		}
		attempts--
		time.Sleep(time.Second)
	}
}

func runApiTest() error {
	tempDir, err := ioutil.TempDir("", "deploy")
	if err != nil {
		return nil
	}
	defer os.RemoveAll(tempDir)
	if *kubeVersions == "" {
		return apiTest("")
	}
	kubeVersionsList := strings.Split(*kubeVersions, ",")
	for _, kubeVersion := range kubeVersionsList {
		return apiTest(kubeVersion)
	}
	return nil
}

func TestHeapster(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping heapster kubernetes integration test.")
	}
	require.NoError(t, runApiTest())
}
<|endoftext|>


<commit_before>package openweathermap

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

// New returns a new WeatherData pointer with the supplied
// arguments.
func New(unit string) (*WeatherData, error) {
	unitChoice := strings.ToLower(unit)
	for _, i := range dataUnits {
		if strings.Contains(unitChoice, i) {
			return &WeatherData{Units: unitChoice}, nil
		}
	}
	return nil, errors.New("ERROR: unit of measure not available")
}

// CurrentByName will provide the current weather with the
// provided location name.
func (w *WeatherData) CurrentByName(location string) {
	response, err := http.Get(fmt.Sprintf(fmt.Sprintf(baseUrl, "q=%s&units=%s"), location, w.Units))
	if err != nil {
		log.Fatalln(err)
	}
	defer response.Body.Close()

	result, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
	}

	err = json.Unmarshal(result, &w)
	if err != nil {
		log.Fatalln(err)
	}
}

// CurrentByCoordinates will provide the current weather with the
// provided location coordinates.
func (w *WeatherData) CurrentByCoordinates(location *Coordinates) {
	response, err := http.Get(fmt.Sprintf(fmt.Sprintf(baseUrl, "lat=%f&lon=%f&units=%s"), location.Latitude, location.Longitude, w.Units))
	if err != nil {
		log.Fatalln(err)
	}
	defer response.Body.Close()

	result, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
	}

	err = json.Unmarshal(result, &w)
	if err != nil {
		log.Fatalln(err)
	}
}

// CurrentByID will provide the current weather with the
// provided location ID.
func (w *WeatherData) CurrentByID(id int) {
	response, err := http.Get(fmt.Sprintf(fmt.Sprintf(baseUrl, "id=%d&units=%s"), id, w.Units))
	if err != nil {
		log.Fatalln(err)
	}
	defer response.Body.Close()

	result, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
	}

	err = json.Unmarshal(result, &w)
	if err != nil {
		log.Fatalln(err)
	}
}

func (w *WeatherData) GetByArea() {}
<commit_msg>fix major bug caused by struct rename<commit_after>package openweathermap

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

// New returns a new WeatherData pointer with the supplied
// arguments.
func New(unit string) (*CurrentWeatherData, error) {
	unitChoice := strings.ToLower(unit)
	for _, i := range dataUnits {
		if strings.Contains(unitChoice, i) {
			return &CurrentWeatherData{Units: unitChoice}, nil
		}
	}
	return nil, errors.New("ERROR: unit of measure not available")
}

// CurrentByName will provide the current weather with the
// provided location name.
func (w *CurrentWeatherData) CurrentByName(location string) {
	response, err := http.Get(fmt.Sprintf(fmt.Sprintf(baseUrl, "q=%s&units=%s"), location, w.Units))
	if err != nil {
		log.Fatalln(err)
	}
	defer response.Body.Close()

	result, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
	}

	err = json.Unmarshal(result, &w)
	if err != nil {
		log.Fatalln(err)
	}
}

// CurrentByCoordinates will provide the current weather with the
// provided location coordinates.
func (w *CurrentWeatherData) CurrentByCoordinates(location *Coordinates) {
	response, err := http.Get(fmt.Sprintf(fmt.Sprintf(baseUrl, "lat=%f&lon=%f&units=%s"), location.Latitude, location.Longitude, w.Units))
	if err != nil {
		log.Fatalln(err)
	}
	defer response.Body.Close()

	result, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
	}

	err = json.Unmarshal(result, &w)
	if err != nil {
		log.Fatalln(err)
	}
}

// CurrentByID will provide the current weather with the
// provided location ID.
func (w *CurrentWeatherData) CurrentByID(id int) {
	response, err := http.Get(fmt.Sprintf(fmt.Sprintf(baseUrl, "id=%d&units=%s"), id, w.Units))
	if err != nil {
		log.Fatalln(err)
	}
	defer response.Body.Close()

	result, err := ioutil.ReadAll(response.Body)
	if err != nil {
		fmt.Println(err)
	}

	err = json.Unmarshal(result, &w)
	if err != nil {
		log.Fatalln(err)
	}
}

func (w *CurrentWeatherData) GetByArea() {}
<|endoftext|>


<commit_before>package xxh32_test

import (
	"encoding/binary"
	"hash/crc32"
	"hash/fnv"
	"testing"

	"github.com/pierrec/lz4/v4/internal/xxh32"
)

type test struct {
	sum  uint32
	data string
}

var testdata = []test{
	{0x02cc5d05, ""},
	{0x550d7456, "a"},
	{0x4999fc53, "ab"},
	{0x32d153ff, "abc"},
	{0xa3643705, "abcd"},
	{0x9738f19b, "abcde"},
	{0x8b7cd587, "abcdef"},
	{0x9dd093b3, "abcdefg"},
	{0x0bb3c6bb, "abcdefgh"},
	{0xd03c13fd, "abcdefghi"},
	{0x8b988cfe, "abcdefghij"},
	{0x9d2d8b62, "abcdefghijklmnop"},
	{0x42ae804d, "abcdefghijklmnopqrstuvwxyz0123456789"},
	{0x62b4ed00, "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."},
}

func TestZeroBlockSize(t *testing.T) {
	var xxh xxh32.XXHZero
	if s := xxh.BlockSize(); s <= 0 {
		t.Errorf("invalid BlockSizeIndex: %d", s)
	}
}

func TestZeroSize(t *testing.T) {
	var xxh xxh32.XXHZero
	if s := xxh.Size(); s != 4 {
		t.Errorf("invalid Size: got %d expected 4", s)
	}
}

func TestZeroData(t *testing.T) {
	for _, td := range testdata {
		var xxh xxh32.XXHZero
		data := []byte(td.data)
		_, _ = xxh.Write(data)

		if got, want := xxh.Sum32(), td.sum; got != want {
			t.Fatalf("got %d; want %d", got, want)
		}
		if got, want := xxh32.ChecksumZero(data), td.sum; got != want {
			t.Fatalf("got %d; want %d", got, want)
		}
	}
}

func TestZeroSplitData(t *testing.T) {
	for _, td := range testdata {
		var xxh xxh32.XXHZero
		data := []byte(td.data)
		l := len(data) / 2
		_, _ = xxh.Write(data[0:l])
		_, _ = xxh.Write(data[l:])

		if got, want := xxh.Sum32(), td.sum; got != want {
			t.Fatalf("got %d; want %d", got, want)
		}
	}
}

func TestZeroSum(t *testing.T) {
	for _, td := range testdata {
		var xxh xxh32.XXHZero
		data := []byte(td.data)
		_, _ = xxh.Write(data)
		b := xxh.Sum(data)
		h := binary.LittleEndian.Uint32(b[len(data):])
		if got, want := h, td.sum; got != want {
			t.Fatalf("got %d; want %d", got, want)
		}
	}
}

func TestZeroChecksum(t *testing.T) {
	for _, td := range testdata {
		data := []byte(td.data)
		h := xxh32.ChecksumZero(data)
		if got, want := h, td.sum; got != want {
			t.Fatalf("got %d; want %d", got, want)
		}
	}
}

func TestZeroReset(t *testing.T) {
	var xxh xxh32.XXHZero
	for _, td := range testdata {
		_, _ = xxh.Write([]byte(td.data))
		h := xxh.Sum32()
		if got, want := h, td.sum; got != want {
			t.Fatalf("got %d; want %d", got, want)
		}
		xxh.Reset()
	}
}

///////////////////////////////////////////////////////////////////////////////
// Benchmarks
//
var testdata1 = []byte(testdata[len(testdata)-1].data)

func Benchmark_XXH32(b *testing.B) {
	var h xxh32.XXHZero
	for n := 0; n < b.N; n++ {
		_, _ = h.Write(testdata1)
		h.Sum32()
		h.Reset()
	}
}

func Benchmark_XXH32_Checksum(b *testing.B) {
	for n := 0; n < b.N; n++ {
		xxh32.ChecksumZero(testdata1)
	}
}

func Benchmark_CRC32(b *testing.B) {
	t := crc32.MakeTable(0)
	for i := 0; i < b.N; i++ {
		crc32.Checksum(testdata1, t)
	}
}

func Benchmark_Fnv32(b *testing.B) {
	h := fnv.New32()
	for i := 0; i < b.N; i++ {
		_, _ = h.Write(testdata1)
		h.Sum32()
		h.Reset()
	}
}
<commit_msg>xxh32: Print test failures in hex for easy searching<commit_after>package xxh32_test

import (
	"encoding/binary"
	"hash/crc32"
	"hash/fnv"
	"testing"

	"github.com/pierrec/lz4/v4/internal/xxh32"
)

type test struct {
	sum  uint32
	data string
}

var testdata = []test{
	{0x02cc5d05, ""},
	{0x550d7456, "a"},
	{0x4999fc53, "ab"},
	{0x32d153ff, "abc"},
	{0xa3643705, "abcd"},
	{0x9738f19b, "abcde"},
	{0x8b7cd587, "abcdef"},
	{0x9dd093b3, "abcdefg"},
	{0x0bb3c6bb, "abcdefgh"},
	{0xd03c13fd, "abcdefghi"},
	{0x8b988cfe, "abcdefghij"},
	{0x9d2d8b62, "abcdefghijklmnop"},
	{0x42ae804d, "abcdefghijklmnopqrstuvwxyz0123456789"},
	{0x62b4ed00, "Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."},
}

func TestZeroBlockSize(t *testing.T) {
	var xxh xxh32.XXHZero
	if s := xxh.BlockSize(); s <= 0 {
		t.Errorf("invalid BlockSizeIndex: %d", s)
	}
}

func TestZeroSize(t *testing.T) {
	var xxh xxh32.XXHZero
	if s := xxh.Size(); s != 4 {
		t.Errorf("invalid Size: got %d expected 4", s)
	}
}

func TestZeroData(t *testing.T) {
	for _, td := range testdata {
		var xxh xxh32.XXHZero
		data := []byte(td.data)
		_, _ = xxh.Write(data)

		if got, want := xxh.Sum32(), td.sum; got != want {
			t.Fatalf("got %x; want %x", got, want)
		}
		if got, want := xxh32.ChecksumZero(data), td.sum; got != want {
			t.Fatalf("got %x; want %x", got, want)
		}
	}
}

func TestZeroSplitData(t *testing.T) {
	for _, td := range testdata {
		var xxh xxh32.XXHZero
		data := []byte(td.data)
		l := len(data) / 2
		_, _ = xxh.Write(data[0:l])
		_, _ = xxh.Write(data[l:])

		if got, want := xxh.Sum32(), td.sum; got != want {
			t.Fatalf("got %x; want %x", got, want)
		}
	}
}

func TestZeroSum(t *testing.T) {
	for _, td := range testdata {
		var xxh xxh32.XXHZero
		data := []byte(td.data)
		_, _ = xxh.Write(data)
		b := xxh.Sum(data)
		h := binary.LittleEndian.Uint32(b[len(data):])
		if got, want := h, td.sum; got != want {
			t.Fatalf("got %x; want %x", got, want)
		}
	}
}

func TestZeroChecksum(t *testing.T) {
	for _, td := range testdata {
		data := []byte(td.data)
		h := xxh32.ChecksumZero(data)
		if got, want := h, td.sum; got != want {
			t.Fatalf("got %x; want %x", got, want)
		}
	}
}

func TestZeroReset(t *testing.T) {
	var xxh xxh32.XXHZero
	for _, td := range testdata {
		_, _ = xxh.Write([]byte(td.data))
		h := xxh.Sum32()
		if got, want := h, td.sum; got != want {
			t.Fatalf("got %x; want %x", got, want)
		}
		xxh.Reset()
	}
}

///////////////////////////////////////////////////////////////////////////////
// Benchmarks
//
var testdata1 = []byte(testdata[len(testdata)-1].data)

func Benchmark_XXH32(b *testing.B) {
	var h xxh32.XXHZero
	for n := 0; n < b.N; n++ {
		_, _ = h.Write(testdata1)
		h.Sum32()
		h.Reset()
	}
}

func Benchmark_XXH32_Checksum(b *testing.B) {
	for n := 0; n < b.N; n++ {
		xxh32.ChecksumZero(testdata1)
	}
}

func Benchmark_CRC32(b *testing.B) {
	t := crc32.MakeTable(0)
	for i := 0; i < b.N; i++ {
		crc32.Checksum(testdata1, t)
	}
}

func Benchmark_Fnv32(b *testing.B) {
	h := fnv.New32()
	for i := 0; i < b.N; i++ {
		_, _ = h.Write(testdata1)
		h.Sum32()
		h.Reset()
	}
}
<|endoftext|>


<commit_before>package decor

import (
	"fmt"
	"math"
	"time"
	"unicode/utf8"
)

const (
	// DidentRight specifies identation direction.
	// |foo   |b     | With DidentRight
	// |   foo|     b| Without DidentRight
	DidentRight = 1 << iota

	// DwidthSync same column width sync participation bit.
	// Effective on multiple bars only.
	DwidthSync

	// DextraSpace adds extra space, makes sense with DwidthSync only.
	// When DidentRight bit set, the space will be added to the right,
	// otherwise to the left.
	DextraSpace

	// DSyncSpace is shortcut for DwidthSync|DextraSpace
	DSyncSpace = DwidthSync | DextraSpace

	// DSyncSpace is shortcut for DwidthSync|DextraSpace|DidentRight
	DSyncSpaceR = DwidthSync | DextraSpace | DidentRight
)

// Statistics represents statistics of the progress bar.
// Cantains: Total, Current, TimeElapsed, TimePerItemEstimate
type Statistics struct {
	ID                  int
	Completed           bool
	Total               int64
	Current             int64
	StartTime           time.Time
	TimeElapsed         time.Duration
	TimePerItemEstimate time.Duration
}

// Eta returns exponential-weighted-moving-average ETA estimator
func (s *Statistics) Eta() time.Duration {
	return time.Duration(s.Total-s.Current) * s.TimePerItemEstimate
}

// DecoratorFunc is a function that can be prepended and appended to the progress bar
type DecoratorFunc func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string

// OnComplete returns decorator, which wraps provided `fn` decorator, with sole
// purpose to display final on complete message.
//
//	`fn` DecoratorFunc to wrap
//
//	`width` width to apply, if `DwidthSync` bit is not set
//
//	`conf` bit set config, [DidentRight|DwidthSync|DextraSpace]
func OnComplete(fn DecoratorFunc, message string, width, conf int) DecoratorFunc {
	msgDecorator := StaticName(message, width, conf)
	return func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		if s.Completed {
			return msgDecorator(s, widthAccumulator, widthDistributor)
		}
		return fn(s, widthAccumulator, widthDistributor)
	}
}

// StaticName returns static name/message decorator.
//
//	`name` string to display
//
//	`width` width to apply, if `DwidthSync` bit is not set
//
//	`conf` bit set config, [DidentRight|DwidthSync|DextraSpace]
func StaticName(name string, width, conf int) DecoratorFunc {
	nameFn := func(*Statistics) string {
		return name
	}
	return DynamicName(nameFn, width, conf)
}

// DynamicName returns dynamic name/message decorator.
//
//	`messageFn` callback function to get dynamic string message
//
//	`width` width to apply, if `DwidthSync` bit is not set
//
//	`conf` bit set config, [DidentRight|DwidthSync|DextraSpace]
func DynamicName(messageFn func(*Statistics) string, width, conf int) DecoratorFunc {
	format := "%%"
	if (conf & DidentRight) != 0 {
		format += "-"
	}
	format += "%ds"
	return func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		name := messageFn(s)
		if (conf & DwidthSync) != 0 {
			widthAccumulator <- utf8.RuneCountInString(name)
			max := <-widthDistributor
			if (conf & DextraSpace) != 0 {
				max++
			}
			return fmt.Sprintf(fmt.Sprintf(format, max), name)
		}
		return fmt.Sprintf(fmt.Sprintf(format, width), name)
	}
}

// CountersNoUnit returns raw counters decorator
//
//	`pairFormat` printf compatible verbs for current and total, like "%f" or "%d"
//
//	`width` width to apply, if `DwidthSync` bit is not set
//
//	`conf` bit set config, [DidentRight|DwidthSync|DextraSpace]
func CountersNoUnit(pairFormat string, width, conf int) DecoratorFunc {
	return counters(pairFormat, 0, width, conf)
}

// CountersKibiByte returns human friendly byte counters decorator, where counters unit is multiple by 1024.
//
//	`pairFormat` printf compatible verbs for current and total, like "%f" or "%d"
//
//	`width` width to apply, if `DwidthSync` bit is not set
//
//	`conf` bit set config, [DidentRight|DwidthSync|DextraSpace]
//
// pairFormat example:
//
//	"%.1f / %.1f" = "1.0MiB / 12.0MiB" or "% .1f / % .1f" = "1.0 MiB / 12.0 MiB"
func CountersKibiByte(pairFormat string, width, conf int) DecoratorFunc {
	return counters(pairFormat, unitKiB, width, conf)
}

// CountersKiloByte returns human friendly byte counters decorator, where counters unit is multiple by 1000.
//
//	`pairFormat` printf compatible verbs for current and total, like "%f" or "%d"
//
//	`width` width to apply, if `DwidthSync` bit is not set
//
//	`conf` bit set config, [DidentRight|DwidthSync|DextraSpace]
//
// pairFormat example:
//
//	"%.1f / %.1f" = "1.0MB / 12.0MB" or "% .1f / % .1f" = "1.0 MB / 12.0 MB"
func CountersKiloByte(pairFormat string, width, conf int) DecoratorFunc {
	return counters(pairFormat, unitKB, width, conf)
}

func counters(pairFormat string, unit counterUnit, width, conf int) DecoratorFunc {
	format := "%%"
	if (conf & DidentRight) != 0 {
		format += "-"
	}
	format += "%ds"
	return func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		var str string
		switch unit {
		case unitKiB:
			str = fmt.Sprintf(pairFormat, CounterKiB(s.Current), CounterKiB(s.Total))
		case unitKB:
			str = fmt.Sprintf(pairFormat, CounterKB(s.Current), CounterKB(s.Total))
		default:
			str = fmt.Sprintf(pairFormat, s.Current, s.Total)
		}
		if (conf & DwidthSync) != 0 {
			widthAccumulator <- utf8.RuneCountInString(str)
			max := <-widthDistributor
			if (conf & DextraSpace) != 0 {
				max++
			}
			return fmt.Sprintf(fmt.Sprintf(format, max), str)
		}
		return fmt.Sprintf(fmt.Sprintf(format, width), str)
	}
}

// ETA returns exponential-weighted-moving-average ETA decorator.
//
//	`width` width to apply, if `DwidthSync` bit is not set
//
//	`conf` bit set config, [DidentRight|DwidthSync|DextraSpace]
func ETA(width, conf int) DecoratorFunc {
	format := "%%"
	if (conf & DidentRight) != 0 {
		format += "-"
	}
	format += "%ds"
	return func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		str := fmt.Sprint(time.Duration(s.Eta().Seconds()) * time.Second)
		if (conf & DwidthSync) != 0 {
			widthAccumulator <- utf8.RuneCountInString(str)
			max := <-widthDistributor
			if (conf & DextraSpace) != 0 {
				max++
			}
			return fmt.Sprintf(fmt.Sprintf(format, max), str)
		}
		return fmt.Sprintf(fmt.Sprintf(format, width), str)
	}
}

// Elapsed returns elapsed time decorator.
//
//	`width` width to apply, if `DwidthSync` bit is not set
//
//	`conf` bit set config, [DidentRight|DwidthSync|DextraSpace]
func Elapsed(width, conf int) DecoratorFunc {
	format := "%%"
	if (conf & DidentRight) != 0 {
		format += "-"
	}
	format += "%ds"
	return func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		str := fmt.Sprint(time.Duration(s.TimeElapsed.Seconds()) * time.Second)
		if (conf & DwidthSync) != 0 {
			widthAccumulator <- utf8.RuneCountInString(str)
			max := <-widthDistributor
			if (conf & DextraSpace) != 0 {
				max++
			}
			return fmt.Sprintf(fmt.Sprintf(format, max), str)
		}
		return fmt.Sprintf(fmt.Sprintf(format, width), str)
	}
}

// Percentage returns percentage decorator.
//
//	`width` width to apply, if `DwidthSync` bit is not set
//
//	`conf` bit set config, [DidentRight|DwidthSync|DextraSpace]
func Percentage(width, conf int) DecoratorFunc {
	format := "%%"
	if (conf & DidentRight) != 0 {
		format += "-"
	}
	format += "%ds"
	return func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		str := fmt.Sprintf("%d %%", CalcPercentage(s.Total, s.Current, 100))
		if (conf & DwidthSync) != 0 {
			widthAccumulator <- utf8.RuneCountInString(str)
			max := <-widthDistributor
			if (conf & DextraSpace) != 0 {
				max++
			}
			return fmt.Sprintf(fmt.Sprintf(format, max), str)
		}
		return fmt.Sprintf(fmt.Sprintf(format, width), str)
	}
}

// CalcPercentage is a helper function, to calculate percentage.
func CalcPercentage(total, current, width int64) (perc int64) {
	if total <= 0 {
		return 0
	}
	if current > total {
		current = total
	}

	num := float64(width) * float64(current) / float64(total)
	ceil := math.Ceil(num)
	diff := ceil - num
	// num = 2.34 will return 2
	// num = 2.44 will return 3
	if math.Max(diff, 0.6) == diff {
		return int64(num)
	}
	return int64(ceil)
}
<commit_msg>DwidthSync description<commit_after>package decor

import (
	"fmt"
	"math"
	"time"
	"unicode/utf8"
)

const (
	// DidentRight specifies identation direction.
	// |foo   |b     | With DidentRight
	// |   foo|     b| Without DidentRight
	DidentRight = 1 << iota

	// DwidthSync bit enables same column width synchronization.
	// Effective on multiple bars only.
	DwidthSync

	// DextraSpace adds extra space, makes sense with DwidthSync only.
	// When DidentRight bit set, the space will be added to the right,
	// otherwise to the left.
	DextraSpace

	// DSyncSpace is shortcut for DwidthSync|DextraSpace
	DSyncSpace = DwidthSync | DextraSpace

	// DSyncSpace is shortcut for DwidthSync|DextraSpace|DidentRight
	DSyncSpaceR = DwidthSync | DextraSpace | DidentRight
)

// Statistics represents statistics of the progress bar.
// Cantains: Total, Current, TimeElapsed, TimePerItemEstimate
type Statistics struct {
	ID                  int
	Completed           bool
	Total               int64
	Current             int64
	StartTime           time.Time
	TimeElapsed         time.Duration
	TimePerItemEstimate time.Duration
}

// Eta returns exponential-weighted-moving-average ETA estimator
func (s *Statistics) Eta() time.Duration {
	return time.Duration(s.Total-s.Current) * s.TimePerItemEstimate
}

// DecoratorFunc is a function that can be prepended and appended to the progress bar
type DecoratorFunc func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string

// OnComplete returns decorator, which wraps provided `fn` decorator, with sole
// purpose to display final on complete message.
//
//	`fn` DecoratorFunc to wrap
//
//	`width` width to apply, if `DwidthSync` bit is not set
//
//	`conf` bit set config, [DidentRight|DwidthSync|DextraSpace]
func OnComplete(fn DecoratorFunc, message string, width, conf int) DecoratorFunc {
	msgDecorator := StaticName(message, width, conf)
	return func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		if s.Completed {
			return msgDecorator(s, widthAccumulator, widthDistributor)
		}
		return fn(s, widthAccumulator, widthDistributor)
	}
}

// StaticName returns static name/message decorator.
//
//	`name` string to display
//
//	`width` width to apply, if `DwidthSync` bit is not set
//
//	`conf` bit set config, [DidentRight|DwidthSync|DextraSpace]
func StaticName(name string, width, conf int) DecoratorFunc {
	nameFn := func(*Statistics) string {
		return name
	}
	return DynamicName(nameFn, width, conf)
}

// DynamicName returns dynamic name/message decorator.
//
//	`messageFn` callback function to get dynamic string message
//
//	`width` width to apply, if `DwidthSync` bit is not set
//
//	`conf` bit set config, [DidentRight|DwidthSync|DextraSpace]
func DynamicName(messageFn func(*Statistics) string, width, conf int) DecoratorFunc {
	format := "%%"
	if (conf & DidentRight) != 0 {
		format += "-"
	}
	format += "%ds"
	return func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		name := messageFn(s)
		if (conf & DwidthSync) != 0 {
			widthAccumulator <- utf8.RuneCountInString(name)
			max := <-widthDistributor
			if (conf & DextraSpace) != 0 {
				max++
			}
			return fmt.Sprintf(fmt.Sprintf(format, max), name)
		}
		return fmt.Sprintf(fmt.Sprintf(format, width), name)
	}
}

// CountersNoUnit returns raw counters decorator
//
//	`pairFormat` printf compatible verbs for current and total, like "%f" or "%d"
//
//	`width` width to apply, if `DwidthSync` bit is not set
//
//	`conf` bit set config, [DidentRight|DwidthSync|DextraSpace]
func CountersNoUnit(pairFormat string, width, conf int) DecoratorFunc {
	return counters(pairFormat, 0, width, conf)
}

// CountersKibiByte returns human friendly byte counters decorator, where counters unit is multiple by 1024.
//
//	`pairFormat` printf compatible verbs for current and total, like "%f" or "%d"
//
//	`width` width to apply, if `DwidthSync` bit is not set
//
//	`conf` bit set config, [DidentRight|DwidthSync|DextraSpace]
//
// pairFormat example:
//
//	"%.1f / %.1f" = "1.0MiB / 12.0MiB" or "% .1f / % .1f" = "1.0 MiB / 12.0 MiB"
func CountersKibiByte(pairFormat string, width, conf int) DecoratorFunc {
	return counters(pairFormat, unitKiB, width, conf)
}

// CountersKiloByte returns human friendly byte counters decorator, where counters unit is multiple by 1000.
//
//	`pairFormat` printf compatible verbs for current and total, like "%f" or "%d"
//
//	`width` width to apply, if `DwidthSync` bit is not set
//
//	`conf` bit set config, [DidentRight|DwidthSync|DextraSpace]
//
// pairFormat example:
//
//	"%.1f / %.1f" = "1.0MB / 12.0MB" or "% .1f / % .1f" = "1.0 MB / 12.0 MB"
func CountersKiloByte(pairFormat string, width, conf int) DecoratorFunc {
	return counters(pairFormat, unitKB, width, conf)
}

func counters(pairFormat string, unit counterUnit, width, conf int) DecoratorFunc {
	format := "%%"
	if (conf & DidentRight) != 0 {
		format += "-"
	}
	format += "%ds"
	return func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		var str string
		switch unit {
		case unitKiB:
			str = fmt.Sprintf(pairFormat, CounterKiB(s.Current), CounterKiB(s.Total))
		case unitKB:
			str = fmt.Sprintf(pairFormat, CounterKB(s.Current), CounterKB(s.Total))
		default:
			str = fmt.Sprintf(pairFormat, s.Current, s.Total)
		}
		if (conf & DwidthSync) != 0 {
			widthAccumulator <- utf8.RuneCountInString(str)
			max := <-widthDistributor
			if (conf & DextraSpace) != 0 {
				max++
			}
			return fmt.Sprintf(fmt.Sprintf(format, max), str)
		}
		return fmt.Sprintf(fmt.Sprintf(format, width), str)
	}
}

// ETA returns exponential-weighted-moving-average ETA decorator.
//
//	`width` width to apply, if `DwidthSync` bit is not set
//
//	`conf` bit set config, [DidentRight|DwidthSync|DextraSpace]
func ETA(width, conf int) DecoratorFunc {
	format := "%%"
	if (conf & DidentRight) != 0 {
		format += "-"
	}
	format += "%ds"
	return func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		str := fmt.Sprint(time.Duration(s.Eta().Seconds()) * time.Second)
		if (conf & DwidthSync) != 0 {
			widthAccumulator <- utf8.RuneCountInString(str)
			max := <-widthDistributor
			if (conf & DextraSpace) != 0 {
				max++
			}
			return fmt.Sprintf(fmt.Sprintf(format, max), str)
		}
		return fmt.Sprintf(fmt.Sprintf(format, width), str)
	}
}

// Elapsed returns elapsed time decorator.
//
//	`width` width to apply, if `DwidthSync` bit is not set
//
//	`conf` bit set config, [DidentRight|DwidthSync|DextraSpace]
func Elapsed(width, conf int) DecoratorFunc {
	format := "%%"
	if (conf & DidentRight) != 0 {
		format += "-"
	}
	format += "%ds"
	return func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		str := fmt.Sprint(time.Duration(s.TimeElapsed.Seconds()) * time.Second)
		if (conf & DwidthSync) != 0 {
			widthAccumulator <- utf8.RuneCountInString(str)
			max := <-widthDistributor
			if (conf & DextraSpace) != 0 {
				max++
			}
			return fmt.Sprintf(fmt.Sprintf(format, max), str)
		}
		return fmt.Sprintf(fmt.Sprintf(format, width), str)
	}
}

// Percentage returns percentage decorator.
//
//	`width` width to apply, if `DwidthSync` bit is not set
//
//	`conf` bit set config, [DidentRight|DwidthSync|DextraSpace]
func Percentage(width, conf int) DecoratorFunc {
	format := "%%"
	if (conf & DidentRight) != 0 {
		format += "-"
	}
	format += "%ds"
	return func(s *Statistics, widthAccumulator chan<- int, widthDistributor <-chan int) string {
		str := fmt.Sprintf("%d %%", CalcPercentage(s.Total, s.Current, 100))
		if (conf & DwidthSync) != 0 {
			widthAccumulator <- utf8.RuneCountInString(str)
			max := <-widthDistributor
			if (conf & DextraSpace) != 0 {
				max++
			}
			return fmt.Sprintf(fmt.Sprintf(format, max), str)
		}
		return fmt.Sprintf(fmt.Sprintf(format, width), str)
	}
}

// CalcPercentage is a helper function, to calculate percentage.
func CalcPercentage(total, current, width int64) (perc int64) {
	if total <= 0 {
		return 0
	}
	if current > total {
		current = total
	}

	num := float64(width) * float64(current) / float64(total)
	ceil := math.Ceil(num)
	diff := ceil - num
	// num = 2.34 will return 2
	// num = 2.44 will return 3
	if math.Max(diff, 0.6) == diff {
		return int64(num)
	}
	return int64(ceil)
}
<|endoftext|>


<commit_before>package cfclient

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
)

type DomainsResponse struct {
	Count     int              `json:"total_results"`
	Pages     int              `json:"total_pages"`
	NextUrl   string           `json:"next_url"`
	Resources []DomainResource `json:"resources"`
}

type DomainResource struct {
	Meta   Meta   `json:"metadata"`
	Entity Domain `json:"entity"`
}

type Domain struct {
	Guid                   string `json:"guid"`
	Name                   string `json:"name"`
	OwningOrganizationGuid string `json:"owning_organization_guid"`
	OwningOrganizationUrl  string `json:"owning_organization_url"`
	SharedOrganizationsUrl string `json:"shared_organizations_url"`
	c                      *Client
}

func (c *Client) ListDomainsByQuery(query url.Values) ([]Domain, error) {
	var domains []Domain
	requestUrl := "/v2/private_domains" + query.Encode()
	for {
		var domainResp DomainsResponse
		r := c.NewRequest("GET", requestUrl)
		resp, err := c.DoRequest(r)
		if err != nil {
			return nil, fmt.Errorf("Error requesting domains %v", err)
		}
		resBody, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, fmt.Errorf("Error reading domains request: %v", err)
		}

		err = json.Unmarshal(resBody, &domainResp)
		if err != nil {
			return nil, fmt.Errorf("Error unmarshaling domains %v", err)
		}
		for _, domain := range domainResp.Resources {
			domain.Entity.Guid = domain.Meta.Guid
			domain.Entity.c = c
			domains = append(domains, domain.Entity)
		}
		requestUrl = domainResp.NextUrl
		if requestUrl == "" {
			break
		}
	}
	return domains, nil
}

func (c *Client) ListDomains() ([]Domain, error) {
	return c.ListDomainsByQuery(nil)
}

func (c *Client) GetDomainByName(name string) (Domain, error) {
	q := url.Values{}
	q.Set("q", "name:"+name)
	domains, err := c.ListDomainsByQuery(q)
	if err != nil {
		return Domain{}, nil
	}
	if len(domains) == 0 {
		return Domain{}, fmt.Errorf("Unable to find domain %s", name)
	}
	return domains[0], nil
}

func (c *Client) CreateDomain(name, orgGuid string) (*Domain, error) {
	req := c.NewRequest("POST", "/v2/private_domains")
	req.obj = map[string]interface{}{
		"name": name,
		"owning_organization_guid": orgGuid,
	}
	resp, err := c.DoRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusCreated {
		return nil, fmt.Errorf("CF API returned with status code %d", resp.StatusCode)
	}
	return respBodyToDomain(resp.Body, c)
}

func (c *Client) DeleteDomain(guid string) error {
	resp, err := c.DoRequest(c.NewRequest("DELETE", fmt.Sprintf("/v2/private_domains/%s", guid)))
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusNoContent {
		return fmt.Errorf("CF API returned with status code %d", resp.StatusCode)
	}
	return nil
}

func respBodyToDomain(body io.ReadCloser, c *Client) (*Domain, error) {
	bodyRaw, err := ioutil.ReadAll(body)
	if err != nil {
		return nil, err
	}
	domainRes := DomainResource{}
	err = json.Unmarshal([]byte(bodyRaw), &domainRes)
	if err != nil {
		return nil, err
	}
	domain := domainRes.Entity
	domain.Guid = domainRes.Meta.Guid
	domain.c = c
	return &domain, nil
}
<commit_msg>Fix typo on domains url. (#56)<commit_after>package cfclient

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
)

type DomainsResponse struct {
	Count     int              `json:"total_results"`
	Pages     int              `json:"total_pages"`
	NextUrl   string           `json:"next_url"`
	Resources []DomainResource `json:"resources"`
}

type DomainResource struct {
	Meta   Meta   `json:"metadata"`
	Entity Domain `json:"entity"`
}

type Domain struct {
	Guid                   string `json:"guid"`
	Name                   string `json:"name"`
	OwningOrganizationGuid string `json:"owning_organization_guid"`
	OwningOrganizationUrl  string `json:"owning_organization_url"`
	SharedOrganizationsUrl string `json:"shared_organizations_url"`
	c                      *Client
}

func (c *Client) ListDomainsByQuery(query url.Values) ([]Domain, error) {
	var domains []Domain
	requestUrl := "/v2/private_domains?" + query.Encode()
	for {
		var domainResp DomainsResponse
		r := c.NewRequest("GET", requestUrl)
		resp, err := c.DoRequest(r)
		if err != nil {
			return nil, fmt.Errorf("Error requesting domains %v", err)
		}
		resBody, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return nil, fmt.Errorf("Error reading domains request: %v", err)
		}

		err = json.Unmarshal(resBody, &domainResp)
		if err != nil {
			return nil, fmt.Errorf("Error unmarshaling domains %v", err)
		}
		for _, domain := range domainResp.Resources {
			domain.Entity.Guid = domain.Meta.Guid
			domain.Entity.c = c
			domains = append(domains, domain.Entity)
		}
		requestUrl = domainResp.NextUrl
		if requestUrl == "" {
			break
		}
	}
	return domains, nil
}

func (c *Client) ListDomains() ([]Domain, error) {
	return c.ListDomainsByQuery(nil)
}

func (c *Client) GetDomainByName(name string) (Domain, error) {
	q := url.Values{}
	q.Set("q", "name:"+name)
	domains, err := c.ListDomainsByQuery(q)
	if err != nil {
		return Domain{}, nil
	}
	if len(domains) == 0 {
		return Domain{}, fmt.Errorf("Unable to find domain %s", name)
	}
	return domains[0], nil
}

func (c *Client) CreateDomain(name, orgGuid string) (*Domain, error) {
	req := c.NewRequest("POST", "/v2/private_domains")
	req.obj = map[string]interface{}{
		"name": name,
		"owning_organization_guid": orgGuid,
	}
	resp, err := c.DoRequest(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusCreated {
		return nil, fmt.Errorf("CF API returned with status code %d", resp.StatusCode)
	}
	return respBodyToDomain(resp.Body, c)
}

func (c *Client) DeleteDomain(guid string) error {
	resp, err := c.DoRequest(c.NewRequest("DELETE", fmt.Sprintf("/v2/private_domains/%s", guid)))
	if err != nil {
		return err
	}
	if resp.StatusCode != http.StatusNoContent {
		return fmt.Errorf("CF API returned with status code %d", resp.StatusCode)
	}
	return nil
}

func respBodyToDomain(body io.ReadCloser, c *Client) (*Domain, error) {
	bodyRaw, err := ioutil.ReadAll(body)
	if err != nil {
		return nil, err
	}
	domainRes := DomainResource{}
	err = json.Unmarshal([]byte(bodyRaw), &domainRes)
	if err != nil {
		return nil, err
	}
	domain := domainRes.Entity
	domain.Guid = domainRes.Meta.Guid
	domain.c = c
	return &domain, nil
}
<|endoftext|>


<commit_before>package terraform

import (
	"log"

	"github.com/hashicorp/terraform/addrs"
	"github.com/hashicorp/terraform/states"

	"github.com/hashicorp/terraform/configs"
	"github.com/hashicorp/terraform/dag"
)

// GraphNodeDestroyer must be implemented by nodes that destroy resources.
type GraphNodeDestroyer interface {
	dag.Vertex

	// DestroyAddr is the address of the resource that is being
	// destroyed by this node. If this returns nil, then this node
	// is not destroying anything.
	DestroyAddr() *addrs.AbsResourceInstance
}

// GraphNodeCreator must be implemented by nodes that create OR update resources.
type GraphNodeCreator interface {
	// CreateAddr is the address of the resource being created or updated
	CreateAddr() *addrs.AbsResourceInstance
}

// DestroyEdgeTransformer is a GraphTransformer that creates the proper
// references for destroy resources. Destroy resources are more complex
// in that they must be depend on the destruction of resources that
// in turn depend on the CREATION of the node being destroy.
//
// That is complicated. Visually:
//
//   B_d -> A_d -> A -> B
//
// Notice that A destroy depends on B destroy, while B create depends on
// A create. They're inverted. This must be done for example because often
// dependent resources will block parent resources from deleting. Concrete
// example: VPC with subnets, the VPC can't be deleted while there are
// still subnets.
type DestroyEdgeTransformer struct {
	// These are needed to properly build the graph of dependencies
	// to determine what a destroy node depends on. Any of these can be nil.
	Config *configs.Config
	State  *states.State

	// If configuration is present then Schemas is required in order to
	// obtain schema information from providers and provisioners in order
	// to properly resolve implicit dependencies.
	Schemas *Schemas
}

func (t *DestroyEdgeTransformer) Transform(g *Graph) error {
	// Build a map of what is being destroyed (by address string) to
	// the list of destroyers.
	destroyers := make(map[string][]GraphNodeDestroyer)

	// Record the creators, which will need to depend on the destroyers if they
	// are only being updated.
	creators := make(map[string]GraphNodeCreator)

	// destroyersByResource records each destroyer by the ConfigResource
	// address.  We use this because dependencies are only referenced as
	// resources and have no index or module instance information, but we will
	// want to connect all the individual instances for correct ordering.
	destroyersByResource := make(map[string][]GraphNodeDestroyer)
	for _, v := range g.Vertices() {
		switch n := v.(type) {
		case GraphNodeDestroyer:
			addrP := n.DestroyAddr()
			if addrP == nil {
				log.Printf("[WARN] DestroyEdgeTransformer: %q (%T) has no destroy address", dag.VertexName(n), v)
				continue
			}
			addr := *addrP

			key := addr.String()
			log.Printf("[TRACE] DestroyEdgeTransformer: %q (%T) destroys %s", dag.VertexName(n), v, key)
			destroyers[key] = append(destroyers[key], n)

			resAddr := addr.ContainingResource().Config().String()
			destroyersByResource[resAddr] = append(destroyersByResource[resAddr], n)
		case GraphNodeCreator:
			addr := n.CreateAddr()
			creators[addr.String()] = n
		}
	}

	// If we aren't destroying anything, there will be no edges to make
	// so just exit early and avoid future work.
	if len(destroyers) == 0 {
		return nil
	}

	// Connect destroy despendencies as stored in the state
	for _, ds := range destroyers {
		for _, des := range ds {
			ri, ok := des.(GraphNodeResourceInstance)
			if !ok {
				continue
			}

			for _, resAddr := range ri.StateDependencies() {
				for _, desDep := range destroyersByResource[resAddr.String()] {
					if !graphNodesAreResourceInstancesInDifferentInstancesOfSameModule(desDep, des) {
						log.Printf("[TRACE] DestroyEdgeTransformer: %s has stored dependency of %s\n", dag.VertexName(desDep), dag.VertexName(des))
						g.Connect(dag.BasicEdge(desDep, des))
					} else {
						log.Printf("[TRACE] DestroyEdgeTransformer: skipping %s => %s inter-module-instance dependency\n", dag.VertexName(desDep), dag.VertexName(des))
					}
				}
			}
		}
	}

	// connect creators to any destroyers on which they may depend
	for _, c := range creators {
		ri, ok := c.(GraphNodeResourceInstance)
		if !ok {
			continue
		}

		for _, resAddr := range ri.StateDependencies() {
			for _, desDep := range destroyersByResource[resAddr.String()] {
				if !graphNodesAreResourceInstancesInDifferentInstancesOfSameModule(c, desDep) {
					log.Printf("[TRACE] DestroyEdgeTransformer: %s has stored dependency of %s\n", dag.VertexName(c), dag.VertexName(desDep))
					g.Connect(dag.BasicEdge(c, desDep))
				} else {
					log.Printf("[TRACE] DestroyEdgeTransformer: skipping %s => %s inter-module-instance dependency\n", dag.VertexName(c), dag.VertexName(desDep))
				}
			}
		}
	}

	// Go through and connect creators to destroyers. Going along with
	// our example, this makes: A_d => A
	for _, v := range g.Vertices() {
		cn, ok := v.(GraphNodeCreator)
		if !ok {
			continue
		}

		addr := cn.CreateAddr()
		if addr == nil {
			continue
		}

		for _, d := range destroyers[addr.String()] {
			// For illustrating our example
			a_d := d.(dag.Vertex)
			a := v

			log.Printf(
				"[TRACE] DestroyEdgeTransformer: connecting creator %q with destroyer %q",
				dag.VertexName(a), dag.VertexName(a_d))

			g.Connect(dag.BasicEdge(a, a_d))
		}
	}

	return nil
}

// Remove any nodes that aren't needed when destroying modules.
// Variables, outputs, locals, and expanders may not be able to evaluate
// correctly, so we can remove these if nothing depends on them. The module
// closers also need to disable their use of expansion if the module itself is
// no longer present.
type pruneUnusedNodesTransformer struct {
}

func (t *pruneUnusedNodesTransformer) Transform(g *Graph) error {
	// We need a reverse depth first walk of modules, processing them in order
	// from the leaf modules to the root. This allows us to remove unneeded
	// dependencies from child modules, freeing up nodes in the parent module
	// to also be removed.

	nodes := g.Vertices()

	for removed := true; removed; {
		removed = false

		for i := 0; i < len(nodes); i++ {
			// run this in a closure, so we can return early rather than
			// dealing with complex looping and labels
			func() {
				n := nodes[i]
				switch n := n.(type) {
				case graphNodeTemporaryValue:
					// root module outputs indicate they are not temporary by
					// returning false here.
					if !n.temporaryValue() {
						return
					}

					// temporary values, which consist of variables, locals,
					// and outputs, must be kept if anything refers to them.
					for _, v := range g.UpEdges(n) {
						// keep any value which is connected through a
						// reference
						if _, ok := v.(GraphNodeReferencer); ok {
							return
						}
					}

				case graphNodeExpandsInstances:
					// Any nodes that expand instances are kept when their
					// instances may need to be evaluated.
					for _, v := range g.UpEdges(n) {
						switch v.(type) {
						case graphNodeExpandsInstances:
							// expanders can always depend on module expansion
							// themselves
							return
						case GraphNodeResourceInstance:
							// resource instances always depend on their
							// resource node, which is an expander
							return
						}
					}

				case GraphNodeProvider:
					// Providers that may have been required by expansion nodes
					// that we no longer need can also be removed.
					if g.UpEdges(n).Len() > 0 {
						return
					}

				default:
					return
				}

				log.Printf("[DEBUG] pruneUnusedNodes: %s is no longer needed, removing", dag.VertexName(n))
				g.Remove(n)
				removed = true

				// remove the node from our iteration as well
				last := len(nodes) - 1
				nodes[i], nodes[last] = nodes[last], nodes[i]
				nodes = nodes[:last]
			}()
		}
	}

	return nil
}
<commit_msg>connect destroyers to all stored create deps<commit_after>package terraform

import (
	"log"

	"github.com/hashicorp/terraform/addrs"
	"github.com/hashicorp/terraform/states"

	"github.com/hashicorp/terraform/configs"
	"github.com/hashicorp/terraform/dag"
)

// GraphNodeDestroyer must be implemented by nodes that destroy resources.
type GraphNodeDestroyer interface {
	dag.Vertex

	// DestroyAddr is the address of the resource that is being
	// destroyed by this node. If this returns nil, then this node
	// is not destroying anything.
	DestroyAddr() *addrs.AbsResourceInstance
}

// GraphNodeCreator must be implemented by nodes that create OR update resources.
type GraphNodeCreator interface {
	// CreateAddr is the address of the resource being created or updated
	CreateAddr() *addrs.AbsResourceInstance
}

// DestroyEdgeTransformer is a GraphTransformer that creates the proper
// references for destroy resources. Destroy resources are more complex
// in that they must be depend on the destruction of resources that
// in turn depend on the CREATION of the node being destroy.
//
// That is complicated. Visually:
//
//   B_d -> A_d -> A -> B
//
// Notice that A destroy depends on B destroy, while B create depends on
// A create. They're inverted. This must be done for example because often
// dependent resources will block parent resources from deleting. Concrete
// example: VPC with subnets, the VPC can't be deleted while there are
// still subnets.
type DestroyEdgeTransformer struct {
	// These are needed to properly build the graph of dependencies
	// to determine what a destroy node depends on. Any of these can be nil.
	Config *configs.Config
	State  *states.State

	// If configuration is present then Schemas is required in order to
	// obtain schema information from providers and provisioners in order
	// to properly resolve implicit dependencies.
	Schemas *Schemas
}

func (t *DestroyEdgeTransformer) Transform(g *Graph) error {
	// Build a map of what is being destroyed (by address string) to
	// the list of destroyers.
	destroyers := make(map[string][]GraphNodeDestroyer)

	// Record the creators, which will need to depend on the destroyers if they
	// are only being updated.
	creators := make(map[string][]GraphNodeCreator)

	// destroyersByResource records each destroyer by the ConfigResource
	// address.  We use this because dependencies are only referenced as
	// resources and have no index or module instance information, but we will
	// want to connect all the individual instances for correct ordering.
	destroyersByResource := make(map[string][]GraphNodeDestroyer)
	for _, v := range g.Vertices() {
		switch n := v.(type) {
		case GraphNodeDestroyer:
			addrP := n.DestroyAddr()
			if addrP == nil {
				log.Printf("[WARN] DestroyEdgeTransformer: %q (%T) has no destroy address", dag.VertexName(n), v)
				continue
			}
			addr := *addrP

			key := addr.String()
			log.Printf("[TRACE] DestroyEdgeTransformer: %q (%T) destroys %s", dag.VertexName(n), v, key)
			destroyers[key] = append(destroyers[key], n)

			resAddr := addr.ContainingResource().Config().String()
			destroyersByResource[resAddr] = append(destroyersByResource[resAddr], n)
		case GraphNodeCreator:
			addr := n.CreateAddr().ContainingResource().Config().String()
			creators[addr] = append(creators[addr], n)
		}
	}

	// If we aren't destroying anything, there will be no edges to make
	// so just exit early and avoid future work.
	if len(destroyers) == 0 {
		return nil
	}

	// Connect destroy despendencies as stored in the state
	for _, ds := range destroyers {
		for _, des := range ds {
			ri, ok := des.(GraphNodeResourceInstance)
			if !ok {
				continue
			}

			for _, resAddr := range ri.StateDependencies() {
				for _, desDep := range destroyersByResource[resAddr.String()] {
					if !graphNodesAreResourceInstancesInDifferentInstancesOfSameModule(desDep, des) {
						log.Printf("[TRACE] DestroyEdgeTransformer: %s has stored dependency of %s\n", dag.VertexName(desDep), dag.VertexName(des))
						g.Connect(dag.BasicEdge(desDep, des))
					} else {
						log.Printf("[TRACE] DestroyEdgeTransformer: skipping %s => %s inter-module-instance dependency\n", dag.VertexName(desDep), dag.VertexName(des))
					}
				}

				// We can have some create or update nodes which were
				// dependents of the destroy node. If they have no destroyer
				// themselves, make the connection directly from the creator.
				for _, createDep := range creators[resAddr.String()] {
					if !graphNodesAreResourceInstancesInDifferentInstancesOfSameModule(createDep, des) {
						log.Printf("[DEBUG] DestroyEdgeTransformer: %s has stored dependency of %s\n", dag.VertexName(createDep), dag.VertexName(des))
						g.Connect(dag.BasicEdge(createDep, des))
					} else {
						log.Printf("[TRACE] DestroyEdgeTransformer: skipping %s => %s inter-module-instance dependency\n", dag.VertexName(createDep), dag.VertexName(des))
					}
				}
			}
		}
	}

	// connect creators to any destroyers on which they may depend
	for _, cs := range creators {
		for _, c := range cs {
			ri, ok := c.(GraphNodeResourceInstance)
			if !ok {
				continue
			}

			for _, resAddr := range ri.StateDependencies() {
				for _, desDep := range destroyersByResource[resAddr.String()] {
					if !graphNodesAreResourceInstancesInDifferentInstancesOfSameModule(c, desDep) {
						log.Printf("[TRACE] DestroyEdgeTransformer: %s has stored dependency of %s\n", dag.VertexName(c), dag.VertexName(desDep))
						g.Connect(dag.BasicEdge(c, desDep))
					} else {
						log.Printf("[TRACE] DestroyEdgeTransformer: skipping %s => %s inter-module-instance dependency\n", dag.VertexName(c), dag.VertexName(desDep))
					}
				}
			}
		}
	}

	// Go through and connect creators to destroyers. Going along with
	// our example, this makes: A_d => A
	for _, v := range g.Vertices() {
		cn, ok := v.(GraphNodeCreator)
		if !ok {
			continue
		}

		addr := cn.CreateAddr()
		if addr == nil {
			continue
		}

		for _, d := range destroyers[addr.String()] {
			// For illustrating our example
			a_d := d.(dag.Vertex)
			a := v

			log.Printf(
				"[TRACE] DestroyEdgeTransformer: connecting creator %q with destroyer %q",
				dag.VertexName(a), dag.VertexName(a_d))

			g.Connect(dag.BasicEdge(a, a_d))
		}
	}

	return nil
}

// Remove any nodes that aren't needed when destroying modules.
// Variables, outputs, locals, and expanders may not be able to evaluate
// correctly, so we can remove these if nothing depends on them. The module
// closers also need to disable their use of expansion if the module itself is
// no longer present.
type pruneUnusedNodesTransformer struct {
}

func (t *pruneUnusedNodesTransformer) Transform(g *Graph) error {
	// We need a reverse depth first walk of modules, processing them in order
	// from the leaf modules to the root. This allows us to remove unneeded
	// dependencies from child modules, freeing up nodes in the parent module
	// to also be removed.

	nodes := g.Vertices()

	for removed := true; removed; {
		removed = false

		for i := 0; i < len(nodes); i++ {
			// run this in a closure, so we can return early rather than
			// dealing with complex looping and labels
			func() {
				n := nodes[i]
				switch n := n.(type) {
				case graphNodeTemporaryValue:
					// root module outputs indicate they are not temporary by
					// returning false here.
					if !n.temporaryValue() {
						return
					}

					// temporary values, which consist of variables, locals,
					// and outputs, must be kept if anything refers to them.
					for _, v := range g.UpEdges(n) {
						// keep any value which is connected through a
						// reference
						if _, ok := v.(GraphNodeReferencer); ok {
							return
						}
					}

				case graphNodeExpandsInstances:
					// Any nodes that expand instances are kept when their
					// instances may need to be evaluated.
					for _, v := range g.UpEdges(n) {
						switch v.(type) {
						case graphNodeExpandsInstances:
							// expanders can always depend on module expansion
							// themselves
							return
						case GraphNodeResourceInstance:
							// resource instances always depend on their
							// resource node, which is an expander
							return
						}
					}

				case GraphNodeProvider:
					// Providers that may have been required by expansion nodes
					// that we no longer need can also be removed.
					if g.UpEdges(n).Len() > 0 {
						return
					}

				default:
					return
				}

				log.Printf("[DEBUG] pruneUnusedNodes: %s is no longer needed, removing", dag.VertexName(n))
				g.Remove(n)
				removed = true

				// remove the node from our iteration as well
				last := len(nodes) - 1
				nodes[i], nodes[last] = nodes[last], nodes[i]
				nodes = nodes[:last]
			}()
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package reg

// xlattice_go/reg/registry.go

// This file contains functions and structures used to describe
// and manage the cluster data managed by the registry.

import (
	"fmt"
	xl "github.com/jddixon/xlattice_go"
	xf "github.com/jddixon/xlattice_go/crypto/filters"
	xi "github.com/jddixon/xlattice_go/nodeID"
	xt "github.com/jddixon/xlattice_go/transport"
	xu "github.com/jddixon/xlattice_go/util"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

var _ = fmt.Print

type Registry struct {
	LogFile string
	Logger  *log.Logger // volatile, not serialized

	// registry data
	m, k     uint          // serialized
	Clusters []*RegCluster // serialized

	idFilter xf.BloomSHAI

	ClustersByName map[string]*RegCluster // volatile, not serialized
	ClustersByID   *xi.IDMap              // -ditto-
	RegMembersByID *xi.IDMap              // -ditto-
	mu             sync.RWMutex           // -ditto-

	// the extended XLattice node, so id, lfs, keys, etc
	RegNode
}

func NewRegistry(clusters []*RegCluster,
	// node *xn.Node, ckPriv, skPriv *rsa.PrivateKey,
	rn *RegNode,
	opt *RegOptions) (
	reg *Registry, err error) {

	var (
		idFilter      xf.BloomSHAI
		m             *xi.IDMap
		serverVersion xu.DecimalVersion
	)
	serverVersion, err = xu.ParseDecimalVersion(xl.VERSION)
	if err == nil && rn == nil {
		err = NilRegNode
	}
	if err == nil {
		if opt.BackingFile == "" {
			idFilter, err = xf.NewBloomSHA(opt.M, opt.K)
		} else {
			idFilter, err = xf.NewMappedBloomSHA(opt.M, opt.K, opt.BackingFile)
		}
		if err == nil {
			m, err = xi.NewNewIDMap()
		}
	}
	if err == nil {
		// registry's own ID added to Bloom filter
		err = idFilter.Insert(rn.GetNodeID().Value())
	}
	if err == nil {
		logger := opt.Logger
		if logger == nil {
			logger = log.New(os.Stderr, "", log.Ldate|log.Ltime)
		}
		reg = &Registry{
			idFilter:       idFilter,
			Clusters:       clusters,
			ClustersByName: make(map[string]*RegCluster),
			ClustersByID:   m,
			Logger:         logger,
			RegNode:        *rn,
		}
		if clusters != nil {
			// XXX need to populate the indexes here
		}
		myLFS := rn.GetLFS()
		if myLFS != "" {
			var ep []xt.EndPointI
			for i := 0; i < rn.SizeEndPoints(); i++ {
				ep = append(ep, rn.GetEndPoint(i))
			}
			regCred := &RegCred{
				Name:        rn.GetName(),
				ID:          rn.GetNodeID(),
				CommsPubKey: rn.GetCommsPublicKey(),
				SigPubKey:   rn.GetSigPublicKey(),
				EndPoints:   ep,
				Version:     serverVersion,
			}
			serialized := regCred.String() // shd have terminating CRLF
			logger.Print(serialized)
			pathToFile := filepath.Join(myLFS, "regCred.dat")
			err = ioutil.WriteFile(pathToFile, []byte(serialized), 0640)
		}
	}
	return
}

func (reg *Registry) ContainsID(n *xi.NodeID) (found bool, err error) {
	found, _, err =  reg.idFilter.Member(n.Value())
	return
}
func (reg *Registry) InsertID(n *xi.NodeID) (err error) {
	b := n.Value()
	found, _, err := reg.idFilter.Member(b)
	if err == nil && found {
		err = IDAlreadyInUse
	}
	if err == nil {
		reg.idFilter.Insert(b)
	}
	return
}
func (reg *Registry) IDCount() uint {
	return reg.idFilter.Size()
}

// XXX RegMembersByID is not being updated!  This is the redundant and so
// possibly inconsistent index of members of registry clusters

func (reg *Registry) AddCluster(cluster *RegCluster) (index int, err error) {

	if cluster == nil {
		err = NilCluster
	} else {
		name := cluster.Name
		id := cluster.ID // []byte

		reg.mu.Lock()
		defer reg.mu.Unlock()

		if _, ok := reg.ClustersByName[name]; ok {
			err = NameAlreadyInUse
		} else {
			var whatever interface{}
			whatever, err = reg.ClustersByID.Find(id)
			if err == nil && whatever != nil {
				err = IDAlreadyInUse
			}
		}
		if err == nil {
			index = len(reg.Clusters)
			reg.Clusters = append(reg.Clusters, cluster)
			reg.ClustersByName[name] = cluster
			err = reg.ClustersByID.Insert(
				cluster.GetNodeID().Value(), cluster)
		}
	}
	if err != nil {
		index = -1
	}
	return
}

// This function generates a good-quality random NodeID (a 32-byte
// value) that is not already known to the registry and then adds
// the new NodeID to the registry's Bloom filter.
func (reg *Registry) UniqueNodeID() (nodeID *xi.NodeID, err error) {

	nodeID, err = xi.New(nil)
	found, err := reg.ContainsID(nodeID)
	for err == nil && found {
		nodeID, err = xi.New(nil)
		found, err = reg.ContainsID(nodeID)
	}
	if err == nil {
		err = reg.idFilter.Insert(nodeID.Value())
	}
	return
}

// SERIALIZATION ====================================================

// Tentatively the registry is serialized separately from the regNode
// and so consists of a sequence of serialized clusters

func (reg *Registry) String() (s string) {
	return strings.Join(reg.Strings(), "\n")
}

// If we change the serialization so that there is no closing brace,
// it will be possible to simply append cluster serializations to the
// registry configuration file while the registry is running.

func (reg *Registry) Strings() (ss []string) {
	ss = []string{"registry {"}
	ss = append(ss, fmt.Sprintf("    LogFile: %s", reg.LogFile))
	ss = append(ss, "}")

	for i := 0; i < len(reg.Clusters); i++ {
		cs := reg.Clusters[i].Strings()
		for j := 0; j < len(cs); j++ {
			ss = append(ss, cs[j])
		}
	}
	return
}

func ParseRegistry(s string) (reg *Registry, rest []string, err error) {

	// XXX STUB
	return
}
<commit_msg>admin_client.go<commit_after>package reg

// xlattice_go/reg/registry.go

// This file contains functions and structures used to describe
// and manage the cluster data managed by the registry.

import (
	"fmt"
	xl "github.com/jddixon/xlattice_go"
	xf "github.com/jddixon/xlattice_go/crypto/filters"
	xi "github.com/jddixon/xlattice_go/nodeID"
	xt "github.com/jddixon/xlattice_go/transport"
	xu "github.com/jddixon/xlattice_go/util"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

var _ = fmt.Print

type Registry struct {
	LogFile string
	Logger  *log.Logger // volatile, not serialized

	// registry data
	m, k     uint          // serialized
	Clusters []*RegCluster // serialized

	idFilter xf.BloomSHAI

	ClustersByName map[string]*RegCluster // volatile, not serialized
	ClustersByID   *xi.IDMap              // -ditto-
	RegMembersByID *xi.IDMap              // -ditto-
	mu             sync.RWMutex           // -ditto-

	// the extended XLattice node, so id, lfs, keys, etc
	RegNode
}

func NewRegistry(clusters []*RegCluster,
	// node *xn.Node, ckPriv, skPriv *rsa.PrivateKey,
	rn *RegNode,
	opt *RegOptions) (
	reg *Registry, err error) {

	var (
		idFilter      xf.BloomSHAI
		m             *xi.IDMap
		serverVersion xu.DecimalVersion
	)
	serverVersion, err = xu.ParseDecimalVersion(xl.VERSION)
	if err == nil && rn == nil {
		err = NilRegNode
	}
	if err == nil {
		if opt.BackingFile == "" {
			idFilter, err = xf.NewBloomSHA(opt.M, opt.K)
		} else {
			idFilter, err = xf.NewMappedBloomSHA(opt.M, opt.K, opt.BackingFile)
		}
		if err == nil {
			m, err = xi.NewNewIDMap()
		}
	}
	if err == nil {
		// registry's own ID added to Bloom filter
		err = idFilter.Insert(rn.GetNodeID().Value())
	}
	if err == nil {
		logger := opt.Logger
		if logger == nil {
			logger = log.New(os.Stderr, "", log.Ldate|log.Ltime)
		}
		reg = &Registry{
			idFilter:       idFilter,
			Clusters:       clusters,
			ClustersByName: make(map[string]*RegCluster),
			ClustersByID:   m,
			Logger:         logger,
			RegNode:        *rn,
		}
		if clusters != nil {
			// XXX need to populate the indexes here
		}
		myLFS := rn.GetLFS()
		if myLFS != "" {
			var ep []xt.EndPointI
			for i := 0; i < rn.SizeEndPoints(); i++ {
				ep = append(ep, rn.GetEndPoint(i))
			}
			regCred := &RegCred{
				Name:        rn.GetName(),
				ID:          rn.GetNodeID(),
				CommsPubKey: rn.GetCommsPublicKey(),
				SigPubKey:   rn.GetSigPublicKey(),
				EndPoints:   ep,
				Version:     serverVersion,
			}
			serialized := regCred.String() // shd have terminating CRLF
			logger.Print(serialized)
			pathToFile := filepath.Join(myLFS, "regCred.dat")
			err = ioutil.WriteFile(pathToFile, []byte(serialized), 0640)
		}
	}
	return
}

func (reg *Registry) ContainsID(n *xi.NodeID) (found bool, err error) {
	found, _, err = reg.idFilter.IsMember(n.Value())
	return
}
func (reg *Registry) InsertID(n *xi.NodeID) (err error) {
	b := n.Value()
	found, _, err := reg.idFilter.IsMember(b)
	if err == nil && found {
		err = IDAlreadyInUse
	}
	if err == nil {
		reg.idFilter.Insert(b)
	}
	return
}
func (reg *Registry) IDCount() uint {
	return reg.idFilter.Size()
}

// XXX RegMembersByID is not being updated!  This is the redundant and so
// possibly inconsistent index of members of registry clusters

func (reg *Registry) AddCluster(cluster *RegCluster) (index int, err error) {

	if cluster == nil {
		err = NilCluster
	} else {
		name := cluster.Name
		id := cluster.ID // []byte

		reg.mu.Lock()
		defer reg.mu.Unlock()

		if _, ok := reg.ClustersByName[name]; ok {
			err = NameAlreadyInUse
		} else {
			var whatever interface{}
			whatever, err = reg.ClustersByID.Find(id)
			if err == nil && whatever != nil {
				err = IDAlreadyInUse
			}
		}
		if err == nil {
			index = len(reg.Clusters)
			reg.Clusters = append(reg.Clusters, cluster)
			reg.ClustersByName[name] = cluster
			err = reg.ClustersByID.Insert(
				cluster.GetNodeID().Value(), cluster)
		}
	}
	if err != nil {
		index = -1
	}
	return
}

// This function generates a good-quality random NodeID (a 32-byte
// value) that is not already known to the registry and then adds
// the new NodeID to the registry's Bloom filter.
func (reg *Registry) UniqueNodeID() (nodeID *xi.NodeID, err error) {

	nodeID, err = xi.New(nil)
	found, err := reg.ContainsID(nodeID)
	for err == nil && found {
		nodeID, err = xi.New(nil)
		found, err = reg.ContainsID(nodeID)
	}
	if err == nil {
		err = reg.idFilter.Insert(nodeID.Value())
	}
	return
}

// SERIALIZATION ====================================================

// Tentatively the registry is serialized separately from the regNode
// and so consists of a sequence of serialized clusters

func (reg *Registry) String() (s string) {
	return strings.Join(reg.Strings(), "\n")
}

// If we change the serialization so that there is no closing brace,
// it will be possible to simply append cluster serializations to the
// registry configuration file while the registry is running.

func (reg *Registry) Strings() (ss []string) {
	ss = []string{"registry {"}
	ss = append(ss, fmt.Sprintf("    LogFile: %s", reg.LogFile))
	ss = append(ss, "}")

	for i := 0; i < len(reg.Clusters); i++ {
		cs := reg.Clusters[i].Strings()
		for j := 0; j < len(cs); j++ {
			ss = append(ss, cs[j])
		}
	}
	return
}

func ParseRegistry(s string) (reg *Registry, rest []string, err error) {

	// XXX STUB
	return
}
<|endoftext|>