<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package testapi provides a helper for retrieving the KUBE_TEST_API environment variable.
package testapi

import (
	"fmt"
	"mime"
	"os"
	"reflect"
	"strings"

	"k8s.io/kubernetes/federation/apis/federation"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/meta"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apimachinery/registered"
	"k8s.io/kubernetes/pkg/apis/apps"
	"k8s.io/kubernetes/pkg/apis/autoscaling"
	"k8s.io/kubernetes/pkg/apis/batch"
	"k8s.io/kubernetes/pkg/apis/extensions"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/runtime/serializer/recognizer"

	_ "k8s.io/kubernetes/federation/apis/federation/install"
	_ "k8s.io/kubernetes/pkg/api/install"
	_ "k8s.io/kubernetes/pkg/apis/apps/install"
	_ "k8s.io/kubernetes/pkg/apis/autoscaling/install"
	_ "k8s.io/kubernetes/pkg/apis/batch/install"
	_ "k8s.io/kubernetes/pkg/apis/componentconfig/install"
	_ "k8s.io/kubernetes/pkg/apis/extensions/install"
	_ "k8s.io/kubernetes/pkg/apis/metrics/install"
)

var (
	Groups      = make(map[string]TestGroup)
	Default     TestGroup
	Autoscaling TestGroup
	Batch       TestGroup
	Extensions  TestGroup
	Apps        TestGroup
	Federation  TestGroup

	serializer        runtime.SerializerInfo
	storageSerializer runtime.SerializerInfo
)

type TestGroup struct {
	externalGroupVersion unversioned.GroupVersion
	internalGroupVersion unversioned.GroupVersion
	internalTypes        map[string]reflect.Type
}

func init() {
	if apiMediaType := os.Getenv("KUBE_TEST_API_TYPE"); len(apiMediaType) > 0 {
		var ok bool
		mediaType, options, err := mime.ParseMediaType(apiMediaType)
		if err != nil {
			panic(err)
		}
		serializer, ok = api.Codecs.SerializerForMediaType(mediaType, options)
		if !ok {
			panic(fmt.Sprintf("no serializer for %s", apiMediaType))
		}
	}

	if storageMediaType := StorageMediaType(); len(storageMediaType) > 0 {
		var ok bool
		mediaType, options, err := mime.ParseMediaType(storageMediaType)
		if err != nil {
			panic(err)
		}
		storageSerializer, ok = api.Codecs.SerializerForMediaType(mediaType, options)
		if !ok {
			panic(fmt.Sprintf("no serializer for %s", storageMediaType))
		}
	}

	kubeTestAPI := os.Getenv("KUBE_TEST_API")
	if len(kubeTestAPI) != 0 {
		testGroupVersions := strings.Split(kubeTestAPI, ",")
		for _, gvString := range testGroupVersions {
			groupVersion, err := unversioned.ParseGroupVersion(gvString)
			if err != nil {
				panic(fmt.Sprintf("Error parsing groupversion %v: %v", gvString, err))
			}

			internalGroupVersion := unversioned.GroupVersion{Group: groupVersion.Group, Version: runtime.APIVersionInternal}
			Groups[groupVersion.Group] = TestGroup{
				externalGroupVersion: groupVersion,
				internalGroupVersion: internalGroupVersion,
				internalTypes:        api.Scheme.KnownTypes(internalGroupVersion),
			}
		}
	}

	if _, ok := Groups[api.GroupName]; !ok {
		Groups[api.GroupName] = TestGroup{
			externalGroupVersion: unversioned.GroupVersion{Group: api.GroupName, Version: registered.GroupOrDie(api.GroupName).GroupVersion.Version},
			internalGroupVersion: api.SchemeGroupVersion,
			internalTypes:        api.Scheme.KnownTypes(api.SchemeGroupVersion),
		}
	}
	if _, ok := Groups[extensions.GroupName]; !ok {
		Groups[extensions.GroupName] = TestGroup{
			externalGroupVersion: unversioned.GroupVersion{Group: extensions.GroupName, Version: registered.GroupOrDie(extensions.GroupName).GroupVersion.Version},
			internalGroupVersion: extensions.SchemeGroupVersion,
			internalTypes:        api.Scheme.KnownTypes(extensions.SchemeGroupVersion),
		}
	}
	if _, ok := Groups[autoscaling.GroupName]; !ok {
		internalTypes := make(map[string]reflect.Type)
		for k, t := range api.Scheme.KnownTypes(extensions.SchemeGroupVersion) {
			if k == "Scale" {
				continue
			}
			internalTypes[k] = t
		}
		Groups[autoscaling.GroupName] = TestGroup{
			externalGroupVersion: unversioned.GroupVersion{Group: autoscaling.GroupName, Version: registered.GroupOrDie(autoscaling.GroupName).GroupVersion.Version},
			internalGroupVersion: extensions.SchemeGroupVersion,
			internalTypes:        internalTypes,
		}
	}
	if _, ok := Groups[autoscaling.GroupName+"IntraGroup"]; !ok {
		internalTypes := make(map[string]reflect.Type)
		for k, t := range api.Scheme.KnownTypes(extensions.SchemeGroupVersion) {
			if k == "Scale" {
				internalTypes[k] = t
				break
			}
		}
		Groups[autoscaling.GroupName] = TestGroup{
			externalGroupVersion: unversioned.GroupVersion{Group: autoscaling.GroupName, Version: registered.GroupOrDie(autoscaling.GroupName).GroupVersion.Version},
			internalGroupVersion: autoscaling.SchemeGroupVersion,
			internalTypes:        internalTypes,
		}
	}
	if _, ok := Groups[batch.GroupName]; !ok {
		Groups[batch.GroupName] = TestGroup{
			externalGroupVersion: unversioned.GroupVersion{Group: batch.GroupName, Version: registered.GroupOrDie(batch.GroupName).GroupVersion.Version},
			internalGroupVersion: batch.SchemeGroupVersion,
			internalTypes:        api.Scheme.KnownTypes(batch.SchemeGroupVersion),
		}
	}
	if _, ok := Groups[apps.GroupName]; !ok {
		Groups[apps.GroupName] = TestGroup{
			externalGroupVersion: unversioned.GroupVersion{Group: apps.GroupName, Version: registered.GroupOrDie(apps.GroupName).GroupVersion.Version},
			internalGroupVersion: extensions.SchemeGroupVersion,
			internalTypes:        api.Scheme.KnownTypes(extensions.SchemeGroupVersion),
		}
	}
	if _, ok := Groups[federation.GroupName]; !ok {
		Groups[federation.GroupName] = TestGroup{
			externalGroupVersion: unversioned.GroupVersion{Group: federation.GroupName, Version: registered.GroupOrDie(federation.GroupName).GroupVersion.Version},
			internalGroupVersion: federation.SchemeGroupVersion,
			internalTypes:        api.Scheme.KnownTypes(federation.SchemeGroupVersion),
		}
	}

	Default = Groups[api.GroupName]
	Autoscaling = Groups[autoscaling.GroupName]
	Batch = Groups[batch.GroupName]
	Apps = Groups[apps.GroupName]
	Extensions = Groups[extensions.GroupName]
	Federation = Groups[federation.GroupName]
}

func (g TestGroup) ContentConfig() (string, *unversioned.GroupVersion, runtime.Codec) {
	return "application/json", g.GroupVersion(), g.Codec()
}

func (g TestGroup) GroupVersion() *unversioned.GroupVersion {
	copyOfGroupVersion := g.externalGroupVersion
	return &copyOfGroupVersion
}

// InternalGroupVersion returns the group,version used to identify the internal
// types for this API
func (g TestGroup) InternalGroupVersion() unversioned.GroupVersion {
	return g.internalGroupVersion
}

// InternalTypes returns a map of internal API types' kind names to their Go types.
func (g TestGroup) InternalTypes() map[string]reflect.Type {
	return g.internalTypes
}

// Codec returns the codec for the API version to test against, as set by the
// KUBE_TEST_API_TYPE env var.
func (g TestGroup) Codec() runtime.Codec {
	if serializer.Serializer == nil {
		return api.Codecs.LegacyCodec(g.externalGroupVersion)
	}
	return api.Codecs.CodecForVersions(serializer, api.Codecs.UniversalDeserializer(), []unversioned.GroupVersion{g.externalGroupVersion}, nil)
}

// NegotiatedSerializer returns the negotiated serializer for the server.
func (g TestGroup) NegotiatedSerializer() runtime.NegotiatedSerializer {
	return api.Codecs
}

func StorageMediaType() string {
	return os.Getenv("KUBE_TEST_API_STORAGE_TYPE")
}

// StorageCodec returns the codec for the API version to store in etcd, as set by the
// KUBE_TEST_API_STORAGE_TYPE env var.
func (g TestGroup) StorageCodec() runtime.Codec {
	s := storageSerializer.Serializer

	if s == nil {
		return api.Codecs.LegacyCodec(g.externalGroupVersion)
	}

	// etcd2 only supports string data - we must wrap any result before returning
	// TODO: remove for etcd3 / make parameterizable
	if !storageSerializer.EncodesAsText {
		s = runtime.NewBase64Serializer(s)
	}
	ds := recognizer.NewDecoder(s, api.Codecs.UniversalDeserializer())

	return api.Codecs.CodecForVersions(s, ds, []unversioned.GroupVersion{g.externalGroupVersion}, nil)
}

// Converter returns the api.Scheme for the API version to test against, as set by the
// KUBE_TEST_API env var.
func (g TestGroup) Converter() runtime.ObjectConvertor {
	interfaces, err := registered.GroupOrDie(g.externalGroupVersion.Group).InterfacesFor(g.externalGroupVersion)
	if err != nil {
		panic(err)
	}
	return interfaces.ObjectConvertor
}

// MetadataAccessor returns the MetadataAccessor for the API version to test against,
// as set by the KUBE_TEST_API env var.
func (g TestGroup) MetadataAccessor() meta.MetadataAccessor {
	interfaces, err := registered.GroupOrDie(g.externalGroupVersion.Group).InterfacesFor(g.externalGroupVersion)
	if err != nil {
		panic(err)
	}
	return interfaces.MetadataAccessor
}

// SelfLink returns a self link that will appear to be for the version Version().
// 'resource' should be the resource path, e.g. "pods" for the Pod type. 'name' should be
// empty for lists.
func (g TestGroup) SelfLink(resource, name string) string {
	if g.externalGroupVersion.Group == api.GroupName {
		if name == "" {
			return fmt.Sprintf("/api/%s/%s", g.externalGroupVersion.Version, resource)
		}
		return fmt.Sprintf("/api/%s/%s/%s", g.externalGroupVersion.Version, resource, name)
	} else {
		// TODO: will need a /apis prefix once we have proper multi-group
		// support
		if name == "" {
			return fmt.Sprintf("/apis/%s/%s/%s", g.externalGroupVersion.Group, g.externalGroupVersion.Version, resource)
		}
		return fmt.Sprintf("/apis/%s/%s/%s/%s", g.externalGroupVersion.Group, g.externalGroupVersion.Version, resource, name)
	}
}

// Returns the appropriate path for the given prefix (watch, proxy, redirect, etc), resource, namespace and name.
// For ex, this is of the form:
// /api/v1/watch/namespaces/foo/pods/pod0 for v1.
func (g TestGroup) ResourcePathWithPrefix(prefix, resource, namespace, name string) string {
	var path string
	if g.externalGroupVersion.Group == api.GroupName {
		path = "/api/" + g.externalGroupVersion.Version
	} else {
		// TODO: switch back once we have proper multiple group support
		// path = "/apis/" + g.Group + "/" + Version(group...)
		path = "/apis/" + g.externalGroupVersion.Group + "/" + g.externalGroupVersion.Version
	}

	if prefix != "" {
		path = path + "/" + prefix
	}
	if namespace != "" {
		path = path + "/namespaces/" + namespace
	}
	// Resource names are lower case.
	resource = strings.ToLower(resource)
	if resource != "" {
		path = path + "/" + resource
	}
	if name != "" {
		path = path + "/" + name
	}
	return path
}

// Returns the appropriate path for the given resource, namespace and name.
// For example, this is of the form:
// /api/v1/namespaces/foo/pods/pod0 for v1.
func (g TestGroup) ResourcePath(resource, namespace, name string) string {
	return g.ResourcePathWithPrefix("", resource, namespace, name)
}

func (g TestGroup) RESTMapper() meta.RESTMapper {
	return registered.RESTMapper()
}

// ExternalGroupVersions returns all external group versions allowed for the server.
func ExternalGroupVersions() []unversioned.GroupVersion {
	versions := []unversioned.GroupVersion{}
	for _, g := range Groups {
		gv := g.GroupVersion()
		versions = append(versions, *gv)
	}
	return versions
}

// Get codec based on runtime.Object
func GetCodecForObject(obj runtime.Object) (runtime.Codec, error) {
	kind, err := api.Scheme.ObjectKind(obj)
	if err != nil {
		return nil, fmt.Errorf("unexpected encoding error: %v", err)
	}

	for _, group := range Groups {
		if group.GroupVersion().Group != kind.Group {
			continue
		}

		if api.Scheme.Recognizes(kind) {
			return group.Codec(), nil
		}
	}
	// Codec used for unversioned types
	if api.Scheme.Recognizes(kind) {
		serializer, ok := api.Codecs.SerializerForFileExtension("json")
		if !ok {
			return nil, fmt.Errorf("no serializer registered for json")
		}
		return serializer, nil
	}
	return nil, fmt.Errorf("unexpected kind: %v", kind)
}

func NewTestGroup(external, internal unversioned.GroupVersion, internalTypes map[string]reflect.Type) TestGroup {
	return TestGroup{external, internal, internalTypes}
}
<commit_msg>enable batch/v2alpha1 tests<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package testapi provides a helper for retrieving the KUBE_TEST_API environment variable.
package testapi

import (
	"fmt"
	"mime"
	"os"
	"reflect"
	"strings"

	"k8s.io/kubernetes/federation/apis/federation"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/meta"
	"k8s.io/kubernetes/pkg/api/unversioned"
	"k8s.io/kubernetes/pkg/apimachinery/registered"
	"k8s.io/kubernetes/pkg/apis/apps"
	"k8s.io/kubernetes/pkg/apis/autoscaling"
	"k8s.io/kubernetes/pkg/apis/batch"
	"k8s.io/kubernetes/pkg/apis/extensions"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/runtime/serializer/recognizer"

	_ "k8s.io/kubernetes/federation/apis/federation/install"
	_ "k8s.io/kubernetes/pkg/api/install"
	_ "k8s.io/kubernetes/pkg/apis/apps/install"
	_ "k8s.io/kubernetes/pkg/apis/autoscaling/install"
	_ "k8s.io/kubernetes/pkg/apis/batch/install"
	_ "k8s.io/kubernetes/pkg/apis/componentconfig/install"
	_ "k8s.io/kubernetes/pkg/apis/extensions/install"
	_ "k8s.io/kubernetes/pkg/apis/metrics/install"
)

var (
	Groups      = make(map[string]TestGroup)
	Default     TestGroup
	Autoscaling TestGroup
	Batch       TestGroup
	Extensions  TestGroup
	Apps        TestGroup
	Federation  TestGroup

	serializer        runtime.SerializerInfo
	storageSerializer runtime.SerializerInfo
)

type TestGroup struct {
	// the first element in the group is meant to be the preferred version
	externalGroupVersions []unversioned.GroupVersion
	internalGroupVersion  unversioned.GroupVersion
	internalTypes         map[string]reflect.Type
}

func init() {
	if apiMediaType := os.Getenv("KUBE_TEST_API_TYPE"); len(apiMediaType) > 0 {
		var ok bool
		mediaType, options, err := mime.ParseMediaType(apiMediaType)
		if err != nil {
			panic(err)
		}
		serializer, ok = api.Codecs.SerializerForMediaType(mediaType, options)
		if !ok {
			panic(fmt.Sprintf("no serializer for %s", apiMediaType))
		}
	}

	if storageMediaType := StorageMediaType(); len(storageMediaType) > 0 {
		var ok bool
		mediaType, options, err := mime.ParseMediaType(storageMediaType)
		if err != nil {
			panic(err)
		}
		storageSerializer, ok = api.Codecs.SerializerForMediaType(mediaType, options)
		if !ok {
			panic(fmt.Sprintf("no serializer for %s", storageMediaType))
		}
	}

	kubeTestAPI := os.Getenv("KUBE_TEST_API")
	if len(kubeTestAPI) != 0 {
		testGroupVersions := strings.Split(kubeTestAPI, ",")
		for _, gvString := range testGroupVersions {
			groupVersion, err := unversioned.ParseGroupVersion(gvString)
			if err != nil {
				panic(fmt.Sprintf("Error parsing groupversion %v: %v", gvString, err))
			}

			internalGroupVersion := unversioned.GroupVersion{Group: groupVersion.Group, Version: runtime.APIVersionInternal}
			if group, ok := Groups[groupVersion.Group]; !ok {
				Groups[groupVersion.Group] = TestGroup{
					externalGroupVersions: []unversioned.GroupVersion{groupVersion},
					internalGroupVersion:  internalGroupVersion,
					internalTypes:         api.Scheme.KnownTypes(internalGroupVersion),
				}
			} else {
				group.externalGroupVersions = append(group.externalGroupVersions, groupVersion)
				Groups[groupVersion.Group] = group
			}
		}
	}

	if _, ok := Groups[api.GroupName]; !ok {
		Groups[api.GroupName] = TestGroup{
			externalGroupVersions: []unversioned.GroupVersion{{Group: api.GroupName, Version: registered.GroupOrDie(api.GroupName).GroupVersion.Version}},
			internalGroupVersion:  api.SchemeGroupVersion,
			internalTypes:         api.Scheme.KnownTypes(api.SchemeGroupVersion),
		}
	}
	if _, ok := Groups[extensions.GroupName]; !ok {
		Groups[extensions.GroupName] = TestGroup{
			externalGroupVersions: []unversioned.GroupVersion{{Group: extensions.GroupName, Version: registered.GroupOrDie(extensions.GroupName).GroupVersion.Version}},
			internalGroupVersion:  extensions.SchemeGroupVersion,
			internalTypes:         api.Scheme.KnownTypes(extensions.SchemeGroupVersion),
		}
	}
	if _, ok := Groups[autoscaling.GroupName]; !ok {
		internalTypes := make(map[string]reflect.Type)
		for k, t := range api.Scheme.KnownTypes(extensions.SchemeGroupVersion) {
			if k == "Scale" {
				continue
			}
			internalTypes[k] = t
		}
		Groups[autoscaling.GroupName] = TestGroup{
			externalGroupVersions: []unversioned.GroupVersion{{Group: autoscaling.GroupName, Version: registered.GroupOrDie(autoscaling.GroupName).GroupVersion.Version}},
			internalGroupVersion:  extensions.SchemeGroupVersion,
			internalTypes:         internalTypes,
		}
	}
	if _, ok := Groups[autoscaling.GroupName+"IntraGroup"]; !ok {
		internalTypes := make(map[string]reflect.Type)
		for k, t := range api.Scheme.KnownTypes(extensions.SchemeGroupVersion) {
			if k == "Scale" {
				internalTypes[k] = t
				break
			}
		}
		Groups[autoscaling.GroupName] = TestGroup{
			externalGroupVersions: []unversioned.GroupVersion{{Group: autoscaling.GroupName, Version: registered.GroupOrDie(autoscaling.GroupName).GroupVersion.Version}},
			internalGroupVersion:  autoscaling.SchemeGroupVersion,
			internalTypes:         internalTypes,
		}
	}
	if _, ok := Groups[batch.GroupName]; !ok {
		var gvs []unversioned.GroupVersion
		for _, gv := range registered.GroupOrDie(batch.GroupName).GroupVersions {
			gvs = append(gvs, gv)
		}
		Groups[batch.GroupName] = TestGroup{
			externalGroupVersions: gvs,
			internalGroupVersion:  batch.SchemeGroupVersion,
			internalTypes:         api.Scheme.KnownTypes(batch.SchemeGroupVersion),
		}
	}
	if _, ok := Groups[apps.GroupName]; !ok {
		Groups[apps.GroupName] = TestGroup{
			externalGroupVersions: []unversioned.GroupVersion{{Group: apps.GroupName, Version: registered.GroupOrDie(apps.GroupName).GroupVersion.Version}},
			internalGroupVersion:  extensions.SchemeGroupVersion,
			internalTypes:         api.Scheme.KnownTypes(extensions.SchemeGroupVersion),
		}
	}
	if _, ok := Groups[federation.GroupName]; !ok {
		Groups[federation.GroupName] = TestGroup{
			externalGroupVersions: []unversioned.GroupVersion{{Group: federation.GroupName, Version: registered.GroupOrDie(federation.GroupName).GroupVersion.Version}},
			internalGroupVersion:  federation.SchemeGroupVersion,
			internalTypes:         api.Scheme.KnownTypes(federation.SchemeGroupVersion),
		}
	}

	Default = Groups[api.GroupName]
	Autoscaling = Groups[autoscaling.GroupName]
	Batch = Groups[batch.GroupName]
	Apps = Groups[apps.GroupName]
	Extensions = Groups[extensions.GroupName]
	Federation = Groups[federation.GroupName]
}

func (g TestGroup) ContentConfig() (string, *unversioned.GroupVersion, runtime.Codec) {
	return "application/json", g.GroupVersion(), g.Codec()
}

func (g TestGroup) GroupVersion() *unversioned.GroupVersion {
	copyOfGroupVersion := g.externalGroupVersions[0]
	return &copyOfGroupVersion
}

func (g TestGroup) GroupVersions() []unversioned.GroupVersion {
	return append([]unversioned.GroupVersion{}, g.externalGroupVersions...)
}

// InternalGroupVersion returns the group,version used to identify the internal
// types for this API
func (g TestGroup) InternalGroupVersion() unversioned.GroupVersion {
	return g.internalGroupVersion
}

// InternalTypes returns a map of internal API types' kind names to their Go types.
func (g TestGroup) InternalTypes() map[string]reflect.Type {
	return g.internalTypes
}

// Codec returns the codec for the API version to test against, as set by the
// KUBE_TEST_API_TYPE env var.
func (g TestGroup) Codec() runtime.Codec {
	if serializer.Serializer == nil {
		return api.Codecs.LegacyCodec(g.externalGroupVersions[0])
	}
	return api.Codecs.CodecForVersions(serializer, api.Codecs.UniversalDeserializer(), g.externalGroupVersions, nil)
}

// NegotiatedSerializer returns the negotiated serializer for the server.
func (g TestGroup) NegotiatedSerializer() runtime.NegotiatedSerializer {
	return api.Codecs
}

func StorageMediaType() string {
	return os.Getenv("KUBE_TEST_API_STORAGE_TYPE")
}

// StorageCodec returns the codec for the API version to store in etcd, as set by the
// KUBE_TEST_API_STORAGE_TYPE env var.
func (g TestGroup) StorageCodec() runtime.Codec {
	s := storageSerializer.Serializer

	if s == nil {
		return api.Codecs.LegacyCodec(g.externalGroupVersions[0])
	}

	// etcd2 only supports string data - we must wrap any result before returning
	// TODO: remove for etcd3 / make parameterizable
	if !storageSerializer.EncodesAsText {
		s = runtime.NewBase64Serializer(s)
	}
	ds := recognizer.NewDecoder(s, api.Codecs.UniversalDeserializer())

	return api.Codecs.CodecForVersions(s, ds, g.externalGroupVersions, nil)
}

// Converter returns the api.Scheme for the API version to test against, as set by the
// KUBE_TEST_API env var.
func (g TestGroup) Converter() runtime.ObjectConvertor {
	interfaces, err := registered.GroupOrDie(g.externalGroupVersions[0].Group).InterfacesFor(g.externalGroupVersions[0])
	if err != nil {
		panic(err)
	}
	return interfaces.ObjectConvertor
}

// MetadataAccessor returns the MetadataAccessor for the API version to test against,
// as set by the KUBE_TEST_API env var.
func (g TestGroup) MetadataAccessor() meta.MetadataAccessor {
	interfaces, err := registered.GroupOrDie(g.externalGroupVersions[0].Group).InterfacesFor(g.externalGroupVersions[0])
	if err != nil {
		panic(err)
	}
	return interfaces.MetadataAccessor
}

// SelfLink returns a self link that will appear to be for the version Version().
// 'resource' should be the resource path, e.g. "pods" for the Pod type. 'name' should be
// empty for lists.
func (g TestGroup) SelfLink(resource, name string) string {
	if g.externalGroupVersions[0].Group == api.GroupName {
		if name == "" {
			return fmt.Sprintf("/api/%s/%s", g.externalGroupVersions[0].Version, resource)
		}
		return fmt.Sprintf("/api/%s/%s/%s", g.externalGroupVersions[0].Version, resource, name)
	} else {
		// TODO: will need a /apis prefix once we have proper multi-group
		// support
		if name == "" {
			return fmt.Sprintf("/apis/%s/%s/%s", g.externalGroupVersions[0].Group, g.externalGroupVersions[0].Version, resource)
		}
		return fmt.Sprintf("/apis/%s/%s/%s/%s", g.externalGroupVersions[0].Group, g.externalGroupVersions[0].Version, resource, name)
	}
}

// Returns the appropriate path for the given prefix (watch, proxy, redirect, etc), resource, namespace and name.
// For ex, this is of the form:
// /api/v1/watch/namespaces/foo/pods/pod0 for v1.
func (g TestGroup) ResourcePathWithPrefix(prefix, resource, namespace, name string) string {
	var path string
	if g.externalGroupVersions[0].Group == api.GroupName {
		path = "/api/" + g.externalGroupVersions[0].Version
	} else {
		// TODO: switch back once we have proper multiple group support
		// path = "/apis/" + g.Group + "/" + Version(group...)
		path = "/apis/" + g.externalGroupVersions[0].Group + "/" + g.externalGroupVersions[0].Version
	}

	if prefix != "" {
		path = path + "/" + prefix
	}
	if namespace != "" {
		path = path + "/namespaces/" + namespace
	}
	// Resource names are lower case.
	resource = strings.ToLower(resource)
	if resource != "" {
		path = path + "/" + resource
	}
	if name != "" {
		path = path + "/" + name
	}
	return path
}

// Returns the appropriate path for the given resource, namespace and name.
// For example, this is of the form:
// /api/v1/namespaces/foo/pods/pod0 for v1.
func (g TestGroup) ResourcePath(resource, namespace, name string) string {
	return g.ResourcePathWithPrefix("", resource, namespace, name)
}

func (g TestGroup) RESTMapper() meta.RESTMapper {
	return registered.RESTMapper()
}

// ExternalGroupVersions returns all external group versions allowed for the server.
func ExternalGroupVersions() []unversioned.GroupVersion {
	versions := []unversioned.GroupVersion{}
	for _, g := range Groups {
		gv := g.GroupVersion()
		versions = append(versions, *gv)
	}
	return versions
}

// Get codec based on runtime.Object
func GetCodecForObject(obj runtime.Object) (runtime.Codec, error) {
	kind, err := api.Scheme.ObjectKind(obj)
	if err != nil {
		return nil, fmt.Errorf("unexpected encoding error: %v", err)
	}

	for _, group := range Groups {
		if group.GroupVersion().Group != kind.Group {
			continue
		}

		if api.Scheme.Recognizes(kind) {
			return group.Codec(), nil
		}
	}
	// Codec used for unversioned types
	if api.Scheme.Recognizes(kind) {
		serializer, ok := api.Codecs.SerializerForFileExtension("json")
		if !ok {
			return nil, fmt.Errorf("no serializer registered for json")
		}
		return serializer, nil
	}
	return nil, fmt.Errorf("unexpected kind: %v", kind)
}

func NewTestGroup(external, internal unversioned.GroupVersion, internalTypes map[string]reflect.Type) TestGroup {
	return TestGroup{
		externalGroupVersions: []unversioned.GroupVersion{external},
		internalGroupVersion:  internal,
		internalTypes:         internalTypes,
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"k8s.io/apimachinery/pkg/apimachinery/announced"
	"k8s.io/apimachinery/pkg/apimachinery/registered"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/apimachinery/pkg/version"
	"k8s.io/apiserver/pkg/registry/rest"
	genericapiserver "k8s.io/apiserver/pkg/server"

	"k8s.io/sample-apiserver/pkg/apis/wardle"
	"k8s.io/sample-apiserver/pkg/apis/wardle/install"
	"k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1"
	wardleregistry "k8s.io/sample-apiserver/pkg/registry"
	fischerstorage "k8s.io/sample-apiserver/pkg/registry/wardle/fischer"
	flunderstorage "k8s.io/sample-apiserver/pkg/registry/wardle/flunder"
)

var (
	groupFactoryRegistry = make(announced.APIGroupFactoryRegistry)
	registry             = registered.NewOrDie("")
	Scheme               = runtime.NewScheme()
	Codecs               = serializer.NewCodecFactory(Scheme)
)

func init() {
	install.Install(groupFactoryRegistry, registry, Scheme)

	// we need to add the options to empty v1
	// TODO fix the server code to avoid this
	metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"})

	// TODO: keep the generic API server from wanting this
	unversioned := schema.GroupVersion{Group: "", Version: "v1"}
	Scheme.AddUnversionedTypes(unversioned,
		&metav1.Status{},
		&metav1.APIVersions{},
		&metav1.APIGroupList{},
		&metav1.APIGroup{},
		&metav1.APIResourceList{},
	)
}

type ExtraConfig struct {
	// Place you custom config here.
}

type Config struct {
	GenericConfig *genericapiserver.RecommendedConfig
	ExtraConfig   ExtraConfig
}

// WardleServer contains state for a Kubernetes cluster master/api server.
type WardleServer struct {
	GenericAPIServer *genericapiserver.GenericAPIServer
}

type completedConfig struct {
	GenericConfig genericapiserver.CompletedConfig
	ExtraConfig   *ExtraConfig
}

type CompletedConfig struct {
	// Embed a private pointer that cannot be instantiated outside of this package.
	*completedConfig
}

// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
func (cfg *Config) Complete() CompletedConfig {
	c := completedConfig{
		cfg.GenericConfig.Complete(),
		&cfg.ExtraConfig,
	}

	c.GenericConfig.Version = &version.Info{
		Major: "1",
		Minor: "0",
	}

	return CompletedConfig{&c}
}

// New returns a new instance of WardleServer from the given config.
func (c completedConfig) New() (*WardleServer, error) {
	genericServer, err := c.GenericConfig.New("sample-apiserver", genericapiserver.EmptyDelegate)
	if err != nil {
		return nil, err
	}

	s := &WardleServer{
		GenericAPIServer: genericServer,
	}

	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(wardle.GroupName, registry, Scheme, metav1.ParameterCodec, Codecs)
	apiGroupInfo.GroupMeta.GroupVersion = v1alpha1.SchemeGroupVersion
	v1alpha1storage := map[string]rest.Storage{}
	v1alpha1storage["flunders"] = wardleregistry.RESTInPeace(flunderstorage.NewREST(Scheme, c.GenericConfig.RESTOptionsGetter))
	v1alpha1storage["fischers"] = wardleregistry.RESTInPeace(fischerstorage.NewREST(Scheme, c.GenericConfig.RESTOptionsGetter))
	apiGroupInfo.VersionedResourcesStorageMap["v1alpha1"] = v1alpha1storage

	if err := s.GenericAPIServer.InstallAPIGroup(&apiGroupInfo); err != nil {
		return nil, err
	}

	return s, nil
}
<commit_msg>apiserver: enforce shared RequestContextMapper in delegation chain<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package apiserver

import (
	"k8s.io/apimachinery/pkg/apimachinery/announced"
	"k8s.io/apimachinery/pkg/apimachinery/registered"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/apimachinery/pkg/version"
	"k8s.io/apiserver/pkg/registry/rest"
	genericapiserver "k8s.io/apiserver/pkg/server"

	"k8s.io/sample-apiserver/pkg/apis/wardle"
	"k8s.io/sample-apiserver/pkg/apis/wardle/install"
	"k8s.io/sample-apiserver/pkg/apis/wardle/v1alpha1"
	wardleregistry "k8s.io/sample-apiserver/pkg/registry"
	fischerstorage "k8s.io/sample-apiserver/pkg/registry/wardle/fischer"
	flunderstorage "k8s.io/sample-apiserver/pkg/registry/wardle/flunder"
)

var (
	groupFactoryRegistry = make(announced.APIGroupFactoryRegistry)
	registry             = registered.NewOrDie("")
	Scheme               = runtime.NewScheme()
	Codecs               = serializer.NewCodecFactory(Scheme)
)

func init() {
	install.Install(groupFactoryRegistry, registry, Scheme)

	// we need to add the options to empty v1
	// TODO fix the server code to avoid this
	metav1.AddToGroupVersion(Scheme, schema.GroupVersion{Version: "v1"})

	// TODO: keep the generic API server from wanting this
	unversioned := schema.GroupVersion{Group: "", Version: "v1"}
	Scheme.AddUnversionedTypes(unversioned,
		&metav1.Status{},
		&metav1.APIVersions{},
		&metav1.APIGroupList{},
		&metav1.APIGroup{},
		&metav1.APIResourceList{},
	)
}

type ExtraConfig struct {
	// Place you custom config here.
}

type Config struct {
	GenericConfig *genericapiserver.RecommendedConfig
	ExtraConfig   ExtraConfig
}

// WardleServer contains state for a Kubernetes cluster master/api server.
type WardleServer struct {
	GenericAPIServer *genericapiserver.GenericAPIServer
}

type completedConfig struct {
	GenericConfig genericapiserver.CompletedConfig
	ExtraConfig   *ExtraConfig
}

type CompletedConfig struct {
	// Embed a private pointer that cannot be instantiated outside of this package.
	*completedConfig
}

// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
func (cfg *Config) Complete() CompletedConfig {
	c := completedConfig{
		cfg.GenericConfig.Complete(),
		&cfg.ExtraConfig,
	}

	c.GenericConfig.Version = &version.Info{
		Major: "1",
		Minor: "0",
	}

	return CompletedConfig{&c}
}

// New returns a new instance of WardleServer from the given config.
func (c completedConfig) New() (*WardleServer, error) {
	genericServer, err := c.GenericConfig.New("sample-apiserver", genericapiserver.NewEmptyDelegate())
	if err != nil {
		return nil, err
	}

	s := &WardleServer{
		GenericAPIServer: genericServer,
	}

	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(wardle.GroupName, registry, Scheme, metav1.ParameterCodec, Codecs)
	apiGroupInfo.GroupMeta.GroupVersion = v1alpha1.SchemeGroupVersion
	v1alpha1storage := map[string]rest.Storage{}
	v1alpha1storage["flunders"] = wardleregistry.RESTInPeace(flunderstorage.NewREST(Scheme, c.GenericConfig.RESTOptionsGetter))
	v1alpha1storage["fischers"] = wardleregistry.RESTInPeace(fischerstorage.NewREST(Scheme, c.GenericConfig.RESTOptionsGetter))
	apiGroupInfo.VersionedResourcesStorageMap["v1alpha1"] = v1alpha1storage

	if err := s.GenericAPIServer.InstallAPIGroup(&apiGroupInfo); err != nil {
		return nil, err
	}

	return s, nil
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package database

import (
	"math"

	"github.com/google/exposure-notifications-verification-server/pkg/pagination"
	"github.com/jinzhu/gorm"
)

// Paginate is a helper that paginates a gorm query into the given result. In
// addition to reflecting into the provided result, it returns a pagination
// struct.
//
// If page is 0, it defaults to 1. If limit is 0, it defaults to the global
// pagination default limit.
func Paginate(query *gorm.DB, result interface{}, page, limit uint64) (*pagination.Paginator, error) {
	return PaginateFn(query, page, limit, func(query *gorm.DB, offset uint64) error {
		return query.
			Limit(limit).
			Offset(offset).
			Find(result).Error
	})
}

// PaginateFn paginates with a custom function for returning results.
func PaginateFn(query *gorm.DB, page, limit uint64, populateFn func(query *gorm.DB, offset uint64) error) (*pagination.Paginator, error) {
	if page < 1 {
		page = 1
	}
	if limit < 1 {
		limit = pagination.DefaultLimit
	}

	var total uint64
	if err := query.
		Count(&total).
		Error; err != nil {
		return nil, err
	}

	// Calculate offset.
	offset := uint64((page - 1) * limit)

	// Get the list of records, limiting and offsetting as needed.
	if err := populateFn(query, offset); err != nil {
		return nil, err
	}

	// Calculate how many future pages exist.
	var nextPages uint64
	if current := offset + limit; current < total {
		remaining := total - current
		nextPages = uint64(math.Ceil(float64(remaining) / float64(limit)))
		if nextPages > 10 {
			nextPages = 10
		}
	}

	prevPages := page - 1
	if prevPages > 10 {
		prevPages = 10
	}

	var paginator pagination.Paginator
	paginator.CurrPage = &pagination.Page{
		Number: page,
	}

	for i := prevPages; i > 0; i-- {
		paginator.PrevPages = append(paginator.PrevPages, &pagination.Page{
			Number: page - i,
		})
	}
	if prevPages > 0 {
		paginator.PrevPage = &pagination.Page{
			Number: page - 1,
		}
	}

	for i := uint64(1); i <= nextPages; i++ {
		paginator.NextPages = append(paginator.NextPages, &pagination.Page{
			Number: page + i,
		})
	}
	if nextPages > 0 {
		paginator.NextPage = &pagination.Page{
			Number: page + 1,
		}
	}

	return &paginator, nil
}
<commit_msg>Only show 5 next/prev pages (#1017)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package database

import (
	"math"

	"github.com/google/exposure-notifications-verification-server/pkg/pagination"
	"github.com/jinzhu/gorm"
)

const (
	// maxPrevPages and maxNextPages control the "window" to show.
	maxPrevPages = 5
	maxNextPages = 5
)

// Paginate is a helper that paginates a gorm query into the given result. In
// addition to reflecting into the provided result, it returns a pagination
// struct.
//
// If page is 0, it defaults to 1. If limit is 0, it defaults to the global
// pagination default limit.
func Paginate(query *gorm.DB, result interface{}, page, limit uint64) (*pagination.Paginator, error) {
	return PaginateFn(query, page, limit, func(query *gorm.DB, offset uint64) error {
		return query.
			Limit(limit).
			Offset(offset).
			Find(result).Error
	})
}

// PaginateFn paginates with a custom function for returning results.
func PaginateFn(query *gorm.DB, page, limit uint64, populateFn func(query *gorm.DB, offset uint64) error) (*pagination.Paginator, error) {
	if page < 1 {
		page = 1
	}
	if limit < 1 {
		limit = pagination.DefaultLimit
	}

	var total uint64
	if err := query.
		Count(&total).
		Error; err != nil {
		return nil, err
	}

	// Calculate offset.
	offset := uint64((page - 1) * limit)

	// Get the list of records, limiting and offsetting as needed.
	if err := populateFn(query, offset); err != nil {
		return nil, err
	}

	// Calculate how many future pages exist.
	var nextPages uint64
	if current := offset + limit; current < total {
		remaining := total - current
		nextPages = uint64(math.Ceil(float64(remaining) / float64(limit)))
		if nextPages > maxNextPages {
			nextPages = maxNextPages
		}
	}

	prevPages := page - 1
	if prevPages > maxPrevPages {
		prevPages = maxPrevPages
	}

	var paginator pagination.Paginator
	paginator.CurrPage = &pagination.Page{
		Number: page,
	}

	for i := prevPages; i > 0; i-- {
		paginator.PrevPages = append(paginator.PrevPages, &pagination.Page{
			Number: page - i,
		})
	}
	if prevPages > 0 {
		paginator.PrevPage = &pagination.Page{
			Number: page - 1,
		}
	}

	for i := uint64(1); i <= nextPages; i++ {
		paginator.NextPages = append(paginator.NextPages, &pagination.Page{
			Number: page + i,
		})
	}
	if nextPages > 0 {
		paginator.NextPage = &pagination.Page{
			Number: page + 1,
		}
	}

	return &paginator, nil
}
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestState_basic(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccState_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckStateValue(
						"terraform_remote_state.foo", "foo", "bar"),
				),
			},
		},
	})
}

func TestState_complexOutputs(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccState_complexOutputs,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckStateValue("terraform_remote_state.foo", "backend", "_local"),
					testAccCheckStateValue("terraform_remote_state.foo", "config.path", "./test-fixtures/complex_outputs.tfstate"),
					testAccCheckStateValue("terraform_remote_state.foo", "computed_set.#", "2"),
					testAccCheckStateValue("terraform_remote_state.foo", `map.%`, "2"),
					testAccCheckStateValue("terraform_remote_state.foo", `map.key`, "test"),
				),
			},
		},
	})
}

func testAccCheckStateValue(id, name, value string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[id]
		if !ok {
			return fmt.Errorf("Not found: %s", id)
		}
		if rs.Primary.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		v := rs.Primary.Attributes[name]
		if v != value {
			return fmt.Errorf(
				"Value for %s is %s, not %s", name, v, value)
		}

		return nil
	}
}

const testAccState_basic = `
resource "terraform_remote_state" "foo" {
	backend = "_local"

	config {
		path = "./test-fixtures/basic.tfstate"
	}
}`

const testAccState_complexOutputs = `
resource "terraform_remote_state" "foo" {
	backend = "_local"

	config {
		path = "./test-fixtures/complex_outputs.tfstate"
	}
}`
<commit_msg>provider/template: convert resources to data sources<commit_after>package terraform

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestState_basic(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccState_basic,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckStateValue(
						"data.terraform_remote_state.foo", "foo", "bar"),
				),
			},
		},
	})
}

func TestState_complexOutputs(t *testing.T) {
	resource.UnitTest(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccState_complexOutputs,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckStateValue("terraform_remote_state.foo", "backend", "_local"),
					testAccCheckStateValue("terraform_remote_state.foo", "config.path", "./test-fixtures/complex_outputs.tfstate"),
					testAccCheckStateValue("terraform_remote_state.foo", "computed_set.#", "2"),
					testAccCheckStateValue("terraform_remote_state.foo", `map.%`, "2"),
					testAccCheckStateValue("terraform_remote_state.foo", `map.key`, "test"),
				),
			},
		},
	})
}

func testAccCheckStateValue(id, name, value string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[id]
		if !ok {
			return fmt.Errorf("Not found: %s", id)
		}
		if rs.Primary.ID == "" {
			return fmt.Errorf("No ID is set")
		}

		v := rs.Primary.Attributes[name]
		if v != value {
			return fmt.Errorf(
				"Value for %s is %s, not %s", name, v, value)
		}

		return nil
	}
}

const testAccState_basic = `
data "terraform_remote_state" "foo" {
	backend = "_local"

	config {
		path = "./test-fixtures/basic.tfstate"
	}
}`

const testAccState_complexOutputs = `
resource "terraform_remote_state" "foo" {
	backend = "_local"

	config {
		path = "./test-fixtures/complex_outputs.tfstate"
	}
}`
<|endoftext|>


<commit_before>package hermetic

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"

	log "github.com/sirupsen/logrus"
	git "gopkg.in/src-d/go-git.v4"
	"gopkg.in/src-d/go-git.v4/plumbing/object"
)

// A Repo abstracts the basic Git and GitHub operations required to submit a GitHub pull request against
// an upstream repository
//
// Besides two obvious cleanup operations (delete forked GitHub repository, delete local Git repository), A
// Repo implementation should expose just  enough functionality to add files to the repository and create
// a GitHub Pull Request containing only those files
type Repo interface {
	Add(fromLocation string, toLocationAfterRoot string) error
	CreatePR(description string, body string) (string, error) // push local changes, hit PR API
	DeleteLocal() error
	DeleteRemote() error
}

const (
	DefaultBranchName  = "master"
	UpstreamRemoteName = "upstream"
	OriginRemoteName   = "origin"
)

type repository struct {
	gitRepo           *git.Repository
	branchName        string
	deleteGithubRepo  deleteUserRepoFunc
	createPullRequest createPullRequestFunc
	pushLocalChanges  pushRepoChangesFunc
	token             tokenProvider
	locker            *sync.Mutex
	localPath         string
	remoteDeleted     bool
	localDeleted      bool
}

func (r *repository) Add(fromLocation string, toLocation string) error {
	r.locker.Lock()
	defer r.locker.Unlock()

	from, err := os.Open(fromLocation)
	if err != nil {
		return err
	}

	worktree, err := r.gitRepo.Worktree()
	if err != nil {
		return err
	}

	dst := filepath.Join(r.localPath, toLocation)

	to, err := os.Create(dst)
	if err != nil {
		return err
	}

	_, err = io.Copy(from, to)
	if err != nil {
		return err
	}

	err = to.Close()
	if err != nil {
		return err
	}

	err = from.Close()
	if err != nil {
		return err
	}

	_, err = worktree.Add(toLocation)
	if err != nil {
		return err
	}

	return nil
}

func (r *repository) CreatePR(prTitle string, prDescription string) (string, error) {
	r.locker.Lock()
	defer r.locker.Unlock()

	if r.localDeleted || r.remoteDeleted {
		log.Errorf("requested PR creation after deleting local or remote repository. Local repository deleted: %t. Remote repository deleted: %t", r.localDeleted, r.remoteDeleted)
		return "", fmt.Errorf("cannot create pull request if either local or remote Git repository has been deleted. Local deleted: %t. Remote deleted: %t", r.localDeleted, r.remoteDeleted)
	}

	// make sure we're on the right branch
	worktree, err := r.gitRepo.Worktree()
	if err != nil {
		return "", err
	}

	commitMsg := fmt.Sprintf("%s\n\n%s", prTitle, prDescription)
	_, err = worktree.Commit(commitMsg, &git.CommitOptions{
		// TODO read principal user info and set KEP tool as committer
		Author: &object.Signature{
			Name:  arbitraryBasicAuthUsername,
			Email: kepToolEmail,
			When:  time.Now(),
		},
	})

	if err != nil {
		log.Errorf("committing files before pushing: %s", err)
		return "", err
	}

	// push local changes
	err = r.pushLocalChanges()
	if err != nil {
		log.Errorf("pushing local changes: %s", err)
		return "", err
	}

	// use GitHub API to create PR
	prUrl, err := r.createPullRequest(prTitle, prDescription)
	if err != nil {
		log.Errorf("creating GitHub Pull Request: %s", err)
		return "", err
	}

	return prUrl, nil
}

func (r *repository) DeleteLocal() error {
	r.locker.Lock()
	defer r.locker.Unlock()

	if r.localDeleted {
		log.Info("local Git repository should already be deleted, skipping request")
		return nil
	}

	err := os.RemoveAll(r.localPath)
	if err != nil {
		log.Errorf("deleting local Git repository: %s", err)
		return err
	}

	r.localDeleted = true

	return nil
}

func (r *repository) DeleteRemote() error {
	r.locker.Lock()
	defer r.locker.Unlock()

	if r.remoteDeleted {
		log.Info("GitHub repository should already be deleted, skipping request")
		return nil
	}

	err := r.deleteGithubRepo()
	if err != nil {
		log.Errorf("deleting GitHub fork: %s", err)
		return err
	}

	r.remoteDeleted = true

	return nil
}

type tokenProvider interface {
	Value() (string, error)
}
<commit_msg>cleanup: language describing implementing a hermetic.Repository<commit_after>package hermetic

import (
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sync"
	"time"

	log "github.com/sirupsen/logrus"
	git "gopkg.in/src-d/go-git.v4"
	"gopkg.in/src-d/go-git.v4/plumbing/object"
)

// A Repo abstracts the basic Git and GitHub operations required to submit a GitHub pull request against
// an upstream repository
//
// Besides two cleanup operations (delete forked GitHub repository, delete local Git repository), A
// Repo implementation should expose just  enough functionality to add files to the repository and create
// a GitHub Pull Request containing only those files
type Repo interface {
	Add(fromLocation string, toLocationAfterRoot string) error
	CreatePR(description string, body string) (string, error) // push local changes, hit PR API
	DeleteLocal() error
	DeleteRemote() error
}

const (
	DefaultBranchName  = "master"
	UpstreamRemoteName = "upstream"
	OriginRemoteName   = "origin"
)

type repository struct {
	gitRepo           *git.Repository
	branchName        string
	deleteGithubRepo  deleteUserRepoFunc
	createPullRequest createPullRequestFunc
	pushLocalChanges  pushRepoChangesFunc
	token             tokenProvider
	locker            *sync.Mutex
	localPath         string
	remoteDeleted     bool
	localDeleted      bool
}

func (r *repository) Add(fromLocation string, toLocation string) error {
	r.locker.Lock()
	defer r.locker.Unlock()

	from, err := os.Open(fromLocation)
	if err != nil {
		return err
	}

	worktree, err := r.gitRepo.Worktree()
	if err != nil {
		return err
	}

	dst := filepath.Join(r.localPath, toLocation)

	to, err := os.Create(dst)
	if err != nil {
		return err
	}

	_, err = io.Copy(from, to)
	if err != nil {
		return err
	}

	err = to.Close()
	if err != nil {
		return err
	}

	err = from.Close()
	if err != nil {
		return err
	}

	_, err = worktree.Add(toLocation)
	if err != nil {
		return err
	}

	return nil
}

func (r *repository) CreatePR(prTitle string, prDescription string) (string, error) {
	r.locker.Lock()
	defer r.locker.Unlock()

	if r.localDeleted || r.remoteDeleted {
		log.Errorf("requested PR creation after deleting local or remote repository. Local repository deleted: %t. Remote repository deleted: %t", r.localDeleted, r.remoteDeleted)
		return "", fmt.Errorf("cannot create pull request if either local or remote Git repository has been deleted. Local deleted: %t. Remote deleted: %t", r.localDeleted, r.remoteDeleted)
	}

	// make sure we're on the right branch
	worktree, err := r.gitRepo.Worktree()
	if err != nil {
		return "", err
	}

	commitMsg := fmt.Sprintf("%s\n\n%s", prTitle, prDescription)
	_, err = worktree.Commit(commitMsg, &git.CommitOptions{
		// TODO read principal user info and set KEP tool as committer
		Author: &object.Signature{
			Name:  arbitraryBasicAuthUsername,
			Email: kepToolEmail,
			When:  time.Now(),
		},
	})

	if err != nil {
		log.Errorf("committing files before pushing: %s", err)
		return "", err
	}

	// push local changes
	err = r.pushLocalChanges()
	if err != nil {
		log.Errorf("pushing local changes: %s", err)
		return "", err
	}

	// use GitHub API to create PR
	prUrl, err := r.createPullRequest(prTitle, prDescription)
	if err != nil {
		log.Errorf("creating GitHub Pull Request: %s", err)
		return "", err
	}

	return prUrl, nil
}

func (r *repository) DeleteLocal() error {
	r.locker.Lock()
	defer r.locker.Unlock()

	if r.localDeleted {
		log.Info("local Git repository should already be deleted, skipping request")
		return nil
	}

	err := os.RemoveAll(r.localPath)
	if err != nil {
		log.Errorf("deleting local Git repository: %s", err)
		return err
	}

	r.localDeleted = true

	return nil
}

func (r *repository) DeleteRemote() error {
	r.locker.Lock()
	defer r.locker.Unlock()

	if r.remoteDeleted {
		log.Info("GitHub repository should already be deleted, skipping request")
		return nil
	}

	err := r.deleteGithubRepo()
	if err != nil {
		log.Errorf("deleting GitHub fork: %s", err)
		return err
	}

	r.remoteDeleted = true

	return nil
}

type tokenProvider interface {
	Value() (string, error)
}
<|endoftext|>


<commit_before>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2018 Red Hat, Inc.
 *
 */

package hostdisk

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"syscall"

	"kubevirt.io/client-go/log"
	ephemeraldiskutils "kubevirt.io/kubevirt/pkg/ephemeral-disk-utils"

	k8sv1 "k8s.io/api/core/v1"

	v1 "kubevirt.io/client-go/api/v1"
	"kubevirt.io/client-go/kubecli"
	"kubevirt.io/kubevirt/pkg/util"
	"kubevirt.io/kubevirt/pkg/util/types"
)

var pvcBaseDir = "/var/run/kubevirt-private/vmi-disks"

const (
	EventReasonToleratedSmallPV = "ToleratedSmallPV"
	EventTypeToleratedSmallPV   = k8sv1.EventTypeNormal
)

// Used by tests.
func setDiskDirectory(dir string) error {
	pvcBaseDir = dir
	return os.MkdirAll(dir, 0755)
}

func ReplacePVCByHostDisk(vmi *v1.VirtualMachineInstance, clientset kubecli.KubevirtClient) error {
	// If PVC is defined and it's not a BlockMode PVC, then it is replaced by HostDisk
	// Filesystem PersistenVolumeClaim is mounted into pod as directory from node filesystem
	for i := range vmi.Spec.Volumes {
		if volumeSource := &vmi.Spec.Volumes[i].VolumeSource; volumeSource.PersistentVolumeClaim != nil {

			pvc, exists, isBlockVolumePVC, err := types.IsPVCBlockFromClient(clientset, vmi.Namespace, volumeSource.PersistentVolumeClaim.ClaimName)
			if err != nil {
				return err
			} else if !exists {
				return fmt.Errorf("persistentvolumeclaim %v not found", volumeSource.PersistentVolumeClaim.ClaimName)
			} else if isBlockVolumePVC {
				continue
			}
			isSharedPvc := types.IsPVCShared(pvc)

			file := getPVCDiskImgPath(vmi.Spec.Volumes[i].Name, "disk.img")
			volumeSource.HostDisk = &v1.HostDisk{
				Path:     file,
				Type:     v1.HostDiskExistsOrCreate,
				Capacity: pvc.Status.Capacity[k8sv1.ResourceStorage],
				Shared:   &isSharedPvc,
			}
			// PersistenVolumeClaim is replaced by HostDisk
			volumeSource.PersistentVolumeClaim = nil
		}
	}
	return nil
}

func dirBytesAvailable(path string) (uint64, error) {
	var stat syscall.Statfs_t
	err := syscall.Statfs(path, &stat)
	if err != nil {
		return 0, err
	}
	return stat.Bavail * uint64(stat.Bsize), nil
}

func createSparseRaw(fullPath string, size int64) (err error) {
	offset := size - 1
	f, err := os.Create(fullPath)
	if err != nil {
		return err
	}
	defer util.CloseIOAndCheckErr(f, &err)
	_, err = f.WriteAt([]byte{0}, offset)
	if err != nil {
		return err
	}
	return nil
}

func getPVCDiskImgPath(volumeName string, diskName string) string {
	return path.Join(pvcBaseDir, volumeName, diskName)
}

func GetMountedHostDiskPath(volumeName string, path string) string {
	return getPVCDiskImgPath(volumeName, filepath.Base(path))
}

func GetMountedHostDiskDir(volumeName string) string {
	return getPVCDiskImgPath(volumeName, "")
}

type DiskImgCreator struct {
	dirBytesAvailableFunc  func(path string) (uint64, error)
	notifier               k8sNotifier
	lessPVCSpaceToleration int
}

type k8sNotifier interface {
	SendK8sEvent(vmi *v1.VirtualMachineInstance, severity string, reason string, message string) error
}

func NewHostDiskCreator(notifier k8sNotifier, lessPVCSpaceToleration int) DiskImgCreator {
	return DiskImgCreator{
		dirBytesAvailableFunc:  dirBytesAvailable,
		notifier:               notifier,
		lessPVCSpaceToleration: lessPVCSpaceToleration,
	}
}

func (hdc *DiskImgCreator) setlessPVCSpaceToleration(toleration int) {
	hdc.lessPVCSpaceToleration = toleration
}

func (hdc DiskImgCreator) Create(vmi *v1.VirtualMachineInstance) error {
	for _, volume := range vmi.Spec.Volumes {
		if hostDisk := volume.VolumeSource.HostDisk; hostDisk != nil && hostDisk.Type == v1.HostDiskExistsOrCreate && hostDisk.Path != "" {
			diskPath := GetMountedHostDiskPath(volume.Name, hostDisk.Path)
			diskDir := GetMountedHostDiskDir(volume.Name)
			if _, err := os.Stat(diskPath); os.IsNotExist(err) {
				availableSize, err := hdc.dirBytesAvailableFunc(diskDir)
				if err != nil {
					return err
				}
				requestedSize, _ := hostDisk.Capacity.AsInt64()
				diskSize := requestedSize
				if uint64(requestedSize) > availableSize {
					// Some storage provisioners provision less space than requested, due to filesystem overhead etc.
					// We tolerate some difference in requested and available capacity up to some degree.
					// This can be configured with the "pvc-tolerate-less-space-up-to-percent" parameter in the kubevirt-config ConfigMap.
					// It is provided as argument to virt-launcher.
					toleratedSize := requestedSize * (100 - int64(hdc.lessPVCSpaceToleration)) / 100
					if uint64(toleratedSize) > availableSize {
						return fmt.Errorf("unable to create %s, not enough space, demanded size %d B is bigger than available space %d B, also after taking %v %% toleration into account",
							hostDisk.Path, uint64(requestedSize), availableSize, hdc.lessPVCSpaceToleration)
					}
					diskSize = int64(availableSize)

					msg := fmt.Sprintf("PV size too small: expected %v B, found %v B. Using it anyway, it is within %v %% toleration", requestedSize, availableSize, hdc.lessPVCSpaceToleration)
					log.Log.Info(msg)
					err = hdc.notifier.SendK8sEvent(vmi, EventTypeToleratedSmallPV, EventReasonToleratedSmallPV, msg)
					if err != nil {
						log.Log.Reason(err).Warningf("Couldn't send k8s event for tolerated PV size: %v", err)
					}
				}
				err = createSparseRaw(diskPath, int64(diskSize))
				if err != nil {
					log.Log.Reason(err).Errorf("Couldn't create a sparse raw file for disk path: %s, error: %v", diskPath, err)
					return err
				}
			} else if err != nil {
				return err
			}
			if err := ephemeraldiskutils.DefaultOwnershipManager.SetFileOwnership(diskPath); err != nil {
				log.Log.Reason(err).Errorf("Couldn't set Ownership on %s: %v", diskPath, err)
				return err
			}
		}
	}
	return nil
}
<commit_msg>change ownership of the image provided by a filesystem PVC to qemu<commit_after>/*
 * This file is part of the KubeVirt project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * Copyright 2018 Red Hat, Inc.
 *
 */

package hostdisk

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"syscall"

	"kubevirt.io/client-go/log"
	ephemeraldiskutils "kubevirt.io/kubevirt/pkg/ephemeral-disk-utils"

	k8sv1 "k8s.io/api/core/v1"

	v1 "kubevirt.io/client-go/api/v1"
	"kubevirt.io/client-go/kubecli"
	"kubevirt.io/kubevirt/pkg/util"
	"kubevirt.io/kubevirt/pkg/util/types"
)

var pvcBaseDir = "/var/run/kubevirt-private/vmi-disks"

const (
	EventReasonToleratedSmallPV = "ToleratedSmallPV"
	EventTypeToleratedSmallPV   = k8sv1.EventTypeNormal
)

// Used by tests.
func setDiskDirectory(dir string) error {
	pvcBaseDir = dir
	return os.MkdirAll(dir, 0755)
}

func ReplacePVCByHostDisk(vmi *v1.VirtualMachineInstance, clientset kubecli.KubevirtClient) error {
	// If PVC is defined and it's not a BlockMode PVC, then it is replaced by HostDisk
	// Filesystem PersistenVolumeClaim is mounted into pod as directory from node filesystem
	for i := range vmi.Spec.Volumes {
		if volumeSource := &vmi.Spec.Volumes[i].VolumeSource; volumeSource.PersistentVolumeClaim != nil {

			pvc, exists, isBlockVolumePVC, err := types.IsPVCBlockFromClient(clientset, vmi.Namespace, volumeSource.PersistentVolumeClaim.ClaimName)
			if err != nil {
				return err
			} else if !exists {
				return fmt.Errorf("persistentvolumeclaim %v not found", volumeSource.PersistentVolumeClaim.ClaimName)
			} else if isBlockVolumePVC {
				continue
			}
			isSharedPvc := types.IsPVCShared(pvc)

			file := getPVCDiskImgPath(vmi.Spec.Volumes[i].Name, "disk.img")
			volumeSource.HostDisk = &v1.HostDisk{
				Path:     file,
				Type:     v1.HostDiskExistsOrCreate,
				Capacity: pvc.Status.Capacity[k8sv1.ResourceStorage],
				Shared:   &isSharedPvc,
			}
			// PersistenVolumeClaim is replaced by HostDisk
			volumeSource.PersistentVolumeClaim = nil
			// Set ownership of the disk.img to qemu
			if err := ephemeraldiskutils.DefaultOwnershipManager.SetFileOwnership(file); err != nil && !os.IsNotExist(err) {
				log.Log.Reason(err).Errorf("Couldn't set Ownership on %s: %v", file, err)
				return err
			}
		}
	}
	return nil
}

func dirBytesAvailable(path string) (uint64, error) {
	var stat syscall.Statfs_t
	err := syscall.Statfs(path, &stat)
	if err != nil {
		return 0, err
	}
	return stat.Bavail * uint64(stat.Bsize), nil
}

func createSparseRaw(fullPath string, size int64) (err error) {
	offset := size - 1
	f, err := os.Create(fullPath)
	if err != nil {
		return err
	}
	defer util.CloseIOAndCheckErr(f, &err)
	_, err = f.WriteAt([]byte{0}, offset)
	if err != nil {
		return err
	}
	return nil
}

func getPVCDiskImgPath(volumeName string, diskName string) string {
	return path.Join(pvcBaseDir, volumeName, diskName)
}

func GetMountedHostDiskPath(volumeName string, path string) string {
	return getPVCDiskImgPath(volumeName, filepath.Base(path))
}

func GetMountedHostDiskDir(volumeName string) string {
	return getPVCDiskImgPath(volumeName, "")
}

type DiskImgCreator struct {
	dirBytesAvailableFunc  func(path string) (uint64, error)
	notifier               k8sNotifier
	lessPVCSpaceToleration int
}

type k8sNotifier interface {
	SendK8sEvent(vmi *v1.VirtualMachineInstance, severity string, reason string, message string) error
}

func NewHostDiskCreator(notifier k8sNotifier, lessPVCSpaceToleration int) DiskImgCreator {
	return DiskImgCreator{
		dirBytesAvailableFunc:  dirBytesAvailable,
		notifier:               notifier,
		lessPVCSpaceToleration: lessPVCSpaceToleration,
	}
}

func (hdc *DiskImgCreator) setlessPVCSpaceToleration(toleration int) {
	hdc.lessPVCSpaceToleration = toleration
}

func (hdc DiskImgCreator) Create(vmi *v1.VirtualMachineInstance) error {
	for _, volume := range vmi.Spec.Volumes {
		if hostDisk := volume.VolumeSource.HostDisk; hostDisk != nil && hostDisk.Type == v1.HostDiskExistsOrCreate && hostDisk.Path != "" {
			diskPath := GetMountedHostDiskPath(volume.Name, hostDisk.Path)
			diskDir := GetMountedHostDiskDir(volume.Name)
			if _, err := os.Stat(diskPath); os.IsNotExist(err) {
				availableSize, err := hdc.dirBytesAvailableFunc(diskDir)
				if err != nil {
					return err
				}
				requestedSize, _ := hostDisk.Capacity.AsInt64()
				diskSize := requestedSize
				if uint64(requestedSize) > availableSize {
					// Some storage provisioners provision less space than requested, due to filesystem overhead etc.
					// We tolerate some difference in requested and available capacity up to some degree.
					// This can be configured with the "pvc-tolerate-less-space-up-to-percent" parameter in the kubevirt-config ConfigMap.
					// It is provided as argument to virt-launcher.
					toleratedSize := requestedSize * (100 - int64(hdc.lessPVCSpaceToleration)) / 100
					if uint64(toleratedSize) > availableSize {
						return fmt.Errorf("unable to create %s, not enough space, demanded size %d B is bigger than available space %d B, also after taking %v %% toleration into account",
							hostDisk.Path, uint64(requestedSize), availableSize, hdc.lessPVCSpaceToleration)
					}
					diskSize = int64(availableSize)

					msg := fmt.Sprintf("PV size too small: expected %v B, found %v B. Using it anyway, it is within %v %% toleration", requestedSize, availableSize, hdc.lessPVCSpaceToleration)
					log.Log.Info(msg)
					err = hdc.notifier.SendK8sEvent(vmi, EventTypeToleratedSmallPV, EventReasonToleratedSmallPV, msg)
					if err != nil {
						log.Log.Reason(err).Warningf("Couldn't send k8s event for tolerated PV size: %v", err)
					}
				}
				err = createSparseRaw(diskPath, int64(diskSize))
				if err != nil {
					log.Log.Reason(err).Errorf("Couldn't create a sparse raw file for disk path: %s, error: %v", diskPath, err)
					return err
				}
			} else if err != nil {
				return err
			}
			// Change file ownership to the qemu user.
			if err := ephemeraldiskutils.DefaultOwnershipManager.SetFileOwnership(diskPath); err != nil {
				log.Log.Reason(err).Errorf("Couldn't set Ownership on %s: %v", diskPath, err)
				return err
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"github.com/zrepl/zrepl/sshbytestream"
	"github.com/zrepl/zrepl/util"
	// "bytes"
	_ "bufio"
	// "strings"
	"io"
	"log"
	"os"
	_ "time"
)

func main() {

	mode := flag.String("mode", "", "incoming|outgoing")
	incomingFile := flag.String("incoming.file", "", "file to deliver to callers")
	outgoingHost := flag.String("outgoing.sshHost", "", "ssh host")
	outgoingUser := flag.String("outgoing.sshUser", "", "ssh user")
	outgoingIdentity := flag.String("outgoing.sshIdentity", "", "ssh private key")
	outgoingPort := flag.Uint("outgoing.sshPort", 22, "ssh port")
	outgoingFile := flag.String("outgoing.File", "", "")
	flag.Parse()

	switch {
	case (*mode == "incoming"):

		conn, err := sshbytestream.Incoming()
		if err != nil {
			panic(err)
		}

		file, err := os.Open(*incomingFile)
		if err != nil {
			panic(err)
		}

		chunker := chunking.NewChunker(file)

		_, err = io.Copy(conn, &chunker)
		if err != nil && err != io.EOF {
			panic(err)
		}

		log.Printf("Chunk Count: %d\n", chunker.ChunkCount)

	case *mode == "outgoing":

		conn, err := sshbytestream.Outgoing(sshbytestream.SSHTransport{
			Host:         *outgoingHost,
			User:         *outgoingUser,
			IdentityFile: *outgoingIdentity,
			Port:         uint16(*outgoingPort),
		})
		if err != nil {
			panic(err)
		}

		f, err := os.OpenFile(*outgoingFile, os.O_WRONLY, 0600)
		if err != nil {
			panic(err)
		}

		unchunker := chunking.NewUnchunker(conn)

		_, err = io.Copy(f, unchunker)
		if err != nil {
			panic(err)
		}

		conn.Close()

		log.Printf("Chunk Count: %d\n", unchunker.ChunkCount)

		os.Exit(0)

	default:
		panic("unsupported mode!")

	}

}
<commit_msg>create file if it doesn't exist<commit_after>package main

import (
	"flag"
	"github.com/zrepl/zrepl/sshbytestream"
	"github.com/zrepl/zrepl/util"
	// "bytes"
	_ "bufio"
	// "strings"
	"io"
	"log"
	"os"
	_ "time"
)

func main() {

	mode := flag.String("mode", "", "incoming|outgoing")
	incomingFile := flag.String("incoming.file", "", "file to deliver to callers")
	outgoingHost := flag.String("outgoing.sshHost", "", "ssh host")
	outgoingUser := flag.String("outgoing.sshUser", "", "ssh user")
	outgoingIdentity := flag.String("outgoing.sshIdentity", "", "ssh private key")
	outgoingPort := flag.Uint("outgoing.sshPort", 22, "ssh port")
	outgoingFile := flag.String("outgoing.File", "", "")
	flag.Parse()

	switch {
	case (*mode == "incoming"):

		conn, err := sshbytestream.Incoming()
		if err != nil {
			panic(err)
		}

		file, err := os.Open(*incomingFile)
		if err != nil {
			panic(err)
		}

		chunker := chunking.NewChunker(file)

		_, err = io.Copy(conn, &chunker)
		if err != nil && err != io.EOF {
			panic(err)
		}

		log.Printf("Chunk Count: %d\n", chunker.ChunkCount)

	case *mode == "outgoing":

		conn, err := sshbytestream.Outgoing(sshbytestream.SSHTransport{
			Host:         *outgoingHost,
			User:         *outgoingUser,
			IdentityFile: *outgoingIdentity,
			Port:         uint16(*outgoingPort),
		})
		if err != nil {
			panic(err)
		}

		f, err := os.OpenFile(*outgoingFile, os.O_CREATE|os.O_WRONLY, 0600)
		if err != nil {
			panic(err)
		}

		unchunker := chunking.NewUnchunker(conn)

		_, err = io.Copy(f, unchunker)
		if err != nil {
			panic(err)
		}

		conn.Close()

		log.Printf("Chunk Count: %d\n", unchunker.ChunkCount)

		os.Exit(0)

	default:
		panic("unsupported mode!")

	}

}
<|endoftext|>


<commit_before>package dns

import (
	"context"
	"fmt"
	"log"
	"time"

	"k8s.io/client-go/kubernetes"
	corev1listers "k8s.io/client-go/listers/core/v1"

	"github.com/jetstack-experimental/cert-manager/pkg/apis/certmanager/v1alpha1"
	"github.com/jetstack-experimental/cert-manager/pkg/issuer/acme/dns/clouddns"
	"github.com/jetstack-experimental/cert-manager/pkg/issuer/acme/dns/cloudflare"
	"github.com/jetstack-experimental/cert-manager/pkg/issuer/acme/dns/route53"
	"github.com/jetstack-experimental/cert-manager/pkg/issuer/acme/dns/util"
)

const (
	cloudDNSServiceAccountKey = "service-account.json"
)

type solver interface {
	Present(domain, token, key string) error
	CleanUp(domain, token, key string) error
	Timeout() (timeout, interval time.Duration)
}

type Solver struct {
	issuer       *v1alpha1.Issuer
	client       kubernetes.Interface
	secretLister corev1listers.SecretLister
}

func (s *Solver) Present(ctx context.Context, crt *v1alpha1.Certificate, domain, token, key string) error {
	slv, err := s.solverFor(crt, domain)
	if err != nil {
		return err
	}
	log.Printf("presenting key: %s", key)
	return slv.Present(domain, token, key)
}

func (s *Solver) Wait(ctx context.Context, crt *v1alpha1.Certificate, domain, token, key string) error {
	slv, err := s.solverFor(crt, domain)
	if err != nil {
		return err
	}

	type boolErr struct {
		bool
		error
	}

	fqdn, value, ttl := util.DNS01Record(domain, key)

	log.Printf("[%s] Checking DNS record propagation using %+v", domain, util.RecursiveNameservers)

	timeout, interval := slv.Timeout()
	ctx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()
	for {
		select {
		case r := <-func() <-chan boolErr {
			out := make(chan boolErr, 1)
			go func() {
				defer close(out)
				ok, err := util.PreCheckDNS(fqdn, value)
				out <- boolErr{ok, err}
			}()
			return out
		}():
			if r.bool {
				// TODO: move this to somewhere else
				// TODO: make this wait for whatever the record *was*, not is now
				log.Printf("sleeping for dns record for '%s' ttl %ds before returning from Wait", fqdn, ttl)
				time.Sleep(time.Second * time.Duration(ttl))
				return nil
			}
			log.Printf("[%s] dns record not yet propegated", domain)
			time.Sleep(interval)
		case <-ctx.Done():
			return ctx.Err()
		}
	}
}

func (s *Solver) CleanUp(ctx context.Context, crt *v1alpha1.Certificate, domain, token, key string) error {
	slv, err := s.solverFor(crt, domain)
	if err != nil {
		return err
	}
	return slv.CleanUp(domain, token, key)
}

func (s *Solver) solverFor(crt *v1alpha1.Certificate, domain string) (solver, error) {
	var cfg *v1alpha1.ACMECertificateDNS01Config
	if cfg = crt.Spec.ACME.ConfigForDomain(domain).DNS01; cfg == nil ||
		cfg.Provider == "" ||
		s.issuer.Spec.ACME == nil ||
		s.issuer.Spec.ACME.DNS01 == nil {
		return nil, fmt.Errorf("no dns01 config found for domain '%s'", domain)
	}

	providerConfig, err := s.issuer.Spec.ACME.DNS01.Provider(cfg.Provider)
	if err != nil {
		return nil, fmt.Errorf("invalid provider config specified for domain '%s': %s", domain, err.Error())
	}

	var impl solver
	switch {
	case providerConfig.CloudDNS != nil:
		saSecret, err := s.secretLister.Secrets(s.issuer.Namespace).Get(providerConfig.CloudDNS.ServiceAccount.Name)
		if err != nil {
			return nil, fmt.Errorf("error getting clouddns service account: %s", err.Error())
		}
		saBytes := saSecret.Data[providerConfig.CloudDNS.ServiceAccount.Key]

		impl, err = clouddns.NewDNSProviderServiceAccountBytes(providerConfig.CloudDNS.Project, saBytes)
		if err != nil {
			return nil, fmt.Errorf("error instantiating google clouddns challenge solver: %s", err.Error())
		}
	case providerConfig.Cloudflare != nil:
		apiKeySecret, err := s.secretLister.Secrets(s.issuer.Namespace).Get(providerConfig.Cloudflare.APIKey.Name)
		if err != nil {
			return nil, fmt.Errorf("error getting clouddns service account: %s", err.Error())
		}

		email := providerConfig.Cloudflare.Email
		apiKey := string(apiKeySecret.Data[providerConfig.Cloudflare.APIKey.Key])

		impl, err = cloudflare.NewDNSProviderCredentials(email, apiKey)
		if err != nil {
			return nil, fmt.Errorf("error instantiating cloudflare challenge solver: %s", err.Error())
		}
	case providerConfig.Route53 != nil:
		secretAccessKeySecret, err := s.secretLister.Secrets(s.issuer.Namespace).Get(providerConfig.Route53.SecretAccessKey.Name)
		if err != nil {
			return nil, fmt.Errorf("error getting route53 secret access key: %s", err.Error())
		}

		secretAccessKeyBytes, ok := secretAccessKeySecret.Data[providerConfig.Cloudflare.APIKey.Key]
		if !ok {
			return nil, fmt.Errorf("error getting route53 secret access key: key '%s' not found in secret", providerConfig.Route53.SecretAccessKey.Key)
		}

		impl, err = route53.NewDNSProviderAccessKey(
			providerConfig.Route53.AccessKeyID,
			string(secretAccessKeyBytes),
			providerConfig.Route53.HostedZoneID,
			providerConfig.Route53.Region,
		)
		if err != nil {
			return nil, fmt.Errorf("error instantiating route53 challenge solver: %s", err.Error())
		}
	default:
		return nil, fmt.Errorf("no dns provider config specified for domain '%s'", domain)
	}

	return impl, nil
}

func NewSolver(issuer *v1alpha1.Issuer, client kubernetes.Interface, secretLister corev1listers.SecretLister) *Solver {
	return &Solver{issuer, client, secretLister}
}
<commit_msg>Fix secret access key link<commit_after>package dns

import (
	"context"
	"fmt"
	"log"
	"time"

	"k8s.io/client-go/kubernetes"
	corev1listers "k8s.io/client-go/listers/core/v1"

	"github.com/jetstack-experimental/cert-manager/pkg/apis/certmanager/v1alpha1"
	"github.com/jetstack-experimental/cert-manager/pkg/issuer/acme/dns/clouddns"
	"github.com/jetstack-experimental/cert-manager/pkg/issuer/acme/dns/cloudflare"
	"github.com/jetstack-experimental/cert-manager/pkg/issuer/acme/dns/route53"
	"github.com/jetstack-experimental/cert-manager/pkg/issuer/acme/dns/util"
)

const (
	cloudDNSServiceAccountKey = "service-account.json"
)

type solver interface {
	Present(domain, token, key string) error
	CleanUp(domain, token, key string) error
	Timeout() (timeout, interval time.Duration)
}

type Solver struct {
	issuer       *v1alpha1.Issuer
	client       kubernetes.Interface
	secretLister corev1listers.SecretLister
}

func (s *Solver) Present(ctx context.Context, crt *v1alpha1.Certificate, domain, token, key string) error {
	slv, err := s.solverFor(crt, domain)
	if err != nil {
		return err
	}
	log.Printf("presenting key: %s", key)
	return slv.Present(domain, token, key)
}

func (s *Solver) Wait(ctx context.Context, crt *v1alpha1.Certificate, domain, token, key string) error {
	slv, err := s.solverFor(crt, domain)
	if err != nil {
		return err
	}

	type boolErr struct {
		bool
		error
	}

	fqdn, value, ttl := util.DNS01Record(domain, key)

	log.Printf("[%s] Checking DNS record propagation using %+v", domain, util.RecursiveNameservers)

	timeout, interval := slv.Timeout()
	ctx, cancel := context.WithTimeout(ctx, timeout)
	defer cancel()
	for {
		select {
		case r := <-func() <-chan boolErr {
			out := make(chan boolErr, 1)
			go func() {
				defer close(out)
				ok, err := util.PreCheckDNS(fqdn, value)
				out <- boolErr{ok, err}
			}()
			return out
		}():
			if r.bool {
				// TODO: move this to somewhere else
				// TODO: make this wait for whatever the record *was*, not is now
				log.Printf("sleeping for dns record for '%s' ttl %ds before returning from Wait", fqdn, ttl)
				time.Sleep(time.Second * time.Duration(ttl))
				return nil
			}
			log.Printf("[%s] dns record not yet propegated", domain)
			time.Sleep(interval)
		case <-ctx.Done():
			return ctx.Err()
		}
	}
}

func (s *Solver) CleanUp(ctx context.Context, crt *v1alpha1.Certificate, domain, token, key string) error {
	slv, err := s.solverFor(crt, domain)
	if err != nil {
		return err
	}
	return slv.CleanUp(domain, token, key)
}

func (s *Solver) solverFor(crt *v1alpha1.Certificate, domain string) (solver, error) {
	var cfg *v1alpha1.ACMECertificateDNS01Config
	if cfg = crt.Spec.ACME.ConfigForDomain(domain).DNS01; cfg == nil ||
		cfg.Provider == "" ||
		s.issuer.Spec.ACME == nil ||
		s.issuer.Spec.ACME.DNS01 == nil {
		return nil, fmt.Errorf("no dns01 config found for domain '%s'", domain)
	}

	providerConfig, err := s.issuer.Spec.ACME.DNS01.Provider(cfg.Provider)
	if err != nil {
		return nil, fmt.Errorf("invalid provider config specified for domain '%s': %s", domain, err.Error())
	}

	var impl solver
	switch {
	case providerConfig.CloudDNS != nil:
		saSecret, err := s.secretLister.Secrets(s.issuer.Namespace).Get(providerConfig.CloudDNS.ServiceAccount.Name)
		if err != nil {
			return nil, fmt.Errorf("error getting clouddns service account: %s", err.Error())
		}
		saBytes := saSecret.Data[providerConfig.CloudDNS.ServiceAccount.Key]

		impl, err = clouddns.NewDNSProviderServiceAccountBytes(providerConfig.CloudDNS.Project, saBytes)
		if err != nil {
			return nil, fmt.Errorf("error instantiating google clouddns challenge solver: %s", err.Error())
		}
	case providerConfig.Cloudflare != nil:
		apiKeySecret, err := s.secretLister.Secrets(s.issuer.Namespace).Get(providerConfig.Cloudflare.APIKey.Name)
		if err != nil {
			return nil, fmt.Errorf("error getting clouddns service account: %s", err.Error())
		}

		email := providerConfig.Cloudflare.Email
		apiKey := string(apiKeySecret.Data[providerConfig.Cloudflare.APIKey.Key])

		impl, err = cloudflare.NewDNSProviderCredentials(email, apiKey)
		if err != nil {
			return nil, fmt.Errorf("error instantiating cloudflare challenge solver: %s", err.Error())
		}
	case providerConfig.Route53 != nil:
		secretAccessKeySecret, err := s.secretLister.Secrets(s.issuer.Namespace).Get(providerConfig.Route53.SecretAccessKey.Name)
		if err != nil {
			return nil, fmt.Errorf("error getting route53 secret access key: %s", err.Error())
		}

		secretAccessKeyBytes, ok := secretAccessKeySecret.Data[providerConfig.Route53.SecretAccessKey.Key]
		if !ok {
			return nil, fmt.Errorf("error getting route53 secret access key: key '%s' not found in secret", providerConfig.Route53.SecretAccessKey.Key)
		}

		impl, err = route53.NewDNSProviderAccessKey(
			providerConfig.Route53.AccessKeyID,
			string(secretAccessKeyBytes),
			providerConfig.Route53.HostedZoneID,
			providerConfig.Route53.Region,
		)
		if err != nil {
			return nil, fmt.Errorf("error instantiating route53 challenge solver: %s", err.Error())
		}
	default:
		return nil, fmt.Errorf("no dns provider config specified for domain '%s'", domain)
	}

	return impl, nil
}

func NewSolver(issuer *v1alpha1.Issuer, client kubernetes.Interface, secretLister corev1listers.SecretLister) *Solver {
	return &Solver{issuer, client, secretLister}
}
<|endoftext|>


<commit_before>package landscaper

import (
	"encoding/json"
	"errors"
	"reflect"

	"github.com/Sirupsen/logrus"
	"github.com/pmezard/go-difflib/difflib"
	"google.golang.org/grpc"
	"k8s.io/helm/pkg/helm"
)

// Executor is responsible for applying a desired landscape to the actual landscape
type Executor interface {
	Apply([]*Component, []*Component) error

	CreateComponent(*Component) error
	UpdateComponent(*Component) error
	DeleteComponent(*Component) error
}

type executor struct {
	env             *Environment
	secretsProvider SecretsProvider
}

// NewExecutor is a factory method to create a new Executor
func NewExecutor(env *Environment, secretsProvider SecretsProvider) Executor {
	return &executor{
		env:             env,
		secretsProvider: secretsProvider,
	}
}

// Apply transforms the current state into the desired state
func (e *executor) Apply(desired, current []*Component) error {
	create, update, delete := diff(desired, current)

	logrus.WithFields(logrus.Fields{"create": len(create), "update": len(update), "delete": len(delete)}).Info("Apply desired state")

	if err := logDifferences(current, create, update, delete, logrus.Infof); err != nil {
		return err
	}

	for _, cmp := range delete {
		if err := e.DeleteComponent(cmp); err != nil {
			logrus.WithFields(logrus.Fields{"error": err}).Error("DeleteComponent failed")
			return err
		}
	}

	for _, cmp := range create {
		if err := e.CreateComponent(cmp); err != nil {
			logrus.WithFields(logrus.Fields{"error": err}).Error("CreateComponent failed")
			return err
		}
	}

	for _, cmp := range update {
		if err := e.UpdateComponent(cmp); err != nil {
			logrus.WithFields(logrus.Fields{"error": err}).Error("UpdateComponent failed")
			return err
		}
	}

	logrus.WithFields(logrus.Fields{"created": len(create), "updated": len(update), "deleted": len(delete)}).Info("Applied desired state sucessfully")
	return nil
}

// CreateComponent creates the given Component
func (e *executor) CreateComponent(cmp *Component) error {
	// We need to ensure the chart is available on the local system. LoadChart will ensure
	// this is the case by downloading the chart if it is not there yet
	chartRef, err := cmp.FullChartRef()
	if err != nil {
		return err
	}
	_, chartPath, err := e.env.ChartLoader.Load(chartRef)
	if err != nil {
		return err
	}

	rawValues, err := cmp.Configuration.YAML()
	if err != nil {
		return err
	}

	logrus.WithFields(logrus.Fields{
		"release":   cmp.Name,
		"chart":     cmp.Release.Chart,
		"chartPath": chartPath,
		"values":    cmp.Configuration,
		"dryrun":    e.env.DryRun,
	}).Debug("Create component")

	if len(cmp.Secrets) > 0 && !e.env.DryRun {
		err = e.secretsProvider.Write(cmp.Name, cmp.SecretValues)
		if err != nil {
			return err
		}
	}

	_, err = e.env.HelmClient().InstallRelease(
		chartPath,
		e.env.Namespace,
		helm.ValueOverrides([]byte(rawValues)),
		helm.ReleaseName(cmp.Name),
		helm.InstallDryRun(e.env.DryRun),
		helm.InstallReuseName(true),
	)
	if err != nil {
		return errors.New(grpc.ErrorDesc(err))
	}

	return nil
}

// UpdateComponent updates the given Component
func (e *executor) UpdateComponent(cmp *Component) error {
	// We need to ensure the chart is available on the local system. LoadChart will ensure
	// this is the case by downloading the chart if it is not there yet
	chartRef, err := cmp.FullChartRef()
	if err != nil {
		return err
	}
	_, chartPath, err := e.env.ChartLoader.Load(chartRef)
	if err != nil {
		return err
	}

	rawValues, err := cmp.Configuration.YAML()
	if err != nil {
		return err
	}

	if !e.env.DryRun {
		err = e.secretsProvider.Delete(cmp.Name)

		if len(cmp.Secrets) > 0 {
			err = e.secretsProvider.Write(cmp.Name, cmp.SecretValues)
			if err != nil {
				return err
			}
		}
	}

	logrus.WithFields(logrus.Fields{
		"release":   cmp.Name,
		"chart":     cmp.Release.Chart,
		"chartPath": chartPath,
		"values":    cmp.Configuration,
		"dryrun":    e.env.DryRun,
	}).Debug("Update component")

	_, err = e.env.HelmClient().UpdateRelease(
		cmp.Name,
		chartPath,
		helm.UpdateValueOverrides([]byte(rawValues)),
		helm.UpgradeDryRun(e.env.DryRun),
	)
	if err != nil {
		return errors.New(grpc.ErrorDesc(err))
	}

	return nil
}

// DeleteComponent removes the given Component
func (e *executor) DeleteComponent(cmp *Component) error {
	logrus.WithFields(logrus.Fields{
		"release": cmp.Name,
		"values":  cmp.Configuration,
		"dryrun":  e.env.DryRun,
	}).Debug("Delete component")

	if len(cmp.Secrets) > 0 && !e.env.DryRun {
		err := e.secretsProvider.Delete(cmp.Name)
		if err != nil {
			return err
		}
	}

	if !e.env.DryRun {
		_, err := e.env.HelmClient().DeleteRelease(
			cmp.Name,
			helm.DeletePurge(true),
			helm.DeleteDryRun(e.env.DryRun),
		)
		if err != nil {
			return errors.New(grpc.ErrorDesc(err))
		}
	}

	return nil
}

// diff takes desired and current components, and returns the components to create, update and delete to get from current to desired
func diff(desired, current []*Component) (create, update, delete []*Component) {
	desiredMap := make(map[string]*Component)
	currentMap := make(map[string]*Component)

	for _, c := range desired {
		desiredMap[c.Name] = c
	}
	for _, c := range current {
		currentMap[c.Name] = c
	}

	for name, desiredCmp := range desiredMap {
		if currentCmp, ok := currentMap[name]; ok {
			if !desiredCmp.Equals(currentCmp) {
				update = append(update, desiredCmp)
			}
		} else {
			create = append(create, desiredCmp)
		}
	}

	for name, currentCmp := range currentMap {
		if _, ok := desiredMap[name]; !ok {
			delete = append(delete, currentCmp)
		}
	}

	return create, update, delete
}

// componentDiffText returns a diff as text. current and desired can be nil and indicate non-existence (e.g. current nil and desired non-nil means: create)
func componentDiffText(current, desired *Component) (string, error) {
	cText, dText := []string{}, []string{}
	cName, dName := "<none>", "<none>"
	if current != nil {
		cs, err := json.MarshalIndent(current, "", "  ")
		if err != nil {
			return "", err
		}
		cText = difflib.SplitLines(string(cs))
		cName = current.Name
	}
	if desired != nil {
		ds, err := json.MarshalIndent(desired, "", "  ")
		if err != nil {
			return "", err
		}
		dText = difflib.SplitLines(string(ds))
		dName = desired.Name
	}

	return difflib.GetUnifiedDiffString(difflib.UnifiedDiff{
		A:        cText,
		FromFile: "Current " + cName,
		B:        dText,
		ToFile:   "Desired " + dName,
		Context:  3,
	})
}

// logDifferences logs the Create, Update and Delete w.r.t. current to logf
func logDifferences(current, creates, updates, deletes []*Component, logf func(format string, args ...interface{})) error {
	currentMap := make(map[string]*Component)
	for _, c := range current {
		currentMap[c.Name] = c
	}

	log := func(action string, current, desired *Component) error {
		diff, err := componentDiffText(current, desired)
		if err != nil {
			return err
		}
		logf("%s", action)
		if diff != "" {
			logf("Diff:\n%s", diff)
		}
		if !reflect.DeepEqual(current.SecretValues, desired.SecretValues) {
			logrus.Info("Diff: secrets have changed, not shown here")
		}
		return nil
	}

	for _, d := range creates {
		if err := log("Create: "+d.Name, nil, d); err != nil {
			return err
		}
	}

	for _, d := range updates {
		c := currentMap[d.Name]
		if err := log("Update: "+d.Name, c, d); err != nil {
			return err
		}
	}

	for _, d := range deletes {
		logf("Delete: %s", d.Name)
	}

	return nil
}
<commit_msg>:bug: Don't dereference nulls<commit_after>package landscaper

import (
	"encoding/json"
	"errors"
	"reflect"

	"github.com/Sirupsen/logrus"
	"github.com/pmezard/go-difflib/difflib"
	"google.golang.org/grpc"
	"k8s.io/helm/pkg/helm"
)

// Executor is responsible for applying a desired landscape to the actual landscape
type Executor interface {
	Apply([]*Component, []*Component) error

	CreateComponent(*Component) error
	UpdateComponent(*Component) error
	DeleteComponent(*Component) error
}

type executor struct {
	env             *Environment
	secretsProvider SecretsProvider
}

// NewExecutor is a factory method to create a new Executor
func NewExecutor(env *Environment, secretsProvider SecretsProvider) Executor {
	return &executor{
		env:             env,
		secretsProvider: secretsProvider,
	}
}

// Apply transforms the current state into the desired state
func (e *executor) Apply(desired, current []*Component) error {
	create, update, delete := diff(desired, current)

	logrus.WithFields(logrus.Fields{"create": len(create), "update": len(update), "delete": len(delete)}).Info("Apply desired state")

	if err := logDifferences(current, create, update, delete, logrus.Infof); err != nil {
		return err
	}

	for _, cmp := range delete {
		if err := e.DeleteComponent(cmp); err != nil {
			logrus.WithFields(logrus.Fields{"error": err}).Error("DeleteComponent failed")
			return err
		}
	}

	for _, cmp := range create {
		if err := e.CreateComponent(cmp); err != nil {
			logrus.WithFields(logrus.Fields{"error": err}).Error("CreateComponent failed")
			return err
		}
	}

	for _, cmp := range update {
		if err := e.UpdateComponent(cmp); err != nil {
			logrus.WithFields(logrus.Fields{"error": err}).Error("UpdateComponent failed")
			return err
		}
	}

	logrus.WithFields(logrus.Fields{"created": len(create), "updated": len(update), "deleted": len(delete)}).Info("Applied desired state sucessfully")
	return nil
}

// CreateComponent creates the given Component
func (e *executor) CreateComponent(cmp *Component) error {
	// We need to ensure the chart is available on the local system. LoadChart will ensure
	// this is the case by downloading the chart if it is not there yet
	chartRef, err := cmp.FullChartRef()
	if err != nil {
		return err
	}
	_, chartPath, err := e.env.ChartLoader.Load(chartRef)
	if err != nil {
		return err
	}

	rawValues, err := cmp.Configuration.YAML()
	if err != nil {
		return err
	}

	logrus.WithFields(logrus.Fields{
		"release":   cmp.Name,
		"chart":     cmp.Release.Chart,
		"chartPath": chartPath,
		"values":    cmp.Configuration,
		"dryrun":    e.env.DryRun,
	}).Debug("Create component")

	if len(cmp.Secrets) > 0 && !e.env.DryRun {
		err = e.secretsProvider.Write(cmp.Name, cmp.SecretValues)
		if err != nil {
			return err
		}
	}

	_, err = e.env.HelmClient().InstallRelease(
		chartPath,
		e.env.Namespace,
		helm.ValueOverrides([]byte(rawValues)),
		helm.ReleaseName(cmp.Name),
		helm.InstallDryRun(e.env.DryRun),
		helm.InstallReuseName(true),
	)
	if err != nil {
		return errors.New(grpc.ErrorDesc(err))
	}

	return nil
}

// UpdateComponent updates the given Component
func (e *executor) UpdateComponent(cmp *Component) error {
	// We need to ensure the chart is available on the local system. LoadChart will ensure
	// this is the case by downloading the chart if it is not there yet
	chartRef, err := cmp.FullChartRef()
	if err != nil {
		return err
	}
	_, chartPath, err := e.env.ChartLoader.Load(chartRef)
	if err != nil {
		return err
	}

	rawValues, err := cmp.Configuration.YAML()
	if err != nil {
		return err
	}

	if !e.env.DryRun {
		err = e.secretsProvider.Delete(cmp.Name)

		if len(cmp.Secrets) > 0 {
			err = e.secretsProvider.Write(cmp.Name, cmp.SecretValues)
			if err != nil {
				return err
			}
		}
	}

	logrus.WithFields(logrus.Fields{
		"release":   cmp.Name,
		"chart":     cmp.Release.Chart,
		"chartPath": chartPath,
		"values":    cmp.Configuration,
		"dryrun":    e.env.DryRun,
	}).Debug("Update component")

	_, err = e.env.HelmClient().UpdateRelease(
		cmp.Name,
		chartPath,
		helm.UpdateValueOverrides([]byte(rawValues)),
		helm.UpgradeDryRun(e.env.DryRun),
	)
	if err != nil {
		return errors.New(grpc.ErrorDesc(err))
	}

	return nil
}

// DeleteComponent removes the given Component
func (e *executor) DeleteComponent(cmp *Component) error {
	logrus.WithFields(logrus.Fields{
		"release": cmp.Name,
		"values":  cmp.Configuration,
		"dryrun":  e.env.DryRun,
	}).Debug("Delete component")

	if len(cmp.Secrets) > 0 && !e.env.DryRun {
		err := e.secretsProvider.Delete(cmp.Name)
		if err != nil {
			return err
		}
	}

	if !e.env.DryRun {
		_, err := e.env.HelmClient().DeleteRelease(
			cmp.Name,
			helm.DeletePurge(true),
			helm.DeleteDryRun(e.env.DryRun),
		)
		if err != nil {
			return errors.New(grpc.ErrorDesc(err))
		}
	}

	return nil
}

// diff takes desired and current components, and returns the components to create, update and delete to get from current to desired
func diff(desired, current []*Component) (create, update, delete []*Component) {
	desiredMap := make(map[string]*Component)
	currentMap := make(map[string]*Component)

	for _, c := range desired {
		desiredMap[c.Name] = c
	}
	for _, c := range current {
		currentMap[c.Name] = c
	}

	for name, desiredCmp := range desiredMap {
		if currentCmp, ok := currentMap[name]; ok {
			if !desiredCmp.Equals(currentCmp) {
				update = append(update, desiredCmp)
			}
		} else {
			create = append(create, desiredCmp)
		}
	}

	for name, currentCmp := range currentMap {
		if _, ok := desiredMap[name]; !ok {
			delete = append(delete, currentCmp)
		}
	}

	return create, update, delete
}

// componentDiffText returns a diff as text. current and desired can be nil and indicate non-existence (e.g. current nil and desired non-nil means: create)
func componentDiffText(current, desired *Component) (string, error) {
	cText, dText := []string{}, []string{}
	cName, dName := "<none>", "<none>"
	if current != nil {
		cs, err := json.MarshalIndent(current, "", "  ")
		if err != nil {
			return "", err
		}
		cText = difflib.SplitLines(string(cs))
		cName = current.Name
	}
	if desired != nil {
		ds, err := json.MarshalIndent(desired, "", "  ")
		if err != nil {
			return "", err
		}
		dText = difflib.SplitLines(string(ds))
		dName = desired.Name
	}

	return difflib.GetUnifiedDiffString(difflib.UnifiedDiff{
		A:        cText,
		FromFile: "Current " + cName,
		B:        dText,
		ToFile:   "Desired " + dName,
		Context:  3,
	})
}

// logDifferences logs the Create, Update and Delete w.r.t. current to logf
func logDifferences(current, creates, updates, deletes []*Component, logf func(format string, args ...interface{})) error {
	currentMap := make(map[string]*Component)
	for _, c := range current {
		currentMap[c.Name] = c
	}

	log := func(action string, current, desired *Component) error {
		diff, err := componentDiffText(current, desired)
		if err != nil {
			return err
		}
		logf("%s", action)
		if diff != "" {
			logf("Diff:\n%s", diff)
		}
		if current != nil && desired != nil && !reflect.DeepEqual(current.SecretValues, desired.SecretValues) {
			logrus.Info("Diff: secrets have changed, not shown here")
		}
		return nil
	}

	for _, d := range creates {
		if err := log("Create: "+d.Name, nil, d); err != nil {
			return err
		}
	}

	for _, d := range updates {
		c := currentMap[d.Name]
		if err := log("Update: "+d.Name, c, d); err != nil {
			return err
		}
	}

	for _, d := range deletes {
		logf("Delete: %s", d.Name)
	}

	return nil
}
<|endoftext|>


<commit_before>package system

import (
	"bytes"

	"golang.org/x/sys/unix"
)

const (
	// Value is larger than the maximum size allowed
	E2BIG unix.Errno = unix.E2BIG

	// Operation not supported
	EOPNOTSUPP unix.Errno = unix.EOPNOTSUPP
)

// Lgetxattr retrieves the value of the extended attribute identified by attr
// and associated with the given path in the file system.
// Returns a []byte slice if the xattr is set and nil otherwise.
func Lgetxattr(path string, attr string) ([]byte, error) {
	// Start with a 128 length byte array
	dest := make([]byte, 128)
	sz, errno := unix.Lgetxattr(path, attr, dest)

	for errno == unix.ERANGE {
		// Buffer too small, use zero-sized buffer to get the actual size
		sz, errno = unix.Lgetxattr(path, attr, []byte{})
		if errno != nil {
			return nil, errno
		}
		dest = make([]byte, sz)
		sz, errno = unix.Lgetxattr(path, attr, dest)
	}

	switch {
	case errno == unix.ENODATA:
		return nil, nil
	case errno != nil:
		return nil, errno
	}

	return dest[:sz], nil
}

// Lsetxattr sets the value of the extended attribute identified by attr
// and associated with the given path in the file system.
func Lsetxattr(path string, attr string, data []byte, flags int) error {
	return unix.Lsetxattr(path, attr, data, flags)
}

// Llistxattr lists extended attributes associated with the given path
// in the file system.
func Llistxattr(path string) ([]string, error) {
	var dest []byte

	for {
		sz, err := unix.Llistxattr(path, dest)
		if err != nil {
			return nil, err
		}

		if sz > len(dest) {
			dest = make([]byte, sz)
		} else {
			dest = dest[:sz]
			break
		}
	}

	var attrs []string
	for _, token := range bytes.Split(dest, []byte{0}) {
		if len(token) > 0 {
			attrs = append(attrs, string(token))
		}
	}

	return attrs, nil
}
<commit_msg>pkg/system: improve Llistxattr<commit_after>package system

import (
	"bytes"

	"golang.org/x/sys/unix"
)

const (
	// Value is larger than the maximum size allowed
	E2BIG unix.Errno = unix.E2BIG

	// Operation not supported
	EOPNOTSUPP unix.Errno = unix.EOPNOTSUPP
)

// Lgetxattr retrieves the value of the extended attribute identified by attr
// and associated with the given path in the file system.
// Returns a []byte slice if the xattr is set and nil otherwise.
func Lgetxattr(path string, attr string) ([]byte, error) {
	// Start with a 128 length byte array
	dest := make([]byte, 128)
	sz, errno := unix.Lgetxattr(path, attr, dest)

	for errno == unix.ERANGE {
		// Buffer too small, use zero-sized buffer to get the actual size
		sz, errno = unix.Lgetxattr(path, attr, []byte{})
		if errno != nil {
			return nil, errno
		}
		dest = make([]byte, sz)
		sz, errno = unix.Lgetxattr(path, attr, dest)
	}

	switch {
	case errno == unix.ENODATA:
		return nil, nil
	case errno != nil:
		return nil, errno
	}

	return dest[:sz], nil
}

// Lsetxattr sets the value of the extended attribute identified by attr
// and associated with the given path in the file system.
func Lsetxattr(path string, attr string, data []byte, flags int) error {
	return unix.Lsetxattr(path, attr, data, flags)
}

// Llistxattr lists extended attributes associated with the given path
// in the file system.
func Llistxattr(path string) ([]string, error) {
	dest := make([]byte, 128)
	sz, errno := unix.Llistxattr(path, dest)

	for errno == unix.ERANGE {
		// Buffer too small, use zero-sized buffer to get the actual size
		sz, errno = unix.Llistxattr(path, []byte{})
		if errno != nil {
			return nil, errno
		}

		dest = make([]byte, sz)
		sz, errno = unix.Llistxattr(path, dest)
	}
	if errno != nil {
		return nil, errno
	}

	var attrs []string
	for _, token := range bytes.Split(dest[:sz], []byte{0}) {
		if len(token) > 0 {
			attrs = append(attrs, string(token))
		}
	}

	return attrs, nil
}
<|endoftext|>