<commit_before>package maintenance_helper

import (
	"context"
	"fmt"

	"github.com/werf/logboek"

	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/cli-runtime/pkg/resource"
)

func Migrate2To3(ctx context.Context, helm2ReleaseName, helm3ReleaseName, helm3Namespace string, maintenanceHelper *MaintenanceHelper) error {
	existingHelm3Releases, err := maintenanceHelper.GetHelm3ReleasesList(ctx)
	if err != nil {
		return fmt.Errorf("error getting existing helm 3 releases to perform check: %s", err)
	}

	foundHelm3Release := false
	for _, releaseName := range existingHelm3Releases {
		if releaseName == helm3ReleaseName {
			foundHelm3Release = true
			break
		}
	}

	if foundHelm3Release {
		return fmt.Errorf("found already existing helm 3 release %q", helm3ReleaseName)
	}

	existingReleases, err := maintenanceHelper.GetHelm2ReleasesList(ctx)
	if err != nil {
		return fmt.Errorf("error getting existing helm 2 releases to perform check: %s", err)
	}

	foundHelm2Release := false
	for _, releaseName := range existingReleases {
		if releaseName == helm2ReleaseName {
			foundHelm2Release = true
			break
		}
	}

	if !foundHelm2Release {
		return fmt.Errorf("not found helm 2 release %q", helm2ReleaseName)
	}

	releaseData, err := maintenanceHelper.GetHelm2ReleaseData(ctx, helm2ReleaseName)
	if err != nil {
		return fmt.Errorf("unable to get helm 2 release %q info: %s", helm2ReleaseName, err)
	}

	logboek.Context(ctx).LogOptionalLn()
	if err := logboek.Context(ctx).Default().LogProcess("Creating helm 3 release %q", helm3ReleaseName).DoError(func() error {
		if err := maintenanceHelper.CreateHelm3ReleaseMetadataFromHelm2Release(ctx, helm3ReleaseName, helm3Namespace, releaseData); err != nil {
			return fmt.Errorf("unable to create helm 3 release %q: %s", helm3ReleaseName, err)
		}
		return nil
	}); err != nil {
		return err
	}

	infos, err := maintenanceHelper.BuildHelm2ResourcesInfos(releaseData)
	if err != nil {
		return fmt.Errorf("error building resources infos for release %q: %s", helm2ReleaseName, err)
	}

	var metadataAccessor = meta.NewAccessor()

	logboek.Context(ctx).LogOptionalLn()
	if err := logboek.Context(ctx).Default().LogProcess("Migrating %d resources of the release %q", len(infos), helm2ReleaseName).DoError(func() error {
		for _, info := range infos {
			logboek.Context(ctx).Default().LogF("%s\n", info.ObjectName())

			helper := resource.NewHelper(info.Client, info.Mapping)

			obj, err := helper.Get(info.Namespace, info.Name)
			if err != nil {
				return fmt.Errorf("error getting resource %s spec from %q namespace: %s", info.ObjectName(), info.Namespace, err)
			}

			annotations, err := metadataAccessor.Annotations(obj)
			if err != nil {
				return fmt.Errorf("error accessing annotations of %s: %s", info.ObjectName(), err)
			}
			if annotations == nil {
				annotations = make(map[string]string)
			}
			annotations["meta.helm.sh/release-name"] = helm3ReleaseName
			annotations["meta.helm.sh/release-namespace"] = helm3Namespace
			if err := metadataAccessor.SetAnnotations(obj, annotations); err != nil {
				return fmt.Errorf("error setting annotations of %s: %s", info.ObjectName(), err)
			}

			labels, err := metadataAccessor.Labels(obj)
			if err != nil {
				return fmt.Errorf("error accessing labels of %s: %s", info.ObjectName(), err)
			}
			if labels == nil {
				labels = make(map[string]string)
			}
			labels["app.kubernetes.io/managed-by"] = "Helm"
			if err := metadataAccessor.SetLabels(obj, labels); err != nil {
				return fmt.Errorf("error setting labels of %s: %s", info.ObjectName(), err)
			}

			if _, err := helper.Replace(info.Namespace, info.Name, false, obj); err != nil {
				return fmt.Errorf("error replacing %s: %s", info.ObjectName(), err)
			}
		}
		return nil
	}); err != nil {
		return err
	}

	logboek.Context(ctx).LogOptionalLn()
	if err := logboek.Context(ctx).Default().LogProcess("Deleting helm 2 metadata for release %q", helm2ReleaseName).DoError(func() error {
		if err := maintenanceHelper.DeleteHelm2ReleaseMetadata(ctx, helm2ReleaseName); err != nil {
			return fmt.Errorf("unable to delete helm 2 release storage metadata for the release %q: %s", helm2ReleaseName, err)
		}
		return nil
	}); err != nil {
		return err
	}

	logboek.Context(ctx).LogOptionalLn()
	logboek.Context(ctx).Default().LogFDetails(" + Successfully migrated helm 2 release %q into helm 3 release %q in the %q namespace\n", helm2ReleaseName, helm3ReleaseName, helm3Namespace)

	return nil
}
<commit_msg>[helm 2 to 3] Fix: ignore resources from helm 2 release which do not exist in kubernetes<commit_after>package maintenance_helper

import (
	"context"
	"fmt"

	apierrors "k8s.io/apimachinery/pkg/api/errors"

	"github.com/werf/logboek"

	"k8s.io/apimachinery/pkg/api/meta"
	"k8s.io/cli-runtime/pkg/resource"
)

func Migrate2To3(ctx context.Context, helm2ReleaseName, helm3ReleaseName, helm3Namespace string, maintenanceHelper *MaintenanceHelper) error {
	existingHelm3Releases, err := maintenanceHelper.GetHelm3ReleasesList(ctx)
	if err != nil {
		return fmt.Errorf("error getting existing helm 3 releases to perform check: %s", err)
	}

	foundHelm3Release := false
	for _, releaseName := range existingHelm3Releases {
		if releaseName == helm3ReleaseName {
			foundHelm3Release = true
			break
		}
	}

	if foundHelm3Release {
		return fmt.Errorf("found already existing helm 3 release %q", helm3ReleaseName)
	}

	existingReleases, err := maintenanceHelper.GetHelm2ReleasesList(ctx)
	if err != nil {
		return fmt.Errorf("error getting existing helm 2 releases to perform check: %s", err)
	}

	foundHelm2Release := false
	for _, releaseName := range existingReleases {
		if releaseName == helm2ReleaseName {
			foundHelm2Release = true
			break
		}
	}

	if !foundHelm2Release {
		return fmt.Errorf("not found helm 2 release %q", helm2ReleaseName)
	}

	releaseData, err := maintenanceHelper.GetHelm2ReleaseData(ctx, helm2ReleaseName)
	if err != nil {
		return fmt.Errorf("unable to get helm 2 release %q info: %s", helm2ReleaseName, err)
	}

	infos, err := maintenanceHelper.BuildHelm2ResourcesInfos(releaseData)
	if err != nil {
		return fmt.Errorf("error building resources infos for release %q: %s", helm2ReleaseName, err)
	}

	var metadataAccessor = meta.NewAccessor()

	logboek.Context(ctx).LogOptionalLn()
	if err := logboek.Context(ctx).Default().LogProcess("Migrating %d resources of the release %q", len(infos), helm2ReleaseName).DoError(func() error {
		for _, info := range infos {
			logboek.Context(ctx).Default().LogF("%s\n", info.ObjectName())

			helper := resource.NewHelper(info.Client, info.Mapping)

			obj, err := helper.Get(info.Namespace, info.Name)
			if apierrors.IsNotFound(err) {
				logboek.Context(ctx).Default().LogF("    %s not found: ignoring\n", info.ObjectName())
				continue
			} else if err != nil {
				return fmt.Errorf("error getting resource %s spec from %q namespace: %s", info.ObjectName(), info.Namespace, err)
			}

			annotations, err := metadataAccessor.Annotations(obj)
			if err != nil {
				return fmt.Errorf("error accessing annotations of %s: %s", info.ObjectName(), err)
			}
			if annotations == nil {
				annotations = make(map[string]string)
			}
			annotations["meta.helm.sh/release-name"] = helm3ReleaseName
			annotations["meta.helm.sh/release-namespace"] = helm3Namespace
			if err := metadataAccessor.SetAnnotations(obj, annotations); err != nil {
				return fmt.Errorf("error setting annotations of %s: %s", info.ObjectName(), err)
			}

			labels, err := metadataAccessor.Labels(obj)
			if err != nil {
				return fmt.Errorf("error accessing labels of %s: %s", info.ObjectName(), err)
			}
			if labels == nil {
				labels = make(map[string]string)
			}
			labels["app.kubernetes.io/managed-by"] = "Helm"
			if err := metadataAccessor.SetLabels(obj, labels); err != nil {
				return fmt.Errorf("error setting labels of %s: %s", info.ObjectName(), err)
			}

			if _, err := helper.Replace(info.Namespace, info.Name, false, obj); err != nil {
				return fmt.Errorf("error replacing %s: %s", info.ObjectName(), err)
			}
		}
		return nil
	}); err != nil {
		return err
	}

	logboek.Context(ctx).LogOptionalLn()
	if err := logboek.Context(ctx).Default().LogProcess("Creating helm 3 release %q", helm3ReleaseName).DoError(func() error {
		if err := maintenanceHelper.CreateHelm3ReleaseMetadataFromHelm2Release(ctx, helm3ReleaseName, helm3Namespace, releaseData); err != nil {
			return fmt.Errorf("unable to create helm 3 release %q: %s", helm3ReleaseName, err)
		}
		return nil
	}); err != nil {
		return err
	}

	logboek.Context(ctx).LogOptionalLn()
	if err := logboek.Context(ctx).Default().LogProcess("Deleting helm 2 metadata for release %q", helm2ReleaseName).DoError(func() error {
		if err := maintenanceHelper.DeleteHelm2ReleaseMetadata(ctx, helm2ReleaseName); err != nil {
			return fmt.Errorf("unable to delete helm 2 release storage metadata for the release %q: %s", helm2ReleaseName, err)
		}
		return nil
	}); err != nil {
		return err
	}

	logboek.Context(ctx).LogOptionalLn()
	logboek.Context(ctx).Default().LogFDetails(" + Successfully migrated helm 2 release %q into helm 3 release %q in the %q namespace\n", helm2ReleaseName, helm3ReleaseName, helm3Namespace)

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2020 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cryptolib

import (
	"encoding/json"

	"github.com/pkg/errors"
)

// AtomicContainerSig represents a JSON-encoded Attestation payload. It
// conforms to Red Hat's Atomic Host attestation signature format defined here:
// https://github.com/aweiteka/image/blob/e5a20d98fe698732df2b142846d007b45873627f/docs/signature.md
type atomicContainerSig struct {
	Critical critical          `json:"critical"`
	Optional map[string]string `json:"optional,omitempty"`
}

type critical struct {
	Identity identity `json:"identity"`
	Image    image    `json:"image"`
	Type     string   `json:"type"`
}

type identity struct {
	DockerRef string `json:"docker-reference"`
}

type image struct {
	Digest string `json:"docker-manifest-digest"`
}

// TODO(https://github.com/grafeas/kritis/issues/503): Decide whether
// AuthenticatedAttestation is a useful abstraction.
// AuthenticatedAttestation contains data that is extracted from an Attestation
// only after its signature has been verified. The contents of an Attestation
// payload should never be analyzed directly, as it may or may not be verified.
// Instead, these should be extracted into an AuthenticatedAttestation and
// analyzed from there.
type authenticatedAttestation struct {
	ImageName   string
	ImageDigest string
}

type authenticatedAttCheckerImpl struct{}

// Check that the data within the Attestation payload matches what we expect.
// NOTE: This is a simple comparison for plain attestations, but it is more
// complex for rich attestations.
func (c authenticatedAttCheckerImpl) checkAuthenticatedAttestation(payload []byte, imageName string, imageDigest string, convert convertFunc) error {
	authAtt, err := convert(payload)
	if err != nil {
		return err
	}
	if authAtt.ImageName != imageName {
		return errors.New("incorrect image name in Attestation payload")
	}
	if authAtt.ImageDigest != imageDigest {
		return errors.New("incorrect image digest in Attestation payload")
	}
	return nil
}
func convertAuthenticatedAttestation(payload []byte) (*authenticatedAttestation, error) {
	atomicSig := &atomicContainerSig{}
	if err := json.Unmarshal(payload, atomicSig); err != nil {
		return nil, errors.Wrap(err, "error parsing attestation payload")
	}

	return &authenticatedAttestation{
		ImageName:   atomicSig.Critical.Identity.DockerRef,
		ImageDigest: atomicSig.Critical.Image.Digest,
	}, nil
}
<commit_msg>Address round 3 comments<commit_after>/*
Copyright 2020 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cryptolib

import (
	"encoding/json"

	"github.com/pkg/errors"
)

// AtomicContainerSig represents a JSON-encoded Attestation payload. It
// conforms to Red Hat's Atomic Host attestation signature format defined here:
// https://github.com/aweiteka/image/blob/e5a20d98fe698732df2b142846d007b45873627f/docs/signature.md
type atomicContainerSig struct {
	Critical critical          `json:"critical"`
	Optional map[string]string `json:"optional,omitempty"`
}

type critical struct {
	Identity identity `json:"identity"`
	Image    image    `json:"image"`
	Type     string   `json:"type"`
}

type identity struct {
	DockerRef string `json:"docker-reference"`
}

type image struct {
	Digest string `json:"docker-manifest-digest"`
}

// TODO(https://github.com/grafeas/kritis/issues/503): Decide whether
// AuthenticatedAttestation is a useful abstraction.
// AuthenticatedAttestation contains data that is extracted from an Attestation
// only after its signature has been verified. The contents of an Attestation
// payload should never be analyzed directly, as it may or may not be verified.
// Instead, these should be extracted into an AuthenticatedAttestation and
// analyzed from there.
type authenticatedAttestation struct {
	ImageName   string
	ImageDigest string
}

type authenticatedAttCheckerImpl struct{}

// Check that the data within the Attestation payload matches what we expect.
// NOTE: This is a simple comparison for plain attestations, but it is more
// complex for rich attestations.
func (c authenticatedAttCheckerImpl) checkAuthenticatedAttestation(payload []byte, imageName string, imageDigest string, convert convertFunc) error {
	authAtt, err := convert(payload)
	if err != nil {
		return err
	}
	if authAtt.ImageName != imageName {
		return errors.New("incorrect image name in Attestation payload")
	}
	if authAtt.ImageDigest != imageDigest {
		return errors.New("incorrect image digest in Attestation payload")
	}
	return nil
}

func convertAuthenticatedAttestation(payload []byte) (*authenticatedAttestation, error) {
	atomicSig := &atomicContainerSig{}
	if err := json.Unmarshal(payload, atomicSig); err != nil {
		return nil, errors.Wrap(err, "error parsing attestation payload")
	}
	return &authenticatedAttestation{
		ImageName:   atomicSig.Critical.Identity.DockerRef,
		ImageDigest: atomicSig.Critical.Image.Digest,
	}, nil
}
<|endoftext|>


<commit_before>package cloudconfig_test

import (
	"errors"
	"fmt"

	. "github.com/enaml-ops/omg-cli/cloudconfigs/aws/cloud-config"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/enaml-ops/enaml"
	"github.com/enaml-ops/enaml/cloudproperties/aws"
)

var _ = Describe("given CloudConfig Deployment for AWS", func() {
	var awsConfig *enaml.CloudConfigManifest
	BeforeEach(func() {
		awsConfig = NewAWSCloudConfig(awscloudproperties.USWest, map[string]string{"us-east-1b": "subnet-asdfasdfa", "us-east-2c": "subnet-4444444"}, []string{"security-group-for-something-secure"})
	})

	Context("when AZs are defined", func() {
		It("then each AZ definition should map to a unique aws AZ", func() {
			err := checkUniqueAZs(awsConfig.AZs)
			Ω(err).ShouldNot(HaveOccurred())
		})
	})

	Context("when a user of the iaas would like to define vm types", func() {
		It("then there should be 2 vm type options available", func() {
			Ω(len(awsConfig.VMTypes)).Should(Equal(2))
		})

		It("then they should have the option of a small VM configuration", func() {
			_, err := getVmTypeByName(SmallVMName, awsConfig.VMTypes)
			Ω(err).ShouldNot(HaveOccurred())
		})

		Context("when the vmtype is small", func() {
			var vm enaml.VMType
			BeforeEach(func() {
				vm, _ = getVmTypeByName(SmallVMName, awsConfig.VMTypes)
			})
			It("then it should use a t2.micro size aws instance", func() {
				Ω(vm.CloudProperties.(awscloudproperties.VMType).InstanceType).Should(Equal(SmallVMSize))
			})

			It("then it should use a properly configured ephemeral disk", func() {
				properSmallDiskSize := SmallEphemeralDiskSize
				properDiskType := SmallDiskType
				Ω(vm.CloudProperties.(awscloudproperties.VMType).EphemeralDisk.Size).Should(Equal(properSmallDiskSize))
				Ω(vm.CloudProperties.(awscloudproperties.VMType).EphemeralDisk.DiskType).Should(Equal(properDiskType))
			})
		})

		It("then they should have the option of a large VM configuration", func() {
			_, err := getVmTypeByName(MediumVMName, awsConfig.VMTypes)
			Ω(err).ShouldNot(HaveOccurred())
		})

		Context("when the vmtype is large", func() {
			var vm enaml.VMType
			BeforeEach(func() {
				vm, _ = getVmTypeByName(MediumVMName, awsConfig.VMTypes)
			})
			It("then it should use a m3.medium size aws instance", func() {
				Ω(vm.CloudProperties.(awscloudproperties.VMType).InstanceType).Should(Equal(MediumVMSize))
			})

			It("then it should use a properly configured ephemeral disk", func() {
				properMediumDiskSize := MediumEphemeralDiskSize
				properDiskType := MediumDiskType
				Ω(vm.CloudProperties.(awscloudproperties.VMType).EphemeralDisk.Size).Should(Equal(properMediumDiskSize))
				Ω(vm.CloudProperties.(awscloudproperties.VMType).EphemeralDisk.DiskType).Should(Equal(properDiskType))
			})
		})
	})

	Context("when a user of the iaas would like to assign disk", func() {
		It("then all disk types should be properly configured", func() {
			for _, v := range awsConfig.DiskTypes {
				Ω(v.Name).ShouldNot(BeEmpty())
				Ω(v.DiskSize).Should(BeNumerically(">", 0))
				Ω(v.CloudProperties).ShouldNot(BeNil())
			}
		})
		It("then they should have the option of a small capacity configuration", func() {
			err := checkDiskTypeExists(awsConfig.DiskTypes, DiskSmallName)
			Ω(err).ShouldNot(HaveOccurred())
		})
		It("then they should have the option of a medium capacity configuration", func() {
			err := checkDiskTypeExists(awsConfig.DiskTypes, DiskMediumName)
			Ω(err).ShouldNot(HaveOccurred())
		})
		It("then they should have the option of a large capacity configuration", func() {
			err := checkDiskTypeExists(awsConfig.DiskTypes, DiskLargeName)
			Ω(err).ShouldNot(HaveOccurred())
		})
	})

	Context("when a user of the iaas would like to assign a network", func() {
		It("then they should have a private and a vip network", func() {
			var networkList []string
			for _, v := range awsConfig.Networks {
				switch v.(type) {
				case enaml.ManualNetwork:
					networkList = append(networkList, v.(enaml.ManualNetwork).Name)
				case enaml.VIPNetwork:
					networkList = append(networkList, v.(enaml.VIPNetwork).Name)
				case enaml.DynamicNetwork:
					networkList = append(networkList, v.(enaml.DynamicNetwork).Name)
				}
			}
			Ω(networkList).Should(ContainElement(PrivateNetworkName))
			Ω(networkList).Should(ContainElement(VIPNetworkName))
		})

		Context("when a user of the iaas is assigning the private network", func() {
			var privateNetwork enaml.ManualNetwork
			BeforeEach(func() {
				for _, v := range awsConfig.Networks {
					var name string
					switch v.(type) {
					case enaml.ManualNetwork:
						name = v.(enaml.ManualNetwork).Name
					case enaml.VIPNetwork:
						name = v.(enaml.VIPNetwork).Name
					case enaml.DynamicNetwork:
						name = v.(enaml.DynamicNetwork).Name
					}
					if name == PrivateNetworkName {
						privateNetwork = v.(enaml.ManualNetwork)
					}
				}
			})

			It("then they should have one subnet for each configured AZ", func() {
				Ω(len(privateNetwork.Subnets)).Should(Equal(len(awsConfig.AZs)))
			})

			It("then each subnet should be configured w/ required fields", func() {
				for _, v := range privateNetwork.Subnets {
					Ω(v.Range).ShouldNot(BeEmpty())
					Ω(v.AZ).ShouldNot(BeEmpty())
					Ω(v.Gateway).ShouldNot(BeEmpty())
					Ω(v.DNS).ShouldNot(BeEmpty())
				}
			})
		})
	})
})

func getVmTypeByName(name string, vmTypes []enaml.VMType) (res enaml.VMType, err error) {
	err = errors.New("no type found")
	for _, k := range vmTypes {
		if k.Name == name {
			err = nil
			res = k
		}
	}
	return
}

func checkUniqueAZs(azs []enaml.AZ) error {
	exists := make(map[string]int)
	for _, v := range azs {
		awsAZ := v.CloudProperties.(awscloudproperties.AZ).AvailabilityZoneName
		if _, alreadyExists := exists[awsAZ]; alreadyExists {
			return errors.New(fmt.Sprintf("duplicate az assignment to: %s", awsAZ))
		}
		exists[awsAZ] = 1
	}
	return nil
}

func checkDiskTypeExists(dsk []enaml.DiskType, name string) (err error) {
	err = errors.New(name + " capacity configuration not found")
	for _, v := range dsk {
		if v.Name == name {
			err = nil
			break
		}
	}
	return
}
<commit_msg>Fix tests to use new plugins dir<commit_after>package cloudconfig_test

import (
	"errors"
	"fmt"

	"github.com/enaml-ops/enaml"
	"github.com/enaml-ops/enaml/cloudproperties/aws"
	. "github.com/enaml-ops/omg-cli/plugins/cloudconfigs/aws/cloud-config"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("given CloudConfig Deployment for AWS", func() {
	var awsConfig *enaml.CloudConfigManifest
	BeforeEach(func() {
		awsConfig = NewAWSCloudConfig(awscloudproperties.USWest, map[string]string{"us-east-1b": "subnet-asdfasdfa", "us-east-2c": "subnet-4444444"}, []string{"security-group-for-something-secure"})
	})

	Context("when AZs are defined", func() {
		It("then each AZ definition should map to a unique aws AZ", func() {
			err := checkUniqueAZs(awsConfig.AZs)
			Ω(err).ShouldNot(HaveOccurred())
		})
	})

	Context("when a user of the iaas would like to define vm types", func() {
		It("then there should be 2 vm type options available", func() {
			Ω(len(awsConfig.VMTypes)).Should(Equal(2))
		})

		It("then they should have the option of a small VM configuration", func() {
			_, err := getVmTypeByName(SmallVMName, awsConfig.VMTypes)
			Ω(err).ShouldNot(HaveOccurred())
		})

		Context("when the vmtype is small", func() {
			var vm enaml.VMType
			BeforeEach(func() {
				vm, _ = getVmTypeByName(SmallVMName, awsConfig.VMTypes)
			})
			It("then it should use a t2.micro size aws instance", func() {
				Ω(vm.CloudProperties.(awscloudproperties.VMType).InstanceType).Should(Equal(SmallVMSize))
			})

			It("then it should use a properly configured ephemeral disk", func() {
				properSmallDiskSize := SmallEphemeralDiskSize
				properDiskType := SmallDiskType
				Ω(vm.CloudProperties.(awscloudproperties.VMType).EphemeralDisk.Size).Should(Equal(properSmallDiskSize))
				Ω(vm.CloudProperties.(awscloudproperties.VMType).EphemeralDisk.DiskType).Should(Equal(properDiskType))
			})
		})

		It("then they should have the option of a large VM configuration", func() {
			_, err := getVmTypeByName(MediumVMName, awsConfig.VMTypes)
			Ω(err).ShouldNot(HaveOccurred())
		})

		Context("when the vmtype is large", func() {
			var vm enaml.VMType
			BeforeEach(func() {
				vm, _ = getVmTypeByName(MediumVMName, awsConfig.VMTypes)
			})
			It("then it should use a m3.medium size aws instance", func() {
				Ω(vm.CloudProperties.(awscloudproperties.VMType).InstanceType).Should(Equal(MediumVMSize))
			})

			It("then it should use a properly configured ephemeral disk", func() {
				properMediumDiskSize := MediumEphemeralDiskSize
				properDiskType := MediumDiskType
				Ω(vm.CloudProperties.(awscloudproperties.VMType).EphemeralDisk.Size).Should(Equal(properMediumDiskSize))
				Ω(vm.CloudProperties.(awscloudproperties.VMType).EphemeralDisk.DiskType).Should(Equal(properDiskType))
			})
		})
	})

	Context("when a user of the iaas would like to assign disk", func() {
		It("then all disk types should be properly configured", func() {
			for _, v := range awsConfig.DiskTypes {
				Ω(v.Name).ShouldNot(BeEmpty())
				Ω(v.DiskSize).Should(BeNumerically(">", 0))
				Ω(v.CloudProperties).ShouldNot(BeNil())
			}
		})
		It("then they should have the option of a small capacity configuration", func() {
			err := checkDiskTypeExists(awsConfig.DiskTypes, DiskSmallName)
			Ω(err).ShouldNot(HaveOccurred())
		})
		It("then they should have the option of a medium capacity configuration", func() {
			err := checkDiskTypeExists(awsConfig.DiskTypes, DiskMediumName)
			Ω(err).ShouldNot(HaveOccurred())
		})
		It("then they should have the option of a large capacity configuration", func() {
			err := checkDiskTypeExists(awsConfig.DiskTypes, DiskLargeName)
			Ω(err).ShouldNot(HaveOccurred())
		})
	})

	Context("when a user of the iaas would like to assign a network", func() {
		It("then they should have a private and a vip network", func() {
			var networkList []string
			for _, v := range awsConfig.Networks {
				switch v.(type) {
				case enaml.ManualNetwork:
					networkList = append(networkList, v.(enaml.ManualNetwork).Name)
				case enaml.VIPNetwork:
					networkList = append(networkList, v.(enaml.VIPNetwork).Name)
				case enaml.DynamicNetwork:
					networkList = append(networkList, v.(enaml.DynamicNetwork).Name)
				}
			}
			Ω(networkList).Should(ContainElement(PrivateNetworkName))
			Ω(networkList).Should(ContainElement(VIPNetworkName))
		})

		Context("when a user of the iaas is assigning the private network", func() {
			var privateNetwork enaml.ManualNetwork
			BeforeEach(func() {
				for _, v := range awsConfig.Networks {
					var name string
					switch v.(type) {
					case enaml.ManualNetwork:
						name = v.(enaml.ManualNetwork).Name
					case enaml.VIPNetwork:
						name = v.(enaml.VIPNetwork).Name
					case enaml.DynamicNetwork:
						name = v.(enaml.DynamicNetwork).Name
					}
					if name == PrivateNetworkName {
						privateNetwork = v.(enaml.ManualNetwork)
					}
				}
			})

			It("then they should have one subnet for each configured AZ", func() {
				Ω(len(privateNetwork.Subnets)).Should(Equal(len(awsConfig.AZs)))
			})

			It("then each subnet should be configured w/ required fields", func() {
				for _, v := range privateNetwork.Subnets {
					Ω(v.Range).ShouldNot(BeEmpty())
					Ω(v.AZ).ShouldNot(BeEmpty())
					Ω(v.Gateway).ShouldNot(BeEmpty())
					Ω(v.DNS).ShouldNot(BeEmpty())
				}
			})
		})
	})
})

func getVmTypeByName(name string, vmTypes []enaml.VMType) (res enaml.VMType, err error) {
	err = errors.New("no type found")
	for _, k := range vmTypes {
		if k.Name == name {
			err = nil
			res = k
		}
	}
	return
}

func checkUniqueAZs(azs []enaml.AZ) error {
	exists := make(map[string]int)
	for _, v := range azs {
		awsAZ := v.CloudProperties.(awscloudproperties.AZ).AvailabilityZoneName
		if _, alreadyExists := exists[awsAZ]; alreadyExists {
			return errors.New(fmt.Sprintf("duplicate az assignment to: %s", awsAZ))
		}
		exists[awsAZ] = 1
	}
	return nil
}

func checkDiskTypeExists(dsk []enaml.DiskType, name string) (err error) {
	err = errors.New(name + " capacity configuration not found")
	for _, v := range dsk {
		if v.Name == name {
			err = nil
			break
		}
	}
	return
}
<|endoftext|>


<commit_before>package resumable // import "github.com/docker/docker/registry/resumable"

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"gotest.tools/assert"
	is "gotest.tools/assert/cmp"
)

func TestResumableRequestHeaderSimpleErrors(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "Hello, world !")
	}))
	defer ts.Close()

	client := &http.Client{}

	var req *http.Request
	req, err := http.NewRequest(http.MethodGet, ts.URL, nil)
	assert.NilError(t, err)

	resreq := &requestReader{}
	_, err = resreq.Read([]byte{})
	assert.Check(t, is.Error(err, "client and request can't be nil"))

	resreq = &requestReader{
		client:    client,
		request:   req,
		totalSize: -1,
	}
	_, err = resreq.Read([]byte{})
	assert.Check(t, is.Error(err, "failed to auto detect content length"))
}

// Not too much failures, bails out after some wait
func TestResumableRequestHeaderNotTooMuchFailures(t *testing.T) {
	client := &http.Client{}

	var badReq *http.Request
	badReq, err := http.NewRequest(http.MethodGet, "I'm not an url", nil)
	assert.NilError(t, err)

	resreq := &requestReader{
		client:       client,
		request:      badReq,
		failures:     0,
		maxFailures:  2,
		waitDuration: 10 * time.Millisecond,
	}
	read, err := resreq.Read([]byte{})
	assert.NilError(t, err)
	assert.Check(t, is.Equal(0, read))
}

// Too much failures, returns the error
func TestResumableRequestHeaderTooMuchFailures(t *testing.T) {
	client := &http.Client{}

	var badReq *http.Request
	badReq, err := http.NewRequest(http.MethodGet, "I'm not an url", nil)
	assert.NilError(t, err)

	resreq := &requestReader{
		client:      client,
		request:     badReq,
		failures:    0,
		maxFailures: 1,
	}
	defer resreq.Close()

	expectedError := `Get I%27m%20not%20an%20url: unsupported protocol scheme ""`
	read, err := resreq.Read([]byte{})
	assert.Check(t, is.Error(err, expectedError))
	assert.Check(t, is.Equal(0, read))
}

type errorReaderCloser struct{}

func (errorReaderCloser) Close() error { return nil }

func (errorReaderCloser) Read(p []byte) (n int, err error) {
	return 0, fmt.Errorf("An error occurred")
}

// If an unknown error is encountered, return 0, nil and log it
func TestResumableRequestReaderWithReadError(t *testing.T) {
	var req *http.Request
	req, err := http.NewRequest(http.MethodGet, "", nil)
	assert.NilError(t, err)

	client := &http.Client{}

	response := &http.Response{
		Status:        "500 Internal Server",
		StatusCode:    http.StatusInternalServerError,
		ContentLength: 0,
		Close:         true,
		Body:          errorReaderCloser{},
	}

	resreq := &requestReader{
		client:          client,
		request:         req,
		currentResponse: response,
		lastRange:       1,
		totalSize:       1,
	}
	defer resreq.Close()

	buf := make([]byte, 1)
	read, err := resreq.Read(buf)
	assert.NilError(t, err)

	assert.Check(t, is.Equal(0, read))
}

func TestResumableRequestReaderWithEOFWith416Response(t *testing.T) {
	var req *http.Request
	req, err := http.NewRequest(http.MethodGet, "", nil)
	assert.NilError(t, err)

	client := &http.Client{}

	response := &http.Response{
		Status:        "416 Requested Range Not Satisfiable",
		StatusCode:    http.StatusRequestedRangeNotSatisfiable,
		ContentLength: 0,
		Close:         true,
		Body:          ioutil.NopCloser(strings.NewReader("")),
	}

	resreq := &requestReader{
		client:          client,
		request:         req,
		currentResponse: response,
		lastRange:       1,
		totalSize:       1,
	}
	defer resreq.Close()

	buf := make([]byte, 1)
	_, err = resreq.Read(buf)
	assert.Check(t, is.Error(err, io.EOF.Error()))
}

func TestResumableRequestReaderWithServerDoesntSupportByteRanges(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("Range") == "" {
			t.Fatalf("Expected a Range HTTP header, got nothing")
		}
	}))
	defer ts.Close()

	var req *http.Request
	req, err := http.NewRequest(http.MethodGet, ts.URL, nil)
	assert.NilError(t, err)

	client := &http.Client{}

	resreq := &requestReader{
		client:    client,
		request:   req,
		lastRange: 1,
	}
	defer resreq.Close()

	buf := make([]byte, 2)
	_, err = resreq.Read(buf)
	assert.Check(t, is.Error(err, "the server doesn't support byte ranges"))
}

func TestResumableRequestReaderWithZeroTotalSize(t *testing.T) {
	srvtxt := "some response text data"

	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, srvtxt)
	}))
	defer ts.Close()

	var req *http.Request
	req, err := http.NewRequest(http.MethodGet, ts.URL, nil)
	assert.NilError(t, err)

	client := &http.Client{}
	retries := uint32(5)

	resreq := NewRequestReader(client, req, retries, 0)
	defer resreq.Close()

	data, err := ioutil.ReadAll(resreq)
	assert.NilError(t, err)

	resstr := strings.TrimSuffix(string(data), "\n")
	assert.Check(t, is.Equal(srvtxt, resstr))
}

func TestResumableRequestReader(t *testing.T) {
	srvtxt := "some response text data"

	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, srvtxt)
	}))
	defer ts.Close()

	var req *http.Request
	req, err := http.NewRequest(http.MethodGet, ts.URL, nil)
	assert.NilError(t, err)

	client := &http.Client{}
	retries := uint32(5)
	imgSize := int64(len(srvtxt))

	resreq := NewRequestReader(client, req, retries, imgSize)
	defer resreq.Close()

	data, err := ioutil.ReadAll(resreq)
	assert.NilError(t, err)

	resstr := strings.TrimSuffix(string(data), "\n")
	assert.Check(t, is.Equal(srvtxt, resstr))
}

func TestResumableRequestReaderWithInitialResponse(t *testing.T) {
	srvtxt := "some response text data"

	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, srvtxt)
	}))
	defer ts.Close()

	var req *http.Request
	req, err := http.NewRequest(http.MethodGet, ts.URL, nil)
	assert.NilError(t, err)

	client := &http.Client{}
	retries := uint32(5)
	imgSize := int64(len(srvtxt))

	res, err := client.Do(req)
	assert.NilError(t, err)

	resreq := NewRequestReaderWithInitialResponse(client, req, retries, imgSize, res)
	defer resreq.Close()

	data, err := ioutil.ReadAll(resreq)
	assert.NilError(t, err)

	resstr := strings.TrimSuffix(string(data), "\n")
	assert.Check(t, is.Equal(srvtxt, resstr))
}
<commit_msg>TestResumableRequestHeaderTooMuchFailures: prepare for Go 1.14 changes<commit_after>package resumable // import "github.com/docker/docker/registry/resumable"

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"gotest.tools/assert"
	is "gotest.tools/assert/cmp"
)

func TestResumableRequestHeaderSimpleErrors(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "Hello, world !")
	}))
	defer ts.Close()

	client := &http.Client{}

	var req *http.Request
	req, err := http.NewRequest(http.MethodGet, ts.URL, nil)
	assert.NilError(t, err)

	resreq := &requestReader{}
	_, err = resreq.Read([]byte{})
	assert.Check(t, is.Error(err, "client and request can't be nil"))

	resreq = &requestReader{
		client:    client,
		request:   req,
		totalSize: -1,
	}
	_, err = resreq.Read([]byte{})
	assert.Check(t, is.Error(err, "failed to auto detect content length"))
}

// Not too much failures, bails out after some wait
func TestResumableRequestHeaderNotTooMuchFailures(t *testing.T) {
	client := &http.Client{}

	var badReq *http.Request
	badReq, err := http.NewRequest(http.MethodGet, "I'm not an url", nil)
	assert.NilError(t, err)

	resreq := &requestReader{
		client:       client,
		request:      badReq,
		failures:     0,
		maxFailures:  2,
		waitDuration: 10 * time.Millisecond,
	}
	read, err := resreq.Read([]byte{})
	assert.NilError(t, err)
	assert.Check(t, is.Equal(0, read))
}

// Too much failures, returns the error
func TestResumableRequestHeaderTooMuchFailures(t *testing.T) {
	client := &http.Client{}

	var badReq *http.Request
	badReq, err := http.NewRequest(http.MethodGet, "I'm not an url", nil)
	assert.NilError(t, err)

	resreq := &requestReader{
		client:      client,
		request:     badReq,
		failures:    0,
		maxFailures: 1,
	}
	defer resreq.Close()

	read, err := resreq.Read([]byte{})
	assert.Assert(t, err != nil)
	assert.Check(t, is.ErrorContains(err, "unsupported protocol scheme"))
	assert.Check(t, is.ErrorContains(err, "I%27m%20not%20an%20url"))
	assert.Check(t, is.Equal(0, read))
}

type errorReaderCloser struct{}

func (errorReaderCloser) Close() error { return nil }

func (errorReaderCloser) Read(p []byte) (n int, err error) {
	return 0, fmt.Errorf("An error occurred")
}

// If an unknown error is encountered, return 0, nil and log it
func TestResumableRequestReaderWithReadError(t *testing.T) {
	var req *http.Request
	req, err := http.NewRequest(http.MethodGet, "", nil)
	assert.NilError(t, err)

	client := &http.Client{}

	response := &http.Response{
		Status:        "500 Internal Server",
		StatusCode:    http.StatusInternalServerError,
		ContentLength: 0,
		Close:         true,
		Body:          errorReaderCloser{},
	}

	resreq := &requestReader{
		client:          client,
		request:         req,
		currentResponse: response,
		lastRange:       1,
		totalSize:       1,
	}
	defer resreq.Close()

	buf := make([]byte, 1)
	read, err := resreq.Read(buf)
	assert.NilError(t, err)

	assert.Check(t, is.Equal(0, read))
}

func TestResumableRequestReaderWithEOFWith416Response(t *testing.T) {
	var req *http.Request
	req, err := http.NewRequest(http.MethodGet, "", nil)
	assert.NilError(t, err)

	client := &http.Client{}

	response := &http.Response{
		Status:        "416 Requested Range Not Satisfiable",
		StatusCode:    http.StatusRequestedRangeNotSatisfiable,
		ContentLength: 0,
		Close:         true,
		Body:          ioutil.NopCloser(strings.NewReader("")),
	}

	resreq := &requestReader{
		client:          client,
		request:         req,
		currentResponse: response,
		lastRange:       1,
		totalSize:       1,
	}
	defer resreq.Close()

	buf := make([]byte, 1)
	_, err = resreq.Read(buf)
	assert.Check(t, is.Error(err, io.EOF.Error()))
}

func TestResumableRequestReaderWithServerDoesntSupportByteRanges(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("Range") == "" {
			t.Fatalf("Expected a Range HTTP header, got nothing")
		}
	}))
	defer ts.Close()

	var req *http.Request
	req, err := http.NewRequest(http.MethodGet, ts.URL, nil)
	assert.NilError(t, err)

	client := &http.Client{}

	resreq := &requestReader{
		client:    client,
		request:   req,
		lastRange: 1,
	}
	defer resreq.Close()

	buf := make([]byte, 2)
	_, err = resreq.Read(buf)
	assert.Check(t, is.Error(err, "the server doesn't support byte ranges"))
}

func TestResumableRequestReaderWithZeroTotalSize(t *testing.T) {
	srvtxt := "some response text data"

	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, srvtxt)
	}))
	defer ts.Close()

	var req *http.Request
	req, err := http.NewRequest(http.MethodGet, ts.URL, nil)
	assert.NilError(t, err)

	client := &http.Client{}
	retries := uint32(5)

	resreq := NewRequestReader(client, req, retries, 0)
	defer resreq.Close()

	data, err := ioutil.ReadAll(resreq)
	assert.NilError(t, err)

	resstr := strings.TrimSuffix(string(data), "\n")
	assert.Check(t, is.Equal(srvtxt, resstr))
}

func TestResumableRequestReader(t *testing.T) {
	srvtxt := "some response text data"

	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, srvtxt)
	}))
	defer ts.Close()

	var req *http.Request
	req, err := http.NewRequest(http.MethodGet, ts.URL, nil)
	assert.NilError(t, err)

	client := &http.Client{}
	retries := uint32(5)
	imgSize := int64(len(srvtxt))

	resreq := NewRequestReader(client, req, retries, imgSize)
	defer resreq.Close()

	data, err := ioutil.ReadAll(resreq)
	assert.NilError(t, err)

	resstr := strings.TrimSuffix(string(data), "\n")
	assert.Check(t, is.Equal(srvtxt, resstr))
}

func TestResumableRequestReaderWithInitialResponse(t *testing.T) {
	srvtxt := "some response text data"

	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, srvtxt)
	}))
	defer ts.Close()

	var req *http.Request
	req, err := http.NewRequest(http.MethodGet, ts.URL, nil)
	assert.NilError(t, err)

	client := &http.Client{}
	retries := uint32(5)
	imgSize := int64(len(srvtxt))

	res, err := client.Do(req)
	assert.NilError(t, err)

	resreq := NewRequestReaderWithInitialResponse(client, req, retries, imgSize, res)
	defer resreq.Close()

	data, err := ioutil.ReadAll(resreq)
	assert.NilError(t, err)

	resstr := strings.TrimSuffix(string(data), "\n")
	assert.Check(t, is.Equal(srvtxt, resstr))
}
<|endoftext|>


<commit_before>package internal

import (
	"fmt"
	"github.com/stretchr/testify/assert"
	"github.com/yunify/metadata-proxy/backends"
	"github.com/yunify/metadata-proxy/store"
	"math/rand"
	"testing"
)

func FillTestData(prefix string, storeClient backends.StoreClient) map[string]string {
	testData := make(map[string]string)
	for i := 0; i < 10; i++ {
		for j := 0; j < 10; j++ {
			key := fmt.Sprintf("%s/%v/%v", prefix, i, j)
			val := fmt.Sprintf("%v-%v", i, j)
			testData[key] = val
		}
	}
	storeClient.SetValues(testData)
	return testData
}

func RandomUpdate(testData map[string]string, storeClient backends.StoreClient, times int) {
	length := len(testData)
	keys := make([]string, 0, length)
	for k := range testData {
		keys = append(keys, k)
	}
	for i := 0; i < times; i++ {
		idx := rand.Intn(length)
		key := keys[idx]
		val := testData[key]
		newVal := fmt.Sprintf("%s-%v", val, 0)

		storeClient.SetValues(map[string]string{key: newVal})
		testData[key] = newVal
	}
}

func RandomDelete(testData map[string]string, storeClient backends.StoreClient) string {
	length := len(testData)
	keys := make([]string, 0, length)
	for k := range testData {
		keys = append(keys, k)
	}
	idx := rand.Intn(length)
	key := keys[idx]
	storeClient.Delete(key)
	delete(testData, key)
	return key
}

func ValidTestData(t *testing.T, testData map[string]string, metastore store.Store) {
	for k, v := range testData {
		storeVal, _ := metastore.Get(k)
		assert.Equal(t, v, storeVal)
	}
}
<commit_msg>testhelper<commit_after><|endoftext|>


<commit_before>package formbuilder

import (
	"fmt"

	"github.com/coscms/forms"
	"github.com/coscms/forms/common"
	"github.com/coscms/forms/config"
	"github.com/coscms/forms/fields"
	"github.com/webx-top/com"
	"github.com/webx-top/echo"
)

// NewConfig 表单配置
func NewConfig(theme, tmpl, method, action string) *config.Config {
	cfg := forms.NewConfig()
	cfg.Theme = theme
	cfg.Template = tmpl
	cfg.Method = method
	cfg.Action = action
	return cfg
}

// NewSnippet 表单片段
func NewSnippet(theme ...string) *forms.Form {
	cfg := forms.NewConfig()
	if len(theme) > 0 {
		cfg.Theme = theme[0]
	}
	cfg.Template = common.TmplDir(cfg.Theme) + `/allfields.html`
	form := forms.NewWithConfig(cfg)
	return form
}

func ClearCache() {
	common.ClearCachedConfig()
	common.ClearCachedTemplate()
}

func DelCachedConfig(file string) bool {
	return common.DelCachedConfig(file)
}

func AddChoiceByKV(field fields.FieldInterface, kvData *echo.KVData, checkedKeys ...string) fields.FieldInterface {
	for _, kv := range kvData.Slice() {
		var checked bool
		if kv.H != nil {
			checked = kv.H.Bool(`checked`) || kv.H.Bool(`selected`)
		}
		if len(checkedKeys) > 0 {
			checked = com.InSlice(kv.K, checkedKeys)
		}
		field.AddChoice(kv.K, kv.V, checked)
	}
	return field
}

func SetChoiceByKV(field fields.FieldInterface, kvData *echo.KVData, checkedKeys ...string) fields.FieldInterface {
	choices := []fields.InputChoice{}
	if len(checkedKeys) == 0 {
		panic(fmt.Sprintf(`%T: %s`, field, echo.Dump(field, false)))
		switch f := field.(type) {
		case *fields.Field:
			if len(f.Value) > 0 {
				checkedKeys = append(checkedKeys, f.Value)
			}
		}
	}
	for _, kv := range kvData.Slice() {
		var checked bool
		if kv.H != nil {
			checked = kv.H.Bool(`checked`) || kv.H.Bool(`selected`)
		}
		if len(checkedKeys) > 0 {
			checked = com.InSlice(kv.K, checkedKeys)
		}
		choices = append(choices, fields.InputChoice{
			ID:      kv.K,
			Val:     kv.V,
			Checked: checked,
		})
	}

	field.SetChoices(choices)
	return field
}
<commit_msg>update<commit_after>package formbuilder

import (
	"github.com/coscms/forms"
	"github.com/coscms/forms/common"
	"github.com/coscms/forms/config"
	"github.com/coscms/forms/fields"
	"github.com/webx-top/com"
	"github.com/webx-top/echo"
)

// NewConfig 表单配置
func NewConfig(theme, tmpl, method, action string) *config.Config {
	cfg := forms.NewConfig()
	cfg.Theme = theme
	cfg.Template = tmpl
	cfg.Method = method
	cfg.Action = action
	return cfg
}

// NewSnippet 表单片段
func NewSnippet(theme ...string) *forms.Form {
	cfg := forms.NewConfig()
	if len(theme) > 0 {
		cfg.Theme = theme[0]
	}
	cfg.Template = common.TmplDir(cfg.Theme) + `/allfields.html`
	form := forms.NewWithConfig(cfg)
	return form
}

func ClearCache() {
	common.ClearCachedConfig()
	common.ClearCachedTemplate()
}

func DelCachedConfig(file string) bool {
	return common.DelCachedConfig(file)
}

func AddChoiceByKV(field fields.FieldInterface, kvData *echo.KVData, checkedKeys ...string) fields.FieldInterface {
	for _, kv := range kvData.Slice() {
		var checked bool
		if kv.H != nil {
			checked = kv.H.Bool(`checked`) || kv.H.Bool(`selected`)
		}
		if len(checkedKeys) > 0 {
			checked = com.InSlice(kv.K, checkedKeys)
		}
		field.AddChoice(kv.K, kv.V, checked)
	}
	return field
}

func SetChoiceByKV(field fields.FieldInterface, kvData *echo.KVData, checkedKeys ...string) fields.FieldInterface {
	choices := []fields.InputChoice{}
	if len(checkedKeys) == 0 {
		switch f := field.(type) {
		case *fields.Field:
			if len(f.Value) > 0 {
				checkedKeys = append(checkedKeys, f.Value)
			}
		}
	}
	for _, kv := range kvData.Slice() {
		var checked bool
		if kv.H != nil {
			checked = kv.H.Bool(`checked`) || kv.H.Bool(`selected`)
		}
		if len(checkedKeys) > 0 {
			checked = com.InSlice(kv.K, checkedKeys)
		}
		choices = append(choices, fields.InputChoice{
			ID:      kv.K,
			Val:     kv.V,
			Checked: checked,
		})
	}

	field.SetChoices(choices)
	return field
}
<|endoftext|>


<commit_before>package itertools

import "math/rand"

type seq struct {
	i []int
}

// Creates sequence of values from [0, n)
func newSeq(n int) seq {
	return seq{make([]int, n, n)}
}

func (me seq) Index(i int) (ret int) {
	ret = me.i[i]
	if ret == 0 {
		ret = i
	}
	return
}

func (me seq) Len() int {
	return len(me.i)
}

// Remove the nth value from the sequence.
func (me *seq) Delete(index int) {
	me.i[index] = me.Index(me.Len() - 1)
	me.i = me.i[:me.Len()-1]
}

func ForPerm(n int, callback func(i int) (more bool)) bool {
	s := newSeq(n)
	for s.Len() > 0 {
		r := rand.Intn(s.Len())
		if !callback(s.Index(r)) {
			return false
		}
		s.Delete(r)
	}
	return true
}
<commit_msg>itertools: Rename seq.Delete to DeleteIndex to better reflect its purpose<commit_after>package itertools

import "math/rand"

type seq struct {
	i []int
}

// Creates sequence of values from [0, n)
func newSeq(n int) seq {
	return seq{make([]int, n, n)}
}

func (me seq) Index(i int) (ret int) {
	ret = me.i[i]
	if ret == 0 {
		ret = i
	}
	return
}

func (me seq) Len() int {
	return len(me.i)
}

// Remove the nth value from the sequence.
func (me *seq) DeleteIndex(index int) {
	me.i[index] = me.Index(me.Len() - 1)
	me.i = me.i[:me.Len()-1]
}

func ForPerm(n int, callback func(i int) (more bool)) bool {
	s := newSeq(n)
	for s.Len() > 0 {
		r := rand.Intn(s.Len())
		if !callback(s.Index(r)) {
			return false
		}
		s.DeleteIndex(r)
	}
	return true
}
<|endoftext|>


<commit_before>package dnsd

import (
	"strings"
	"sync"

	"github.com/HouzuoGuo/laitos/inet"
	"github.com/HouzuoGuo/laitos/lalog"
)

const (
	/*
		MaxNameEntriesToExtract is the maximum number of entries to be extracted from one instance of hosts file.
		The limit prevents an exceedingly long third party host file from taking too much memory.
	*/
	MaxNameEntriesToExtract = 50000
)

// HostsFileURLs is a collection of URLs where up-to-date ad/malware/spyware blacklist hosts files are published.
var HostsFileURLs = []string{
	"http://winhelp2002.mvps.org/hosts.txt",
	"http://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext",
	"http://www.malwaredomainlist.com/hostslist/hosts.txt",
	"http://someonewhocares.org/hosts/hosts",
	"https://hosts.oisd.nl/light",
	"https://raw.githubusercontent.com/blocklistproject/Lists/master/ransomware.txt",
	"https://raw.githubusercontent.com/blocklistproject/Lists/master/scam.txt",
	"https://raw.githubusercontent.com/blocklistproject/Lists/master/tracking.txt",
}

/*
Whitelist is an array of domain names that often appear in black lists, but cause inconvenience when blocked. These
names are removed from downloaded black lists.
*/
var Whitelist = []string{
	/*
		2018-06-24 - youtube app on iPhone fails to save watch history, some sources suggest that this domain name is
		the culprit.
	*/
	"s.youtube.com",
	/*
		2019-05-23 - facebook app and instagram app take ~1 minute to load due to blocked connectivity to their graph
		API domains.
	*/
	"graph.facebook.com",
	"graph.instagram.com",
}

/*
DownloadAllBlacklists attempts to download all hosts files and return combined list of domain names to block.
The special cases of white listed names are removed from return value.
*/
func DownloadAllBlacklists(logger lalog.Logger) []string {
	wg := new(sync.WaitGroup)
	wg.Add(len(HostsFileURLs))

	// Download all lists in parallel
	lists := make([][]string, len(HostsFileURLs))
	for i, url := range HostsFileURLs {
		go func(i int, url string) {
			resp, err := inet.DoHTTP(inet.HTTPRequest{TimeoutSec: BlackListDownloadTimeoutSec}, url)
			if err == nil {
				names := ExtractNamesFromHostsContent(string(resp.Body))
				logger.Info("DownloadAllBlacklists", url, err, "downloaded %d names, please obey the license in which the list author publishes the data.", len(names))
				lists[i] = names
			} else {
				logger.Warning("DownloadAllBlacklists", url, err, "failed to download blacklist")
				lists[i] = []string{}
			}
			defer wg.Done()
		}(i, url)
	}
	wg.Wait()
	// Calculate unique set of domain names
	set := map[string]struct{}{}
	for _, list := range lists {
		for _, str := range list {
			set[str] = struct{}{}
		}
	}
	// Remove white listed names
	for _, toRemove := range Whitelist {
		delete(set, toRemove)
	}

	ret := make([]string, 0, len(set))
	for str := range set {
		ret = append(ret, str)
	}
	logger.Info("DownloadAllBlacklists", "", nil, "downloaded %d unique names in total", len(ret))
	return ret
}

/*
ExtractNamesFromHostsContent extracts domain names from hosts file content. It will not return empty lines, comments, and potentially
illegal domain names.
*/
func ExtractNamesFromHostsContent(content string) []string {
	ret := make([]string, 0, 16384)
	for _, line := range strings.Split(content, "\n") {
		if strings.ContainsRune(line, 0) {
			/*
				If attempting to resolve this name that contains NULL byte on Windows, it will unfortunately trigger an
				internal panic in Go's DNS resolution routine.
			*/
			continue
		}
		line = strings.TrimSpace(line)
		if len(line) == 0 || line[0] == '#' {
			// Skip blank and comments
			continue
		}
		// Find the second field
		space := strings.IndexRune(line, ' ')
		if space == -1 {
			// Skip malformed line
			continue
		}
		line = strings.TrimSpace(line[space:])
		nameEnd := strings.IndexRune(line, '#')
		// Name may be followed by a comment
		if nameEnd == -1 {
			nameEnd = len(line)
		}
		// Extract the name itself. Matching of black list name always takes place in lower case.
		aName := strings.ToLower(strings.TrimSpace(line[:nameEnd]))
		if aName == "" || strings.HasSuffix(aName, "localhost") || strings.HasSuffix(aName, "localdomain") ||
			len(aName) < 4 || len(aName) > 253 {
			// Skip empty names, local names, and overly short names
			// Also, domain name length may not exceed 253 characters according to various technical documents in the public domain.
			continue
		}
		ret = append(ret, aName)
		if len(ret) > MaxNameEntriesToExtract {
			// Avoid taking in too many names
			break
		}
	}
	return ret
}
<commit_msg>DNS server - whitelist "commonly whitelisted domains" from pi-hole's collection<commit_after>package dnsd

import (
	"strings"
	"sync"

	"github.com/HouzuoGuo/laitos/inet"
	"github.com/HouzuoGuo/laitos/lalog"
)

const (
	/*
		MaxNameEntriesToExtract is the maximum number of entries to be extracted from one instance of hosts file.
		The limit prevents an exceedingly long third party host file from taking too much memory.
	*/
	MaxNameEntriesToExtract = 50000
)

// HostsFileURLs is a collection of URLs where up-to-date ad/malware/spyware blacklist hosts files are published.
var HostsFileURLs = []string{
	"http://winhelp2002.mvps.org/hosts.txt",
	"http://pgl.yoyo.org/adservers/serverlist.php?hostformat=hosts&showintro=0&mimetype=plaintext",
	"http://www.malwaredomainlist.com/hostslist/hosts.txt",
	"http://someonewhocares.org/hosts/hosts",
	"https://hosts.oisd.nl/light",
	"https://raw.githubusercontent.com/blocklistproject/Lists/master/ransomware.txt",
	"https://raw.githubusercontent.com/blocklistproject/Lists/master/scam.txt",
	"https://raw.githubusercontent.com/blocklistproject/Lists/master/tracking.txt",
}

/*
Whitelist is an array of domain names that often appear in black lists, but cause inconvenience when blocked. These
names are removed from downloaded black lists.
*/
var Whitelist = []string{
	/*
		2018-06-24 - youtube app on iPhone fails to save watch history, some sources suggest that this domain name is
		the culprit.
	*/
	"s.youtube.com",
	/*
		2019-05-23 - facebook app and instagram app take ~1 minute to load due to blocked connectivity to their graph
		API domains.
	*/
	"graph.facebook.com",
	"graph.instagram.com",
	/*
		"Common whitelisted domains"(https://discourse.pi-hole.net/t/commonly-whitelisted-domains/212) as seen on 2020-09-08
	*/
	"106c06cd218b007d-b1e8a1331f68446599e96a4b46a050f5.ams.plex.services", "79423.analytics.edgekey.net", "android.clients.google.com", "api.facebook.com", "app.plex.tv", "appleid.apple.com",
	"apps.skype.com", "appspot-preview.l.google.com", "apresolve.spotify.com", "apt.sonarr.tv", "assets.adobedtm.com", "attestation.xboxlive.com", "b-api.facebook.com", "b-graph.facebook.com",
	"bigzipfiles.facebook.com", "c.s-microsoft.com", "captive.apple.com", "cdn.fbsbx.com", "cdn.vidible.tv", "cert.mgt.xboxlive.com", "chevrolet.com", "client-s.gateway.messenger.live.com",
	"clientconfig.passport.net", "clients2.google.com", "clients3.google.com", "clients4.google.com", "code.bildstatic.de", "connect.facebook.com", "connectivitycheck.android.com", "connectivitycheck.gstatic.com",
	"cpms.spop10.ams.plex.bz", "cpms35.spop10.ams.plex.bz", "creative.ak.fbcdn.net", "ctldl.windowsupdate.com", "dashboard.plex.tv", "de.ioam.de", "def-vef.xboxlive.com", "delivery.vidible.tv",
	"dev.virtualearth.net", "device.auth.xboxlive.com", "directvapplications.hb.omtrdc.net", "directvnow.com", "displaycatalog.mp.microsoft.com", "dl.delivery.mp.microsoft.com", "dl.dropboxusercontent.com",
	"dl.google.com", "download.sonarr.tv", "ecn.dev.virtualearth.net", "edge-mqtt.facebook.com", "edge.api.brightcove.com", "eds.xboxlive.com", "entitlement.auth.adobe.com", "external-lhr0-1.xx.fbcdn.net",
	"external-lhr1-1.xx.fbcdn.net", "external-lhr10-1.xx.fbcdn.net", "external-lhr2-1.xx.fbcdn.net", "external-lhr3-1.xx.fbcdn.net", "external-lhr4-1.xx.fbcdn.net", "external-lhr5-1.xx.fbcdn.net", "external-lhr6-1.xx.fbcdn.net",
	"external-lhr7-1.xx.fbcdn.net", "external-lhr8-1.xx.fbcdn.net", "external-lhr9-1.xx.fbcdn.net", "fb.me", "fbcdn-creative-a.akamaihd.net", "fe3.delivery.dsp.mp.microsoft.com.nsatc.net", "firestore.googleapis.com",
	"forums.sonarr.tv", "fpdownload.adobe.com", "g.live.com", "geo-prod.do.dsp.mp.microsoft.com", "gfwsl.geforce.com", "googleapis.l.google.com", "graph.facebook.com", "gravatar.com", "gsp1.apple.com", "help.ui.xboxlive.com",
	"i.s-microsoft.com", "imagesak.secureserver.net", "img.vidible.tv", "ipv6.msftncsi.com", "itunes.apple.com", "js.maxmind.com", "json.bild.de", "l.facebook.com", "licensing.xboxlive.com", "livepassdl.conviva.com",
	"login.live.com", "login.microsoftonline.com", "m.hotmail.com", "m.weeklyad.target.com", "meta-db-worker02.pop.ric.plex.bz", "meta.plex.bz", "meta.plex.tv", "metrics.plex.tv", "mqtt.c10r.facebook.com",
	"msftncsi.com", "nexus.ensighten.com", "nine.plugins.plexapp.com", "node.plexapp.com", "notify.xboxlive.com", "ns1.dropbox.com", "ns2.dropbox.com", "o1.email.plex.tv", "o2.sg0.plex.tv", "officeclient.microsoft.com",
	"outlook.office365.com", "placehold.it", "placeholdit.imgix.net", "plex.tv", "portal.fb.com", "pricelist.skype.com", "prod.telemetry.ros.rockstargames.com", "products.office.com", "proxy.plex.bz",
	"proxy.plex.tv", "proxy02.pop.ord.plex.bz", "pubsub.plex.bz", "pubsub.plex.tv", "reminders-pa.googleapis.com", "s.gateway.messenger.live.com", "s.marketwatch.com", "s.mzstatic.com", "s.youtube.com",
	"s.zkcdn.net", "sa.symcb.com", "scontent-lhr3-1.xx.fbcdn.net", "scontent.fgdl5-1.fna.fbcdn.net", "scontent.xx.fbcdn.net", "script.ioam.de", "services.sonarr.tv", "settings-win.data.microsoft.com",
	"skyhook.sonarr.tv", "sls.update.microsoft.com.akadns.net", "spclient.wg.spotify.com", "ssl.google-analytics.com", "staging.plex.tv", "star-mini.c10r.facebook.com", "star.c10r.facebook.com", "status.plex.tv",
	"s1.symcb.com", "s2.symcb.com", "s3.symcb.com", "s4.symcb.com", "s5.symcb.com", "t0.ssl.ak.dynamic.tiles.virtualearth.net", "t0.ssl.ak.tiles.virtualearth.net", "tags.tiqcdn.com", "themoviedb.com", "thetvdb.com",
	"title.auth.xboxlive.com", "title.mgt.xboxlive.com", "tracking-protection.cdn.mozilla.net", "tracking.epicgames.com", "tvdb2.plex.tv", "tvthemes.plexapp.com", "tvthemes.plexapp.com.cdn.cloudflare.net", "ui.skype.com",
	"upload.facebook.com", "v10.events.data.microsoft.com", "v10.vortex-win.data.microsoft.com", "v20.events.data.microsoft.com", "video-stats.l.google.com", "videos.vidible.tv", "weeklyad.target.com",
	"weeklyad.target.com.edgesuite.net", "widget-cdn.rpxnow.com", "www.apple.com", "www.appleiphonecell.com", "www.asadcdn.com", "www.google-analytics.com", "www.msftncsi.com", "www.plex.tv", "www.xboxlive.com",
	"xbox.ipv6.microsoft.com", "xboxexperiencesprod.experimentation.xboxlive.com", "xflight.xboxlive.com", "xkms.xboxlive.com", "xsts.auth.xboxlive.com",
}

/*
DownloadAllBlacklists attempts to download all hosts files and return combined list of domain names to block.
The special cases of white listed names are removed from return value.
*/
func DownloadAllBlacklists(logger lalog.Logger) []string {
	wg := new(sync.WaitGroup)
	wg.Add(len(HostsFileURLs))

	// Download all lists in parallel
	lists := make([][]string, len(HostsFileURLs))
	for i, url := range HostsFileURLs {
		go func(i int, url string) {
			resp, err := inet.DoHTTP(inet.HTTPRequest{TimeoutSec: BlackListDownloadTimeoutSec}, url)
			if err == nil {
				names := ExtractNamesFromHostsContent(string(resp.Body))
				logger.Info("DownloadAllBlacklists", url, err, "downloaded %d names, please obey the license in which the list author publishes the data.", len(names))
				lists[i] = names
			} else {
				logger.Warning("DownloadAllBlacklists", url, err, "failed to download blacklist")
				lists[i] = []string{}
			}
			defer wg.Done()
		}(i, url)
	}
	wg.Wait()
	// Calculate unique set of domain names
	set := map[string]struct{}{}
	for _, list := range lists {
		for _, str := range list {
			set[str] = struct{}{}
		}
	}
	// Remove white listed names
	for _, toRemove := range Whitelist {
		delete(set, toRemove)
	}

	ret := make([]string, 0, len(set))
	for str := range set {
		ret = append(ret, str)
	}
	logger.Info("DownloadAllBlacklists", "", nil, "downloaded %d unique names in total", len(ret))
	return ret
}

/*
ExtractNamesFromHostsContent extracts domain names from hosts file content. It will not return empty lines, comments, and potentially
illegal domain names.
*/
func ExtractNamesFromHostsContent(content string) []string {
	ret := make([]string, 0, 16384)
	for _, line := range strings.Split(content, "\n") {
		if strings.ContainsRune(line, 0) {
			/*
				If attempting to resolve this name that contains NULL byte on Windows, it will unfortunately trigger an
				internal panic in Go's DNS resolution routine.
			*/
			continue
		}
		line = strings.TrimSpace(line)
		if len(line) == 0 || line[0] == '#' {
			// Skip blank and comments
			continue
		}
		// Find the second field
		space := strings.IndexRune(line, ' ')
		if space == -1 {
			// Skip malformed line
			continue
		}
		line = strings.TrimSpace(line[space:])
		nameEnd := strings.IndexRune(line, '#')
		// Name may be followed by a comment
		if nameEnd == -1 {
			nameEnd = len(line)
		}
		// Extract the name itself. Matching of black list name always takes place in lower case.
		aName := strings.ToLower(strings.TrimSpace(line[:nameEnd]))
		if aName == "" || strings.HasSuffix(aName, "localhost") || strings.HasSuffix(aName, "localdomain") ||
			len(aName) < 4 || len(aName) > 253 {
			// Skip empty names, local names, and overly short names
			// Also, domain name length may not exceed 253 characters according to various technical documents in the public domain.
			continue
		}
		ret = append(ret, aName)
		if len(ret) > MaxNameEntriesToExtract {
			// Avoid taking in too many names
			break
		}
	}
	return ret
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"code.google.com/p/go.tools/go/vcs"
)

// builderEnv represents the environment that a Builder will run tests in.
type builderEnv interface {
	// setup sets up the builder environment and returns the directory to run the buildCmd in.
	setup(repo *Repo, workpath, hash string, envv []string) (string, error)
}

// goEnv represents the builderEnv for the main Go repo.
type goEnv struct {
	goos, goarch string
}

func (b *Builder) envv() []string {
	if runtime.GOOS == "windows" {
		return b.envvWindows()
	}

	var e []string
	if *buildTool == "go" {
		e = []string{
			"GOOS=" + b.goos,
			"GOARCH=" + b.goarch,
			"GOROOT_FINAL=/usr/local/go",
		}
		switch b.goos {
		case "android", "nacl":
			// Cross compile.
		default:
			// If we are building, for example, linux/386 on a linux/amd64 machine we want to
			// make sure that the whole build is done as a if this were compiled on a real
			// linux/386 machine. In other words, we want to not do a cross compilation build.
			// To do this we set GOHOSTOS and GOHOSTARCH to override the detection in make.bash.
			//
			// The exception to this rule is when we are doing nacl/android builds. These are by
			// definition always cross compilation, and we have support built into cmd/go to be
			// able to handle this case.
			e = append(e, "GOHOSTOS="+b.goos, "GOHOSTARCH="+b.goarch)
		}
	}

	for _, k := range extraEnv() {
		if s, ok := getenvOk(k); ok {
			e = append(e, k+"="+s)
		}
	}
	return e
}

func (b *Builder) envvWindows() []string {
	var start map[string]string
	if *buildTool == "go" {
		start = map[string]string{
			"GOOS":         b.goos,
			"GOHOSTOS":     b.goos,
			"GOARCH":       b.goarch,
			"GOHOSTARCH":   b.goarch,
			"GOROOT_FINAL": `c:\go`,
			"GOBUILDEXIT":  "1", // exit all.bat with completion status.
		}
	}

	for _, name := range extraEnv() {
		if s, ok := getenvOk(name); ok {
			start[name] = s
		}
	}
	if b.goos == "windows" {
		switch b.goarch {
		case "amd64":
			start["PATH"] = `c:\TDM-GCC-64\bin;` + start["PATH"]
		case "386":
			start["PATH"] = `c:\TDM-GCC-32\bin;` + start["PATH"]
		}
	}
	skip := map[string]bool{
		"GOBIN":   true,
		"GOPATH":  true,
		"GOROOT":  true,
		"INCLUDE": true,
		"LIB":     true,
	}
	var e []string
	for name, v := range start {
		e = append(e, name+"="+v)
		skip[name] = true
	}
	for _, kv := range os.Environ() {
		s := strings.SplitN(kv, "=", 2)
		name := strings.ToUpper(s[0])
		switch {
		case name == "":
			// variables, like "=C:=C:\", just copy them
			e = append(e, kv)
		case !skip[name]:
			e = append(e, kv)
			skip[name] = true
		}
	}
	return e
}

// setup for a goEnv clones the main go repo to workpath/go at the provided hash
// and returns the path workpath/go/src, the location of all go build scripts.
func (env *goEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
	goworkpath := filepath.Join(workpath, "go")
	if err := repo.Export(goworkpath, hash); err != nil {
		return "", fmt.Errorf("error exporting repository: %s", err)
	}
	if err := ioutil.WriteFile(filepath.Join(goworkpath, "VERSION"), []byte(hash), 0644); err != nil {
		return "", fmt.Errorf("error writing VERSION file: %s", err)
	}
	return filepath.Join(goworkpath, "src"), nil
}

// gccgoEnv represents the builderEnv for the gccgo compiler.
type gccgoEnv struct{}

// setup for a gccgoEnv clones the gofrontend repo to workpath/go at the hash
// and clones the latest GCC branch to repo.Path/gcc. The gccgo sources are
// replaced with the updated sources in the gofrontend repo and gcc gets
// gets configured and built in workpath/gcc-objdir. The path to
// workpath/gcc-objdir is returned.
func (env *gccgoEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
	gccpath := filepath.Join(repo.Path, "gcc")

	// get a handle to Git vcs.Cmd for pulling down GCC from the mirror.
	git := vcs.ByCmd("git")

	// only pull down gcc if we don't have a local copy.
	if _, err := os.Stat(gccpath); err != nil {
		if err := timeout(*cmdTimeout, func() error {
			// pull down a working copy of GCC.
			return git.Create(gccpath, *gccPath)
		}); err != nil {
			return "", err
		}
	}

	if err := git.Download(gccpath); err != nil {
		return "", err
	}

	// get the modified files for this commit.

	var buf bytes.Buffer
	if err := run(exec.Command("hg", "status", "--no-status", "--change", hash),
		allOutput(&buf), runDir(repo.Path), runEnv(envv)); err != nil {
		return "", fmt.Errorf("Failed to find the modified files for %s: %s", hash, err)
	}
	modifiedFiles := strings.Split(buf.String(), "\n")
	var isMirrored bool
	for _, f := range modifiedFiles {
		if strings.HasPrefix(f, "go/") || strings.HasPrefix(f, "libgo/") {
			isMirrored = true
			break
		}
	}

	// use git log to find the corresponding commit to sync to in the gcc mirror.
	// If the files modified in the gofrontend are mirrored to gcc, we expect a
	// commit with a similar description in the gcc mirror. If the files modified are
	// not mirrored, e.g. in support/, we can sync to the most recent gcc commit that
	// occurred before those files were modified to verify gccgo's status at that point.
	logCmd := []string{
		"git",
		"log",
		"-1",
		"--format=%H",
	}
	var errMsg string
	if isMirrored {
		commitDesc, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{desc|firstline|escape}")
		if err != nil {
			return "", err
		}

		quotedDesc := regexp.QuoteMeta(string(commitDesc))
		logCmd = append(logCmd, "--grep", quotedDesc, "--regexp-ignore-case", "--extended-regexp")
		errMsg = fmt.Sprintf("Failed to find a commit with a similar description to '%s'", string(commitDesc))
	} else {
		commitDate, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{date|rfc3339date}")
		if err != nil {
			return "", err
		}

		logCmd = append(logCmd, "--before", string(commitDate))
		errMsg = fmt.Sprintf("Failed to find a commit before '%s'", string(commitDate))
	}

	buf.Reset()
	if err := run(exec.Command(gccpath, logCmd...), runEnv(envv), allOutput(&buf), runDir(gccpath)); err != nil {
		return "", fmt.Errorf("%s: %s", errMsg, err)
	}
	gccRev := buf.String()
	if gccRev == "" {
		return "", fmt.Errorf(errMsg)
	}

	// checkout gccRev
	// TODO(cmang): Fix this to work in parallel mode.
	if err := run(exec.Command("git", "reset", "--hard", strings.TrimSpace(gccRev)), runEnv(envv), runDir(gccpath)); err != nil {
		return "", fmt.Errorf("Failed to checkout commit at revision %s: %s", gccRev, err)
	}

	// make objdir to work in
	gccobjdir := filepath.Join(workpath, "gcc-objdir")
	if err := os.Mkdir(gccobjdir, mkdirPerm); err != nil {
		return "", err
	}

	// configure GCC with substituted gofrontend and libgo
	if err := run(exec.Command(filepath.Join(gccpath, "configure"),
		"--enable-languages=c,c++,go",
		"--disable-bootstrap",
		"--disable-multilib",
	), runEnv(envv), runDir(gccobjdir)); err != nil {
		return "", fmt.Errorf("Failed to configure GCC: %v", err)
	}

	// build gcc
	if err := run(exec.Command("make"), runTimeout(*buildTimeout), runEnv(envv), runDir(gccobjdir)); err != nil {
		return "", fmt.Errorf("Failed to build GCC: %s", err)
	}

	return gccobjdir, nil
}

func getenvOk(k string) (v string, ok bool) {
	v = os.Getenv(k)
	if v != "" {
		return v, true
	}
	keq := k + "="
	for _, kv := range os.Environ() {
		if kv == keq {
			return "", true
		}
	}
	return "", false
}

// extraEnv returns environment variables that need to be copied from
// the gobuilder's environment to the envv of its subprocesses.
func extraEnv() []string {
	extra := []string{
		"GOARM",
		"GO386",
		"CGO_ENABLED",
		"CC",
		"CC_FOR_TARGET",
		"PATH",
		"TMPDIR",
		"USER",
	}
	if runtime.GOOS == "plan9" {
		extra = append(extra, "objtype", "cputype", "path")
	}
	return extra
}
<commit_msg>dashboard/builder: fix gccgo `git log` execution<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"code.google.com/p/go.tools/go/vcs"
)

// builderEnv represents the environment that a Builder will run tests in.
type builderEnv interface {
	// setup sets up the builder environment and returns the directory to run the buildCmd in.
	setup(repo *Repo, workpath, hash string, envv []string) (string, error)
}

// goEnv represents the builderEnv for the main Go repo.
type goEnv struct {
	goos, goarch string
}

func (b *Builder) envv() []string {
	if runtime.GOOS == "windows" {
		return b.envvWindows()
	}

	var e []string
	if *buildTool == "go" {
		e = []string{
			"GOOS=" + b.goos,
			"GOARCH=" + b.goarch,
			"GOROOT_FINAL=/usr/local/go",
		}
		switch b.goos {
		case "android", "nacl":
			// Cross compile.
		default:
			// If we are building, for example, linux/386 on a linux/amd64 machine we want to
			// make sure that the whole build is done as a if this were compiled on a real
			// linux/386 machine. In other words, we want to not do a cross compilation build.
			// To do this we set GOHOSTOS and GOHOSTARCH to override the detection in make.bash.
			//
			// The exception to this rule is when we are doing nacl/android builds. These are by
			// definition always cross compilation, and we have support built into cmd/go to be
			// able to handle this case.
			e = append(e, "GOHOSTOS="+b.goos, "GOHOSTARCH="+b.goarch)
		}
	}

	for _, k := range extraEnv() {
		if s, ok := getenvOk(k); ok {
			e = append(e, k+"="+s)
		}
	}
	return e
}

func (b *Builder) envvWindows() []string {
	var start map[string]string
	if *buildTool == "go" {
		start = map[string]string{
			"GOOS":         b.goos,
			"GOHOSTOS":     b.goos,
			"GOARCH":       b.goarch,
			"GOHOSTARCH":   b.goarch,
			"GOROOT_FINAL": `c:\go`,
			"GOBUILDEXIT":  "1", // exit all.bat with completion status.
		}
	}

	for _, name := range extraEnv() {
		if s, ok := getenvOk(name); ok {
			start[name] = s
		}
	}
	if b.goos == "windows" {
		switch b.goarch {
		case "amd64":
			start["PATH"] = `c:\TDM-GCC-64\bin;` + start["PATH"]
		case "386":
			start["PATH"] = `c:\TDM-GCC-32\bin;` + start["PATH"]
		}
	}
	skip := map[string]bool{
		"GOBIN":   true,
		"GOPATH":  true,
		"GOROOT":  true,
		"INCLUDE": true,
		"LIB":     true,
	}
	var e []string
	for name, v := range start {
		e = append(e, name+"="+v)
		skip[name] = true
	}
	for _, kv := range os.Environ() {
		s := strings.SplitN(kv, "=", 2)
		name := strings.ToUpper(s[0])
		switch {
		case name == "":
			// variables, like "=C:=C:\", just copy them
			e = append(e, kv)
		case !skip[name]:
			e = append(e, kv)
			skip[name] = true
		}
	}
	return e
}

// setup for a goEnv clones the main go repo to workpath/go at the provided hash
// and returns the path workpath/go/src, the location of all go build scripts.
func (env *goEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
	goworkpath := filepath.Join(workpath, "go")
	if err := repo.Export(goworkpath, hash); err != nil {
		return "", fmt.Errorf("error exporting repository: %s", err)
	}
	if err := ioutil.WriteFile(filepath.Join(goworkpath, "VERSION"), []byte(hash), 0644); err != nil {
		return "", fmt.Errorf("error writing VERSION file: %s", err)
	}
	return filepath.Join(goworkpath, "src"), nil
}

// gccgoEnv represents the builderEnv for the gccgo compiler.
type gccgoEnv struct{}

// setup for a gccgoEnv clones the gofrontend repo to workpath/go at the hash
// and clones the latest GCC branch to repo.Path/gcc. The gccgo sources are
// replaced with the updated sources in the gofrontend repo and gcc gets
// gets configured and built in workpath/gcc-objdir. The path to
// workpath/gcc-objdir is returned.
func (env *gccgoEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
	gccpath := filepath.Join(repo.Path, "gcc")

	// get a handle to Git vcs.Cmd for pulling down GCC from the mirror.
	git := vcs.ByCmd("git")

	// only pull down gcc if we don't have a local copy.
	if _, err := os.Stat(gccpath); err != nil {
		if err := timeout(*cmdTimeout, func() error {
			// pull down a working copy of GCC.
			return git.Create(gccpath, *gccPath)
		}); err != nil {
			return "", err
		}
	}

	if err := git.Download(gccpath); err != nil {
		return "", err
	}

	// get the modified files for this commit.

	var buf bytes.Buffer
	if err := run(exec.Command("hg", "status", "--no-status", "--change", hash),
		allOutput(&buf), runDir(repo.Path), runEnv(envv)); err != nil {
		return "", fmt.Errorf("Failed to find the modified files for %s: %s", hash, err)
	}
	modifiedFiles := strings.Split(buf.String(), "\n")
	var isMirrored bool
	for _, f := range modifiedFiles {
		if strings.HasPrefix(f, "go/") || strings.HasPrefix(f, "libgo/") {
			isMirrored = true
			break
		}
	}

	// use git log to find the corresponding commit to sync to in the gcc mirror.
	// If the files modified in the gofrontend are mirrored to gcc, we expect a
	// commit with a similar description in the gcc mirror. If the files modified are
	// not mirrored, e.g. in support/, we can sync to the most recent gcc commit that
	// occurred before those files were modified to verify gccgo's status at that point.
	logCmd := []string{
		"log",
		"-1",
		"--format=%H",
	}
	var errMsg string
	if isMirrored {
		commitDesc, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{desc|firstline|escape}")
		if err != nil {
			return "", err
		}

		quotedDesc := regexp.QuoteMeta(string(commitDesc))
		logCmd = append(logCmd, "--grep", quotedDesc, "--regexp-ignore-case", "--extended-regexp")
		errMsg = fmt.Sprintf("Failed to find a commit with a similar description to '%s'", string(commitDesc))
	} else {
		commitDate, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{date|rfc3339date}")
		if err != nil {
			return "", err
		}

		logCmd = append(logCmd, "--before", string(commitDate))
		errMsg = fmt.Sprintf("Failed to find a commit before '%s'", string(commitDate))
	}

	buf.Reset()
	if err := run(exec.Command("git", logCmd...), runEnv(envv), allOutput(&buf), runDir(gccpath)); err != nil {
		return "", fmt.Errorf("%s: %s", errMsg, err)
	}
	gccRev := buf.String()
	if gccRev == "" {
		return "", fmt.Errorf(errMsg)
	}

	// checkout gccRev
	// TODO(cmang): Fix this to work in parallel mode.
	if err := run(exec.Command("git", "reset", "--hard", strings.TrimSpace(gccRev)), runEnv(envv), runDir(gccpath)); err != nil {
		return "", fmt.Errorf("Failed to checkout commit at revision %s: %s", gccRev, err)
	}

	// make objdir to work in
	gccobjdir := filepath.Join(workpath, "gcc-objdir")
	if err := os.Mkdir(gccobjdir, mkdirPerm); err != nil {
		return "", err
	}

	// configure GCC with substituted gofrontend and libgo
	if err := run(exec.Command(filepath.Join(gccpath, "configure"),
		"--enable-languages=c,c++,go",
		"--disable-bootstrap",
		"--disable-multilib",
	), runEnv(envv), runDir(gccobjdir)); err != nil {
		return "", fmt.Errorf("Failed to configure GCC: %v", err)
	}

	// build gcc
	if err := run(exec.Command("make"), runTimeout(*buildTimeout), runEnv(envv), runDir(gccobjdir)); err != nil {
		return "", fmt.Errorf("Failed to build GCC: %s", err)
	}

	return gccobjdir, nil
}

func getenvOk(k string) (v string, ok bool) {
	v = os.Getenv(k)
	if v != "" {
		return v, true
	}
	keq := k + "="
	for _, kv := range os.Environ() {
		if kv == keq {
			return "", true
		}
	}
	return "", false
}

// extraEnv returns environment variables that need to be copied from
// the gobuilder's environment to the envv of its subprocesses.
func extraEnv() []string {
	extra := []string{
		"GOARM",
		"GO386",
		"CGO_ENABLED",
		"CC",
		"CC_FOR_TARGET",
		"PATH",
		"TMPDIR",
		"USER",
	}
	if runtime.GOOS == "plan9" {
		extra = append(extra, "objtype", "cputype", "path")
	}
	return extra
}
<|endoftext|>


<commit_before>package trash

import (
	"io/ioutil"
	"os"
	"path/filepath"
)

// TODO: Use better type than string.
func MoveToTrash(name string) error {
	name = filepath.Clean(name)
	home := os.Getenv("HOME")
	dir, file := filepath.Split(name)
	target := filepath.Join(home, ".Trash", file)

	// TODO: If target name exists in Trash, come up with a unique one (perhaps append a timestamp) instead of overwriting.
	// TODO: Support OS X "Put Back". Figure out how it's done and do it.

	err := os.Rename(name, target)
	if err != nil {
		return err
	}

	// If directory became empty, remove it (recursively up).
	for {
		/*// Ensure it's a directory, not file.
		if fi, err := os.Stat(dir); err != nil || !fi.IsDir() {
			break
		}*/
		// Ensure it's an empty directory.
		if dirEntries, err := ioutil.ReadDir(dir); err != nil || len(dirEntries) != 0 {
			break
		}

		// Remove directory if it's (now) empty.
		err := os.Remove(dir)
		if err != nil {
			break
		}

		dir, _ = filepath.Split(dir)
	}

	return nil
}
<commit_msg>Remove superseded code.<commit_after>package trash

import (
	"io/ioutil"
	"os"
	"path/filepath"
)

// TODO: Use better type than string.
func MoveToTrash(name string) error {
	name = filepath.Clean(name)
	home := os.Getenv("HOME")
	dir, file := filepath.Split(name)
	target := filepath.Join(home, ".Trash", file)

	// TODO: If target name exists in Trash, come up with a unique one (perhaps append a timestamp) instead of overwriting.
	// TODO: Support OS X "Put Back". Figure out how it's done and do it.

	err := os.Rename(name, target)
	if err != nil {
		return err
	}

	// If directory became empty, remove it (recursively up).
	for {
		// Ensure it's an empty directory.
		if dirEntries, err := ioutil.ReadDir(dir); err != nil || len(dirEntries) != 0 {
			break
		}

		// Remove directory if it's (now) empty.
		err := os.Remove(dir)
		if err != nil {
			break
		}

		dir, _ = filepath.Split(dir)
	}

	return nil
}
<|endoftext|>


<commit_before>package db

import "github.com/OpenBazaar/openbazaar-go/pb"

func PaymentCoinForContract(contract *pb.RicardianContract) string {
	if contract.BuyerOrder.Payment.AmountCurrency != nil &&
		contract.BuyerOrder.Payment.AmountCurrency.Code != "" {
		return contract.BuyerOrder.Payment.AmountCurrency.Code
	}
	if contract.BuyerOrder.Payment.Coin != "" {
		return contract.BuyerOrder.Payment.Coin
	}
	if len(contract.VendorListings[0].Metadata.AcceptedCurrencies) > 0 {
		return contract.VendorListings[0].Metadata.AcceptedCurrencies[0]
	}
	return ""
}

func CoinTypeForContract(contract *pb.RicardianContract) string {
	if len(contract.VendorListings) > 0 {
		if contract.VendorListings[0].Metadata.CryptoCurrencyCode != "" {
			return contract.VendorListings[0].Metadata.CryptoCurrencyCode
		}
	}
	return ""
}
<commit_msg>Update CoinTypeForContract utility function<commit_after>package db

import "github.com/OpenBazaar/openbazaar-go/pb"

func PaymentCoinForContract(contract *pb.RicardianContract) string {
	if contract.BuyerOrder.Payment.AmountCurrency != nil &&
		contract.BuyerOrder.Payment.AmountCurrency.Code != "" {
		return contract.BuyerOrder.Payment.AmountCurrency.Code
	}
	if contract.BuyerOrder.Payment.Coin != "" {
		return contract.BuyerOrder.Payment.Coin
	}
	if len(contract.VendorListings[0].Metadata.AcceptedCurrencies) > 0 {
		return contract.VendorListings[0].Metadata.AcceptedCurrencies[0]
	}
	return ""
}

func CoinTypeForContract(contract *pb.RicardianContract) string {
	if len(contract.VendorListings) > 0 && contract.VendorListings[0].Metadata.ContractType == pb.Listing_Metadata_CRYPTOCURRENCY {
		return contract.VendorListings[0].Metadata.CryptoCurrencyCode
	}
	return ""
}
<|endoftext|>


<commit_before>// TODO: create a proper template for these. The strings are nasty
package reports

import (
	"bytes"
	log "github.com/Sirupsen/logrus"
	"github.com/pemcconnell/amald/defs"
	"html/template"
	"math"
	"time"
)

type ReportHTML struct{}

type DataDiff struct {
	AddedApps   map[string]bool
	RemovedApps map[string]bool
	UpdatedApps map[string]bool
}

// findDiffs is designed to scan all of the available data to try and pull out
// some comparitive differences in the data sets.
// data is in order (newest first) which gives us some advantages for marking
// out which data to use for - the capture points will keep getting overwritten
// with the earliest available entry for that day
func findDiffs(data []defs.JsonData) (map[string]DataDiff, error) {
	var err error
	diffdata := make(map[string]defs.JsonData)
	diffreturn := make(map[string]DataDiff)
	for _, d := range data {
		if then, err := time.Parse(time.RFC3339, d.Meta["timestamp"]); err == nil {
			distance := math.Ceil(time.Since(then).Hours()/24) - 1
			log.Debug(distance)
			// no need to inspect past 30 days
			if distance > 30 {
				break
			}
			switch distance {
			case 1:
				diffdata["yesterday"] = d
			case 7:
				diffdata["lastweek"] = d
			case 30:
				diffdata["thirtydays"] = d
			}
		}
	}
	// now we (might) have the relevant data selected, run comparisons against
	// the latest scan [0]
	if _, ok := diffdata["yesterday"]; ok {
		diffreturn["yesterday"] = compareData(diffdata["yesterday"], data[0])
	}
	if _, ok := diffdata["lastweek"]; ok {
		diffreturn["lastweek"] = compareData(diffdata["lastweek"], data[0])
	}
	if _, ok := diffdata["thirtydays"]; ok {
		diffreturn["thirtydays"] = compareData(diffdata["thirtydays"], data[0])
	}
	return diffreturn, err
}

// compareData is designed to inspect and compare two different maps. It
// returns a DataDiff struct detailing the differences that it finds
func compareData(oldscan defs.JsonData, newscan defs.JsonData) DataDiff {
	diffreturn := DataDiff{
		AddedApps:   make(map[string]bool),
		RemovedApps: make(map[string]bool),
		UpdatedApps: make(map[string]bool)}
	// loop through the old data to assess what's changed
	for _, def := range oldscan.Data {
		// does this url appear in both datasets?
		if _, newdata := newscan.Data[def.Url]; newdata {
			// was the lockdown status different?
			if newscan.Data[def.Url].IsLockedDown != def.IsLockedDown {
				log.Debug("url: %s, has changed lockdown status to %s",
					def.Url, newscan.Data[def.Url].IsLockedDown)
				diffreturn.UpdatedApps[def.Url] = newscan.Data[def.Url].IsLockedDown
				// remove item from newscan Data so at the end we can assess
				// what's left (will leave new urls).
				delete(newscan.Data, def.Url)
			}
		} else {
			// url is not in the newscan (removed)
			diffreturn.RemovedApps[def.Url] = newscan.Data[def.Url].IsLockedDown
			// remove item from newscan Data so at the end we can assess what's
			// left (will leave new urls).
			delete(newscan.Data, def.Url)
		}
	}
	// now that we have checked all of the old data, we can see what's left
	// from the new scan, which we will determine as new urls
	for _, data := range newscan.Data {
		diffreturn.AddedApps[data.Url] = data.IsLockedDown
	}
	return diffreturn
}

// GenerateHTML creates an HTML string with all the required data
// in place
func (r *ReportHTML) Generate(data []defs.JsonData) (string, error) {
	diff, err := findDiffs(data)
	if err != nil {
		log.Errorf("failed to diffData: %s", err)
	}
	type TemplateData struct {
		Summary map[string]DataDiff
		List    defs.JsonData
	}
	tmpldata := TemplateData{}
	tmpldata.Summary = diff
	tmpldata.List = data[0]
	var doc bytes.Buffer
	tmpl := template.Must(template.ParseGlob("reports/tmpl/email/*"))
	err = tmpl.ExecuteTemplate(&doc, "email", tmpldata)
	if err != nil {
		log.Fatalf("Error running ParseFiles: %s", err)
	}
	return doc.String(), nil
}
<commit_msg>Fixed a bug which caused urls to be removed from output list<commit_after>// TODO: create a proper template for these. The strings are nasty
package reports

import (
	"bytes"
	log "github.com/Sirupsen/logrus"
	"github.com/pemcconnell/amald/defs"
	"html/template"
	"math"
	"time"
)

type ReportHTML struct{}

type DataDiff struct {
	AddedApps   map[string]bool
	RemovedApps map[string]bool
	UpdatedApps map[string]bool
}

// findDiffs is designed to scan all of the available data to try and pull out
// some comparitive differences in the data sets.
// data is in order (newest first) which gives us some advantages for marking
// out which data to use for - the capture points will keep getting overwritten
// with the earliest available entry for that day
func findDiffs(data []defs.JsonData) (map[string]DataDiff, error) {
	var err error
	diffdata := make(map[string]defs.JsonData)
	diffreturn := make(map[string]DataDiff)
	for _, d := range data {
		if then, err := time.Parse(time.RFC3339, d.Meta["timestamp"]); err == nil {
			distance := math.Ceil(time.Since(then).Hours()/24) - 1
			log.Debug(distance)
			// no need to inspect past 30 days
			if distance > 30 {
				break
			}
			switch distance {
			case 1:
				diffdata["yesterday"] = d
			case 7:
				diffdata["lastweek"] = d
			case 30:
				diffdata["thirtydays"] = d
			}
		}
	}
	// now we (might) have the relevant data selected, run comparisons against
	// the latest scan [0]
	if _, ok := diffdata["yesterday"]; ok {
		diffreturn["yesterday"] = compareData(diffdata["yesterday"], data[0])
	}
	if _, ok := diffdata["lastweek"]; ok {
		diffreturn["lastweek"] = compareData(diffdata["lastweek"], data[0])
	}
	if _, ok := diffdata["thirtydays"]; ok {
		diffreturn["thirtydays"] = compareData(diffdata["thirtydays"], data[0])
	}
	return diffreturn, err
}

// compareData is designed to inspect and compare two different maps. It
// returns a DataDiff struct detailing the differences that it finds
func compareData(oldscan defs.JsonData, newscan defs.JsonData) DataDiff {
	diffreturn := DataDiff{
		AddedApps:   make(map[string]bool),
		RemovedApps: make(map[string]bool),
		UpdatedApps: make(map[string]bool)}
	matchedurls := make(map[string]bool)
	// loop through the old data to assess what's changed
	for _, def := range oldscan.Data {
		// does this url appear in both datasets?
		if _, newdata := newscan.Data[def.Url]; newdata {
			// was the lockdown status different?
			if newscan.Data[def.Url].IsLockedDown != def.IsLockedDown {
				log.Debug("url: %s has changed lockdown status to %s",
					def.Url, newscan.Data[def.Url].IsLockedDown)
				diffreturn.UpdatedApps[def.Url] = newscan.Data[def.Url].IsLockedDown
			}
			// track matched urls so at the end we can assess what's left
			// (will leave new urls).
			matchedurls[def.Url] = true
		} else {
			// url is not in the newscan (removed)
			diffreturn.RemovedApps[def.Url] = newscan.Data[def.Url].IsLockedDown
			log.Debugf("url: %s appears to have been removed", def.Url)
		}
	}
	// now that we have checked all of the old data, we can see what's left
	// from the new scan, which we will determine as new urls
	for _, data := range newscan.Data {
		if _, ok := matchedurls[data.Url]; !ok {
			diffreturn.AddedApps[data.Url] = data.IsLockedDown
			log.Debugf("url: %s appears to be new", data.Url)
		}
	}
	return diffreturn
}

// GenerateHTML creates an HTML string with all the required data
// in place
func (r *ReportHTML) Generate(data []defs.JsonData) (string, error) {
	diff, err := findDiffs(data)
	if err != nil {
		log.Errorf("failed to diffData: %s", err)
	}
	type TemplateData struct {
		Summary map[string]DataDiff
		List    defs.JsonData
	}
	tmpldata := TemplateData{}
	tmpldata.Summary = diff
	tmpldata.List = data[0]
	var doc bytes.Buffer
	tmpl := template.Must(template.ParseGlob("reports/tmpl/email/*"))
	err = tmpl.ExecuteTemplate(&doc, "email", tmpldata)
	if err != nil {
		log.Fatalf("Error running ParseFiles: %s", err)
	}
	return doc.String(), nil
}
<|endoftext|>


<commit_before>package application

import (
	"encoding/json"
	"sort"

	. "github.com/cloudfoundry/cli/cf/i18n"

	"github.com/cloudfoundry/cli/cf/api/applications"
	"github.com/cloudfoundry/cli/cf/command_metadata"
	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	"github.com/cloudfoundry/cli/cf/errors"
	"github.com/cloudfoundry/cli/cf/requirements"
	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/codegangsta/cli"
)

type Env struct {
	ui      terminal.UI
	config  core_config.Reader
	appRepo applications.ApplicationRepository
}

func NewEnv(ui terminal.UI, config core_config.Reader, appRepo applications.ApplicationRepository) (cmd *Env) {
	cmd = new(Env)
	cmd.ui = ui
	cmd.config = config
	cmd.appRepo = appRepo
	return
}

func (cmd *Env) Metadata() command_metadata.CommandMetadata {
	return command_metadata.CommandMetadata{
		Name:        "env",
		ShortName:   "e",
		Description: T("Show all env variables for an app"),
		Usage:       T("CF_NAME env APP"),
	}
}

func (cmd *Env) GetRequirements(requirementsFactory requirements.Factory, c *cli.Context) ([]requirements.Requirement, error) {
	if len(c.Args()) != 1 {
		cmd.ui.FailWithUsage(c)
	}

	return []requirements.Requirement{requirementsFactory.NewLoginRequirement()}, nil
}

func (cmd *Env) Run(c *cli.Context) {
	app, err := cmd.appRepo.Read(c.Args()[0])
	if notFound, ok := err.(*errors.ModelNotFoundError); ok {
		cmd.ui.Failed(notFound.Error())
	}

	cmd.ui.Say(T("Getting env variables for app {{.AppName}} in org {{.OrgName}} / space {{.SpaceName}} as {{.Username}}...",
		map[string]interface{}{
			"AppName":   terminal.EntityNameColor(app.Name),
			"OrgName":   terminal.EntityNameColor(cmd.config.OrganizationFields().Name),
			"SpaceName": terminal.EntityNameColor(cmd.config.SpaceFields().Name),
			"Username":  terminal.EntityNameColor(cmd.config.Username())}))

	env, err := cmd.appRepo.ReadEnv(app.Guid)
	if err != nil {
		cmd.ui.Failed(err.Error())
	}

	cmd.ui.Ok()
	cmd.ui.Say("")

	cmd.displaySystemProvidedEnvironment(env.System)
	cmd.ui.Say("")
	cmd.displayUserProvidedEnvironment(env.Environment)
	cmd.ui.Say("")
	cmd.displayRunningEnvironment(env.Running)
	cmd.ui.Say("")
	cmd.displayStagingEnvironment(env.Staging)
	cmd.ui.Say("")
}

func (cmd *Env) displaySystemProvidedEnvironment(env map[string]interface{}) {
	var vcapServices string
	servicesAsMap, ok := env["VCAP_SERVICES"].(map[string]interface{})
	if ok && len(servicesAsMap) > 0 {
		jsonBytes, err := json.MarshalIndent(env, "", " ")
		if err != nil {
			cmd.ui.Failed(err.Error())
		}
		vcapServices = string(jsonBytes)
	}

	if len(vcapServices) == 0 {
		cmd.ui.Say(T("No system-provided env variables have been set"))
		return
	}
	cmd.ui.Say(T("System-Provided:"))
	cmd.ui.Say(vcapServices)
}

func (cmd *Env) displayUserProvidedEnvironment(envVars map[string]string) {
	if len(envVars) == 0 {
		cmd.ui.Say(T("No user-defined env variables have been set"))
		return
	}

	keys := make([]string, 0, len(envVars))
	for key, _ := range envVars {
		keys = append(keys, key)
	}
	sort.Strings(keys)

	cmd.ui.Say(T("User-Provided:"))
	for _, key := range keys {
		cmd.ui.Say("%s: %s", key, terminal.EntityNameColor(envVars[key]))
	}
}

func (cmd *Env) displayRunningEnvironment(envVars map[string]string) {
	if len(envVars) == 0 {
		cmd.ui.Say(T("No running env variables have been set"))
		return
	}

	keys := make([]string, 0, len(envVars))
	for key, _ := range envVars {
		keys = append(keys, key)
	}
	sort.Strings(keys)

	cmd.ui.Say(T("Running Environment Variable Groups:"))
	for _, key := range keys {
		cmd.ui.Say("%s: %s", key, terminal.EntityNameColor(envVars[key]))
	}
}

func (cmd *Env) displayStagingEnvironment(envVars map[string]string) {
	if len(envVars) == 0 {
		cmd.ui.Say(T("No staging env variables have been set"))
		return
	}

	keys := make([]string, 0, len(envVars))
	for key, _ := range envVars {
		keys = append(keys, key)
	}
	sort.Strings(keys)

	cmd.ui.Say(T("Staging Environment Variable Groups:"))
	for _, key := range keys {
		cmd.ui.Say("%s: %s", key, terminal.EntityNameColor(envVars[key]))
	}
}
<commit_msg>Update to highligh env group headers instead of fields.<commit_after>package application

import (
	"encoding/json"
	"sort"

	. "github.com/cloudfoundry/cli/cf/i18n"

	"github.com/cloudfoundry/cli/cf/api/applications"
	"github.com/cloudfoundry/cli/cf/command_metadata"
	"github.com/cloudfoundry/cli/cf/configuration/core_config"
	"github.com/cloudfoundry/cli/cf/errors"
	"github.com/cloudfoundry/cli/cf/requirements"
	"github.com/cloudfoundry/cli/cf/terminal"
	"github.com/codegangsta/cli"
)

type Env struct {
	ui      terminal.UI
	config  core_config.Reader
	appRepo applications.ApplicationRepository
}

func NewEnv(ui terminal.UI, config core_config.Reader, appRepo applications.ApplicationRepository) (cmd *Env) {
	cmd = new(Env)
	cmd.ui = ui
	cmd.config = config
	cmd.appRepo = appRepo
	return
}

func (cmd *Env) Metadata() command_metadata.CommandMetadata {
	return command_metadata.CommandMetadata{
		Name:        "env",
		ShortName:   "e",
		Description: T("Show all env variables for an app"),
		Usage:       T("CF_NAME env APP"),
	}
}

func (cmd *Env) GetRequirements(requirementsFactory requirements.Factory, c *cli.Context) ([]requirements.Requirement, error) {
	if len(c.Args()) != 1 {
		cmd.ui.FailWithUsage(c)
	}

	return []requirements.Requirement{requirementsFactory.NewLoginRequirement()}, nil
}

func (cmd *Env) Run(c *cli.Context) {
	app, err := cmd.appRepo.Read(c.Args()[0])
	if notFound, ok := err.(*errors.ModelNotFoundError); ok {
		cmd.ui.Failed(notFound.Error())
	}

	cmd.ui.Say(T("Getting env variables for app {{.AppName}} in org {{.OrgName}} / space {{.SpaceName}} as {{.Username}}...",
		map[string]interface{}{
			"AppName":   terminal.EntityNameColor(app.Name),
			"OrgName":   terminal.EntityNameColor(cmd.config.OrganizationFields().Name),
			"SpaceName": terminal.EntityNameColor(cmd.config.SpaceFields().Name),
			"Username":  terminal.EntityNameColor(cmd.config.Username())}))

	env, err := cmd.appRepo.ReadEnv(app.Guid)
	if err != nil {
		cmd.ui.Failed(err.Error())
	}

	cmd.ui.Ok()
	cmd.ui.Say("")

	cmd.displaySystemProvidedEnvironment(env.System)
	cmd.ui.Say("")
	cmd.displayUserProvidedEnvironment(env.Environment)
	cmd.ui.Say("")
	cmd.displayRunningEnvironment(env.Running)
	cmd.ui.Say("")
	cmd.displayStagingEnvironment(env.Staging)
	cmd.ui.Say("")
}

func (cmd *Env) displaySystemProvidedEnvironment(env map[string]interface{}) {
	var vcapServices string
	servicesAsMap, ok := env["VCAP_SERVICES"].(map[string]interface{})
	if ok && len(servicesAsMap) > 0 {
		jsonBytes, err := json.MarshalIndent(env, "", " ")
		if err != nil {
			cmd.ui.Failed(err.Error())
		}
		vcapServices = string(jsonBytes)
	}

	if len(vcapServices) == 0 {
		cmd.ui.Say(T("No system-provided env variables have been set"))
		return
	}
	cmd.ui.Say(terminal.EntityNameColor(T("System-Provided:")))
	cmd.ui.Say(vcapServices)
}

func (cmd *Env) displayUserProvidedEnvironment(envVars map[string]string) {
	if len(envVars) == 0 {
		cmd.ui.Say(T("No user-defined env variables have been set"))
		return
	}

	keys := make([]string, 0, len(envVars))
	for key, _ := range envVars {
		keys = append(keys, key)
	}
	sort.Strings(keys)

	cmd.ui.Say(terminal.EntityNameColor(T("User-Provided:")))
	for _, key := range keys {
		cmd.ui.Say("%s: %s", key, envVars[key])
	}
}

func (cmd *Env) displayRunningEnvironment(envVars map[string]string) {
	if len(envVars) == 0 {
		cmd.ui.Say(T("No running env variables have been set"))
		return
	}

	keys := make([]string, 0, len(envVars))
	for key, _ := range envVars {
		keys = append(keys, key)
	}
	sort.Strings(keys)

	cmd.ui.Say(terminal.EntityNameColor(T("Running Environment Variable Groups:")))
	for _, key := range keys {
		cmd.ui.Say("%s: %s", key, envVars[key])
	}
}

func (cmd *Env) displayStagingEnvironment(envVars map[string]string) {
	if len(envVars) == 0 {
		cmd.ui.Say(T("No staging env variables have been set"))
		return
	}

	keys := make([]string, 0, len(envVars))
	for key, _ := range envVars {
		keys = append(keys, key)
	}
	sort.Strings(keys)

	cmd.ui.Say(terminal.EntityNameColor(T("Staging Environment Variable Groups:")))
	for _, key := range keys {
		cmd.ui.Say("%s: %s", key, envVars[key])
	}
}
<|endoftext|>