<commit_before>// Copyright (c) 2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	ServiceGroupResourcePlural = "servicegroups"
	ServiceGroupLabel          = "service-group"

	TopologyLabel = "topology"

	HabitatLabel = "habitat"
)

type ServiceGroup struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata"`
	Spec              ServiceGroupSpec   `json:"spec"`
	Status            ServiceGroupStatus `json:"status,omitempty"`
}

type ServiceGroupSpec struct {
	// Count is the amount of Services to start in this Service Group.
	Count int `json:"count"`
	// Image is the Docker image of the Habitat Service.
	Image   string  `json:"image"`
	Habitat Habitat `json:"habitat"`
}

type ServiceGroupStatus struct {
	State   ServiceGroupState `json:"state,omitempty"`
	Message string            `json:"message,omitempty"`
}

type ServiceGroupState string

type Habitat struct {
	// Group is the value of the --group flag for the hab client.
	// Optional. Defaults to `default`.
	Group string `json:"group"`
	// Topology is the value of the --topology flag for the hab client.
	Topology `json:"topology"`
	// ConfigSecretName is the name of the Secret containing the config the user has previously created.
	// The file with this name is mounted inside of the pod. Habitat will
	// use it for initial configuration of the service.
	ConfigSecretName string `json:"config"`
	// The name of the secret that contains the ring key.
	// Optional.
	RingSecretName string `json:"ringSecretName,omitempty"`
	// Bind is when one service connects to another forming a producer/consumer relationship.
	// Optional.
	Bind []Bind `json:"bind,omitempty"`
}

type Bind struct {
	// Name is the name of the bind specified in the Habitat configuration files.
	Name string `json:"name"`
	// Service is the name of the service this bind refers to.
	Service string `json:"service"`
	// Group is the group of the service this bind refers to.
	Group string `json:"group"`
}

type Topology string

func (t Topology) String() string {
	return string(t)
}

const (
	ServiceGroupStateCreated   ServiceGroupState = "Created"
	ServiceGroupStateProcessed ServiceGroupState = "Processed"

	TopologyStandalone Topology = "standalone"
	TopologyLeader     Topology = "leader"
)

type ServiceGroupList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata"`
	Items           []ServiceGroup `json:"items"`
}
<commit_msg>Fix serialization of user.toml secret<commit_after>// Copyright (c) 2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	ServiceGroupResourcePlural = "servicegroups"
	ServiceGroupLabel          = "service-group"

	TopologyLabel = "topology"

	HabitatLabel = "habitat"
)

type ServiceGroup struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata"`
	Spec              ServiceGroupSpec   `json:"spec"`
	Status            ServiceGroupStatus `json:"status,omitempty"`
}

type ServiceGroupSpec struct {
	// Count is the amount of Services to start in this Service Group.
	Count int `json:"count"`
	// Image is the Docker image of the Habitat Service.
	Image   string  `json:"image"`
	Habitat Habitat `json:"habitat"`
}

type ServiceGroupStatus struct {
	State   ServiceGroupState `json:"state,omitempty"`
	Message string            `json:"message,omitempty"`
}

type ServiceGroupState string

type Habitat struct {
	// Group is the value of the --group flag for the hab client.
	// Optional. Defaults to `default`.
	Group string `json:"group"`
	// Topology is the value of the --topology flag for the hab client.
	Topology `json:"topology"`
	// ConfigSecretName is the name of the Secret containing the config the user has previously created.
	// The file with this name is mounted inside of the pod. Habitat will
	// use it for initial configuration of the service.
	ConfigSecretName string `json:"configSecretName,omitempty"`
	// The name of the secret that contains the ring key.
	// Optional.
	RingSecretName string `json:"ringSecretName,omitempty"`
	// Bind is when one service connects to another forming a producer/consumer relationship.
	// Optional.
	Bind []Bind `json:"bind,omitempty"`
}

type Bind struct {
	// Name is the name of the bind specified in the Habitat configuration files.
	Name string `json:"name"`
	// Service is the name of the service this bind refers to.
	Service string `json:"service"`
	// Group is the group of the service this bind refers to.
	Group string `json:"group"`
}

type Topology string

func (t Topology) String() string {
	return string(t)
}

const (
	ServiceGroupStateCreated   ServiceGroupState = "Created"
	ServiceGroupStateProcessed ServiceGroupState = "Processed"

	TopologyStandalone Topology = "standalone"
	TopologyLeader     Topology = "leader"
)

type ServiceGroupList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata"`
	Items           []ServiceGroup `json:"items"`
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const ServiceGroupResourcePlural = "servicegroups"

type ServiceGroup struct {
	metav1.TypeMeta   `json:,inline"`
	metav1.ObjectMeta `json:"metadata"`
	Spec              ServiceGroupSpec   `json:"spec"`
	Status            ServiceGroupStatus `json:"status,omitempty"`
}

type ServiceGroupSpec struct {
	// Count is the amount of Services to start in this Service Group.
	Count int
	// Image is the Docker image of the Habitat Service.
	Image string
}

type ServiceGroupStatus struct {
	State   ServiceGroupState `json:"state,omitempty"`
	Message string            `json:"message,omitempty"`
}

type ServiceGroupState string

const (
	ServiceGroupStateCreated   ServiceGroupState = "Created"
	ServiceGroupStateProcessed ServiceGroupState = "Processed"
)

type ServiceGroupList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata"`
	Items           []ServiceGroup `json:"items"`
}
<commit_msg>Add json tags for spec fields<commit_after>// Copyright (c) 2017 Chef Software Inc. and/or applicable contributors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v1

import (
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const ServiceGroupResourcePlural = "servicegroups"

type ServiceGroup struct {
	metav1.TypeMeta   `json:,inline"`
	metav1.ObjectMeta `json:"metadata"`
	Spec              ServiceGroupSpec   `json:"spec"`
	Status            ServiceGroupStatus `json:"status,omitempty"`
}

type ServiceGroupSpec struct {
	// Count is the amount of Services to start in this Service Group.
	Count int `json:"count"`
	// Image is the Docker image of the Habitat Service.
	Image string `json:"image"`
}

type ServiceGroupStatus struct {
	State   ServiceGroupState `json:"state,omitempty"`
	Message string            `json:"message,omitempty"`
}

type ServiceGroupState string

const (
	ServiceGroupStateCreated   ServiceGroupState = "Created"
	ServiceGroupStateProcessed ServiceGroupState = "Processed"
)

type ServiceGroupList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata"`
	Items           []ServiceGroup `json:"items"`
}
<|endoftext|>


<commit_before>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helmreconciler

import (
	"context"
	"fmt"

	"istio.io/pkg/version"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/helm/pkg/manifest"
	kubectl "k8s.io/kubectl/pkg/util"
	"sigs.k8s.io/controller-runtime/pkg/client"

	"istio.io/operator/pkg/apis/istio/v1alpha2"
	"istio.io/operator/pkg/component/controlplane"
	"istio.io/operator/pkg/helm"
	istiomanifest "istio.io/operator/pkg/manifest"
	"istio.io/operator/pkg/name"
	"istio.io/operator/pkg/object"
	"istio.io/operator/pkg/translate"
	"istio.io/operator/pkg/util"
	"istio.io/operator/pkg/validate"
	binversion "istio.io/operator/version"
	"istio.io/pkg/log"
)

func (h *HelmReconciler) renderCharts(in RenderingInput) (ChartManifestsMap, error) {
	icp, ok := in.GetInputConfig().(*v1alpha2.IstioControlPlane)
	if !ok {
		return nil, fmt.Errorf("unexpected type %T in renderCharts", in.GetInputConfig())
	}

	icpSpec := icp.GetSpec()
	if err := validate.CheckIstioControlPlaneSpec(icpSpec, false); err != nil {
		return nil, err
	}

	mergedICPS, err := mergeICPSWithProfile(icpSpec)
	if err != nil {
		return nil, err
	}

	t, err := translate.NewTranslator(binversion.OperatorBinaryVersion.MinorVersion)
	if err != nil {
		return nil, err
	}

	cp := controlplane.NewIstioControlPlane(mergedICPS, t)
	if err := cp.Run(); err != nil {
		return nil, fmt.Errorf("failed to create Istio control plane with spec: \n%v\nerror: %s", mergedICPS, err)
	}

	manifests, errs := cp.RenderManifest()
	if errs != nil {
		err = errs.ToError()
	}

	return toChartManifestsMap(manifests), err
}

// mergeICPSWithProfile overlays the values in icp on top of the defaults for the profile given by icp.profile and
// returns the merged result.
func mergeICPSWithProfile(icp *v1alpha2.IstioControlPlaneSpec) (*v1alpha2.IstioControlPlaneSpec, error) {
	profile := icp.Profile

	// This contains the IstioControlPlane CR.
	baseCRYAML, err := helm.ReadProfileYAML(profile)
	if err != nil {
		return nil, fmt.Errorf("could not read the profile values for %s: %s", profile, err)
	}

	if !helm.IsDefaultProfile(profile) {
		// Profile definitions are relative to the default profile, so read that first.
		dfn, err := helm.DefaultFilenameForProfile(profile)
		if err != nil {
			return nil, err
		}
		defaultYAML, err := helm.ReadProfileYAML(dfn)
		if err != nil {
			return nil, fmt.Errorf("could not read the default profile values for %s: %s", dfn, err)
		}
		baseCRYAML, err = helm.OverlayYAML(defaultYAML, baseCRYAML)
		if err != nil {
			return nil, fmt.Errorf("could not overlay the profile over the default %s: %s", profile, err)
		}
	}

	_, baseYAML, err := unmarshalAndValidateICP(baseCRYAML)
	if err != nil {
		return nil, err
	}

	// Due to the fact that base profile is compiled in before a tag can be created, we must allow an additional
	// override from variables that are set during release build time.
	hub := version.DockerInfo.Hub
	tag := version.DockerInfo.Tag
	if hub != "unknown" && tag != "unknown" {
		buildHubTagOverlayYAML, err := helm.GenerateHubTagOverlay(hub, tag)
		if err != nil {
			return nil, err
		}
		baseYAML, err = helm.OverlayYAML(baseYAML, buildHubTagOverlayYAML)
		if err != nil {
			return nil, err
		}
	}

	overlayYAML, err := util.MarshalWithJSONPB(icp)
	if err != nil {
		return nil, err
	}

	// Merge base and overlay.
	mergedYAML, err := helm.OverlayYAML(baseYAML, overlayYAML)
	if err != nil {
		return nil, fmt.Errorf("could not overlay user config over base: %s", err)
	}
	return unmarshalAndValidateICPSpec(mergedYAML)
}

// unmarshalAndValidateICP unmarshals the IstioControlPlane in the crYAML string and validates it.
// If successful, it returns both a struct and string YAML representations of the IstioControlPlaneSpec embedded in icp.
func unmarshalAndValidateICP(crYAML string) (*v1alpha2.IstioControlPlaneSpec, string, error) {
	// TODO: add GroupVersionKind handling as appropriate.
	if crYAML == "" {
		return &v1alpha2.IstioControlPlaneSpec{}, "", nil
	}
	icps, _, err := istiomanifest.ParseK8SYAMLToIstioControlPlaneSpec(crYAML)
	if err != nil {
		return nil, "", fmt.Errorf("could not parse the overlay file: %s\n\nOriginal YAML:\n%s", err, crYAML)
	}
	if errs := validate.CheckIstioControlPlaneSpec(icps, false); len(errs) != 0 {
		return nil, "", fmt.Errorf("input file failed validation with the following errors: %s\n\nOriginal YAML:\n%s", errs, crYAML)
	}
	icpsYAML, err := util.MarshalWithJSONPB(icps)
	if err != nil {
		return nil, "", fmt.Errorf("could not marshal: %s", err)
	}
	return icps, icpsYAML, nil
}

// unmarshalAndValidateICPSpec unmarshals the IstioControlPlaneSpec in the icpsYAML string and validates it.
// If successful, it returns a struct representation of icpsYAML.
func unmarshalAndValidateICPSpec(icpsYAML string) (*v1alpha2.IstioControlPlaneSpec, error) {
	icps := &v1alpha2.IstioControlPlaneSpec{}
	if err := util.UnmarshalWithJSONPB(icpsYAML, icps); err != nil {
		return nil, fmt.Errorf("could not unmarshal the merged YAML: %s\n\nYAML:\n%s", err, icpsYAML)
	}
	if errs := validate.CheckIstioControlPlaneSpec(icps, true); len(errs) != 0 {
		return nil, fmt.Errorf(errs.Error())
	}
	return icps, nil
}

// ProcessManifest apply the manifest to create or update resources, returns the number of objects processed
func (h *HelmReconciler) ProcessManifest(manifest manifest.Manifest) (int, error) {
	var errs []error
	log.Infof("Processing resources from manifest: %s", manifest.Name)
	objects, err := object.ParseK8sObjectsFromYAMLManifest(manifest.Content)
	if err != nil {
		return 0, err
	}
	for _, obj := range objects {
		err = h.ProcessObject(manifest.Name, obj.UnstructuredObject())
		if err != nil {
			errs = append(errs, err)
		}
	}
	return len(objects), utilerrors.NewAggregate(errs)
}

func (h *HelmReconciler) ProcessObject(chartName string, obj *unstructured.Unstructured) error {
	if obj.GetKind() == "List" {
		allErrors := []error{}
		list, err := obj.ToList()
		if err != nil {
			log.Errorf("error converting List object: %s", err)
			return err
		}
		for _, item := range list.Items {
			err = h.ProcessObject(chartName, &item)
			if err != nil {
				allErrors = append(allErrors, err)
			}
		}
		return utilerrors.NewAggregate(allErrors)
	}

	mutatedObj, err := h.customizer.Listener().BeginResource(chartName, obj)
	if err != nil {
		log.Errorf("error preprocessing object: %s", err)
		return err
	}

	err = kubectl.CreateApplyAnnotation(obj, unstructured.UnstructuredJSONScheme)
	if err != nil {
		log.Errorf("unexpected error adding apply annotation to object: %s", err)
	}

	receiver := &unstructured.Unstructured{}
	receiver.SetGroupVersionKind(mutatedObj.GetObjectKind().GroupVersionKind())
	objectKey, _ := client.ObjectKeyFromObject(mutatedObj)

	var patch Patch

	err = h.client.Get(context.TODO(), objectKey, receiver)
	if err != nil {
		if apierrors.IsNotFound(err) {
			log.Infof("creating resource: %s", objectKey)
			err = h.client.Create(context.TODO(), mutatedObj)
			if err == nil {
				// special handling
				if err = h.customizer.Listener().ResourceCreated(mutatedObj); err != nil {
					log.Errorf("unexpected error occurred during postprocessing of new resource: %s", err)
				}
			} else {
				listenerErr := h.customizer.Listener().ResourceError(mutatedObj, err)
				if listenerErr != nil {
					log.Errorf("unexpected error occurred invoking ResourceError on listener: %s", listenerErr)
				}
			}
		}
	} else if patch, err = h.CreatePatch(receiver, mutatedObj); err == nil && patch != nil {
		log.Info("updating existing resource")
		mutatedObj, err = patch.Apply()
		if err == nil {
			if err = h.customizer.Listener().ResourceUpdated(mutatedObj, receiver); err != nil {
				log.Errorf("unexpected error occurred during postprocessing of updated resource: %s", err)
			}
		} else {
			listenerErr := h.customizer.Listener().ResourceError(obj, err)
			if listenerErr != nil {
				log.Errorf("unexpected error occurred invoking ResourceError on listener: %s", listenerErr)
			}
		}
	}
	if err != nil {
		log.Errorf("error occurred reconciling resource: %s", err)
	}
	return err
}

func toChartManifestsMap(m name.ManifestMap) ChartManifestsMap {
	out := make(ChartManifestsMap)
	for k, v := range m {
		out[string(k)] = []manifest.Manifest{{
			Name:    string(k),
			Content: v,
		}}
	}
	return out
}
<commit_msg>fix invalid image name. (#626)<commit_after>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package helmreconciler

import (
	"context"
	"fmt"

	"istio.io/pkg/version"

	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	"k8s.io/helm/pkg/manifest"
	kubectl "k8s.io/kubectl/pkg/util"
	"sigs.k8s.io/controller-runtime/pkg/client"

	"istio.io/operator/pkg/apis/istio/v1alpha2"
	"istio.io/operator/pkg/component/controlplane"
	"istio.io/operator/pkg/helm"
	istiomanifest "istio.io/operator/pkg/manifest"
	"istio.io/operator/pkg/name"
	"istio.io/operator/pkg/object"
	"istio.io/operator/pkg/translate"
	"istio.io/operator/pkg/util"
	"istio.io/operator/pkg/validate"
	binversion "istio.io/operator/version"
	"istio.io/pkg/log"
)

func (h *HelmReconciler) renderCharts(in RenderingInput) (ChartManifestsMap, error) {
	icp, ok := in.GetInputConfig().(*v1alpha2.IstioControlPlane)
	if !ok {
		return nil, fmt.Errorf("unexpected type %T in renderCharts", in.GetInputConfig())
	}

	icpSpec := icp.GetSpec()
	if err := validate.CheckIstioControlPlaneSpec(icpSpec, false); err != nil {
		return nil, err
	}

	mergedICPS, err := mergeICPSWithProfile(icpSpec)
	if err != nil {
		return nil, err
	}

	t, err := translate.NewTranslator(binversion.OperatorBinaryVersion.MinorVersion)
	if err != nil {
		return nil, err
	}

	cp := controlplane.NewIstioControlPlane(mergedICPS, t)
	if err := cp.Run(); err != nil {
		return nil, fmt.Errorf("failed to create Istio control plane with spec: \n%v\nerror: %s", mergedICPS, err)
	}

	manifests, errs := cp.RenderManifest()
	if errs != nil {
		err = errs.ToError()
	}

	return toChartManifestsMap(manifests), err
}

// mergeICPSWithProfile overlays the values in icp on top of the defaults for the profile given by icp.profile and
// returns the merged result.
func mergeICPSWithProfile(icp *v1alpha2.IstioControlPlaneSpec) (*v1alpha2.IstioControlPlaneSpec, error) {
	profile := icp.Profile

	// This contains the IstioControlPlane CR.
	baseCRYAML, err := helm.ReadProfileYAML(profile)
	if err != nil {
		return nil, fmt.Errorf("could not read the profile values for %s: %s", profile, err)
	}

	if !helm.IsDefaultProfile(profile) {
		// Profile definitions are relative to the default profile, so read that first.
		dfn, err := helm.DefaultFilenameForProfile(profile)
		if err != nil {
			return nil, err
		}
		defaultYAML, err := helm.ReadProfileYAML(dfn)
		if err != nil {
			return nil, fmt.Errorf("could not read the default profile values for %s: %s", dfn, err)
		}
		baseCRYAML, err = helm.OverlayYAML(defaultYAML, baseCRYAML)
		if err != nil {
			return nil, fmt.Errorf("could not overlay the profile over the default %s: %s", profile, err)
		}
	}

	_, baseYAML, err := unmarshalAndValidateICP(baseCRYAML)
	if err != nil {
		return nil, err
	}

	// Due to the fact that base profile is compiled in before a tag can be created, we must allow an additional
	// override from variables that are set during release build time.
	hub := version.DockerInfo.Hub
	tag := version.DockerInfo.Tag
	if hub != "" && hub != "unknown" && tag != "" && tag != "unknown" {
		buildHubTagOverlayYAML, err := helm.GenerateHubTagOverlay(hub, tag)
		if err != nil {
			return nil, err
		}
		baseYAML, err = helm.OverlayYAML(baseYAML, buildHubTagOverlayYAML)
		if err != nil {
			return nil, err
		}
	}

	overlayYAML, err := util.MarshalWithJSONPB(icp)
	if err != nil {
		return nil, err
	}

	// Merge base and overlay.
	mergedYAML, err := helm.OverlayYAML(baseYAML, overlayYAML)
	if err != nil {
		return nil, fmt.Errorf("could not overlay user config over base: %s", err)
	}
	return unmarshalAndValidateICPSpec(mergedYAML)
}

// unmarshalAndValidateICP unmarshals the IstioControlPlane in the crYAML string and validates it.
// If successful, it returns both a struct and string YAML representations of the IstioControlPlaneSpec embedded in icp.
func unmarshalAndValidateICP(crYAML string) (*v1alpha2.IstioControlPlaneSpec, string, error) {
	// TODO: add GroupVersionKind handling as appropriate.
	if crYAML == "" {
		return &v1alpha2.IstioControlPlaneSpec{}, "", nil
	}
	icps, _, err := istiomanifest.ParseK8SYAMLToIstioControlPlaneSpec(crYAML)
	if err != nil {
		return nil, "", fmt.Errorf("could not parse the overlay file: %s\n\nOriginal YAML:\n%s", err, crYAML)
	}
	if errs := validate.CheckIstioControlPlaneSpec(icps, false); len(errs) != 0 {
		return nil, "", fmt.Errorf("input file failed validation with the following errors: %s\n\nOriginal YAML:\n%s", errs, crYAML)
	}
	icpsYAML, err := util.MarshalWithJSONPB(icps)
	if err != nil {
		return nil, "", fmt.Errorf("could not marshal: %s", err)
	}
	return icps, icpsYAML, nil
}

// unmarshalAndValidateICPSpec unmarshals the IstioControlPlaneSpec in the icpsYAML string and validates it.
// If successful, it returns a struct representation of icpsYAML.
func unmarshalAndValidateICPSpec(icpsYAML string) (*v1alpha2.IstioControlPlaneSpec, error) {
	icps := &v1alpha2.IstioControlPlaneSpec{}
	if err := util.UnmarshalWithJSONPB(icpsYAML, icps); err != nil {
		return nil, fmt.Errorf("could not unmarshal the merged YAML: %s\n\nYAML:\n%s", err, icpsYAML)
	}
	if errs := validate.CheckIstioControlPlaneSpec(icps, true); len(errs) != 0 {
		return nil, fmt.Errorf(errs.Error())
	}
	return icps, nil
}

// ProcessManifest apply the manifest to create or update resources, returns the number of objects processed
func (h *HelmReconciler) ProcessManifest(manifest manifest.Manifest) (int, error) {
	var errs []error
	log.Infof("Processing resources from manifest: %s", manifest.Name)
	objects, err := object.ParseK8sObjectsFromYAMLManifest(manifest.Content)
	if err != nil {
		return 0, err
	}
	for _, obj := range objects {
		err = h.ProcessObject(manifest.Name, obj.UnstructuredObject())
		if err != nil {
			errs = append(errs, err)
		}
	}
	return len(objects), utilerrors.NewAggregate(errs)
}

func (h *HelmReconciler) ProcessObject(chartName string, obj *unstructured.Unstructured) error {
	if obj.GetKind() == "List" {
		allErrors := []error{}
		list, err := obj.ToList()
		if err != nil {
			log.Errorf("error converting List object: %s", err)
			return err
		}
		for _, item := range list.Items {
			err = h.ProcessObject(chartName, &item)
			if err != nil {
				allErrors = append(allErrors, err)
			}
		}
		return utilerrors.NewAggregate(allErrors)
	}

	mutatedObj, err := h.customizer.Listener().BeginResource(chartName, obj)
	if err != nil {
		log.Errorf("error preprocessing object: %s", err)
		return err
	}

	err = kubectl.CreateApplyAnnotation(obj, unstructured.UnstructuredJSONScheme)
	if err != nil {
		log.Errorf("unexpected error adding apply annotation to object: %s", err)
	}

	receiver := &unstructured.Unstructured{}
	receiver.SetGroupVersionKind(mutatedObj.GetObjectKind().GroupVersionKind())
	objectKey, _ := client.ObjectKeyFromObject(mutatedObj)

	var patch Patch

	err = h.client.Get(context.TODO(), objectKey, receiver)
	if err != nil {
		if apierrors.IsNotFound(err) {
			log.Infof("creating resource: %s", objectKey)
			err = h.client.Create(context.TODO(), mutatedObj)
			if err == nil {
				// special handling
				if err = h.customizer.Listener().ResourceCreated(mutatedObj); err != nil {
					log.Errorf("unexpected error occurred during postprocessing of new resource: %s", err)
				}
			} else {
				listenerErr := h.customizer.Listener().ResourceError(mutatedObj, err)
				if listenerErr != nil {
					log.Errorf("unexpected error occurred invoking ResourceError on listener: %s", listenerErr)
				}
			}
		}
	} else if patch, err = h.CreatePatch(receiver, mutatedObj); err == nil && patch != nil {
		log.Info("updating existing resource")
		mutatedObj, err = patch.Apply()
		if err == nil {
			if err = h.customizer.Listener().ResourceUpdated(mutatedObj, receiver); err != nil {
				log.Errorf("unexpected error occurred during postprocessing of updated resource: %s", err)
			}
		} else {
			listenerErr := h.customizer.Listener().ResourceError(obj, err)
			if listenerErr != nil {
				log.Errorf("unexpected error occurred invoking ResourceError on listener: %s", listenerErr)
			}
		}
	}
	if err != nil {
		log.Errorf("error occurred reconciling resource: %s", err)
	}
	return err
}

func toChartManifestsMap(m name.ManifestMap) ChartManifestsMap {
	out := make(ChartManifestsMap)
	for k, v := range m {
		out[string(k)] = []manifest.Manifest{{
			Name:    string(k),
			Content: v,
		}}
	}
	return out
}
<|endoftext|>


<commit_before>package dp

import "testing"

var tests = []struct {
	in  *Parameters
	out int
}{
	{
		in: &Parameters{
			Scores:    []int{3, 4, 6},
			Weights:   []int{3, 4, 6},
			MaxWeight: 7,
		},
		out: 7,
	},
}

func TestMax(t *testing.T) {
	for _, v := range tests {
		out := Max(v.in)
		if out != v.out {
			t.Errorf("Got %v, expected %v", out, v.out)
		}
	}
}
<commit_msg>more DP tests<commit_after>package dp

import "testing"

var tests = []struct {
	in  *Parameters
	out int
}{
	{
		in: &Parameters{
			Scores:    []int{3, 4, 6},
			Weights:   []int{3, 4, 6},
			MaxWeight: 7,
		},
		out: 7,
	},
	{
		in: &Parameters{
			Scores:    []int{3, 4, 5, 6},
			Weights:   []int{2, 3, 4, 5},
			MaxWeight: 5,
		},
		out: 7,
	},
	{
		in: &Parameters{
			Scores:    []int{1, 6, 18, 22, 28},
			Weights:   []int{1, 2, 5, 6, 7},
			MaxWeight: 11,
		},
		out: 40,
	},
}

func TestMax(t *testing.T) {
	for _, v := range tests {
		out := Max(v.in)
		if out != v.out {
			t.Errorf("Got %v, expected %v", out, v.out)
		}
	}
}
<|endoftext|>


<commit_before>package thesaurus

import (
	"encoding/json"
	"errors"
	"net/http"
)

type BigHugh struct {
	APIKey string
}

type synonyms struct {
	Noun *words `json:"noun"`
	Verb *words `json:"verb"`
}

type words struct {
	Syn []string `json:"syn"`
}

func (b *BigHugh) Synonyms(term string) ([]string, error) {
	var syns []string
	response, err := http.Get("http://words.bighugelabs.com/api/2/" + b.APIKey + "/" + term + "/json")
	if err != nil {
		return syns, errors.New("bighugh: Failed when looking for synonyms for \"" + term + "\"" + err.Error())
	}
	var data synonyms
	defer response.Body.Close()
	if err := json.NewDecoder(response.Body).Decode(&data); err != nil {
		return syns, err
	}
	syns = append(syns, data.Noun.Syn...)
	syns = append(syns, data.Verb.Syn...)
	return syns, nil
}
<commit_msg>made field access more robust by checking for nils<commit_after>package thesaurus

import (
	"encoding/json"
	"errors"
	"net/http"
)

type BigHugh struct {
	APIKey string
}

type synonyms struct {
	Noun *words `json:"noun"`
	Verb *words `json:"verb"`
}

type words struct {
	Syn []string `json:"syn"`
}

func (b *BigHugh) Synonyms(term string) ([]string, error) {
	var syns []string
	response, err := http.Get("http://words.bighugelabs.com/api/2/" + b.APIKey + "/" + term + "/json")
	if err != nil {
		return syns, errors.New("bighugh: Failed when looking for synonyms for \"" + term + "\"" + err.Error())
	}
	var data synonyms
	defer response.Body.Close()
	if err := json.NewDecoder(response.Body).Decode(&data); err != nil {
		return syns, err
	}
	if data.Noun != nil {
		syns = append(syns, data.Noun.Syn...)
	}
	if data.Verb != nil {
		syns = append(syns, data.Verb.Syn...)
	}
	return syns, nil
}
<|endoftext|>


<commit_before>package currency

import "strings"

type CurrencyConf struct {
	Currency  string
	Label     string
	Symbol    string
	Pairs     []string
	Pools     []string
	Explorers ExplorerConf
	Unit      string
}

type ExplorerConf struct {
	BaseUri        string
	AddressUri     string
	TransactionUri string
}

type CurrencyConfigMapping string

const (
	CurrencyLabelMap    CurrencyConfigMapping = "pub:map:currency:label"
	CurrencySymbolMap   CurrencyConfigMapping = "pub:map:currency:sym"
	CurrencyUnitMap     CurrencyConfigMapping = "pub:map:currency:unit"
	CurrencyExplorerMap CurrencyConfigMapping = "pub:map:currency:explorer"
	CurrencyExchangeMap CurrencyConfigMapping = "pub:list:pair:exchange"
)

type RawCurrencyConf struct {
	Mapping string
	Data    interface{}
}

func parseCurrencyLabelMap(config map[string]CurrencyConf, raw []interface{}) {
	for _, rawLabel := range raw {
		data := rawLabel.([]interface{})
		cur := data[0].(string)
		if val, ok := config[cur]; ok {
			// add value
			val.Label = data[1].(string)
			config[cur] = val
		} else {
			// create new empty config instance
			cfg := CurrencyConf{}
			cfg.Label = data[1].(string)
			cfg.Currency = cur
			config[cur] = cfg
		}
	}
}

func parseCurrencySymbMap(config map[string]CurrencyConf, raw []interface{}) {
	for _, rawLabel := range raw {
		data := rawLabel.([]interface{})
		cur := data[0].(string)
		if val, ok := config[cur]; ok {
			// add value
			val.Symbol = data[1].(string)
			config[cur] = val
		} else {
			// create new empty config instance
			cfg := CurrencyConf{}
			cfg.Symbol = data[1].(string)
			cfg.Currency = cur
			config[cur] = cfg
		}
	}
}

func parseCurrencyUnitMap(config map[string]CurrencyConf, raw []interface{}) {
	for _, rawLabel := range raw {
		data := rawLabel.([]interface{})
		cur := data[0].(string)
		if val, ok := config[cur]; ok {
			// add value
			val.Unit = data[1].(string)
			config[cur] = val
		} else {
			// create new empty config instance
			cfg := CurrencyConf{}
			cfg.Unit = data[1].(string)
			cfg.Currency = cur
			config[cur] = cfg
		}
	}
}

func parseCurrencyExplorerMap(config map[string]CurrencyConf, raw []interface{}) {
	for _, rawLabel := range raw {
		data := rawLabel.([]interface{})
		cur := data[0].(string)
		explorers := data[1].([]interface{})
		var cfg CurrencyConf
		if val, ok := config[cur]; ok {
			cfg = val
		} else {
			// create new empty config instance
			cc := CurrencyConf{}
			cc.Currency = cur
			cfg = cc
		}
		ec := ExplorerConf{
			explorers[0].(string),
			explorers[1].(string),
			explorers[2].(string),
		}
		cfg.Explorers = ec
		config[cur] = cfg
	}
}

func parseCurrencyExchangeMap(config map[string]CurrencyConf, raw []interface{}) {
	for _, rs := range raw {
		symbol := rs.(string)
		var base, quote string

		if len(symbol) > 6 {
			base = strings.Split(symbol, ":")[0]
			quote = strings.Split(symbol, ":")[1]
		} else {
			base = symbol[3:]
			quote = symbol[:3]
		}

		// append if base exists in configs
		if val, ok := config[base]; ok {
			val.Pairs = append(val.Pairs, symbol)
			config[base] = val
		}

		// append if quote exists in configs
		if val, ok := config[quote]; ok {
			val.Pairs = append(val.Pairs, symbol)
			config[quote] = val
		}
	}
}

func NewCurrencyConfFromRaw(raw []RawCurrencyConf) ([]CurrencyConf, error) {
	configMap := make(map[string]CurrencyConf)
	for _, r := range raw {
		switch CurrencyConfigMapping(r.Mapping) {
		case CurrencyLabelMap:
			data := r.Data.([]interface{})
			parseCurrencyLabelMap(configMap, data)
		case CurrencySymbolMap:
			data := r.Data.([]interface{})
			parseCurrencySymbMap(configMap, data)
		case CurrencyUnitMap:
			data := r.Data.([]interface{})
			parseCurrencyUnitMap(configMap, data)
		case CurrencyExplorerMap:
			data := r.Data.([]interface{})
			parseCurrencyExplorerMap(configMap, data)
		case CurrencyExchangeMap:
			data := r.Data.([]interface{})
			parseCurrencyExchangeMap(configMap, data)
		}
	}

	// convert map to array
	configs := make([]CurrencyConf, 0)
	for _, v := range configMap {
		configs = append(configs, v)
	}

	return configs, nil
}
<commit_msg>amending currency type related names to follow go convention<commit_after>package currency

import "strings"

type Conf struct {
	Currency  string
	Label     string
	Symbol    string
	Pairs     []string
	Pools     []string
	Explorers ExplorerConf
	Unit      string
}

type ExplorerConf struct {
	BaseUri        string
	AddressUri     string
	TransactionUri string
}

type ConfigMapping string

const (
	LabelMap    ConfigMapping = "pub:map:currency:label"
	SymbolMap   ConfigMapping = "pub:map:currency:sym"
	UnitMap     ConfigMapping = "pub:map:currency:unit"
	ExplorerMap ConfigMapping = "pub:map:currency:explorer"
	ExchangeMap ConfigMapping = "pub:list:pair:exchange"
)

type RawConf struct {
	Mapping string
	Data    interface{}
}

func parseLabelMap(config map[string]Conf, raw []interface{}) {
	for _, rawLabel := range raw {
		data := rawLabel.([]interface{})
		cur := data[0].(string)
		if val, ok := config[cur]; ok {
			// add value
			val.Label = data[1].(string)
			config[cur] = val
		} else {
			// create new empty config instance
			cfg := Conf{}
			cfg.Label = data[1].(string)
			cfg.Currency = cur
			config[cur] = cfg
		}
	}
}

func parseSymbMap(config map[string]Conf, raw []interface{}) {
	for _, rawLabel := range raw {
		data := rawLabel.([]interface{})
		cur := data[0].(string)
		if val, ok := config[cur]; ok {
			// add value
			val.Symbol = data[1].(string)
			config[cur] = val
		} else {
			// create new empty config instance
			cfg := Conf{}
			cfg.Symbol = data[1].(string)
			cfg.Currency = cur
			config[cur] = cfg
		}
	}
}

func parseUnitMap(config map[string]Conf, raw []interface{}) {
	for _, rawLabel := range raw {
		data := rawLabel.([]interface{})
		cur := data[0].(string)
		if val, ok := config[cur]; ok {
			// add value
			val.Unit = data[1].(string)
			config[cur] = val
		} else {
			// create new empty config instance
			cfg := Conf{}
			cfg.Unit = data[1].(string)
			cfg.Currency = cur
			config[cur] = cfg
		}
	}
}

func parseExplorerMap(config map[string]Conf, raw []interface{}) {
	for _, rawLabel := range raw {
		data := rawLabel.([]interface{})
		cur := data[0].(string)
		explorers := data[1].([]interface{})
		var cfg Conf
		if val, ok := config[cur]; ok {
			cfg = val
		} else {
			// create new empty config instance
			cc := Conf{}
			cc.Currency = cur
			cfg = cc
		}
		ec := ExplorerConf{
			explorers[0].(string),
			explorers[1].(string),
			explorers[2].(string),
		}
		cfg.Explorers = ec
		config[cur] = cfg
	}
}

func parseExchangeMap(config map[string]Conf, raw []interface{}) {
	for _, rs := range raw {
		symbol := rs.(string)
		var base, quote string

		if len(symbol) > 6 {
			base = strings.Split(symbol, ":")[0]
			quote = strings.Split(symbol, ":")[1]
		} else {
			base = symbol[3:]
			quote = symbol[:3]
		}

		// append if base exists in configs
		if val, ok := config[base]; ok {
			val.Pairs = append(val.Pairs, symbol)
			config[base] = val
		}

		// append if quote exists in configs
		if val, ok := config[quote]; ok {
			val.Pairs = append(val.Pairs, symbol)
			config[quote] = val
		}
	}
}

func ConfFromRaw(raw []RawConf) ([]Conf, error) {
	configMap := make(map[string]Conf)
	for _, r := range raw {
		switch ConfigMapping(r.Mapping) {
		case LabelMap:
			data := r.Data.([]interface{})
			parseLabelMap(configMap, data)
		case SymbolMap:
			data := r.Data.([]interface{})
			parseSymbMap(configMap, data)
		case UnitMap:
			data := r.Data.([]interface{})
			parseUnitMap(configMap, data)
		case ExplorerMap:
			data := r.Data.([]interface{})
			parseExplorerMap(configMap, data)
		case ExchangeMap:
			data := r.Data.([]interface{})
			parseExchangeMap(configMap, data)
		}
	}

	// convert map to array
	configs := make([]Conf, 0)
	for _, v := range configMap {
		configs = append(configs, v)
	}

	return configs, nil
}
<|endoftext|>


<commit_before>package pingfed

import (
	"context"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/tidwall/gjson"
	"github.com/versent/saml2aws/v2/pkg/cfg"
	"github.com/versent/saml2aws/v2/pkg/creds"
	"github.com/versent/saml2aws/v2/pkg/page"
	"github.com/versent/saml2aws/v2/pkg/prompter"
	"github.com/versent/saml2aws/v2/pkg/provider"
)

var logger = logrus.WithField("provider", "pingfed")

// Client wrapper around PingFed + PingId enabling authentication and retrieval of assertions
type Client struct {
	provider.ValidateBase

	client     *provider.HTTPClient
	idpAccount *cfg.IDPAccount
}

// New create a new PingFed client
func New(idpAccount *cfg.IDPAccount) (*Client, error) {

	tr := provider.NewDefaultTransport(idpAccount.SkipVerify)

	client, err := provider.NewHTTPClient(tr, provider.BuildHttpClientOpts(idpAccount))
	if err != nil {
		return nil, errors.Wrap(err, "error building http client")
	}

	// assign a response validator to ensure all responses are either success or a redirect
	// this is to avoid have explicit checks for every single response
	client.CheckResponseStatus = provider.SuccessOrRedirectResponseValidator

	return &Client{
		client:     client,
		idpAccount: idpAccount,
	}, nil
}

type ctxKey string

// Authenticate Authenticate to PingFed and return the data from the body of the SAML assertion.
func (ac *Client) Authenticate(loginDetails *creds.LoginDetails) (string, error) {
	u := fmt.Sprintf("%s/idp/startSSO.ping?PartnerSpId=%s", loginDetails.URL, ac.idpAccount.AmazonWebservicesURN)
	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		return "", errors.Wrap(err, "error building request")
	}
	ctx := context.WithValue(context.Background(), ctxKey("login"), loginDetails)
	return ac.follow(ctx, req)
}

func (ac *Client) follow(ctx context.Context, req *http.Request) (string, error) {
	res, err := ac.client.Do(req)
	if err != nil {
		return "", errors.Wrap(err, "error following")
	}
	doc, err := goquery.NewDocumentFromResponse(res)
	if err != nil {
		return "", errors.Wrap(err, "failed to build document from response")
	}

	var handler func(context.Context, *goquery.Document) (context.Context, *http.Request, error)

	if docIsFormRedirectToAWS(doc) {
		logger.WithField("type", "saml-response-to-aws").Debug("doc detect")
		if samlResponse, ok := extractSAMLResponse(doc); ok {
			decodedSamlResponse, err := base64.StdEncoding.DecodeString(samlResponse)
			if err != nil {
				return "", errors.Wrap(err, "failed to decode saml-response")
			}
			logger.WithField("type", "saml-response").WithField("saml-response", string(decodedSamlResponse)).Debug("doc detect")
			return samlResponse, nil
		}
	} else if docIsFormSamlRequest(doc) {
		logger.WithField("type", "saml-request").Debug("doc detect")
		handler = ac.handleFormRedirect
	} else if docIsFormResume(doc) {
		logger.WithField("type", "resume").Debug("doc detect")
		handler = ac.handleFormRedirect
	} else if docIsFormSamlResponse(doc) {
		logger.WithField("type", "saml-response").Debug("doc detect")
		handler = ac.handleFormRedirect
	} else if docIsLogin(doc) {
		logger.WithField("type", "login").Debug("doc detect")
		handler = ac.handleLogin
	} else if docIsOTP(doc) {
		logger.WithField("type", "otp").Debug("doc detect")
		handler = ac.handleOTP
	} else if docIsSwipe(doc) {
		logger.WithField("type", "swipe").Debug("doc detect")
		handler = ac.handleSwipe
	} else if docIsFormRedirect(doc) {
		logger.WithField("type", "form-redirect").Debug("doc detect")
		handler = ac.handleFormRedirect
	} else if docIsWebAuthn(doc) {
		logger.WithField("type", "webauthn").Debug("doc detect")
		handler = ac.handleWebAuthn
	}
	if handler == nil {
		html, _ := doc.Selection.Html()
		logger.WithField("doc", html).Debug("Unknown document type")
		return "", fmt.Errorf("Unknown document type")
	}

	ctx, req, err = handler(ctx, doc)
	if err != nil {
		return "", err
	}
	return ac.follow(ctx, req)
}

func (ac *Client) handleLogin(ctx context.Context, doc *goquery.Document) (context.Context, *http.Request, error) {
	loginDetails, ok := ctx.Value(ctxKey("login")).(*creds.LoginDetails)
	if !ok {
		return ctx, nil, fmt.Errorf("no context value for 'login'")
	}

	form, err := page.NewFormFromDocument(doc, "form")
	if err != nil {
		return ctx, nil, errors.Wrap(err, "error extracting login form")
	}

	form.Values.Set("pf.username", loginDetails.Username)
	form.Values.Set("pf.pass", loginDetails.Password)
	form.URL = makeAbsoluteURL(form.URL, loginDetails.URL)

	req, err := form.BuildRequest()
	return ctx, req, err
}

func (ac *Client) handleOTP(ctx context.Context, doc *goquery.Document) (context.Context, *http.Request, error) {
	form, err := page.NewFormFromDocument(doc, "#otp-form")
	if err != nil {
		return ctx, nil, errors.Wrap(err, "error extracting OTP form")
	}

	token := prompter.StringRequired("Enter passcode")
	form.Values.Set("otp", token)
	req, err := form.BuildRequest()
	return ctx, req, err
}

func (ac *Client) handleSwipe(ctx context.Context, doc *goquery.Document) (context.Context, *http.Request, error) {
	form, err := page.NewFormFromDocument(doc, "#form1")
	if err != nil {
		return ctx, nil, errors.Wrap(err, "error extracting swipe status form")
	}

	// poll status. request must specifically be a GET
	form.Method = "GET"
	req, err := form.BuildRequest()
	if err != nil {
		return ctx, nil, err
	}

	for {
		time.Sleep(3 * time.Second)

		res, err := ac.client.Do(req)
		if err != nil {
			return ctx, nil, errors.Wrap(err, "error polling swipe status")
		}

		body, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return ctx, nil, errors.Wrap(err, "error parsing body from swipe status response")
		}

		resp := string(body)

		pingfedMFAStatusResponse := gjson.Get(resp, "status").String()

		//ASYNC_AUTH_WAIT indicates we keep going
		//OK indicates someone swiped
		//DEVICE_CLAIM_TIMEOUT indicates nobody swiped
		//otherwise loop forever?

		if pingfedMFAStatusResponse == "OK" || pingfedMFAStatusResponse == "DEVICE_CLAIM_TIMEOUT" || pingfedMFAStatusResponse == "TIMEOUT" {
			break
		}
	}

	// now build a request for getting response of MFA
	form, err = page.NewFormFromDocument(doc, "#reponseView")
	if err != nil {
		return ctx, nil, errors.Wrap(err, "error extracting swipe response form")
	}
	req, err = form.BuildRequest()
	return ctx, req, err
}

func (ac *Client) handleFormRedirect(ctx context.Context, doc *goquery.Document) (context.Context, *http.Request, error) {
	form, err := page.NewFormFromDocument(doc, "")
	if err != nil {
		return ctx, nil, errors.Wrap(err, "error extracting redirect form")
	}
	req, err := form.BuildRequest()
	return ctx, req, err
}

func (ac *Client) handleWebAuthn(ctx context.Context, doc *goquery.Document) (context.Context, *http.Request, error) {
	form, err := page.NewFormFromDocument(doc, "")
	if err != nil {
		return ctx, nil, errors.Wrap(err, "error extracting webauthn form")
	}
	form.Values.Set("isWebAuthnSupportedByBrowser", "false")
	req, err := form.BuildRequest()
	return ctx, req, err
}

func docIsLogin(doc *goquery.Document) bool {
	return doc.Has("input[name=\"pf.pass\"]").Size() == 1
}

func docIsOTP(doc *goquery.Document) bool {
	return doc.Has("form#otp-form").Size() == 1
}

func docIsSwipe(doc *goquery.Document) bool {
	return doc.Has("form#form1").Size() == 1 && doc.Has("form#reponseView").Size() == 1
}

func docIsFormRedirect(doc *goquery.Document) bool {
	return doc.Has("input[name=\"ppm_request\"]").Size() == 1
}

func docIsWebAuthn(doc *goquery.Document) bool {
	return doc.Has("input[name=\"isWebAuthnSupportedByBrowser\"]").Size() == 1
}

func docIsFormSamlRequest(doc *goquery.Document) bool {
	return doc.Find("input[name=\"SAMLRequest\"]").Size() == 1
}

func docIsFormSamlResponse(doc *goquery.Document) bool {
	return doc.Find("input[name=\"SAMLResponse\"]").Size() == 1
}

func docIsFormResume(doc *goquery.Document) bool {
	return doc.Find("input[name=\"RelayState\"]").Size() == 1
}

func docIsFormRedirectToAWS(doc *goquery.Document) bool {
	return doc.Find("form[action=\"https://signin.aws.amazon.com/saml\"]").Size() == 1
}

func extractSAMLResponse(doc *goquery.Document) (v string, ok bool) {
	return doc.Find("input[name=\"SAMLResponse\"]").Attr("value")
}

// ensures given url is an absolute URL. if not, it will be combined with the base URL
func makeAbsoluteURL(v string, base string) string {
	if u, err := url.ParseRequestURI(v); err == nil && !u.IsAbs() {
		return fmt.Sprintf("%s%s", base, v)
	}
	return v
}
<commit_msg>add support for `USER` and `PASSWORD` fields in pingfed provider<commit_after>package pingfed

import (
	"context"
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/tidwall/gjson"
	"github.com/versent/saml2aws/v2/pkg/cfg"
	"github.com/versent/saml2aws/v2/pkg/creds"
	"github.com/versent/saml2aws/v2/pkg/page"
	"github.com/versent/saml2aws/v2/pkg/prompter"
	"github.com/versent/saml2aws/v2/pkg/provider"
)

var logger = logrus.WithField("provider", "pingfed")

// Client wrapper around PingFed + PingId enabling authentication and retrieval of assertions
type Client struct {
	provider.ValidateBase

	client     *provider.HTTPClient
	idpAccount *cfg.IDPAccount
}

// New create a new PingFed client
func New(idpAccount *cfg.IDPAccount) (*Client, error) {

	tr := provider.NewDefaultTransport(idpAccount.SkipVerify)

	client, err := provider.NewHTTPClient(tr, provider.BuildHttpClientOpts(idpAccount))
	if err != nil {
		return nil, errors.Wrap(err, "error building http client")
	}

	// assign a response validator to ensure all responses are either success or a redirect
	// this is to avoid have explicit checks for every single response
	client.CheckResponseStatus = provider.SuccessOrRedirectResponseValidator

	return &Client{
		client:     client,
		idpAccount: idpAccount,
	}, nil
}

type ctxKey string

// Authenticate Authenticate to PingFed and return the data from the body of the SAML assertion.
func (ac *Client) Authenticate(loginDetails *creds.LoginDetails) (string, error) {
	u := fmt.Sprintf("%s/idp/startSSO.ping?PartnerSpId=%s", loginDetails.URL, ac.idpAccount.AmazonWebservicesURN)
	req, err := http.NewRequest("GET", u, nil)
	if err != nil {
		return "", errors.Wrap(err, "error building request")
	}
	ctx := context.WithValue(context.Background(), ctxKey("login"), loginDetails)
	return ac.follow(ctx, req)
}

func (ac *Client) follow(ctx context.Context, req *http.Request) (string, error) {
	res, err := ac.client.Do(req)
	if err != nil {
		return "", errors.Wrap(err, "error following")
	}
	doc, err := goquery.NewDocumentFromResponse(res)
	if err != nil {
		return "", errors.Wrap(err, "failed to build document from response")
	}

	var handler func(context.Context, *goquery.Document) (context.Context, *http.Request, error)

	if docIsFormRedirectToAWS(doc) {
		logger.WithField("type", "saml-response-to-aws").Debug("doc detect")
		if samlResponse, ok := extractSAMLResponse(doc); ok {
			decodedSamlResponse, err := base64.StdEncoding.DecodeString(samlResponse)
			if err != nil {
				return "", errors.Wrap(err, "failed to decode saml-response")
			}
			logger.WithField("type", "saml-response").WithField("saml-response", string(decodedSamlResponse)).Debug("doc detect")
			return samlResponse, nil
		}
	} else if docIsFormSamlRequest(doc) {
		logger.WithField("type", "saml-request").Debug("doc detect")
		handler = ac.handleFormRedirect
	} else if docIsFormResume(doc) {
		logger.WithField("type", "resume").Debug("doc detect")
		handler = ac.handleFormRedirect
	} else if docIsFormSamlResponse(doc) {
		logger.WithField("type", "saml-response").Debug("doc detect")
		handler = ac.handleFormRedirect
	} else if docIsLogin(doc) {
		logger.WithField("type", "login").Debug("doc detect")
		handler = ac.handleLogin
	} else if docIsOTP(doc) {
		logger.WithField("type", "otp").Debug("doc detect")
		handler = ac.handleOTP
	} else if docIsSwipe(doc) {
		logger.WithField("type", "swipe").Debug("doc detect")
		handler = ac.handleSwipe
	} else if docIsFormRedirect(doc) {
		logger.WithField("type", "form-redirect").Debug("doc detect")
		handler = ac.handleFormRedirect
	} else if docIsWebAuthn(doc) {
		logger.WithField("type", "webauthn").Debug("doc detect")
		handler = ac.handleWebAuthn
	}
	if handler == nil {
		html, _ := doc.Selection.Html()
		logger.WithField("doc", html).Debug("Unknown document type")
		return "", fmt.Errorf("Unknown document type")
	}

	ctx, req, err = handler(ctx, doc)
	if err != nil {
		return "", err
	}
	return ac.follow(ctx, req)
}

func (ac *Client) handleLogin(ctx context.Context, doc *goquery.Document) (context.Context, *http.Request, error) {
	loginDetails, ok := ctx.Value(ctxKey("login")).(*creds.LoginDetails)
	if !ok {
		return ctx, nil, fmt.Errorf("no context value for 'login'")
	}

	form, err := page.NewFormFromDocument(doc, "form")
	if err != nil {
		return ctx, nil, errors.Wrap(err, "error extracting login form")
	}

	form.Values.Set("pf.username", loginDetails.Username)
	form.Values.Set("pf.pass", loginDetails.Password)
	form.Values.Set("USER", loginDetails.Username)
	form.Values.Set("PASSWORD", loginDetails.Password)
	form.URL = makeAbsoluteURL(form.URL, loginDetails.URL)

	req, err := form.BuildRequest()
	return ctx, req, err
}

func (ac *Client) handleOTP(ctx context.Context, doc *goquery.Document) (context.Context, *http.Request, error) {
	form, err := page.NewFormFromDocument(doc, "#otp-form")
	if err != nil {
		return ctx, nil, errors.Wrap(err, "error extracting OTP form")
	}

	token := prompter.StringRequired("Enter passcode")
	form.Values.Set("otp", token)
	req, err := form.BuildRequest()
	return ctx, req, err
}

func (ac *Client) handleSwipe(ctx context.Context, doc *goquery.Document) (context.Context, *http.Request, error) {
	form, err := page.NewFormFromDocument(doc, "#form1")
	if err != nil {
		return ctx, nil, errors.Wrap(err, "error extracting swipe status form")
	}

	// poll status. request must specifically be a GET
	form.Method = "GET"
	req, err := form.BuildRequest()
	if err != nil {
		return ctx, nil, err
	}

	for {
		time.Sleep(3 * time.Second)

		res, err := ac.client.Do(req)
		if err != nil {
			return ctx, nil, errors.Wrap(err, "error polling swipe status")
		}

		body, err := ioutil.ReadAll(res.Body)
		if err != nil {
			return ctx, nil, errors.Wrap(err, "error parsing body from swipe status response")
		}

		resp := string(body)

		pingfedMFAStatusResponse := gjson.Get(resp, "status").String()

		//ASYNC_AUTH_WAIT indicates we keep going
		//OK indicates someone swiped
		//DEVICE_CLAIM_TIMEOUT indicates nobody swiped
		//otherwise loop forever?

		if pingfedMFAStatusResponse == "OK" || pingfedMFAStatusResponse == "DEVICE_CLAIM_TIMEOUT" || pingfedMFAStatusResponse == "TIMEOUT" {
			break
		}
	}

	// now build a request for getting response of MFA
	form, err = page.NewFormFromDocument(doc, "#reponseView")
	if err != nil {
		return ctx, nil, errors.Wrap(err, "error extracting swipe response form")
	}
	req, err = form.BuildRequest()
	return ctx, req, err
}

func (ac *Client) handleFormRedirect(ctx context.Context, doc *goquery.Document) (context.Context, *http.Request, error) {
	form, err := page.NewFormFromDocument(doc, "")
	if err != nil {
		return ctx, nil, errors.Wrap(err, "error extracting redirect form")
	}
	req, err := form.BuildRequest()
	return ctx, req, err
}

func (ac *Client) handleWebAuthn(ctx context.Context, doc *goquery.Document) (context.Context, *http.Request, error) {
	form, err := page.NewFormFromDocument(doc, "")
	if err != nil {
		return ctx, nil, errors.Wrap(err, "error extracting webauthn form")
	}
	form.Values.Set("isWebAuthnSupportedByBrowser", "false")
	req, err := form.BuildRequest()
	return ctx, req, err
}

func docIsLogin(doc *goquery.Document) bool {
	return doc.Has("input[name=\"pf.pass\"]").Size() == 1 || doc.Has("input[name=\"PASSWORD\"]").Size() == 1
}

func docIsOTP(doc *goquery.Document) bool {
	return doc.Has("form#otp-form").Size() == 1
}

func docIsSwipe(doc *goquery.Document) bool {
	return doc.Has("form#form1").Size() == 1 && doc.Has("form#reponseView").Size() == 1
}

func docIsFormRedirect(doc *goquery.Document) bool {
	return doc.Has("input[name=\"ppm_request\"]").Size() == 1
}

func docIsWebAuthn(doc *goquery.Document) bool {
	return doc.Has("input[name=\"isWebAuthnSupportedByBrowser\"]").Size() == 1
}

func docIsFormSamlRequest(doc *goquery.Document) bool {
	return doc.Find("input[name=\"SAMLRequest\"]").Size() == 1
}

func docIsFormSamlResponse(doc *goquery.Document) bool {
	return doc.Find("input[name=\"SAMLResponse\"]").Size() == 1
}

func docIsFormResume(doc *goquery.Document) bool {
	return doc.Find("input[name=\"RelayState\"]").Size() == 1
}

func docIsFormRedirectToAWS(doc *goquery.Document) bool {
	return doc.Find("form[action=\"https://signin.aws.amazon.com/saml\"]").Size() == 1
}

func extractSAMLResponse(doc *goquery.Document) (v string, ok bool) {
	return doc.Find("input[name=\"SAMLResponse\"]").Attr("value")
}

// ensures given url is an absolute URL. if not, it will be combined with the base URL
func makeAbsoluteURL(v string, base string) string {
	if u, err := url.ParseRequestURI(v); err == nil && !u.IsAbs() {
		return fmt.Sprintf("%s%s", base, v)
	}
	return v
}
<|endoftext|>


<commit_before>/*
 * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package sql

import (
	"errors"
	"fmt"
)

// Aggregation Function name constants
const (
	aggFnAvg   FuncName = "AVG"
	aggFnCount FuncName = "COUNT"
	aggFnMax   FuncName = "MAX"
	aggFnMin   FuncName = "MIN"
	aggFnSum   FuncName = "SUM"
)

var (
	errNonNumericArg = func(fnStr FuncName) error {
		return fmt.Errorf("%s() requires a numeric argument", fnStr)
	}
	errInvalidAggregation = errors.New("Invalid aggregation seen")
)

type aggVal struct {
	runningSum             *Value
	runningCount           int64
	runningMax, runningMin *Value

	// Stores if at least one record has been seen
	seen bool
}

func newAggVal(fn FuncName) *aggVal {
	switch fn {
	case aggFnAvg, aggFnSum:
		return &aggVal{runningSum: FromInt(0)}
	case aggFnMin:
		return &aggVal{runningMin: FromInt(0)}
	case aggFnMax:
		return &aggVal{runningMax: FromInt(0)}
	default:
		return &aggVal{}
	}
}

// evalAggregationNode - performs partial computation using the
// current row and stores the result.
//
// On success, it returns (nil, nil).
func (e *FuncExpr) evalAggregationNode(r Record) error {
	// It is assumed that this function is called only when
	// `e` is an aggregation function.

	var val *Value
	var err error
	funcName := e.getFunctionName()
	if aggFnCount == funcName {
		if e.Count.StarArg {
			// Handle COUNT(*)
			e.aggregate.runningCount++
			return nil
		}

		val, err = e.Count.ExprArg.evalNode(r)
		if err != nil {
			return err
		}
	} else {
		// Evaluate the (only) argument
		val, err = e.SFunc.ArgsList[0].evalNode(r)
		if err != nil {
			return err
		}
	}

	if val.IsNull() {
		// E.g. the column or field does not exist in the
		// record - in all such cases the aggregation is not
		// updated.
		return nil
	}

	argVal := val
	if funcName != aggFnCount {
		// All aggregation functions, except COUNT require a
		// numeric argument.

		// Here, we diverge from Amazon S3 behavior by
		// inferring untyped values are numbers.
		if !argVal.isNumeric() {
			if i, ok := argVal.bytesToInt(); ok {
				argVal.setInt(i)
			} else if f, ok := argVal.bytesToFloat(); ok {
				argVal.setFloat(f)
			} else {
				return errNonNumericArg(funcName)
			}
		}
	}

	// Mark that we have seen one non-null value.
	isFirstRow := false
	if !e.aggregate.seen {
		e.aggregate.seen = true
		isFirstRow = true
	}

	switch funcName {
	case aggFnCount:
		// For all non-null values, the count is incremented.
		e.aggregate.runningCount++

	case aggFnAvg:
		e.aggregate.runningCount++
		err = e.aggregate.runningSum.arithOp(opPlus, argVal)

	case aggFnMin:
		err = e.aggregate.runningMin.minmax(argVal, false, isFirstRow)

	case aggFnMax:
		err = e.aggregate.runningMax.minmax(argVal, true, isFirstRow)

	case aggFnSum:
		err = e.aggregate.runningSum.arithOp(opPlus, argVal)

	default:
		err = errInvalidAggregation
	}

	return err
}

func (e *AliasedExpression) aggregateRow(r Record) error {
	return e.Expression.aggregateRow(r)
}

func (e *Expression) aggregateRow(r Record) error {
	for _, ex := range e.And {
		err := ex.aggregateRow(r)
		if err != nil {
			return err
		}
	}
	return nil
}

func (e *AndCondition) aggregateRow(r Record) error {
	for _, ex := range e.Condition {
		err := ex.aggregateRow(r)
		if err != nil {
			return err
		}
	}
	return nil
}

func (e *Condition) aggregateRow(r Record) error {
	if e.Operand != nil {
		return e.Operand.aggregateRow(r)
	}
	return e.Not.aggregateRow(r)
}

func (e *ConditionOperand) aggregateRow(r Record) error {
	err := e.Operand.aggregateRow(r)
	if err != nil {
		return err
	}

	if e.ConditionRHS == nil {
		return nil
	}

	switch {
	case e.ConditionRHS.Compare != nil:
		return e.ConditionRHS.Compare.Operand.aggregateRow(r)
	case e.ConditionRHS.Between != nil:
		err = e.ConditionRHS.Between.Start.aggregateRow(r)
		if err != nil {
			return err
		}
		return e.ConditionRHS.Between.End.aggregateRow(r)
	case e.ConditionRHS.In != nil:
		for _, elt := range e.ConditionRHS.In.Expressions {
			err = elt.aggregateRow(r)
			if err != nil {
				return err
			}
		}
		return nil
	case e.ConditionRHS.Like != nil:
		err = e.ConditionRHS.Like.Pattern.aggregateRow(r)
		if err != nil {
			return err
		}
		return e.ConditionRHS.Like.EscapeChar.aggregateRow(r)
	default:
		return errInvalidASTNode
	}
}

func (e *Operand) aggregateRow(r Record) error {
	err := e.Left.aggregateRow(r)
	if err != nil {
		return err
	}
	for _, rt := range e.Right {
		err = rt.Right.aggregateRow(r)
		if err != nil {
			return err
		}
	}
	return nil
}

func (e *MultOp) aggregateRow(r Record) error {
	err := e.Left.aggregateRow(r)
	if err != nil {
		return err
	}
	for _, rt := range e.Right {
		err = rt.Right.aggregateRow(r)
		if err != nil {
			return err
		}
	}
	return nil
}

func (e *UnaryTerm) aggregateRow(r Record) error {
	if e.Negated != nil {
		return e.Negated.Term.aggregateRow(r)
	}
	return e.Primary.aggregateRow(r)
}

func (e *PrimaryTerm) aggregateRow(r Record) error {
	switch {
	case e.SubExpression != nil:
		return e.SubExpression.aggregateRow(r)
	case e.FuncCall != nil:
		return e.FuncCall.aggregateRow(r)
	}
	return nil
}

func (e *FuncExpr) aggregateRow(r Record) error {
	switch e.getFunctionName() {
	case aggFnAvg, aggFnSum, aggFnMax, aggFnMin, aggFnCount:
		return e.evalAggregationNode(r)
	default:
		// TODO: traverse arguments and call aggregateRow on
		// them if they could be an ancestor of an
		// aggregation.
	}
	return nil
}

// getAggregate() implementation for each AST node follows. This is
// called after calling aggregateRow() on each input row, to calculate
// the final aggregate result.

func (e *FuncExpr) getAggregate() (*Value, error) {
	switch e.getFunctionName() {
	case aggFnCount:
		return FromInt(e.aggregate.runningCount), nil

	case aggFnAvg:
		if e.aggregate.runningCount == 0 {
			// No rows were seen by AVG.
			return FromNull(), nil
		}
		err := e.aggregate.runningSum.arithOp(opDivide, FromInt(e.aggregate.runningCount))
		return e.aggregate.runningSum, err

	case aggFnMin:
		if !e.aggregate.seen {
			// No rows were seen by MIN
			return FromNull(), nil
		}
		return e.aggregate.runningMin, nil

	case aggFnMax:
		if !e.aggregate.seen {
			// No rows were seen by MAX
			return FromNull(), nil
		}
		return e.aggregate.runningMax, nil

	case aggFnSum:
		// TODO: check if returning 0 when no rows were seen
		// by SUM is expected behavior.
		return e.aggregate.runningSum, nil

	default:
		// TODO:
	}

	return nil, errInvalidAggregation
}
<commit_msg>S3 Select: Aggregate AVG/SUM as float (#8326)<commit_after>/*
 * MinIO Cloud Storage, (C) 2019 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package sql

import (
	"errors"
	"fmt"
)

// Aggregation Function name constants
const (
	aggFnAvg   FuncName = "AVG"
	aggFnCount FuncName = "COUNT"
	aggFnMax   FuncName = "MAX"
	aggFnMin   FuncName = "MIN"
	aggFnSum   FuncName = "SUM"
)

var (
	errNonNumericArg = func(fnStr FuncName) error {
		return fmt.Errorf("%s() requires a numeric argument", fnStr)
	}
	errInvalidAggregation = errors.New("Invalid aggregation seen")
)

type aggVal struct {
	runningSum             *Value
	runningCount           int64
	runningMax, runningMin *Value

	// Stores if at least one record has been seen
	seen bool
}

func newAggVal(fn FuncName) *aggVal {
	switch fn {
	case aggFnAvg, aggFnSum:
		return &aggVal{runningSum: FromFloat(0)}
	case aggFnMin:
		return &aggVal{runningMin: FromInt(0)}
	case aggFnMax:
		return &aggVal{runningMax: FromInt(0)}
	default:
		return &aggVal{}
	}
}

// evalAggregationNode - performs partial computation using the
// current row and stores the result.
//
// On success, it returns (nil, nil).
func (e *FuncExpr) evalAggregationNode(r Record) error {
	// It is assumed that this function is called only when
	// `e` is an aggregation function.

	var val *Value
	var err error
	funcName := e.getFunctionName()
	if aggFnCount == funcName {
		if e.Count.StarArg {
			// Handle COUNT(*)
			e.aggregate.runningCount++
			return nil
		}

		val, err = e.Count.ExprArg.evalNode(r)
		if err != nil {
			return err
		}
	} else {
		// Evaluate the (only) argument
		val, err = e.SFunc.ArgsList[0].evalNode(r)
		if err != nil {
			return err
		}
	}

	if val.IsNull() {
		// E.g. the column or field does not exist in the
		// record - in all such cases the aggregation is not
		// updated.
		return nil
	}

	argVal := val
	if funcName != aggFnCount {
		// All aggregation functions, except COUNT require a
		// numeric argument.

		// Here, we diverge from Amazon S3 behavior by
		// inferring untyped values are numbers.
		if !argVal.isNumeric() {
			if i, ok := argVal.bytesToInt(); ok {
				argVal.setInt(i)
			} else if f, ok := argVal.bytesToFloat(); ok {
				argVal.setFloat(f)
			} else {
				return errNonNumericArg(funcName)
			}
		}
	}

	// Mark that we have seen one non-null value.
	isFirstRow := false
	if !e.aggregate.seen {
		e.aggregate.seen = true
		isFirstRow = true
	}

	switch funcName {
	case aggFnCount:
		// For all non-null values, the count is incremented.
		e.aggregate.runningCount++

	case aggFnAvg, aggFnSum:
		e.aggregate.runningCount++
		// Convert to float.
		f, ok := argVal.ToFloat()
		if !ok {
			return fmt.Errorf("Could not convert value %v (%s) to a number", argVal.value, argVal.GetTypeString())
		}
		argVal.setFloat(f)
		err = e.aggregate.runningSum.arithOp(opPlus, argVal)

	case aggFnMin:
		err = e.aggregate.runningMin.minmax(argVal, false, isFirstRow)

	case aggFnMax:
		err = e.aggregate.runningMax.minmax(argVal, true, isFirstRow)

	default:
		err = errInvalidAggregation
	}

	return err
}

func (e *AliasedExpression) aggregateRow(r Record) error {
	return e.Expression.aggregateRow(r)
}

func (e *Expression) aggregateRow(r Record) error {
	for _, ex := range e.And {
		err := ex.aggregateRow(r)
		if err != nil {
			return err
		}
	}
	return nil
}

func (e *AndCondition) aggregateRow(r Record) error {
	for _, ex := range e.Condition {
		err := ex.aggregateRow(r)
		if err != nil {
			return err
		}
	}
	return nil
}

func (e *Condition) aggregateRow(r Record) error {
	if e.Operand != nil {
		return e.Operand.aggregateRow(r)
	}
	return e.Not.aggregateRow(r)
}

func (e *ConditionOperand) aggregateRow(r Record) error {
	err := e.Operand.aggregateRow(r)
	if err != nil {
		return err
	}

	if e.ConditionRHS == nil {
		return nil
	}

	switch {
	case e.ConditionRHS.Compare != nil:
		return e.ConditionRHS.Compare.Operand.aggregateRow(r)
	case e.ConditionRHS.Between != nil:
		err = e.ConditionRHS.Between.Start.aggregateRow(r)
		if err != nil {
			return err
		}
		return e.ConditionRHS.Between.End.aggregateRow(r)
	case e.ConditionRHS.In != nil:
		for _, elt := range e.ConditionRHS.In.Expressions {
			err = elt.aggregateRow(r)
			if err != nil {
				return err
			}
		}
		return nil
	case e.ConditionRHS.Like != nil:
		err = e.ConditionRHS.Like.Pattern.aggregateRow(r)
		if err != nil {
			return err
		}
		return e.ConditionRHS.Like.EscapeChar.aggregateRow(r)
	default:
		return errInvalidASTNode
	}
}

func (e *Operand) aggregateRow(r Record) error {
	err := e.Left.aggregateRow(r)
	if err != nil {
		return err
	}
	for _, rt := range e.Right {
		err = rt.Right.aggregateRow(r)
		if err != nil {
			return err
		}
	}
	return nil
}

func (e *MultOp) aggregateRow(r Record) error {
	err := e.Left.aggregateRow(r)
	if err != nil {
		return err
	}
	for _, rt := range e.Right {
		err = rt.Right.aggregateRow(r)
		if err != nil {
			return err
		}
	}
	return nil
}

func (e *UnaryTerm) aggregateRow(r Record) error {
	if e.Negated != nil {
		return e.Negated.Term.aggregateRow(r)
	}
	return e.Primary.aggregateRow(r)
}

func (e *PrimaryTerm) aggregateRow(r Record) error {
	switch {
	case e.SubExpression != nil:
		return e.SubExpression.aggregateRow(r)
	case e.FuncCall != nil:
		return e.FuncCall.aggregateRow(r)
	}
	return nil
}

func (e *FuncExpr) aggregateRow(r Record) error {
	switch e.getFunctionName() {
	case aggFnAvg, aggFnSum, aggFnMax, aggFnMin, aggFnCount:
		return e.evalAggregationNode(r)
	default:
		// TODO: traverse arguments and call aggregateRow on
		// them if they could be an ancestor of an
		// aggregation.
	}
	return nil
}

// getAggregate() implementation for each AST node follows. This is
// called after calling aggregateRow() on each input row, to calculate
// the final aggregate result.

func (e *FuncExpr) getAggregate() (*Value, error) {
	switch e.getFunctionName() {
	case aggFnCount:
		return FromInt(e.aggregate.runningCount), nil

	case aggFnAvg:
		if e.aggregate.runningCount == 0 {
			// No rows were seen by AVG.
			return FromNull(), nil
		}
		err := e.aggregate.runningSum.arithOp(opDivide, FromInt(e.aggregate.runningCount))
		return e.aggregate.runningSum, err

	case aggFnMin:
		if !e.aggregate.seen {
			// No rows were seen by MIN
			return FromNull(), nil
		}
		return e.aggregate.runningMin, nil

	case aggFnMax:
		if !e.aggregate.seen {
			// No rows were seen by MAX
			return FromNull(), nil
		}
		return e.aggregate.runningMax, nil

	case aggFnSum:
		// TODO: check if returning 0 when no rows were seen
		// by SUM is expected behavior.
		return e.aggregate.runningSum, nil

	default:
		// TODO:
	}

	return nil, errInvalidAggregation
}
<|endoftext|>


<commit_before>package v1

import (
	"encoding/json"
	"github.com/fritzpay/paymentd/pkg/metadata"
	"github.com/fritzpay/paymentd/pkg/paymentd/principal"
	"github.com/fritzpay/paymentd/pkg/service"
	"github.com/gorilla/mux"
	"gopkg.in/inconshreveable/log15.v2"
	"net/http"
	"time"
)

type PrincipalAdminAPIResponse struct {
	AdminAPIResponse
}

// handler to create or change a principal
//
// PUT creates new principal
// POST can be used to change the principals metadata
func (a *AdminAPI) PrincipalRequest() http.Handler {

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		log := a.log.New(log15.Ctx{"method": "PrincipalRequest"})

		switch r.Method {
		case "PUT":
			a.putNewPrincipal(w, r)
		case "POST":
			a.postChangePrincipal(w, r)
		default:
			ErrMethod.Write(w)
			log.Info("http method not supported", log15.Ctx{"requestMethod": r.Method})
		}
	})
}

// handler to display a specific existing principal
func (a *AdminAPI) PrincipalGetRequest() http.Handler {

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		log := a.log.New(log15.Ctx{"method": "PrincipalGetRequest"})

		// get principal by name
		vars := mux.Vars(r)
		principalName := vars["name"]

		log = log.New(log15.Ctx{"principalName": principalName})

		db := a.ctx.PrincipalDB(service.ReadOnly)
		pr, err := principal.PrincipalByNameDB(db, principalName)
		if err == principal.ErrPrincipalNotFound {
			ErrNotFound.Write(w)
			log.Info("principal not found")
			return
		}
		if err != nil {
			ErrDatabase.Write(w)
			log.Error("DB get by name failed", log15.Ctx{"err": err})
			return
		}
		md, err := metadata.MetadataByPrimaryDB(db, principal.MetadataModel, pr.ID)
		if err != nil {
			ErrDatabase.Write(w)
			log.Error("get metadata failed", log15.Ctx{"err": err})
			return
		}
		if len(md) > 0 {
			pr.Metadata = md.Values()
		}

		// create service response object
		resp := PrincipalAdminAPIResponse{}
		resp.Status = StatusSuccess
		resp.Info = "principal " + pr.Name + " found"
		resp.Response = pr
		err = resp.Write(w)
		if err != nil {
			log.Error("write error", log15.Ctx{"err": err})
			return
		}
	})
}

func (a *AdminAPI) putNewPrincipal(w http.ResponseWriter, r *http.Request) {
	log := a.log.New(log15.Ctx{"method": "putNewPrincipal"})

	// create new principal
	jd := json.NewDecoder(r.Body)
	pr := principal.Principal{}
	err := jd.Decode(&pr)
	r.Body.Close()
	if err != nil {
		ErrReadJson.Write(w)
		log.Error("json decode failed", log15.Ctx{"err": err})
		return
	}

	log = log.New(log15.Ctx{"principalName": pr.Name})

	auth, err := getAuthContainer(r)
	if err != nil {
		log.Crit("context auth error", log15.Ctx{"err": err})
		ErrSystem.Write(w)
		return
	}
	pr.CreatedBy = auth[AuthUserIDKey].(string)
	// set created time
	pr.Created = time.Now()

	// check if principal exists
	db := a.ctx.PrincipalDB()
	_, err = principal.PrincipalByNameDB(db, pr.Name)
	if err != nil && err != principal.ErrPrincipalNotFound {
		// other db error
		log.Error("DB get by name failed", log15.Ctx{"err": err})
		ErrSystem.Write(w)
		return
	} else if err == nil {
		// already exists
		log.Warn("principal already exists")
		ErrConflict.Write(w)
		return
	}

	// insert pr if not exists
	// start dbtx to save metadata and pr together
	tx, err := db.Begin()
	if err != nil {
		log.Crit("Tx begin failed", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}

	// insert pr
	err = principal.InsertPrincipalTx(tx, &pr)
	if err != nil {
		tx.Rollback()
		ErrDatabase.Write(w)
		log.Error("TX insert failed.", log15.Ctx{"err": err})
		return
	}

	// insert metadata
	md := metadata.MetadataFromValues(pr.Metadata, pr.CreatedBy)
	err = metadata.InsertMetadataTx(tx, principal.MetadataModel, pr.ID, md)
	if err != nil {
		tx.Rollback()
		log.Error("TX metadata insert failed.", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}

	//commit tx
	err = tx.Commit()
	if err != nil {
		ErrDatabase.Write(w)
		log.Crit("TX commit failed.", log15.Ctx{"err": err})
		return
	}

	// get data explicit from DB
	pr, err = principal.PrincipalByNameDB(db, pr.Name)
	if err != nil {
		log.Error("DB get by name failed.", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}
	md, err = metadata.MetadataByPrimaryDB(db, principal.MetadataModel, pr.ID)
	if len(md) > 0 {
		pr.Metadata = md.Values()
	}
	if err != nil {
		log.Error("get metadata failed.", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}

	resp := PrincipalAdminAPIResponse{}
	resp.HttpStatus = http.StatusOK
	resp.Status = StatusSuccess
	resp.Response = pr
	resp.Info = "principal " + pr.Name + " created"
	err = resp.Write(w)
	if err != nil {
		log.Error("write error", log15.Ctx{"err": err})
		ErrSystem.Write(w)
		return
	}
}

// post method to add and change the metadata
func (a *AdminAPI) postChangePrincipal(w http.ResponseWriter, r *http.Request) {
	log := a.log.New(log15.Ctx{"method": "postChangePrincipal"})

	// get Metadata from post variables
	jd := json.NewDecoder(r.Body)
	pr := principal.Principal{}
	err := jd.Decode(&pr)
	r.Body.Close()
	if err != nil {
		log.Error("json decode failed", log15.Ctx{"err": err})
		ErrReadJson.Write(w)
		return
	}

	auth, err := getAuthContainer(r)
	if err != nil {
		log.Crit("error getting auth container", log15.Ctx{"err": err})
		ErrSystem.Write(w)
		return
	}
	pr.CreatedBy = auth[AuthUserIDKey].(string)

	log = log.New(log15.Ctx{"principalName": pr.Name})

	// open transaction to add the posted metadata
	tx, err := a.ctx.PrincipalDB().Begin()
	if err != nil {
		log.Crit("Tx begin failed.", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}

	// does principal exist
	pr.ID, err = principal.PrincipalIDByNameTx(tx, pr.Name)
	if err != nil {
		txErr := tx.Rollback()
		if txErr != nil {
			log.Crit("error on rollback", log15.Ctx{"err": err})
		}
		if err == principal.ErrPrincipalNotFound {
			log.Warn("principal not found")
			ErrNotFound.Write(w)
			return
		}
		log.Error("get principal from DB failed", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}

	md := metadata.MetadataFromValues(pr.Metadata, pr.CreatedBy)

	err = metadata.InsertMetadataTx(tx, principal.MetadataModel, pr.ID, md)
	if err != nil {
		txErr := tx.Rollback()
		if txErr != nil {
			log.Crit("error on rollback", log15.Ctx{"err": err})
		}
		log.Error("insert metadata failed", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}
	err = tx.Commit()
	if err != nil {
		log.Crit("error on commit", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}

	// get stored data from db
	db := a.ctx.PrincipalDB(service.ReadOnly)
	pr, err = principal.PrincipalByNameDB(db, pr.Name)
	if err != nil {
		if err == principal.ErrPrincipalNotFound {
			log.Error("principal not found")
			ErrNotFound.Write(w)
			return
		}
		log.Error("DB get by name failed.", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}
	md, err = metadata.MetadataByPrimaryDB(db, principal.MetadataModel, pr.ID)
	if err != nil {
		log.Error("get metadata failed.", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}
	if len(md) > 0 {
		pr.Metadata = md.Values()
	}

	// create response
	resp := PrincipalAdminAPIResponse{}
	resp.Info = "principal " + pr.Name + " changed"
	resp.Status = StatusSuccess
	resp.Response = pr
	err = resp.Write(w)
	if err != nil {
		ErrSystem.Write(w)
		log.Error("write error", log15.Ctx{"err": err})
		return
	}
}
<commit_msg>fix transaction handling on put principal<commit_after>package v1

import (
	"encoding/json"
	"github.com/fritzpay/paymentd/pkg/metadata"
	"github.com/fritzpay/paymentd/pkg/paymentd/principal"
	"github.com/fritzpay/paymentd/pkg/service"
	"github.com/gorilla/mux"
	"gopkg.in/inconshreveable/log15.v2"
	"net/http"
	"time"
)

type PrincipalAdminAPIResponse struct {
	AdminAPIResponse
}

// handler to create or change a principal
//
// PUT creates new principal
// POST can be used to change the principals metadata
func (a *AdminAPI) PrincipalRequest() http.Handler {

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		log := a.log.New(log15.Ctx{"method": "PrincipalRequest"})

		switch r.Method {
		case "PUT":
			a.putNewPrincipal(w, r)
		case "POST":
			a.postChangePrincipal(w, r)
		default:
			ErrMethod.Write(w)
			log.Info("http method not supported", log15.Ctx{"requestMethod": r.Method})
		}
	})
}

// handler to display a specific existing principal
func (a *AdminAPI) PrincipalGetRequest() http.Handler {

	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Content-Type", "application/json")
		log := a.log.New(log15.Ctx{"method": "PrincipalGetRequest"})

		// get principal by name
		vars := mux.Vars(r)
		principalName := vars["name"]

		log = log.New(log15.Ctx{"principalName": principalName})

		db := a.ctx.PrincipalDB(service.ReadOnly)
		pr, err := principal.PrincipalByNameDB(db, principalName)
		if err == principal.ErrPrincipalNotFound {
			ErrNotFound.Write(w)
			log.Info("principal not found")
			return
		}
		if err != nil {
			ErrDatabase.Write(w)
			log.Error("DB get by name failed", log15.Ctx{"err": err})
			return
		}
		md, err := metadata.MetadataByPrimaryDB(db, principal.MetadataModel, pr.ID)
		if err != nil {
			ErrDatabase.Write(w)
			log.Error("get metadata failed", log15.Ctx{"err": err})
			return
		}
		if len(md) > 0 {
			pr.Metadata = md.Values()
		}

		// create service response object
		resp := PrincipalAdminAPIResponse{}
		resp.Status = StatusSuccess
		resp.Info = "principal " + pr.Name + " found"
		resp.Response = pr
		err = resp.Write(w)
		if err != nil {
			log.Error("write error", log15.Ctx{"err": err})
			return
		}
	})
}

func (a *AdminAPI) putNewPrincipal(w http.ResponseWriter, r *http.Request) {
	log := a.log.New(log15.Ctx{"method": "putNewPrincipal"})

	// create new principal
	jd := json.NewDecoder(r.Body)
	pr := principal.Principal{}
	err := jd.Decode(&pr)
	r.Body.Close()
	if err != nil {
		ErrReadJson.Write(w)
		log.Error("json decode failed", log15.Ctx{"err": err})
		return
	}

	log = log.New(log15.Ctx{"principalName": pr.Name})

	auth, err := getAuthContainer(r)
	if err != nil {
		log.Crit("context auth error", log15.Ctx{"err": err})
		ErrSystem.Write(w)
		return
	}
	pr.CreatedBy = auth[AuthUserIDKey].(string)
	// set created time
	pr.Created = time.Now()

	// insert pr if not exists
	// start dbtx to save metadata and pr together
	tx, err := a.ctx.PrincipalDB().Begin()
	if err != nil {
		log.Crit("Tx begin failed", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}

	// check if principal exists
	_, err = principal.PrincipalByNameTx(tx, pr.Name)
	if err != nil && err != principal.ErrPrincipalNotFound {
		// other db error
		log.Error("DB get by name failed", log15.Ctx{"err": err})
		ErrSystem.Write(w)
		return
	} else if err == nil {
		// already exists
		log.Warn("principal already exists")
		ErrConflict.Write(w)
		return
	}

	// insert pr
	err = principal.InsertPrincipalTx(tx, &pr)
	if err != nil {
		tx.Rollback()
		ErrDatabase.Write(w)
		log.Error("TX insert failed.", log15.Ctx{"err": err})
		return
	}

	// insert metadata
	md := metadata.MetadataFromValues(pr.Metadata, pr.CreatedBy)
	err = metadata.InsertMetadataTx(tx, principal.MetadataModel, pr.ID, md)
	if err != nil {
		tx.Rollback()
		log.Error("TX metadata insert failed.", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}

	//commit tx
	err = tx.Commit()
	if err != nil {
		ErrDatabase.Write(w)
		log.Crit("TX commit failed.", log15.Ctx{"err": err})
		return
	}

	// get data explicit from DB
	db := a.ctx.PrincipalDB(service.ReadOnly)
	pr, err = principal.PrincipalByNameDB(db, pr.Name)
	if err != nil {
		log.Error("DB get by name failed.", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}
	md, err = metadata.MetadataByPrimaryDB(db, principal.MetadataModel, pr.ID)
	if len(md) > 0 {
		pr.Metadata = md.Values()
	}
	if err != nil {
		log.Error("get metadata failed.", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}

	resp := PrincipalAdminAPIResponse{}
	resp.HttpStatus = http.StatusOK
	resp.Status = StatusSuccess
	resp.Response = pr
	resp.Info = "principal " + pr.Name + " created"
	err = resp.Write(w)
	if err != nil {
		log.Error("write error", log15.Ctx{"err": err})
		ErrSystem.Write(w)
		return
	}
}

// post method to add and change the metadata
func (a *AdminAPI) postChangePrincipal(w http.ResponseWriter, r *http.Request) {
	log := a.log.New(log15.Ctx{"method": "postChangePrincipal"})

	// get Metadata from post variables
	jd := json.NewDecoder(r.Body)
	pr := principal.Principal{}
	err := jd.Decode(&pr)
	r.Body.Close()
	if err != nil {
		log.Error("json decode failed", log15.Ctx{"err": err})
		ErrReadJson.Write(w)
		return
	}

	auth, err := getAuthContainer(r)
	if err != nil {
		log.Crit("error getting auth container", log15.Ctx{"err": err})
		ErrSystem.Write(w)
		return
	}
	pr.CreatedBy = auth[AuthUserIDKey].(string)

	log = log.New(log15.Ctx{"principalName": pr.Name})

	// open transaction to add the posted metadata
	tx, err := a.ctx.PrincipalDB().Begin()
	if err != nil {
		log.Crit("Tx begin failed.", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}

	// does principal exist
	pr.ID, err = principal.PrincipalIDByNameTx(tx, pr.Name)
	if err != nil {
		txErr := tx.Rollback()
		if txErr != nil {
			log.Crit("error on rollback", log15.Ctx{"err": err})
		}
		if err == principal.ErrPrincipalNotFound {
			log.Warn("principal not found")
			ErrNotFound.Write(w)
			return
		}
		log.Error("get principal from DB failed", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}

	md := metadata.MetadataFromValues(pr.Metadata, pr.CreatedBy)

	err = metadata.InsertMetadataTx(tx, principal.MetadataModel, pr.ID, md)
	if err != nil {
		txErr := tx.Rollback()
		if txErr != nil {
			log.Crit("error on rollback", log15.Ctx{"err": err})
		}
		log.Error("insert metadata failed", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}
	err = tx.Commit()
	if err != nil {
		log.Crit("error on commit", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}

	// get stored data from db
	db := a.ctx.PrincipalDB(service.ReadOnly)
	pr, err = principal.PrincipalByNameDB(db, pr.Name)
	if err != nil {
		if err == principal.ErrPrincipalNotFound {
			log.Error("principal not found")
			ErrNotFound.Write(w)
			return
		}
		log.Error("DB get by name failed.", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}
	md, err = metadata.MetadataByPrimaryDB(db, principal.MetadataModel, pr.ID)
	if err != nil {
		log.Error("get metadata failed.", log15.Ctx{"err": err})
		ErrDatabase.Write(w)
		return
	}
	if len(md) > 0 {
		pr.Metadata = md.Values()
	}

	// create response
	resp := PrincipalAdminAPIResponse{}
	resp.Info = "principal " + pr.Name + " changed"
	resp.Status = StatusSuccess
	resp.Response = pr
	err = resp.Write(w)
	if err != nil {
		ErrSystem.Write(w)
		log.Error("write error", log15.Ctx{"err": err})
		return
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubernetes

import (
	"bufio"
	"io"
	"os"
	"strings"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"gopkg.in/yaml.v2"
	k8syaml "k8s.io/apimachinery/pkg/util/yaml"
)

type yamlObject map[interface{}]interface{}

// These are the required fields for a yaml document to be a valid Kubernetes yaml
var requiredFields = []string{"apiVersion", "kind", "metadata"}

// These are the supported file formats for Kubernetes manifests
var validSuffixes = []string{".yml", ".yaml", ".json"}

// HasKubernetesFileExtension is for determining if a file under a glob pattern
// is deployable file format. It makes no attempt to check whether or not the file
// is actually deployable or has the correct contents.
func HasKubernetesFileExtension(n string) bool {
	for _, s := range validSuffixes {
		if strings.HasSuffix(n, s) {
			return true
		}
	}
	return false
}

// IsKubernetesManifest is for determining if a file is a valid Kubernetes manifest
func IsKubernetesManifest(file string) bool {
	if !HasKubernetesFileExtension(file) {
		return false
	}

	_, err := parseKubernetesObjects(file)
	return err == nil
}

// ParseImagesFromKubernetesYaml parses the kubernetes yamls, and if it finds at least one
// valid Kubernetes object, it will return the images referenced in them.
func ParseImagesFromKubernetesYaml(filepath string) ([]string, error) {
	k8sObjects, err := parseKubernetesObjects(filepath)
	if err != nil {
		return nil, err
	}

	images := []string{}
	for _, k8sObject := range k8sObjects {
		images = append(images, parseImagesFromYaml(k8sObject)...)
	}
	return images, nil
}

// parseKubernetesObjects uses required fields from the k8s spec
// to determine if a provided yaml file is a valid k8s manifest, as detailed in
// https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/#required-fields.
// If so, it will return the parsed objects.
func parseKubernetesObjects(filepath string) ([]yamlObject, error) {
	f, err := os.Open(filepath)
	if err != nil {
		return nil, errors.Wrap(err, "opening config file")
	}
	r := k8syaml.NewYAMLReader(bufio.NewReader(f))

	var k8sObjects []yamlObject

	for {
		doc, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, errors.Wrap(err, "reading config file")
		}

		obj := make(yamlObject)
		if err := yaml.Unmarshal(doc, &obj); err != nil {
			return nil, errors.Wrap(err, "reading Kubernetes YAML")
		}

		if !hasRequiredK8sManifestFields(obj, filepath) {
			continue
		}

		k8sObjects = append(k8sObjects, obj)
	}
	if len(k8sObjects) == 0 {
		return nil, errors.New("no valid Kubernetes objects decoded")
	}
	return k8sObjects, nil
}

func hasRequiredK8sManifestFields(doc map[interface{}]interface{}, fileName string) bool {
	for _, field := range requiredFields {
		if _, ok := doc[field]; !ok {
			logrus.Tracef("%s not present in %s, continuing", field, fileName)
			return false
		}
	}
	return true
}

// adapted from pkg/skaffold/deploy/kubectl/recursiveReplaceImage()
func parseImagesFromYaml(obj interface{}) []string {
	images := []string{}
	switch t := obj.(type) {
	case []interface{}:
		for _, v := range t {
			images = append(images, parseImagesFromYaml(v)...)
		}
	case yamlObject:
		for k, v := range t {
			if k.(string) != "image" {
				images = append(images, parseImagesFromYaml(v)...)
				continue
			}

			images = append(images, v.(string))
		}
	}
	return images
}
<commit_msg>revert extra debugging<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package kubernetes

import (
	"bufio"
	"io"
	"os"
	"strings"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"gopkg.in/yaml.v2"
	k8syaml "k8s.io/apimachinery/pkg/util/yaml"
)

type yamlObject map[interface{}]interface{}

// These are the required fields for a yaml document to be a valid Kubernetes yaml
var requiredFields = []string{"apiVersion", "kind", "metadata"}

// These are the supported file formats for Kubernetes manifests
var validSuffixes = []string{".yml", ".yaml", ".json"}

// HasKubernetesFileExtension is for determining if a file under a glob pattern
// is deployable file format. It makes no attempt to check whether or not the file
// is actually deployable or has the correct contents.
func HasKubernetesFileExtension(n string) bool {
	for _, s := range validSuffixes {
		if strings.HasSuffix(n, s) {
			return true
		}
	}
	return false
}

// IsKubernetesManifest is for determining if a file is a valid Kubernetes manifest
func IsKubernetesManifest(file string) bool {
	if !HasKubernetesFileExtension(file) {
		return false
	}

	_, err := parseKubernetesObjects(file)
	return err == nil
}

// ParseImagesFromKubernetesYaml parses the kubernetes yamls, and if it finds at least one
// valid Kubernetes object, it will return the images referenced in them.
func ParseImagesFromKubernetesYaml(filepath string) ([]string, error) {
	k8sObjects, err := parseKubernetesObjects(filepath)
	if err != nil {
		return nil, err
	}

	images := []string{}
	for _, k8sObject := range k8sObjects {
		images = append(images, parseImagesFromYaml(k8sObject)...)
	}
	return images, nil
}

// parseKubernetesObjects uses required fields from the k8s spec
// to determine if a provided yaml file is a valid k8s manifest, as detailed in
// https://kubernetes.io/docs/concepts/overview/working-with-objects/kubernetes-objects/#required-fields.
// If so, it will return the parsed objects.
func parseKubernetesObjects(filepath string) ([]yamlObject, error) {
	f, err := os.Open(filepath)
	if err != nil {
		return nil, errors.Wrap(err, "opening config file")
	}
	r := k8syaml.NewYAMLReader(bufio.NewReader(f))

	var k8sObjects []yamlObject

	for {
		doc, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, errors.Wrap(err, "reading config file")
		}

		obj := make(yamlObject)
		if err := yaml.Unmarshal(doc, &obj); err != nil {
			return nil, errors.Wrap(err, "reading Kubernetes YAML")
		}

		if !hasRequiredK8sManifestFields(obj) {
			continue
		}

		k8sObjects = append(k8sObjects, obj)
	}
	if len(k8sObjects) == 0 {
		return nil, errors.New("no valid Kubernetes objects decoded")
	}
	return k8sObjects, nil
}

func hasRequiredK8sManifestFields(doc map[interface{}]interface{}) bool {
	for _, field := range requiredFields {
		if _, ok := doc[field]; !ok {
			logrus.Debugf("%s not present in yaml, continuing", field)
			return false
		}
	}
	return true
}

// adapted from pkg/skaffold/deploy/kubectl/recursiveReplaceImage()
func parseImagesFromYaml(obj interface{}) []string {
	images := []string{}
	switch t := obj.(type) {
	case []interface{}:
		for _, v := range t {
			images = append(images, parseImagesFromYaml(v)...)
		}
	case yamlObject:
		for k, v := range t {
			if k.(string) != "image" {
				images = append(images, parseImagesFromYaml(v)...)
				continue
			}

			images = append(images, v.(string))
		}
	}
	return images
}
<|endoftext|>


<commit_before>package onedrive

import (
	"encoding/json"
	"io/ioutil"
	"testing"
)

func TestGetDefaultDriveValid(t *testing.T) {
	setup()
	defer teardown()

	testFile := "fixtures/drive.valid.json"

	fb, err := ioutil.ReadFile(testFile)
	if err != nil {
		t.Fatal(err)
	}

	var expectedDrive Drive
	if err := json.Unmarshal(fb, &expectedDrive); err != nil {
		t.Fatal(err)
	}

	mux.HandleFunc("/drive", fileWrapperHandler(testFile))

	drive, resp, err := oneDrive.Drives.GetDefaultDrive()
	if err != nil {
		t.Fatalf("Problem fetching the default drive: %s", err.Error())
	}

	if resp.StatusCode != 200 {
		t.Fatalf("Expected response to be 200 got %d", resp.StatusCode)
	}

	if drive.ID != expectedDrive.ID {
		t.Fatalf("Expected ID to be %q, but got %q", expectedDrive.ID, drive.ID)
	}

}
<commit_msg>Add test for GetRootDrive.<commit_after>package onedrive

import (
	"encoding/json"
	"io/ioutil"
	"testing"
)

func TestGetDefaultDrive(t *testing.T) {
	setup()
	defer teardown()

	testFile := "fixtures/drive.valid.json"

	fb, err := ioutil.ReadFile(testFile)
	if err != nil {
		t.Fatal(err)
	}

	var expectedDrive Drive
	if err := json.Unmarshal(fb, &expectedDrive); err != nil {
		t.Fatal(err)
	}

	mux.HandleFunc("/drive", fileWrapperHandler(testFile))

	defaultDrive, resp, err := oneDrive.Drives.GetDefaultDrive()
	if err != nil {
		t.Fatalf("Problem fetching the default drive: %s", err.Error())
	}

	if resp.StatusCode != 200 {
		t.Fatalf("Expected response to be 200 got %d", resp.StatusCode)
	}

	if got, want := defaultDrive.ID, expectedDrive.ID; got != want {
		t.Fatalf("Got %q Expected %q", got, want)
	}

	if got, want := defaultDrive.Owner.User.DisplayName, expectedDrive.Owner.User.DisplayName; got != want {
		t.Fatalf("Got %q Expected %q", got, want)
	}

}

func TestGetRootDrive(t *testing.T) {
	setup()
	defer teardown()

	testFile := "fixtures/drive.valid.json"

	fb, err := ioutil.ReadFile(testFile)
	if err != nil {
		t.Fatal(err)
	}

	var expectedDrive Drive
	if err := json.Unmarshal(fb, &expectedDrive); err != nil {
		t.Fatal(err)
	}

	mux.HandleFunc("/drive/root", fileWrapperHandler(testFile))

	rootDrive, resp, err := oneDrive.Drives.GetRootDrive()
	if err != nil {
		t.Fatalf("Problem fetching the root drive: %s", err.Error())
	}

	if resp.StatusCode != 200 {
		t.Fatalf("Expected response to be 200 got %d", resp.StatusCode)
	}

	if got, want := rootDrive.ID, expectedDrive.ID; got != want {
		t.Fatalf("Got %q Expected %q", got, want)
	}

	if got, want := rootDrive.Owner.User.DisplayName, expectedDrive.Owner.User.DisplayName; got != want {
		t.Fatalf("Got %q Expected %q", got, want)
	}

}
<|endoftext|>