<commit_before>package helm

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"time"

	"encoding/base64"

	"github.com/pkg/errors"
	"github.com/rancher/rancher/pkg/controllers/user/helm/common"
	"github.com/rancher/rancher/pkg/templatecontent"
	mgmtv3 "github.com/rancher/types/apis/management.cattle.io/v3"
	"github.com/rancher/types/apis/project.cattle.io/v3"
	"github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	helmName = "helm"
	kubectl  = "kubectl"
	appLabel = "io.cattle.field/appId"
	kcEnv    = "KUBECONFIG"
)

func WriteTempDir(rootDir string, files map[string]string) (string, error) {
	for name, content := range files {
		fp := filepath.Join(rootDir, name)
		if err := os.MkdirAll(filepath.Dir(fp), 0755); err != nil {
			return "", err
		}
		if err := ioutil.WriteFile(fp, []byte(content), 0755); err != nil {
			return "", err
		}
	}
	for name := range files {
		parts := strings.Split(name, "/")
		if len(parts) > 0 {
			return filepath.Join(rootDir, parts[0]), nil
		}
	}
	return "", nil
}

func kubectlApply(template, kubeconfig string, app *v3.App) error {
	file, err := ioutil.TempFile("", "app-template")
	if err != nil {
		return err
	}
	defer os.RemoveAll(file.Name())
	if err := ioutil.WriteFile(file.Name(), []byte(template), 0755); err != nil {
		return err
	}

	command := []string{"apply", "--all", "--overwrite", "-n", app.Spec.TargetNamespace, "-f", file.Name()}
	if app.Spec.Prune {
		command = append(command, "--prune")
	}
	cmd := exec.Command(kubectl, command...)
	cmd.Env = []string{fmt.Sprintf("%s=%s", kcEnv, kubeconfig)}
	sbErr := &bytes.Buffer{}
	cmd.Stdout = os.Stdout
	cmd.Stderr = sbErr
	if err := cmd.Start(); err != nil {
		return err
	}
	if err := cmd.Wait(); err != nil {
		return errors.Wrapf(err, fmt.Sprintf("Kubectl apply failed. Error: %s", sbErr.String()))
	}

	// wait for resources to be created and try 5 times
	// todo: there is a bug around rolebindings that are being deleted if created outside rancher
	start := 250 * time.Millisecond
	for i := 0; i < 5; i++ {
		time.Sleep(start)
		command = []string{"label", "-n", app.Spec.TargetNamespace, "--overwrite", "-f", file.Name(), fmt.Sprintf("%s=%s", appLabel, app.Name)}
		cmd = exec.Command(kubectl, command...)
		cmd.Env = []string{fmt.Sprintf("%s=%s", kcEnv, kubeconfig)}
		buf := &bytes.Buffer{}
		cmd.Stdout = os.Stdout
		cmd.Stderr = buf
		if err := cmd.Start(); err != nil {
			return err
		}
		if err := cmd.Wait(); err != nil {
			if i == 4 {
				logrus.Warnf("tried 4 times and kubectl label failed. Error: %s", buf.String())
				break
			}
			start = start * 2
			continue
		}
		break
	}

	return nil
}

func kubectlDelete(template, kubeconfig, namespace string) error {
	file, err := ioutil.TempFile("", "app-template")
	if err != nil {
		return err
	}
	defer os.RemoveAll(file.Name())
	if err := ioutil.WriteFile(file.Name(), []byte(template), 0755); err != nil {
		return err
	}
	command := []string{"delete", "--all", "-n", namespace, "-f", file.Name()}
	// try three times and succeed
	start := time.Second * 1
	for i := 0; i < 3; i++ {
		cmd := exec.Command(kubectl, command...)
		cmd.Env = []string{fmt.Sprintf("%s=%s", kcEnv, kubeconfig)}
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Start(); err != nil {
			return err
		}
		if err := cmd.Wait(); err != nil {
			time.Sleep(start)
			start = start * 2
			continue
		}
		break
	}
	return nil
}

func convertTemplates(files map[string]string, templateContentClient mgmtv3.TemplateContentInterface) (map[string]string, error) {
	templates := map[string]string{}
	for name, tag := range files {
		data, err := templatecontent.GetTemplateFromTag(tag, templateContentClient)
		if err != nil {
			continue
		}
		templates[name] = data
	}
	return templates, nil
}

func generateTemplates(obj *v3.App, templateVersionClient mgmtv3.TemplateVersionInterface, templateContentClient mgmtv3.TemplateContentInterface) (string, string, error) {
	files := map[string]string{}
	if obj.Spec.ExternalID != "" {
		templateVersionID, err := common.ParseExternalID(obj.Spec.ExternalID)
		if err != nil {
			return "", "", err
		}
		templateVersion, err := templateVersionClient.Get(templateVersionID, metav1.GetOptions{})
		if err != nil {
			return "", "", err
		}
		files, err = convertTemplates(templateVersion.Spec.Files, templateContentClient)
		if err != nil {
			return "", "", err
		}
	} else {
		for k, v := range obj.Spec.Files {
			content, err := base64.StdEncoding.DecodeString(v)
			if err != nil {
				return "", "", err
			}
			files[k] = string(content)
		}
	}
	tempDir, err := ioutil.TempDir("", "helm-")
	if err != nil {
		return "", "", err
	}
	dir, err := WriteTempDir(tempDir, files)
	defer os.RemoveAll(dir)
	if err != nil {
		return "", "", err
	}

	setValues := []string{}
	if obj.Spec.Answers != nil {
		answers := obj.Spec.Answers
		result := []string{}
		for k, v := range answers {
			result = append(result, fmt.Sprintf("%s=%s", k, v))
		}
		setValues = append([]string{"--set-string"}, strings.Join(result, ","))
	}
	commands := append([]string{"template", dir, "--name", obj.Name, "--namespace", obj.Spec.TargetNamespace}, setValues...)

	cmd := exec.Command(helmName, commands...)
	sbOut := &bytes.Buffer{}
	sbErr := &bytes.Buffer{}
	cmd.Stdout = sbOut
	cmd.Stderr = sbErr
	if err := cmd.Start(); err != nil {
		return "", "", errors.Wrapf(err, "helm template failed. %s", sbErr.String())
	}
	if err := cmd.Wait(); err != nil {
		return "", "", errors.Wrapf(err, "helm template failed. %s", sbErr.String())
	}

	// notes.txt
	commands = []string{"template", dir, "--name", obj.Name, "--namespace", obj.Spec.TargetNamespace, "--notes"}
	cmd = exec.Command(helmName, commands...)
	noteOut := &bytes.Buffer{}
	sbErr = &bytes.Buffer{}
	cmd.Stdout = noteOut
	cmd.Stderr = sbErr
	if err := cmd.Start(); err != nil {
		return "", "", errors.Wrapf(err, "helm template --notes failed. %s", sbErr.String())
	}
	if err := cmd.Wait(); err != nil {
		return "", "", errors.Wrapf(err, "helm template --notes failed. %s", sbErr.String())
	}
	template := sbOut.String()
	notes := noteOut.String()
	return template, notes, nil
}
<commit_msg>Revert "use set-string"<commit_after>package helm

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"time"

	"encoding/base64"

	"github.com/pkg/errors"
	"github.com/rancher/rancher/pkg/controllers/user/helm/common"
	"github.com/rancher/rancher/pkg/templatecontent"
	mgmtv3 "github.com/rancher/types/apis/management.cattle.io/v3"
	"github.com/rancher/types/apis/project.cattle.io/v3"
	"github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	helmName = "helm"
	kubectl  = "kubectl"
	appLabel = "io.cattle.field/appId"
	kcEnv    = "KUBECONFIG"
)

func WriteTempDir(rootDir string, files map[string]string) (string, error) {
	for name, content := range files {
		fp := filepath.Join(rootDir, name)
		if err := os.MkdirAll(filepath.Dir(fp), 0755); err != nil {
			return "", err
		}
		if err := ioutil.WriteFile(fp, []byte(content), 0755); err != nil {
			return "", err
		}
	}
	for name := range files {
		parts := strings.Split(name, "/")
		if len(parts) > 0 {
			return filepath.Join(rootDir, parts[0]), nil
		}
	}
	return "", nil
}

func kubectlApply(template, kubeconfig string, app *v3.App) error {
	file, err := ioutil.TempFile("", "app-template")
	if err != nil {
		return err
	}
	defer os.RemoveAll(file.Name())
	if err := ioutil.WriteFile(file.Name(), []byte(template), 0755); err != nil {
		return err
	}

	command := []string{"apply", "--all", "--overwrite", "-n", app.Spec.TargetNamespace, "-f", file.Name()}
	if app.Spec.Prune {
		command = append(command, "--prune")
	}
	cmd := exec.Command(kubectl, command...)
	cmd.Env = []string{fmt.Sprintf("%s=%s", kcEnv, kubeconfig)}
	sbErr := &bytes.Buffer{}
	cmd.Stdout = os.Stdout
	cmd.Stderr = sbErr
	if err := cmd.Start(); err != nil {
		return err
	}
	if err := cmd.Wait(); err != nil {
		return errors.Wrapf(err, fmt.Sprintf("Kubectl apply failed. Error: %s", sbErr.String()))
	}

	// wait for resources to be created and try 5 times
	// todo: there is a bug around rolebindings that are being deleted if created outside rancher
	start := 250 * time.Millisecond
	for i := 0; i < 5; i++ {
		time.Sleep(start)
		command = []string{"label", "-n", app.Spec.TargetNamespace, "--overwrite", "-f", file.Name(), fmt.Sprintf("%s=%s", appLabel, app.Name)}
		cmd = exec.Command(kubectl, command...)
		cmd.Env = []string{fmt.Sprintf("%s=%s", kcEnv, kubeconfig)}
		buf := &bytes.Buffer{}
		cmd.Stdout = os.Stdout
		cmd.Stderr = buf
		if err := cmd.Start(); err != nil {
			return err
		}
		if err := cmd.Wait(); err != nil {
			if i == 4 {
				logrus.Warnf("tried 4 times and kubectl label failed. Error: %s", buf.String())
				break
			}
			start = start * 2
			continue
		}
		break
	}

	return nil
}

func kubectlDelete(template, kubeconfig, namespace string) error {
	file, err := ioutil.TempFile("", "app-template")
	if err != nil {
		return err
	}
	defer os.RemoveAll(file.Name())
	if err := ioutil.WriteFile(file.Name(), []byte(template), 0755); err != nil {
		return err
	}
	command := []string{"delete", "--all", "-n", namespace, "-f", file.Name()}
	// try three times and succeed
	start := time.Second * 1
	for i := 0; i < 3; i++ {
		cmd := exec.Command(kubectl, command...)
		cmd.Env = []string{fmt.Sprintf("%s=%s", kcEnv, kubeconfig)}
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Start(); err != nil {
			return err
		}
		if err := cmd.Wait(); err != nil {
			time.Sleep(start)
			start = start * 2
			continue
		}
		break
	}
	return nil
}

func convertTemplates(files map[string]string, templateContentClient mgmtv3.TemplateContentInterface) (map[string]string, error) {
	templates := map[string]string{}
	for name, tag := range files {
		data, err := templatecontent.GetTemplateFromTag(tag, templateContentClient)
		if err != nil {
			continue
		}
		templates[name] = data
	}
	return templates, nil
}

func generateTemplates(obj *v3.App, templateVersionClient mgmtv3.TemplateVersionInterface, templateContentClient mgmtv3.TemplateContentInterface) (string, string, error) {
	files := map[string]string{}
	if obj.Spec.ExternalID != "" {
		templateVersionID, err := common.ParseExternalID(obj.Spec.ExternalID)
		if err != nil {
			return "", "", err
		}
		templateVersion, err := templateVersionClient.Get(templateVersionID, metav1.GetOptions{})
		if err != nil {
			return "", "", err
		}
		files, err = convertTemplates(templateVersion.Spec.Files, templateContentClient)
		if err != nil {
			return "", "", err
		}
	} else {
		for k, v := range obj.Spec.Files {
			content, err := base64.StdEncoding.DecodeString(v)
			if err != nil {
				return "", "", err
			}
			files[k] = string(content)
		}
	}
	tempDir, err := ioutil.TempDir("", "helm-")
	if err != nil {
		return "", "", err
	}
	dir, err := WriteTempDir(tempDir, files)
	defer os.RemoveAll(dir)
	if err != nil {
		return "", "", err
	}

	setValues := []string{}
	if obj.Spec.Answers != nil {
		answers := obj.Spec.Answers
		result := []string{}
		for k, v := range answers {
			result = append(result, fmt.Sprintf("%s=%s", k, v))
		}
		setValues = append([]string{"--set"}, strings.Join(result, ","))
	}
	commands := append([]string{"template", dir, "--name", obj.Name, "--namespace", obj.Spec.TargetNamespace}, setValues...)

	cmd := exec.Command(helmName, commands...)
	sbOut := &bytes.Buffer{}
	sbErr := &bytes.Buffer{}
	cmd.Stdout = sbOut
	cmd.Stderr = sbErr
	if err := cmd.Start(); err != nil {
		return "", "", errors.Wrapf(err, "helm template failed. %s", sbErr.String())
	}
	if err := cmd.Wait(); err != nil {
		return "", "", errors.Wrapf(err, "helm template failed. %s", sbErr.String())
	}

	// notes.txt
	commands = []string{"template", dir, "--name", obj.Name, "--namespace", obj.Spec.TargetNamespace, "--notes"}
	cmd = exec.Command(helmName, commands...)
	noteOut := &bytes.Buffer{}
	sbErr = &bytes.Buffer{}
	cmd.Stdout = noteOut
	cmd.Stderr = sbErr
	if err := cmd.Start(); err != nil {
		return "", "", errors.Wrapf(err, "helm template --notes failed. %s", sbErr.String())
	}
	if err := cmd.Wait(); err != nil {
		return "", "", errors.Wrapf(err, "helm template --notes failed. %s", sbErr.String())
	}
	template := sbOut.String()
	notes := noteOut.String()
	return template, notes, nil
}
<|endoftext|>


<commit_before>package selfsigned

import (
	"k8s.io/client-go/kubernetes"
	corelisters "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/tools/record"

	"github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha1"
	clientset "github.com/jetstack/cert-manager/pkg/client/clientset/versioned"
	"github.com/jetstack/cert-manager/pkg/issuer"
)

// SelfSigned is an Issuer implementation the simply self-signs Certificates.
type SelfSigned struct {
	issuer        v1alpha1.GenericIssuer
	cl            kubernetes.Interface
	cmclient      clientset.Interface
	recorder      record.EventRecorder
	secretsLister corelisters.SecretLister
}

func NewSelfSigned(issuer v1alpha1.GenericIssuer,
	cl kubernetes.Interface,
	cmclient clientset.Interface,
	recorder record.EventRecorder,
	secretsLister corelisters.SecretLister) (issuer.Interface, error) {
	return &SelfSigned{
		issuer:        issuer,
		cl:            cl,
		cmclient:      cmclient,
		recorder:      recorder,
		secretsLister: secretsLister,
	}, nil
}

const (
	ControllerName = "selfsigned"
)

func init() {
	issuer.Register(ControllerName, func(issuer v1alpha1.GenericIssuer, ctx *issuer.Context) (issuer.Interface, error) {
		return NewSelfSigned(
			issuer,
			ctx.Client,
			ctx.CMClient,
			ctx.Recorder,
			ctx.KubeSharedInformerFactory.Core().V1().Secrets().Lister(),
		)
	})
}
<commit_msg>Update selfsigned issuer<commit_after>package selfsigned

import (
	corelisters "k8s.io/client-go/listers/core/v1"

	"github.com/jetstack/cert-manager/pkg/apis/certmanager/v1alpha1"
	"github.com/jetstack/cert-manager/pkg/controller"
	"github.com/jetstack/cert-manager/pkg/issuer"
)

// SelfSigned is an Issuer implementation the simply self-signs Certificates.
type SelfSigned struct {
	*controller.Context
	issuer v1alpha1.GenericIssuer

	secretsLister corelisters.SecretLister
}

func NewSelfSigned(ctx *controller.Context, issuer v1alpha1.GenericIssuer) (issuer.Interface, error) {
	secretsLister := ctx.KubeSharedInformerFactory.Core().V1().Secrets().Lister()

	return &SelfSigned{
		Context:       ctx,
		issuer:        issuer,
		secretsLister: secretsLister,
	}, nil
}

func init() {
	controller.RegisterIssuer(controller.IssuerSelfSigned, NewSelfSigned)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package openstackmodel

import (
	"strings"

	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/openstacktasks"
)

// NetworkModelBuilder configures network objects
type NetworkModelBuilder struct {
	*OpenstackModelContext
	Lifecycle *fi.Lifecycle
}

var _ fi.ModelBuilder = &NetworkModelBuilder{}

func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
	clusterName := b.ClusterName()

	netName, err := b.GetNetworkName()
	if err != nil {
		return err
	}
	{
		t := &openstacktasks.Network{
			Name:      s(netName),
			ID:        s(b.Cluster.Spec.NetworkID),
			Tag:       s(clusterName),
			Lifecycle: b.Lifecycle,
		}

		c.AddTask(t)
	}

	needRouter := true
	//Do not need router if there is no external network
	if b.Cluster.Spec.CloudConfig.Openstack.Router.ExternalNetwork == nil {
		needRouter = false
	}
	routerName := strings.Replace(clusterName, ".", "-", -1)
	for _, sp := range b.Cluster.Spec.Subnets {
		// assumes that we do not need to create routers if we use existing subnets
		if sp.ProviderID != "" {
			needRouter = false
		}
		subnetName, err := b.findSubnetNameByID(sp.ProviderID, sp.Name)
		if err != nil {
			return err
		}
		t := &openstacktasks.Subnet{
			Name:       s(subnetName),
			Network:    b.LinkToNetwork(),
			CIDR:       s(sp.CIDR),
			DNSServers: make([]*string, 0),
			Lifecycle:  b.Lifecycle,
			Tag:        s(clusterName),
		}
		if b.Cluster.Spec.CloudConfig.Openstack.Router != nil && b.Cluster.Spec.CloudConfig.Openstack.Router.DNSServers != nil {
			dnsSplitted := strings.Split(fi.StringValue(b.Cluster.Spec.CloudConfig.Openstack.Router.DNSServers), ",")
			dnsNameSrv := make([]*string, len(dnsSplitted))
			for i, ns := range dnsSplitted {
				dnsNameSrv[i] = fi.String(ns)
			}
			t.DNSServers = dnsNameSrv
		}
		c.AddTask(t)

		if needRouter {
			t1 := &openstacktasks.RouterInterface{
				Name:      s("ri-" + sp.Name),
				Subnet:    b.LinkToSubnet(s(subnetName)),
				Router:    b.LinkToRouter(s(routerName)),
				Lifecycle: b.Lifecycle,
			}
			c.AddTask(t1)
		}
	}

	if needRouter {
		t := &openstacktasks.Router{
			Name:      s(routerName),
			Lifecycle: b.Lifecycle,
		}

		c.AddTask(t)
	}
	return nil
}
<commit_msg>Fix potential npr<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package openstackmodel

import (
	"strings"

	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/cloudup/openstacktasks"
)

// NetworkModelBuilder configures network objects
type NetworkModelBuilder struct {
	*OpenstackModelContext
	Lifecycle *fi.Lifecycle
}

var _ fi.ModelBuilder = &NetworkModelBuilder{}

func (b *NetworkModelBuilder) Build(c *fi.ModelBuilderContext) error {
	clusterName := b.ClusterName()

	netName, err := b.GetNetworkName()
	if err != nil {
		return err
	}
	{
		t := &openstacktasks.Network{
			Name:      s(netName),
			ID:        s(b.Cluster.Spec.NetworkID),
			Tag:       s(clusterName),
			Lifecycle: b.Lifecycle,
		}

		c.AddTask(t)
	}

	osSpec := b.Cluster.Spec.CloudConfig.Openstack

	needRouter := true
	//Do not need router if there is no external network
	if osSpec.Router == nil || osSpec.Router.ExternalNetwork == nil {
		needRouter = false
	}
	routerName := strings.Replace(clusterName, ".", "-", -1)
	for _, sp := range b.Cluster.Spec.Subnets {
		// assumes that we do not need to create routers if we use existing subnets
		if sp.ProviderID != "" {
			needRouter = false
		}
		subnetName, err := b.findSubnetNameByID(sp.ProviderID, sp.Name)
		if err != nil {
			return err
		}
		t := &openstacktasks.Subnet{
			Name:       s(subnetName),
			Network:    b.LinkToNetwork(),
			CIDR:       s(sp.CIDR),
			DNSServers: make([]*string, 0),
			Lifecycle:  b.Lifecycle,
			Tag:        s(clusterName),
		}
		if osSpec.Router != nil && osSpec.Router.DNSServers != nil {
			dnsSplitted := strings.Split(fi.StringValue(osSpec.Router.DNSServers), ",")
			dnsNameSrv := make([]*string, len(dnsSplitted))
			for i, ns := range dnsSplitted {
				dnsNameSrv[i] = fi.String(ns)
			}
			t.DNSServers = dnsNameSrv
		}
		c.AddTask(t)

		if needRouter {
			t1 := &openstacktasks.RouterInterface{
				Name:      s("ri-" + sp.Name),
				Subnet:    b.LinkToSubnet(s(subnetName)),
				Router:    b.LinkToRouter(s(routerName)),
				Lifecycle: b.Lifecycle,
			}
			c.AddTask(t1)
		}
	}

	if needRouter {
		t := &openstacktasks.Router{
			Name:      s(routerName),
			Lifecycle: b.Lifecycle,
		}

		c.AddTask(t)
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package csi

import (
	"context"
	"os"

	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller"
	"sigs.k8s.io/controller-runtime/pkg/handler"
	"sigs.k8s.io/controller-runtime/pkg/manager"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
	"sigs.k8s.io/controller-runtime/pkg/source"

	"github.com/pkg/errors"
	"github.com/rook/rook/pkg/clusterd"
	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
	"github.com/rook/rook/pkg/operator/ceph/csi/peermap"
	"github.com/rook/rook/pkg/operator/k8sutil"
	v1 "k8s.io/api/core/v1"
	kerrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
)

const (
	controllerName = "rook-ceph-operator-csi-controller"
)

// ReconcileCSI reconciles a ceph-csi driver
type ReconcileCSI struct {
	client           client.Client
	context          *clusterd.Context
	opManagerContext context.Context
	opConfig         opcontroller.OperatorConfig
	multusClusters   []ClusterDetail
}

// ClusterDetail is a struct that holds the information of a cluster, it knows its internals (like
// FSID through clusterInfo) and the Kubernetes object that represents it (like the CephCluster CRD)
type ClusterDetail struct {
	cluster     *cephv1.CephCluster
	clusterInfo *cephclient.ClusterInfo
}

// Add creates a new Ceph CSI Controller and adds it to the Manager. The Manager will set fields on the Controller
// and Start it when the Manager is Started.
func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error {
	return add(opManagerContext, mgr, newReconciler(mgr, context, opManagerContext, opConfig), opConfig)
}

// newReconciler returns a new reconcile.Reconciler
func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler {
	return &ReconcileCSI{
		client:           mgr.GetClient(),
		context:          context,
		opConfig:         opConfig,
		opManagerContext: opManagerContext,
		multusClusters:   []ClusterDetail{},
	}
}

func add(ctx context.Context, mgr manager.Manager, r reconcile.Reconciler, opConfig opcontroller.OperatorConfig) error {
	// Create a new controller
	c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r})
	if err != nil {
		return err
	}
	logger.Infof("%s successfully started", controllerName)

	// Watch for ConfigMap (operator config)
	err = c.Watch(&source.Kind{
		Type: &v1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController(ctx, mgr.GetClient(), opConfig.OperatorNamespace))
	if err != nil {
		return err
	}

	// Watch for CephCluster
	err = c.Watch(&source.Kind{
		Type: &cephv1.CephCluster{TypeMeta: metav1.TypeMeta{Kind: "CephCluster", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController(ctx, mgr.GetClient(), opConfig.OperatorNamespace))
	if err != nil {
		return err
	}

	return nil
}

// Reconcile reads that state of the operator config map and makes changes based on the state read
// The Controller will requeue the Request to be processed again if the returned error is non-nil or
// Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
func (r *ReconcileCSI) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) {
	// workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface
	reconcileResponse, err := r.reconcile(request)
	if err != nil {
		logger.Errorf("failed to reconcile %v", err)
	}

	return reconcileResponse, err
}

func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, error) {
	serverVersion, err := r.context.Clientset.Discovery().ServerVersion()
	if err != nil {
		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get server version")
	}

	// See if there is a CephCluster
	cephClusters := &cephv1.CephClusterList{}
	err = r.client.List(r.opManagerContext, cephClusters, &client.ListOptions{})
	if err != nil {
		if kerrors.IsNotFound(err) {
			logger.Debug("no ceph cluster found not deploying ceph csi driver")
			EnableRBD, EnableCephFS, EnableNFS = false, false, false
			err = r.stopDrivers(serverVersion)
			if err != nil {
				return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to stop Drivers")
			}

			return reconcile.Result{}, nil
		}
		// Error reading the object - requeue the request.
		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to list ceph clusters")
	}

	// // Do not nothing if no ceph cluster is present
	if len(cephClusters.Items) == 0 {
		logger.Debug("no ceph cluster found not deploying ceph csi driver")
		EnableRBD, EnableCephFS, EnableNFS = false, false, false
		err = r.stopDrivers(serverVersion)
		if err != nil {
			return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to stop Drivers")
		}

		return reconcile.Result{}, nil
	}
	for i, cluster := range cephClusters.Items {
		if !cluster.DeletionTimestamp.IsZero() {
			logger.Debugf("ceph cluster %q is being deleting, no need to reconcile the csi driver", request.NamespacedName)
			return reconcile.Result{}, nil
		}

		if !cluster.Spec.External.Enable && cluster.Spec.CleanupPolicy.HasDataDirCleanPolicy() {
			logger.Debugf("ceph cluster %q has cleanup policy, the cluster will soon go away, no need to reconcile the csi driver", cluster.Name)
			return reconcile.Result{}, nil
		}

		// Do we have a multus cluster?
		// If so deploy the plugin holder with the fsid attached
		if cluster.Spec.Network.IsMultus() {
			// Load cluster info for later use in updating the ceph-csi configmap
			clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cluster.Namespace)
			if err != nil {
				// This avoids a requeue with exponential backoff and allows the controller to reconcile
				// more quickly when the cluster is ready.
				if errors.Is(err, opcontroller.ClusterInfoNoClusterNoSecret) {
					return opcontroller.WaitForRequeueIfCephClusterNotReady, nil
				}
				return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to populate cluster info for cluster %q", cluster.Name)
			}

			logger.Debugf("cluster %q is running on multus, deploying the ceph-csi plugin holder", cluster.Name)
			r.multusClusters = append(r.multusClusters, ClusterDetail{cluster: &cephClusters.Items[i], clusterInfo: clusterInfo})
		} else {
			logger.Debugf("not a multus cluster %q, not deploying the ceph-csi plugin holder", request.NamespacedName)
		}
	}

	// Fetch the operator's configmap. We force the NamespaceName to the operator since the request
	// could be a CephCluster. If so the NamespaceName will be the one from the cluster and thus the
	// CM won't be found
	opNamespaceName := types.NamespacedName{Name: opcontroller.OperatorSettingConfigMapName, Namespace: r.opConfig.OperatorNamespace}
	opConfig := &v1.ConfigMap{}
	err = r.client.Get(r.opManagerContext, opNamespaceName, opConfig)
	if err != nil {
		if kerrors.IsNotFound(err) {
			logger.Debug("operator's configmap resource not found. will use default value or env var.")
			r.opConfig.Parameters = make(map[string]string)
		} else {
			// Error reading the object - requeue the request.
			return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get operator's configmap")
		}
	} else {
		// Populate the operator's config
		r.opConfig.Parameters = opConfig.Data
	}

	ownerRef, err := k8sutil.GetDeploymentOwnerReference(r.opManagerContext, r.context.Clientset, os.Getenv(k8sutil.PodNameEnvVar), r.opConfig.OperatorNamespace)
	if err != nil {
		logger.Warningf("could not find deployment owner reference to assign to csi drivers. %v", err)
	}
	if ownerRef != nil {
		blockOwnerDeletion := false
		ownerRef.BlockOwnerDeletion = &blockOwnerDeletion
	}

	ownerInfo := k8sutil.NewOwnerInfoWithOwnerRef(ownerRef, r.opConfig.OperatorNamespace)
	// create an empty config map. config map will be filled with data
	// later when clusters have mons
	err = CreateCsiConfigMap(r.opManagerContext, r.opConfig.OperatorNamespace, r.context.Clientset, ownerInfo)
	if err != nil {
		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed creating csi config map")
	}

	err = peermap.CreateOrUpdateConfig(r.opManagerContext, r.context, &peermap.PeerIDMappings{})
	if err != nil {
		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to create pool ID mapping config map")
	}

	exists, err := checkCsiCephConfigMapExists(r.opManagerContext, r.context.Clientset, r.opConfig.OperatorNamespace)
	if err != nil {
		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get csi ceph.conf configmap")
	}
	CustomCSICephConfigExists = exists

	err = r.validateAndConfigureDrivers(serverVersion, ownerInfo)
	if err != nil {
		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to configure ceph csi")
	}

	return reconcile.Result{}, nil
}
<commit_msg>csi: do not block the reconcile if all clusters are not ready<commit_after>/*
Copyright 2021 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package csi

import (
	"context"
	"os"

	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	cephclient "github.com/rook/rook/pkg/daemon/ceph/client"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller"
	"sigs.k8s.io/controller-runtime/pkg/handler"
	"sigs.k8s.io/controller-runtime/pkg/manager"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
	"sigs.k8s.io/controller-runtime/pkg/source"

	"github.com/pkg/errors"
	"github.com/rook/rook/pkg/clusterd"
	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
	"github.com/rook/rook/pkg/operator/ceph/csi/peermap"
	"github.com/rook/rook/pkg/operator/k8sutil"
	v1 "k8s.io/api/core/v1"
	kerrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
)

const (
	controllerName = "rook-ceph-operator-csi-controller"
)

// ReconcileCSI reconciles a ceph-csi driver
type ReconcileCSI struct {
	client           client.Client
	context          *clusterd.Context
	opManagerContext context.Context
	opConfig         opcontroller.OperatorConfig
	multusClusters   []ClusterDetail
}

// ClusterDetail is a struct that holds the information of a cluster, it knows its internals (like
// FSID through clusterInfo) and the Kubernetes object that represents it (like the CephCluster CRD)
type ClusterDetail struct {
	cluster     *cephv1.CephCluster
	clusterInfo *cephclient.ClusterInfo
}

// Add creates a new Ceph CSI Controller and adds it to the Manager. The Manager will set fields on the Controller
// and Start it when the Manager is Started.
func Add(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) error {
	return add(opManagerContext, mgr, newReconciler(mgr, context, opManagerContext, opConfig), opConfig)
}

// newReconciler returns a new reconcile.Reconciler
func newReconciler(mgr manager.Manager, context *clusterd.Context, opManagerContext context.Context, opConfig opcontroller.OperatorConfig) reconcile.Reconciler {
	return &ReconcileCSI{
		client:           mgr.GetClient(),
		context:          context,
		opConfig:         opConfig,
		opManagerContext: opManagerContext,
		multusClusters:   []ClusterDetail{},
	}
}

func add(ctx context.Context, mgr manager.Manager, r reconcile.Reconciler, opConfig opcontroller.OperatorConfig) error {
	// Create a new controller
	c, err := controller.New(controllerName, mgr, controller.Options{Reconciler: r})
	if err != nil {
		return err
	}
	logger.Infof("%s successfully started", controllerName)

	// Watch for ConfigMap (operator config)
	err = c.Watch(&source.Kind{
		Type: &v1.ConfigMap{TypeMeta: metav1.TypeMeta{Kind: "ConfigMap", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController(ctx, mgr.GetClient(), opConfig.OperatorNamespace))
	if err != nil {
		return err
	}

	// Watch for CephCluster
	err = c.Watch(&source.Kind{
		Type: &cephv1.CephCluster{TypeMeta: metav1.TypeMeta{Kind: "CephCluster", APIVersion: v1.SchemeGroupVersion.String()}}}, &handler.EnqueueRequestForObject{}, predicateController(ctx, mgr.GetClient(), opConfig.OperatorNamespace))
	if err != nil {
		return err
	}

	return nil
}

// Reconcile reads that state of the operator config map and makes changes based on the state read
// The Controller will requeue the Request to be processed again if the returned error is non-nil or
// Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
func (r *ReconcileCSI) Reconcile(context context.Context, request reconcile.Request) (reconcile.Result, error) {
	// workaround because the rook logging mechanism is not compatible with the controller-runtime logging interface
	reconcileResponse, err := r.reconcile(request)
	if err != nil {
		logger.Errorf("failed to reconcile %v", err)
	}

	return reconcileResponse, err
}

func (r *ReconcileCSI) reconcile(request reconcile.Request) (reconcile.Result, error) {
	// reconcileResult is used to communicate the result of the reconciliation back to the caller
	var reconcileResult reconcile.Result

	serverVersion, err := r.context.Clientset.Discovery().ServerVersion()
	if err != nil {
		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get server version")
	}

	// See if there is a CephCluster
	cephClusters := &cephv1.CephClusterList{}
	err = r.client.List(r.opManagerContext, cephClusters, &client.ListOptions{})
	if err != nil {
		if kerrors.IsNotFound(err) {
			logger.Debug("no ceph cluster found not deploying ceph csi driver")
			EnableRBD, EnableCephFS, EnableNFS = false, false, false
			err = r.stopDrivers(serverVersion)
			if err != nil {
				return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to stop Drivers")
			}

			return reconcile.Result{}, nil
		}
		// Error reading the object - requeue the request.
		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to list ceph clusters")
	}

	// // Do not nothing if no ceph cluster is present
	if len(cephClusters.Items) == 0 {
		logger.Debug("no ceph cluster found not deploying ceph csi driver")
		EnableRBD, EnableCephFS, EnableNFS = false, false, false
		err = r.stopDrivers(serverVersion)
		if err != nil {
			return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to stop Drivers")
		}

		return reconcile.Result{}, nil
	}

	for i, cluster := range cephClusters.Items {
		if !cluster.DeletionTimestamp.IsZero() {
			logger.Debugf("ceph cluster %q is being deleting, no need to reconcile the csi driver", request.NamespacedName)
			return reconcile.Result{}, nil
		}

		if !cluster.Spec.External.Enable && cluster.Spec.CleanupPolicy.HasDataDirCleanPolicy() {
			logger.Debugf("ceph cluster %q has cleanup policy, the cluster will soon go away, no need to reconcile the csi driver", cluster.Name)
			return reconcile.Result{}, nil
		}

		// Do we have a multus cluster?
		// If so deploy the plugin holder with the fsid attached
		if cluster.Spec.Network.IsMultus() {
			// Load cluster info for later use in updating the ceph-csi configmap
			clusterInfo, _, _, err := opcontroller.LoadClusterInfo(r.context, r.opManagerContext, cluster.Namespace)
			if err != nil {
				// This avoids a requeue with exponential backoff and allows the controller to reconcile
				// more quickly when the cluster is ready.
				if errors.Is(err, opcontroller.ClusterInfoNoClusterNoSecret) {
					logger.Infof("cluster info for cluster %q is not ready yet, will retry in %s, proceeding with ready clusters", cluster.Name, opcontroller.WaitForRequeueIfCephClusterNotReady.RequeueAfter.String())
					reconcileResult = opcontroller.WaitForRequeueIfCephClusterNotReady
					continue
				}
				return opcontroller.ImmediateRetryResult, errors.Wrapf(err, "failed to load cluster info for cluster %q", cluster.Name)
			}

			logger.Debugf("cluster %q is running on multus, deploying the ceph-csi plugin holder", cluster.Name)
			r.multusClusters = append(r.multusClusters, ClusterDetail{cluster: &cephClusters.Items[i], clusterInfo: clusterInfo})
		} else {
			logger.Debugf("not a multus cluster %q, not deploying the ceph-csi plugin holder", request.NamespacedName)
		}
	}

	// Fetch the operator's configmap. We force the NamespaceName to the operator since the request
	// could be a CephCluster. If so the NamespaceName will be the one from the cluster and thus the
	// CM won't be found
	opNamespaceName := types.NamespacedName{Name: opcontroller.OperatorSettingConfigMapName, Namespace: r.opConfig.OperatorNamespace}
	opConfig := &v1.ConfigMap{}
	err = r.client.Get(r.opManagerContext, opNamespaceName, opConfig)
	if err != nil {
		if kerrors.IsNotFound(err) {
			logger.Debug("operator's configmap resource not found. will use default value or env var.")
			r.opConfig.Parameters = make(map[string]string)
		} else {
			// Error reading the object - requeue the request.
			return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get operator's configmap")
		}
	} else {
		// Populate the operator's config
		r.opConfig.Parameters = opConfig.Data
	}

	ownerRef, err := k8sutil.GetDeploymentOwnerReference(r.opManagerContext, r.context.Clientset, os.Getenv(k8sutil.PodNameEnvVar), r.opConfig.OperatorNamespace)
	if err != nil {
		logger.Warningf("could not find deployment owner reference to assign to csi drivers. %v", err)
	}
	if ownerRef != nil {
		blockOwnerDeletion := false
		ownerRef.BlockOwnerDeletion = &blockOwnerDeletion
	}

	ownerInfo := k8sutil.NewOwnerInfoWithOwnerRef(ownerRef, r.opConfig.OperatorNamespace)
	// create an empty config map. config map will be filled with data
	// later when clusters have mons
	err = CreateCsiConfigMap(r.opManagerContext, r.opConfig.OperatorNamespace, r.context.Clientset, ownerInfo)
	if err != nil {
		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed creating csi config map")
	}

	err = peermap.CreateOrUpdateConfig(r.opManagerContext, r.context, &peermap.PeerIDMappings{})
	if err != nil {
		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to create pool ID mapping config map")
	}

	exists, err := checkCsiCephConfigMapExists(r.opManagerContext, r.context.Clientset, r.opConfig.OperatorNamespace)
	if err != nil {
		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to get csi ceph.conf configmap")
	}
	CustomCSICephConfigExists = exists

	err = r.validateAndConfigureDrivers(serverVersion, ownerInfo)
	if err != nil {
		return opcontroller.ImmediateRetryResult, errors.Wrap(err, "failed to configure ceph csi")
	}

	return reconcileResult, nil
}
<|endoftext|>


<commit_before>package bitbucket

import (
	"context"
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"

	"github.com/estafette/estafette-ci-api/pkg/api"
	"github.com/estafette/estafette-ci-api/pkg/clients/bitbucketapi"
	manifest "github.com/estafette/estafette-ci-manifest"
	"github.com/gin-gonic/gin"
	"github.com/opentracing/opentracing-go"
	"github.com/rs/zerolog/log"
)

func NewHandler(service Service, config *api.APIConfig, bitbucketapiClient bitbucketapi.Client) Handler {
	return Handler{
		config:             config,
		service:            service,
		bitbucketapiClient: bitbucketapiClient,
	}
}

type Handler struct {
	config             *api.APIConfig
	service            Service
	bitbucketapiClient bitbucketapi.Client
}

func (h *Handler) Handle(c *gin.Context) {

	// https://confluence.atlassian.com/bitbucket/manage-webhooks-735643732.html

	eventType := c.GetHeader("X-Event-Key")

	body, err := ioutil.ReadAll(c.Request.Body)
	if err != nil {
		log.Error().Err(err).Msg("Reading body from Bitbucket webhook failed")
		c.Status(http.StatusInternalServerError)
		return
	}

	// unmarshal json body to check if installation is allowed
	var eventCheck bitbucketapi.EventCheck
	err = json.Unmarshal(body, &eventCheck)
	if err != nil {
		log.Error().Err(err).Str("body", string(body)).Msg("Deserializing body for EventCheck failed")
		c.Status(http.StatusBadRequest)
		return
	}

	// verify owner is allowed
	isAllowed, _ := h.service.IsAllowedOwner(eventCheck.GetRepository())
	if !isAllowed {
		log.Warn().Interface("event", eventCheck).Str("body", string(body)).Msg("Bitbucket EventCheck owner is not allowed")
		c.Status(http.StatusUnauthorized)
		return
	}

	switch eventType {
	case "repo:push":
		// unmarshal json body
		var pushEvent bitbucketapi.RepositoryPushEvent

		if eventCheck.Data != nil {
			var pushEventEnvelope bitbucketapi.RepositoryPushEventEnvelope
			err := json.Unmarshal(body, &pushEventEnvelope)
			if err != nil {
				log.Error().Err(err).Str("body", string(body)).Msg("Deserializing body to Bitbucket RepositoryPushEventEnvelope failed")
				c.Status(http.StatusInternalServerError)
				return
			}
			pushEvent = pushEventEnvelope.Data
		} else {
			err := json.Unmarshal(body, &pushEvent)
			if err != nil {
				log.Error().Err(err).Str("body", string(body)).Msg("Deserializing body to Bitbucket RepositoryPushEvent failed")
				c.Status(http.StatusInternalServerError)
				return
			}
		}

		err = h.service.CreateJobForBitbucketPush(c.Request.Context(), pushEvent)
		if err != nil && !errors.Is(err, ErrNonCloneableEvent) && !errors.Is(err, ErrNoManifest) {
			c.Status(http.StatusInternalServerError)
			return
		}

	case
		"repo:fork",
		"repo:transfer",
		"repo:created",
		"repo:commit_comment_created",
		"repo:commit_status_created",
		"repo:commit_status_updated",
		"issue:created",
		"issue:updated",
		"issue:comment_created",
		"pullrequest:created",
		"pullrequest:updated",
		"pullrequest:approved",
		"pullrequest:unapproved",
		"pullrequest:fulfilled",
		"pullrequest:rejected",
		"pullrequest:comment_created",
		"pullrequest:comment_updated",
		"pullrequest:comment_deleted":

	case "repo:updated":
		log.Debug().Str("event", eventType).Str("requestBody", string(body)).Msgf("Bitbucket webhook event of type '%v', logging request body", eventType)

		// unmarshal json body
		var repoUpdatedEvent bitbucketapi.RepoUpdatedEvent
		err := json.Unmarshal(body, &repoUpdatedEvent)
		if err != nil {
			log.Error().Err(err).Str("body", string(body)).Msg("Deserializing body to BitbucketRepoUpdatedEvent failed")
			c.Status(http.StatusInternalServerError)
			return
		}

		if repoUpdatedEvent.IsValidRenameEvent() {
			log.Info().Msgf("Renaming repository from %v/%v/%v to %v/%v/%v", repoUpdatedEvent.GetRepoSource(), repoUpdatedEvent.GetOldRepoOwner(), repoUpdatedEvent.GetOldRepoName(), repoUpdatedEvent.GetRepoSource(), repoUpdatedEvent.GetNewRepoOwner(), repoUpdatedEvent.GetNewRepoName())
			err = h.service.Rename(c.Request.Context(), repoUpdatedEvent.GetRepoSource(), repoUpdatedEvent.GetOldRepoOwner(), repoUpdatedEvent.GetOldRepoName(), repoUpdatedEvent.GetRepoSource(), repoUpdatedEvent.GetNewRepoOwner(), repoUpdatedEvent.GetNewRepoName())
			if err != nil {
				log.Error().Err(err).Msgf("Failed renaming repository from %v/%v/%v to %v/%v/%v", repoUpdatedEvent.GetRepoSource(), repoUpdatedEvent.GetOldRepoOwner(), repoUpdatedEvent.GetOldRepoName(), repoUpdatedEvent.GetRepoSource(), repoUpdatedEvent.GetNewRepoOwner(), repoUpdatedEvent.GetNewRepoName())
				c.Status(http.StatusInternalServerError)
				return
			}
		}

	case "repo:deleted":
		log.Debug().Str("event", eventType).Str("requestBody", string(body)).Msgf("Bitbucket webhook event of type '%v', logging request body", eventType)
		// unmarshal json body
		var repoDeletedEvent bitbucketapi.RepoDeletedEvent
		err := json.Unmarshal(body, &repoDeletedEvent)
		if err != nil {
			log.Error().Err(err).Str("body", string(body)).Msg("Deserializing body to BitbucketRepoDeletedEvent failed")
			c.Status(http.StatusInternalServerError)
			return
		}

		log.Info().Msgf("Archiving repository %v/%v/%v", repoDeletedEvent.GetRepoSource(), repoDeletedEvent.GetRepoOwner(), repoDeletedEvent.GetRepoName())
		err = h.service.Archive(c.Request.Context(), repoDeletedEvent.GetRepoSource(), repoDeletedEvent.GetRepoOwner(), repoDeletedEvent.GetRepoName())
		if err != nil {
			log.Error().Err(err).Msgf("Failed archiving repository %v/%v/%v", repoDeletedEvent.GetRepoSource(), repoDeletedEvent.GetRepoOwner(), repoDeletedEvent.GetRepoName())
			c.Status(http.StatusInternalServerError)
			return
		}

	default:
		log.Warn().Str("event", eventType).Msgf("Unsupported Bitbucket webhook event of type '%v'", eventType)
	}

	// publish event for bots to run
	go func() {
		// create new context to avoid cancellation impacting execution
		span, _ := opentracing.StartSpanFromContext(c.Request.Context(), "bitbucket:AsyncPublishBitbucketEvent")
		ctx := opentracing.ContextWithSpan(context.Background(), span)
		defer span.Finish()

		err = h.service.PublishBitbucketEvent(ctx, manifest.EstafetteBitbucketEvent{
			Event:         eventType,
			Repository:    eventCheck.GetFullRepository(),
			HookUUID:      c.GetHeader("X-Hook-UUID"),
			RequestUUID:   c.GetHeader("X-Request-UUID"),
			AttemptNumber: c.GetHeader("X-Attempt-Number"),
			Payload:       string(body),
		})
		if err != nil {
			log.Error().Err(err).Msgf("Failed PublishBitbucketEvent")
		}
	}()

	c.Status(http.StatusOK)
}

func (h *Handler) Descriptor(c *gin.Context) {

	// https://developer.atlassian.com/cloud/bitbucket/app-descriptor/

	descriptor := Descriptor{
		Key:         h.config.Integrations.Bitbucket.Key,
		Name:        h.config.Integrations.Bitbucket.Name,
		Description: "Estafette - The The resilient and cloud-native CI/CD platform",
		Vendor: DescriptorVendor{
			Name: "Estafette",
			URL:  "https://estafette.io",
		},
		BaseURL: h.config.APIServer.IntegrationsURL,
		Authentication: &DescriptorAuthentication{
			Type: "none",
		},
		Lifecycle: &DescriptorLifecycle{
			Installed:   "/api/integrations/bitbucket/installed",
			Uninstalled: "/api/integrations/bitbucket/uninstalled",
		},
		Scopes:   []string{"repository:write", "pullrequest:write"},
		Contexts: []string{"account"},
		Modules: &DescriptorModules{
			Webhooks: []DescriptorWebhook{
				{
					Event: "repo:push",
					URL:   "/api/integrations/bitbucket/events",
				},
				{
					Event: "repo:updated",
					URL:   "/api/integrations/bitbucket/events",
				},
				{
					Event: "repo:deleted",
					URL:   "/api/integrations/bitbucket/events",
				},
			},
		},
	}

	c.JSON(http.StatusOK, descriptor)
}

func (h *Handler) Installed(c *gin.Context) {

	body, err := ioutil.ReadAll(c.Request.Body)
	if err != nil {
		log.Error().Err(err).Msg("Reading body for Bitbucket App installed event failed")
		c.Status(http.StatusInternalServerError)
		return
	}

	log.Info().Str("body", string(body)).Msg("Bitbucket App installed")

	var installation bitbucketapi.BitbucketAppInstallation
	err = json.Unmarshal(body, &installation)
	if err != nil {
		log.Error().Err(err).Msg("Failed unmarshalling bitbucket app install body")
	} else {
		err = h.bitbucketapiClient.AddInstallation(c.Request.Context(), installation)
		if err != nil {
			log.Error().Err(err).Msg("Failed adding bitbucket app installation")
		}
	}

	c.Status(http.StatusOK)
}

func (h *Handler) Uninstalled(c *gin.Context) {

	body, err := ioutil.ReadAll(c.Request.Body)
	if err != nil {
		log.Error().Err(err).Msg("Reading body for Bitbucket App uninstalled event failed")
		c.Status(http.StatusInternalServerError)
		return
	}

	log.Info().Str("body", string(body)).Msg("Bitbucket App uninstalled")

	var installation bitbucketapi.BitbucketAppInstallation
	err = json.Unmarshal(body, &installation)
	if err != nil {
		log.Error().Err(err).Msg("Failed unmarshalling bitbucket app uninstall body")
	} else {
		err = h.bitbucketapiClient.RemoveInstallation(c.Request.Context(), installation)
		if err != nil {
			log.Error().Err(err).Msg("Failed removing bitbucket app installation")
		}
	}

	c.Status(http.StatusOK)
}
<commit_msg>more debug logging around bitbucket installation configmap add/remove<commit_after>package bitbucket

import (
	"context"
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"

	"github.com/estafette/estafette-ci-api/pkg/api"
	"github.com/estafette/estafette-ci-api/pkg/clients/bitbucketapi"
	manifest "github.com/estafette/estafette-ci-manifest"
	"github.com/gin-gonic/gin"
	"github.com/opentracing/opentracing-go"
	"github.com/rs/zerolog/log"
)

func NewHandler(service Service, config *api.APIConfig, bitbucketapiClient bitbucketapi.Client) Handler {
	return Handler{
		config:             config,
		service:            service,
		bitbucketapiClient: bitbucketapiClient,
	}
}

type Handler struct {
	config             *api.APIConfig
	service            Service
	bitbucketapiClient bitbucketapi.Client
}

func (h *Handler) Handle(c *gin.Context) {

	// https://confluence.atlassian.com/bitbucket/manage-webhooks-735643732.html

	eventType := c.GetHeader("X-Event-Key")

	body, err := ioutil.ReadAll(c.Request.Body)
	if err != nil {
		log.Error().Err(err).Msg("Reading body from Bitbucket webhook failed")
		c.Status(http.StatusInternalServerError)
		return
	}

	// unmarshal json body to check if installation is allowed
	var eventCheck bitbucketapi.EventCheck
	err = json.Unmarshal(body, &eventCheck)
	if err != nil {
		log.Error().Err(err).Str("body", string(body)).Msg("Deserializing body for EventCheck failed")
		c.Status(http.StatusBadRequest)
		return
	}

	// verify owner is allowed
	isAllowed, _ := h.service.IsAllowedOwner(eventCheck.GetRepository())
	if !isAllowed {
		log.Warn().Interface("event", eventCheck).Str("body", string(body)).Msg("Bitbucket EventCheck owner is not allowed")
		c.Status(http.StatusUnauthorized)
		return
	}

	switch eventType {
	case "repo:push":
		// unmarshal json body
		var pushEvent bitbucketapi.RepositoryPushEvent

		if eventCheck.Data != nil {
			var pushEventEnvelope bitbucketapi.RepositoryPushEventEnvelope
			err := json.Unmarshal(body, &pushEventEnvelope)
			if err != nil {
				log.Error().Err(err).Str("body", string(body)).Msg("Deserializing body to Bitbucket RepositoryPushEventEnvelope failed")
				c.Status(http.StatusInternalServerError)
				return
			}
			pushEvent = pushEventEnvelope.Data
		} else {
			err := json.Unmarshal(body, &pushEvent)
			if err != nil {
				log.Error().Err(err).Str("body", string(body)).Msg("Deserializing body to Bitbucket RepositoryPushEvent failed")
				c.Status(http.StatusInternalServerError)
				return
			}
		}

		err = h.service.CreateJobForBitbucketPush(c.Request.Context(), pushEvent)
		if err != nil && !errors.Is(err, ErrNonCloneableEvent) && !errors.Is(err, ErrNoManifest) {
			c.Status(http.StatusInternalServerError)
			return
		}

	case
		"repo:fork",
		"repo:transfer",
		"repo:created",
		"repo:commit_comment_created",
		"repo:commit_status_created",
		"repo:commit_status_updated",
		"issue:created",
		"issue:updated",
		"issue:comment_created",
		"pullrequest:created",
		"pullrequest:updated",
		"pullrequest:approved",
		"pullrequest:unapproved",
		"pullrequest:fulfilled",
		"pullrequest:rejected",
		"pullrequest:comment_created",
		"pullrequest:comment_updated",
		"pullrequest:comment_deleted":

	case "repo:updated":
		log.Debug().Str("event", eventType).Str("requestBody", string(body)).Msgf("Bitbucket webhook event of type '%v', logging request body", eventType)

		// unmarshal json body
		var repoUpdatedEvent bitbucketapi.RepoUpdatedEvent
		err := json.Unmarshal(body, &repoUpdatedEvent)
		if err != nil {
			log.Error().Err(err).Str("body", string(body)).Msg("Deserializing body to BitbucketRepoUpdatedEvent failed")
			c.Status(http.StatusInternalServerError)
			return
		}

		if repoUpdatedEvent.IsValidRenameEvent() {
			log.Info().Msgf("Renaming repository from %v/%v/%v to %v/%v/%v", repoUpdatedEvent.GetRepoSource(), repoUpdatedEvent.GetOldRepoOwner(), repoUpdatedEvent.GetOldRepoName(), repoUpdatedEvent.GetRepoSource(), repoUpdatedEvent.GetNewRepoOwner(), repoUpdatedEvent.GetNewRepoName())
			err = h.service.Rename(c.Request.Context(), repoUpdatedEvent.GetRepoSource(), repoUpdatedEvent.GetOldRepoOwner(), repoUpdatedEvent.GetOldRepoName(), repoUpdatedEvent.GetRepoSource(), repoUpdatedEvent.GetNewRepoOwner(), repoUpdatedEvent.GetNewRepoName())
			if err != nil {
				log.Error().Err(err).Msgf("Failed renaming repository from %v/%v/%v to %v/%v/%v", repoUpdatedEvent.GetRepoSource(), repoUpdatedEvent.GetOldRepoOwner(), repoUpdatedEvent.GetOldRepoName(), repoUpdatedEvent.GetRepoSource(), repoUpdatedEvent.GetNewRepoOwner(), repoUpdatedEvent.GetNewRepoName())
				c.Status(http.StatusInternalServerError)
				return
			}
		}

	case "repo:deleted":
		log.Debug().Str("event", eventType).Str("requestBody", string(body)).Msgf("Bitbucket webhook event of type '%v', logging request body", eventType)
		// unmarshal json body
		var repoDeletedEvent bitbucketapi.RepoDeletedEvent
		err := json.Unmarshal(body, &repoDeletedEvent)
		if err != nil {
			log.Error().Err(err).Str("body", string(body)).Msg("Deserializing body to BitbucketRepoDeletedEvent failed")
			c.Status(http.StatusInternalServerError)
			return
		}

		log.Info().Msgf("Archiving repository %v/%v/%v", repoDeletedEvent.GetRepoSource(), repoDeletedEvent.GetRepoOwner(), repoDeletedEvent.GetRepoName())
		err = h.service.Archive(c.Request.Context(), repoDeletedEvent.GetRepoSource(), repoDeletedEvent.GetRepoOwner(), repoDeletedEvent.GetRepoName())
		if err != nil {
			log.Error().Err(err).Msgf("Failed archiving repository %v/%v/%v", repoDeletedEvent.GetRepoSource(), repoDeletedEvent.GetRepoOwner(), repoDeletedEvent.GetRepoName())
			c.Status(http.StatusInternalServerError)
			return
		}

	default:
		log.Warn().Str("event", eventType).Msgf("Unsupported Bitbucket webhook event of type '%v'", eventType)
	}

	// publish event for bots to run
	go func() {
		// create new context to avoid cancellation impacting execution
		span, _ := opentracing.StartSpanFromContext(c.Request.Context(), "bitbucket:AsyncPublishBitbucketEvent")
		ctx := opentracing.ContextWithSpan(context.Background(), span)
		defer span.Finish()

		err = h.service.PublishBitbucketEvent(ctx, manifest.EstafetteBitbucketEvent{
			Event:         eventType,
			Repository:    eventCheck.GetFullRepository(),
			HookUUID:      c.GetHeader("X-Hook-UUID"),
			RequestUUID:   c.GetHeader("X-Request-UUID"),
			AttemptNumber: c.GetHeader("X-Attempt-Number"),
			Payload:       string(body),
		})
		if err != nil {
			log.Error().Err(err).Msgf("Failed PublishBitbucketEvent")
		}
	}()

	c.Status(http.StatusOK)
}

func (h *Handler) Descriptor(c *gin.Context) {

	// https://developer.atlassian.com/cloud/bitbucket/app-descriptor/

	descriptor := Descriptor{
		Key:         h.config.Integrations.Bitbucket.Key,
		Name:        h.config.Integrations.Bitbucket.Name,
		Description: "Estafette - The The resilient and cloud-native CI/CD platform",
		Vendor: DescriptorVendor{
			Name: "Estafette",
			URL:  "https://estafette.io",
		},
		BaseURL: h.config.APIServer.IntegrationsURL,
		Authentication: &DescriptorAuthentication{
			Type: "none",
		},
		Lifecycle: &DescriptorLifecycle{
			Installed:   "/api/integrations/bitbucket/installed",
			Uninstalled: "/api/integrations/bitbucket/uninstalled",
		},
		Scopes:   []string{"repository:write", "pullrequest:write"},
		Contexts: []string{"account"},
		Modules: &DescriptorModules{
			Webhooks: []DescriptorWebhook{
				{
					Event: "repo:push",
					URL:   "/api/integrations/bitbucket/events",
				},
				{
					Event: "repo:updated",
					URL:   "/api/integrations/bitbucket/events",
				},
				{
					Event: "repo:deleted",
					URL:   "/api/integrations/bitbucket/events",
				},
			},
		},
	}

	c.JSON(http.StatusOK, descriptor)
}

func (h *Handler) Installed(c *gin.Context) {

	body, err := ioutil.ReadAll(c.Request.Body)
	if err != nil {
		log.Error().Err(err).Msg("Reading body for Bitbucket App installed event failed")
		c.Status(http.StatusInternalServerError)
		return
	}

	log.Info().Str("body", string(body)).Msg("Bitbucket App installed")

	var installation bitbucketapi.BitbucketAppInstallation
	err = json.Unmarshal(body, &installation)
	if err != nil {
		log.Error().Err(err).Msg("Failed unmarshalling bitbucket app install body")
	} else {
		log.Info().Interface("installation", installation).Msg("Unmarshalled bitbucket app install body")
		err = h.bitbucketapiClient.AddInstallation(c.Request.Context(), installation)
		if err != nil {
			log.Error().Err(err).Msg("Failed adding bitbucket app installation")
		} else {
			log.Info().Msg("Added bitbucket app installation")
		}
	}

	c.Status(http.StatusOK)
}

func (h *Handler) Uninstalled(c *gin.Context) {

	body, err := ioutil.ReadAll(c.Request.Body)
	if err != nil {
		log.Error().Err(err).Msg("Reading body for Bitbucket App uninstalled event failed")
		c.Status(http.StatusInternalServerError)
		return
	}

	log.Info().Str("body", string(body)).Msg("Bitbucket App uninstalled")

	var installation bitbucketapi.BitbucketAppInstallation
	err = json.Unmarshal(body, &installation)
	if err != nil {
		log.Error().Err(err).Msg("Failed unmarshalling bitbucket app uninstall body")
	} else {
		log.Info().Interface("installation", installation).Msg("Unmarshalled bitbucket app uninstall body")
		err = h.bitbucketapiClient.RemoveInstallation(c.Request.Context(), installation)
		if err != nil {
			log.Error().Err(err).Msg("Failed removing bitbucket app installation")
		} else {
			log.Info().Msg("Removed bitbucket app installation")
		}
	}

	c.Status(http.StatusOK)
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package template (html/template) implements data-driven templates for
generating HTML output safe against code injection. It provides the
same interface as package text/template and should be used instead of
text/template whenever the output is HTML.

The documentation here focuses on the security features of the package.
For information about how to program the templates themselves, see the
documentation for text/template.

Introduction

This package wraps package text/template so you can share its template API
to parse and execute HTML templates safely.

  tmpl, err := template.New("name").Parse(...)
  // Error checking elided
  err = tmpl.Execute(out, data)

If successful, tmpl will now be injection-safe. Otherwise, err is an error
defined in the docs for ErrorCode.

HTML templates treat data values as plain text which should be encoded so they
can be safely embedded in an HTML document. The escaping is contextual, so
actions can appear within JavaScript, CSS, and URI contexts.

The security model used by this package assumes that template authors are
trusted, while Execute's data parameter is not. More details are
provided below.

Example

  import "text/template"
  ...
  t, err := template.New("foo").Parse(`{{define "T"}}Hello, {{.}}!{{end}}`)
  err = t.ExecuteTemplate(out, "T", "")

produces

  Hello, !

but the contextual autoescaping in html/template

  import "html/template"
  ...
  t, err := template.New("foo").Parse(`{{define "T"}}Hello, {{.}}!{{end}}`)
  err = t.ExecuteTemplate(out, "T", "")

produces safe, escaped HTML output

  Hello, <script>alert('you have been pwned')</script>!

Contexts

This package understands HTML, CSS, JavaScript, and URIs. It adds sanitizing
functions to each simple action pipeline, so given the excerpt

  {{.}}

At parse time each {{.}} is overwritten to add escaping functions as necessary.
In this case it becomes

  {{. | htmlescaper}}

where urlescaper, attrescaper, and htmlescaper are aliases for internal escaping
functions.

For these internal escaping functions, if an action pipeline evaluates to
a nil interface value, it is treated as though it were an empty string.

Errors

See the documentation of ErrorCode for details.

A fuller picture

The rest of this package comment may be skipped on first reading; it includes
details necessary to understand escaping contexts and error messages. Most users
will not need to understand these details.

Contexts

Assuming {{.}} is `O'Reilly: How are <i>you</i>?`, the table below shows
how {{.}} appears when used in the context to the left.

  Context                          {{.}} After
  {{.}}                            O'Reilly: How are <i>you</i>?
                  O'Reilly: How are you?
                  O'Reilly: How are %3ci%3eyou%3c/i%3e?
                O'Reilly%3a%20How%20are%3ci%3e...%3f
               O\x27Reilly: How are \x3ci\x3eyou...?
                 "O\x27Reilly: How are \x3ci\x3eyou...?"
       O\x27Reilly: How are \x3ci\x3eyou...\x3f

If used in an unsafe context, then the value might be filtered out:

  Context                          {{.}} After
                   #ZgotmplZ

since "O'Reilly:" is not an allowed protocol like "http:".

If {{.}} is the innocuous word, `left`, then it can appear more widely,

  Context                              {{.}} After
  {{.}}                                left
                      left
                       left
                      left
                  left
          left
               left
         left
    left
     left

Non-string values can be used in JavaScript contexts.
If {{.}} is

  struct{A,B string}{ "foo", "bar" }

in the escaped template

then the template output is

See package json to understand how non-string content is marshaled for
embedding in JavaScript contexts.

Typed Strings

By default, this package assumes that all pipelines produce a plain text string.
It adds escaping pipeline stages necessary to correctly and safely embed that
plain text string in the appropriate context.

When a data value is not plain text, you can make sure it is not over-escaped
by marking it with its type.

Types HTML, JS, URL, and others from content.go can carry safe content that is
exempted from escaping.

The template

  Hello, {{.}}!

can be invoked with

  tmpl.Execute(out, template.HTML(`<b>World</b>`))

to produce

  Hello, World!

instead of the

  Hello, <b>World<b>!

that would have been produced if {{.}} was a regular string.

Security Model

https://rawgit.com/mikesamuel/sanitized-jquery-templates/trunk/safetemplate.html#problem_definition defines "safe" as used by this package.

This package assumes that template authors are trusted, that Execute's data
parameter is not, and seeks to preserve the properties below in the face
of untrusted data:

Structure Preservation Property:
"... when a template author writes an HTML tag in a safe templating language,
the browser will interpret the corresponding portion of the output as a tag
regardless of the values of untrusted data, and similarly for other structures
such as attribute boundaries and JS and CSS string boundaries."

Code Effect Property:
"... only code specified by the template author should run as a result of
injecting the template output into a page and all code specified by the
template author should run as a result of the same."

Least Surprise Property:
"A developer (or code reviewer) familiar with HTML, CSS, and JavaScript, who
knows that contextual autoescaping happens should be able to look at a {{.}}
and correctly infer what sanitization happens."
*/
package template
<commit_msg>html/template: document handling of namespaced and data- attributes<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Package template (html/template) implements data-driven templates for
generating HTML output safe against code injection. It provides the
same interface as package text/template and should be used instead of
text/template whenever the output is HTML.

The documentation here focuses on the security features of the package.
For information about how to program the templates themselves, see the
documentation for text/template.

Introduction

This package wraps package text/template so you can share its template API
to parse and execute HTML templates safely.

  tmpl, err := template.New("name").Parse(...)
  // Error checking elided
  err = tmpl.Execute(out, data)

If successful, tmpl will now be injection-safe. Otherwise, err is an error
defined in the docs for ErrorCode.

HTML templates treat data values as plain text which should be encoded so they
can be safely embedded in an HTML document. The escaping is contextual, so
actions can appear within JavaScript, CSS, and URI contexts.

The security model used by this package assumes that template authors are
trusted, while Execute's data parameter is not. More details are
provided below.

Example

  import "text/template"
  ...
  t, err := template.New("foo").Parse(`{{define "T"}}Hello, {{.}}!{{end}}`)
  err = t.ExecuteTemplate(out, "T", "")

produces

  Hello, !

but the contextual autoescaping in html/template

  import "html/template"
  ...
  t, err := template.New("foo").Parse(`{{define "T"}}Hello, {{.}}!{{end}}`)
  err = t.ExecuteTemplate(out, "T", "")

produces safe, escaped HTML output

  Hello, <script>alert('you have been pwned')</script>!

Contexts

This package understands HTML, CSS, JavaScript, and URIs. It adds sanitizing
functions to each simple action pipeline, so given the excerpt

  {{.}}

At parse time each {{.}} is overwritten to add escaping functions as necessary.
In this case it becomes

  {{. | htmlescaper}}

where urlescaper, attrescaper, and htmlescaper are aliases for internal escaping
functions.

For these internal escaping functions, if an action pipeline evaluates to
a nil interface value, it is treated as though it were an empty string.

Namespaced and data- attributes

Attributes with a namespace are treated as if they had no namespace.
Given the excerpt

At parse time the attribute will be treated as if it were just "href".
So at parse time the template becomes:

Similarly to attributes with namespaces, attributes with a "data-" prefix are
treated as if they had no "data-" prefix. So given

At parse time this becomes

If an attribute has both a namespace and a "data-" prefix, only the namespace
will be removed when determining the context. For example

This is handled as if "my:data-href" was just "data-href" and not "href" as
it would be if the "data-" prefix were to be ignored too. Thus at parse
time this becomes just

As a special case, attributes with the namespace "xmlns" are always treated
as containing URLs. Given the excerpts

At parse time they become:

Errors

See the documentation of ErrorCode for details.

A fuller picture

The rest of this package comment may be skipped on first reading; it includes
details necessary to understand escaping contexts and error messages. Most users
will not need to understand these details.

Contexts

Assuming {{.}} is `O'Reilly: How are <i>you</i>?`, the table below shows
how {{.}} appears when used in the context to the left.

  Context                          {{.}} After
  {{.}}                            O'Reilly: How are <i>you</i>?
                  O'Reilly: How are you?
                  O'Reilly: How are %3ci%3eyou%3c/i%3e?
                O'Reilly%3a%20How%20are%3ci%3e...%3f
               O\x27Reilly: How are \x3ci\x3eyou...?
                 "O\x27Reilly: How are \x3ci\x3eyou...?"
       O\x27Reilly: How are \x3ci\x3eyou...\x3f

If used in an unsafe context, then the value might be filtered out:

  Context                          {{.}} After
                   #ZgotmplZ

since "O'Reilly:" is not an allowed protocol like "http:".

If {{.}} is the innocuous word, `left`, then it can appear more widely,

  Context                              {{.}} After
  {{.}}                                left
                      left
                       left
                      left
                  left
          left
               left
         left
    left
     left

Non-string values can be used in JavaScript contexts.
If {{.}} is

  struct{A,B string}{ "foo", "bar" }

in the escaped template

then the template output is

See package json to understand how non-string content is marshaled for
embedding in JavaScript contexts.

Typed Strings

By default, this package assumes that all pipelines produce a plain text string.
It adds escaping pipeline stages necessary to correctly and safely embed that
plain text string in the appropriate context.

When a data value is not plain text, you can make sure it is not over-escaped
by marking it with its type.

Types HTML, JS, URL, and others from content.go can carry safe content that is
exempted from escaping.

The template

  Hello, {{.}}!

can be invoked with

  tmpl.Execute(out, template.HTML(`<b>World</b>`))

to produce

  Hello, World!

instead of the

  Hello, <b>World<b>!

that would have been produced if {{.}} was a regular string.

Security Model

https://rawgit.com/mikesamuel/sanitized-jquery-templates/trunk/safetemplate.html#problem_definition defines "safe" as used by this package.

This package assumes that template authors are trusted, that Execute's data
parameter is not, and seeks to preserve the properties below in the face
of untrusted data:

Structure Preservation Property:
"... when a template author writes an HTML tag in a safe templating language,
the browser will interpret the corresponding portion of the output as a tag
regardless of the values of untrusted data, and similarly for other structures
such as attribute boundaries and JS and CSS string boundaries."

Code Effect Property:
"... only code specified by the template author should run as a result of
injecting the template output into a page and all code specified by the
template author should run as a result of the same."

Least Surprise Property:
"A developer (or code reviewer) familiar with HTML, CSS, and JavaScript, who
knows that contextual autoescaping happens should be able to look at a {{.}}
and correctly infer what sanitization happens."
*/
package template
<|endoftext|>


<commit_before>package httpjson

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/internal"
	"github.com/influxdata/telegraf/plugins/inputs"
)

type HttpJson struct {
	Name       string
	Servers    []string
	Method     string
	TagKeys    []string
	Parameters map[string]string
	Headers    map[string]string
	client     HTTPClient
}

type HTTPClient interface {
	// Returns the result of an http request
	//
	// Parameters:
	// req: HTTP request object
	//
	// Returns:
	// http.Response:  HTTP respons object
	// error        :  Any error that may have occurred
	MakeRequest(req *http.Request) (*http.Response, error)
}

type RealHTTPClient struct {
	client *http.Client
}

func (c RealHTTPClient) MakeRequest(req *http.Request) (*http.Response, error) {
	return c.client.Do(req)
}

var sampleConfig = `
  # NOTE This plugin only reads numerical measurements, strings and booleans
  # will be ignored.

  # a name for the service being polled
  name = "webserver_stats"

  # URL of each server in the service's cluster
  servers = [
    "http://localhost:9999/stats/",
    "http://localhost:9998/stats/",
  ]

  # HTTP method to use (case-sensitive)
  method = "GET"

  # List of tag names to extract from top-level of JSON server response
  # tag_keys = [
  #   "my_tag_1",
  #   "my_tag_2"
  # ]

  # HTTP parameters (all values must be strings)
  [inputs.httpjson.parameters]
    event_type = "cpu_spike"
    threshold = "0.75"

  # HTTP Header parameters (all values must be strings)
  # [inputs.httpjson.headers]
  #   X-Auth-Token = "my-xauth-token"
  #   apiVersion = "v1"

`

func (h *HttpJson) SampleConfig() string {
	return sampleConfig
}

func (h *HttpJson) Description() string {
	return "Read flattened metrics from one or more JSON HTTP endpoints"
}

// Gathers data for all servers.
func (h *HttpJson) Gather(acc telegraf.Accumulator) error {
	var wg sync.WaitGroup

	errorChannel := make(chan error, len(h.Servers))

	for _, server := range h.Servers {
		wg.Add(1)
		go func(server string) {
			defer wg.Done()
			if err := h.gatherServer(acc, server); err != nil {
				errorChannel <- err
			}
		}(server)
	}

	wg.Wait()
	close(errorChannel)

	// Get all errors and return them as one giant error
	errorStrings := []string{}
	for err := range errorChannel {
		errorStrings = append(errorStrings, err.Error())
	}

	if len(errorStrings) == 0 {
		return nil
	}
	return errors.New(strings.Join(errorStrings, "\n"))
}

// Gathers data from a particular server
// Parameters:
//     acc      : The telegraf Accumulator to use
//     serverURL: endpoint to send request to
//     service  : the service being queried
//
// Returns:
//     error: Any error that may have occurred
func (h *HttpJson) gatherServer(
	acc telegraf.Accumulator,
	serverURL string,
) error {
	resp, responseTime, err := h.sendRequest(serverURL)

	if err != nil {
		return err
	}

	var jsonOut map[string]interface{}
	if err = json.Unmarshal([]byte(resp), &jsonOut); err != nil {
		return errors.New("Error decoding JSON response")
	}

	tags := map[string]string{
		"server": serverURL,
	}

	for _, tag := range h.TagKeys {
		switch v := jsonOut[tag].(type) {
		case string:
			tags[tag] = v
		}
		delete(jsonOut, tag)
	}

	if responseTime >= 0 {
		jsonOut["response_time"] = responseTime
	}
	f := internal.JSONFlattener{}
	err = f.FlattenJSON("", jsonOut)
	if err != nil {
		return err
	}

	var msrmnt_name string
	if h.Name == "" {
		msrmnt_name = "httpjson"
	} else {
		msrmnt_name = "httpjson_" + h.Name
	}
	acc.AddFields(msrmnt_name, f.Fields, tags)
	return nil
}

// Sends an HTTP request to the server using the HttpJson object's HTTPClient
// Parameters:
//     serverURL: endpoint to send request to
//
// Returns:
//     string: body of the response
//     error : Any error that may have occurred
func (h *HttpJson) sendRequest(serverURL string) (string, float64, error) {
	// Prepare URL
	requestURL, err := url.Parse(serverURL)
	if err != nil {
		return "", -1, fmt.Errorf("Invalid server URL \"%s\"", serverURL)
	}

	params := url.Values{}
	for k, v := range h.Parameters {
		params.Add(k, v)
	}
	requestURL.RawQuery = params.Encode()

	// Create + send request
	req, err := http.NewRequest(h.Method, requestURL.String(), nil)
	if err != nil {
		return "", -1, err
	}

	// Add header parameters
	for k, v := range h.Headers {
		req.Header.Add(k, v)
	}

	start := time.Now()
	resp, err := h.client.MakeRequest(req)
	if err != nil {
		return "", -1, err
	}

	defer resp.Body.Close()
	responseTime := time.Since(start).Seconds()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return string(body), responseTime, err
	}

	// Process response
	if resp.StatusCode != http.StatusOK {
		err = fmt.Errorf("Response from url \"%s\" has status code %d (%s), expected %d (%s)",
			requestURL.String(),
			resp.StatusCode,
			http.StatusText(resp.StatusCode),
			http.StatusOK,
			http.StatusText(http.StatusOK))
		return string(body), responseTime, err
	}

	return string(body), responseTime, err
}

func init() {
	inputs.Add("httpjson", func() telegraf.Input {
		return &HttpJson{client: RealHTTPClient{client: &http.Client{}}}
	})
}
<commit_msg>Fix a bug when setting host header in httpjson<commit_after>package httpjson

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/internal"
	"github.com/influxdata/telegraf/plugins/inputs"
)

type HttpJson struct {
	Name       string
	Servers    []string
	Method     string
	TagKeys    []string
	Parameters map[string]string
	Headers    map[string]string
	client     HTTPClient
}

type HTTPClient interface {
	// Returns the result of an http request
	//
	// Parameters:
	// req: HTTP request object
	//
	// Returns:
	// http.Response:  HTTP respons object
	// error        :  Any error that may have occurred
	MakeRequest(req *http.Request) (*http.Response, error)
}

type RealHTTPClient struct {
	client *http.Client
}

func (c RealHTTPClient) MakeRequest(req *http.Request) (*http.Response, error) {
	return c.client.Do(req)
}

var sampleConfig = `
  # NOTE This plugin only reads numerical measurements, strings and booleans
  # will be ignored.

  # a name for the service being polled
  name = "webserver_stats"

  # URL of each server in the service's cluster
  servers = [
    "http://localhost:9999/stats/",
    "http://localhost:9998/stats/",
  ]

  # HTTP method to use (case-sensitive)
  method = "GET"

  # List of tag names to extract from top-level of JSON server response
  # tag_keys = [
  #   "my_tag_1",
  #   "my_tag_2"
  # ]

  # HTTP parameters (all values must be strings)
  [inputs.httpjson.parameters]
    event_type = "cpu_spike"
    threshold = "0.75"

  # HTTP Header parameters (all values must be strings)
  # [inputs.httpjson.headers]
  #   X-Auth-Token = "my-xauth-token"
  #   apiVersion = "v1"

`

func (h *HttpJson) SampleConfig() string {
	return sampleConfig
}

func (h *HttpJson) Description() string {
	return "Read flattened metrics from one or more JSON HTTP endpoints"
}

// Gathers data for all servers.
func (h *HttpJson) Gather(acc telegraf.Accumulator) error {
	var wg sync.WaitGroup

	errorChannel := make(chan error, len(h.Servers))

	for _, server := range h.Servers {
		wg.Add(1)
		go func(server string) {
			defer wg.Done()
			if err := h.gatherServer(acc, server); err != nil {
				errorChannel <- err
			}
		}(server)
	}

	wg.Wait()
	close(errorChannel)

	// Get all errors and return them as one giant error
	errorStrings := []string{}
	for err := range errorChannel {
		errorStrings = append(errorStrings, err.Error())
	}

	if len(errorStrings) == 0 {
		return nil
	}
	return errors.New(strings.Join(errorStrings, "\n"))
}

// Gathers data from a particular server
// Parameters:
//     acc      : The telegraf Accumulator to use
//     serverURL: endpoint to send request to
//     service  : the service being queried
//
// Returns:
//     error: Any error that may have occurred
func (h *HttpJson) gatherServer(
	acc telegraf.Accumulator,
	serverURL string,
) error {
	resp, responseTime, err := h.sendRequest(serverURL)

	if err != nil {
		return err
	}

	var jsonOut map[string]interface{}
	if err = json.Unmarshal([]byte(resp), &jsonOut); err != nil {
		return errors.New("Error decoding JSON response")
	}

	tags := map[string]string{
		"server": serverURL,
	}

	for _, tag := range h.TagKeys {
		switch v := jsonOut[tag].(type) {
		case string:
			tags[tag] = v
		}
		delete(jsonOut, tag)
	}

	if responseTime >= 0 {
		jsonOut["response_time"] = responseTime
	}
	f := internal.JSONFlattener{}
	err = f.FlattenJSON("", jsonOut)
	if err != nil {
		return err
	}

	var msrmnt_name string
	if h.Name == "" {
		msrmnt_name = "httpjson"
	} else {
		msrmnt_name = "httpjson_" + h.Name
	}
	acc.AddFields(msrmnt_name, f.Fields, tags)
	return nil
}

// Sends an HTTP request to the server using the HttpJson object's HTTPClient
// Parameters:
//     serverURL: endpoint to send request to
//
// Returns:
//     string: body of the response
//     error : Any error that may have occurred
func (h *HttpJson) sendRequest(serverURL string) (string, float64, error) {
	// Prepare URL
	requestURL, err := url.Parse(serverURL)
	if err != nil {
		return "", -1, fmt.Errorf("Invalid server URL \"%s\"", serverURL)
	}

	params := url.Values{}
	for k, v := range h.Parameters {
		params.Add(k, v)
	}
	requestURL.RawQuery = params.Encode()

	// Create + send request
	req, err := http.NewRequest(h.Method, requestURL.String(), nil)
	if err != nil {
		return "", -1, err
	}

	// Add header parameters
	for k, v := range h.Headers {
		if strings.ToLower(k) == "host" {
			req.Host = v
		} else {
			req.Header.Add(k, v)
		}
	}

	start := time.Now()
	resp, err := h.client.MakeRequest(req)
	if err != nil {
		return "", -1, err
	}

	defer resp.Body.Close()
	responseTime := time.Since(start).Seconds()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return string(body), responseTime, err
	}

	// Process response
	if resp.StatusCode != http.StatusOK {
		err = fmt.Errorf("Response from url \"%s\" has status code %d (%s), expected %d (%s)",
			requestURL.String(),
			resp.StatusCode,
			http.StatusText(resp.StatusCode),
			http.StatusOK,
			http.StatusText(http.StatusOK))
		return string(body), responseTime, err
	}

	return string(body), responseTime, err
}

func init() {
	inputs.Add("httpjson", func() telegraf.Input {
		return &HttpJson{client: RealHTTPClient{client: &http.Client{}}}
	})
}
<|endoftext|>


<commit_before>// +build freebsd,darwin

package procstat_ostent

import "github.com/shirou/gopsutil/process"

func newProcess(pid int32) (proc, error) {
	np, err := process.NewProcess(pid)
	return &psproc{np}, err
}

type psproc struct{ *process.Process }

func (pp *psproc) Prio() (int32, error) { return 0, nil }
<commit_msg>fix build constraint<commit_after>// +build freebsd darwin

package procstat_ostent

import "github.com/shirou/gopsutil/process"

func newProcess(pid int32) (proc, error) {
	np, err := process.NewProcess(pid)
	return &psproc{np}, err
}

type psproc struct{ *process.Process }

func (pp *psproc) Prio() (int32, error) { return 0, nil }
<|endoftext|>


<commit_before>package acceptance_test

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"time"

	acceptance "github.com/cloudfoundry/bosh-bootloader/acceptance-tests"
	"github.com/cloudfoundry/bosh-bootloader/acceptance-tests/actors"
	"github.com/cloudfoundry/bosh-bootloader/testhelpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Stack Migration", func() {
	var (
		bblStack     actors.BBL
		bblTerraform actors.BBL
		aws          actors.AWS
		boshcli      actors.BOSHCLI
		state        acceptance.State

		f *os.File
	)

	BeforeEach(func() {
		var err error
		configuration, err := acceptance.LoadConfig()
		Expect(err).NotTo(HaveOccurred())

		var bblBinaryLocation string
		if runtime.GOOS == "darwin" {
			bblBinaryLocation = "https://www.github.com/cloudfoundry/bosh-bootloader/releases/download/v3.2.4/bbl-v3.2.4_osx"
		} else {
			bblBinaryLocation = "https://www.github.com/cloudfoundry/bosh-bootloader/releases/download/v3.2.4/bbl-v3.2.4_linux_x86-64"
		}

		resp, err := http.Get(bblBinaryLocation)
		Expect(err).NotTo(HaveOccurred())

		f, err = ioutil.TempFile("", "")
		Expect(err).NotTo(HaveOccurred())

		_, err = io.Copy(f, resp.Body)
		Expect(err).NotTo(HaveOccurred())

		err = os.Chmod(f.Name(), 0700)
		Expect(err).NotTo(HaveOccurred())

		err = f.Close()
		Expect(err).NotTo(HaveOccurred())

		bblStack = actors.NewBBL(configuration.StateFileDir, f.Name(), configuration, "stack-migration-env")
		bblTerraform = actors.NewBBL(configuration.StateFileDir, pathToBBL, configuration, "stack-migration-env")
		aws = actors.NewAWS(configuration)
		boshcli = actors.NewBOSHCLI()
		state = acceptance.NewState(configuration.StateFileDir)
	})

	AfterEach(func() {
		By("destroying with the old bbl", func() {
			session := bblStack.Destroy()
			Eventually(session, 10*time.Minute).Should(gexec.Exit())
		})

		By("destroying with the latest bbl", func() {
			session := bblTerraform.Destroy()
			Eventually(session, 10*time.Minute).Should(gexec.Exit())
		})

		err := os.Remove(f.Name())
		Expect(err).NotTo(HaveOccurred())
	})

	Describe("Up", func() {
		It("is able to bbl up idempotently with a director", func() {
			acceptance.SkipUnless("stack-migration-up")
			var (
				stackName       string
				directorAddress string
				caCertPath      string
			)

			By("bbl'ing up with cloudformation", func() {
				session := bblStack.Up("--iaas", "aws", "--name", bblStack.PredefinedEnvID())
				Eventually(session, 40*time.Minute).Should(gexec.Exit(0))
			})

			By("verifying the stack exists", func() {
				stackName = state.StackName()
				Expect(aws.StackExists(stackName)).To(BeTrue())
			})

			By("verifying the director exists", func() {
				directorAddress = bblStack.DirectorAddress()
				caCertPath = bblStack.SaveDirectorCA()

				exists, err := boshcli.DirectorExists(directorAddress, caCertPath)
				Expect(err).NotTo(HaveOccurred())
				Expect(exists).To(BeTrue())
			})

			By("migrating to terraform with latest bbl", func() {
				session := bblTerraform.Up()
				Eventually(session, 40*time.Minute).Should(gexec.Exit(0))
			})

			By("verifying the stack doesn't exists", func() {
				Expect(aws.StackExists(stackName)).To(BeFalse())
			})

			By("verifying the director still exists", func() {
				exists, err := boshcli.DirectorExists(directorAddress, caCertPath)
				Expect(err).NotTo(HaveOccurred())
				Expect(exists).To(BeTrue())
			})
		})
	})

	Describe("Create LBs", func() {
		It("is able to bbl create-lbs", func() {
			acceptance.SkipUnless("stack-migration-create-lbs")
			var (
				stackName string
				lbNames   []string
			)

			By("bbl'ing up with cloudformation", func() {
				session := bblStack.Up("--iaas", "aws", "--name", bblStack.PredefinedEnvID())
				Eventually(session, 40*time.Minute).Should(gexec.Exit(0))
			})

			By("verifying the stack exists", func() {
				stackName = state.StackName()
				Expect(aws.StackExists(stackName)).To(BeTrue())
			})

			By("verifying there are no LBs", func() {
				lbNames = aws.LoadBalancers(fmt.Sprintf("vpc-%s", bblStack.PredefinedEnvID()))
				Expect(lbNames).To(BeEmpty())
			})

			By("creating a concourse load balancer", func() {
				certPath, err := testhelpers.WriteContentsToTempFile(testhelpers.BBL_CERT)
				Expect(err).NotTo(HaveOccurred())

				chainPath, err := testhelpers.WriteContentsToTempFile(testhelpers.BBL_CHAIN)
				Expect(err).NotTo(HaveOccurred())

				keyPath, err := testhelpers.WriteContentsToTempFile(testhelpers.BBL_KEY)
				Expect(err).NotTo(HaveOccurred())

				session := bblTerraform.CreateLB("concourse", certPath, keyPath, chainPath)
				Eventually(session, 10*time.Minute).Should(gexec.Exit(0))
			})

			By("verifying that no stack exists", func() {
				Expect(aws.StackExists(stackName)).To(BeFalse())
			})

			By("checking that the LB was created", func() {
				vpcName := fmt.Sprintf("%s-vpc", bblStack.PredefinedEnvID())
				Expect(aws.LoadBalancers(vpcName)).To(HaveLen(1))
				Expect(aws.LoadBalancers(vpcName)).To(ConsistOf(
					MatchRegexp(".*-concourse-lb"),
				))
			})
		})

		It("deletes lbs from older bbl", func() {
			acceptance.SkipUnless("stack-migration-delete-lbs")
			var (
				stackName string
				lbNames   []string
			)

			By("bbl'ing up with cloudformation", func() {
				session := bblStack.Up("--iaas", "aws", "--name", bblStack.PredefinedEnvID())
				Eventually(session, 40*time.Minute).Should(gexec.Exit(0))
			})

			By("verifying the stack exists", func() {
				stackName = state.StackName()
				Expect(aws.StackExists(stackName)).To(BeTrue())
			})

			By("verifying there are no LBs", func() {
				lbNames = aws.LoadBalancers(fmt.Sprintf("vpc-%s", bblStack.PredefinedEnvID()))
				Expect(lbNames).To(BeEmpty())
			})

			By("creating cf lbs", func() {
				certPath, err := testhelpers.WriteContentsToTempFile(testhelpers.OTHER_BBL_CERT)
				Expect(err).NotTo(HaveOccurred())

				chainPath, err := testhelpers.WriteContentsToTempFile(testhelpers.OTHER_BBL_CHAIN)
				Expect(err).NotTo(HaveOccurred())

				keyPath, err := testhelpers.WriteContentsToTempFile(testhelpers.OTHER_BBL_KEY)
				Expect(err).NotTo(HaveOccurred())

				session := bblStack.CreateLB("cf", certPath, keyPath, chainPath)
				Eventually(session, 10*time.Minute).Should(gexec.Exit(0))
			})

			By("checking that the LB was created", func() {
				vpcName := fmt.Sprintf("vpc-%s", bblStack.PredefinedEnvID())
				Expect(aws.LoadBalancers(vpcName)).To(HaveLen(2))
				Expect(aws.LoadBalancers(vpcName)).To(ConsistOf(
					MatchRegexp("stack-.*-CFSSHPro-.*"),
					MatchRegexp("stack-.*-CFRouter-.*"),
				))
			})

			By("deleting the LBs", func() {
				session := bblTerraform.DeleteLBs()
				Eventually(session, 15*time.Minute).Should(gexec.Exit(0))
			})

			By("verifying that no stack exists", func() {
				Expect(aws.StackExists(stackName)).To(BeFalse())
			})

			By("confirming that the cf lbs do not exist", func() {
				vpcName := fmt.Sprintf("%s-vpc", bblStack.PredefinedEnvID())
				Expect(aws.LoadBalancers(vpcName)).To(BeEmpty())
			})
		})
	})
})
<commit_msg>Use unique env names for stack migration tests<commit_after>package acceptance_test

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"time"

	acceptance "github.com/cloudfoundry/bosh-bootloader/acceptance-tests"
	"github.com/cloudfoundry/bosh-bootloader/acceptance-tests/actors"
	"github.com/cloudfoundry/bosh-bootloader/testhelpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Stack Migration", func() {
	var (
		bblStack     actors.BBL
		bblTerraform actors.BBL
		aws          actors.AWS
		boshcli      actors.BOSHCLI
		state        acceptance.State

		f *os.File
	)

	BeforeEach(func() {
		var err error
		configuration, err := acceptance.LoadConfig()
		Expect(err).NotTo(HaveOccurred())

		var bblBinaryLocation string
		if runtime.GOOS == "darwin" {
			bblBinaryLocation = "https://www.github.com/cloudfoundry/bosh-bootloader/releases/download/v3.2.4/bbl-v3.2.4_osx"
		} else {
			bblBinaryLocation = "https://www.github.com/cloudfoundry/bosh-bootloader/releases/download/v3.2.4/bbl-v3.2.4_linux_x86-64"
		}

		resp, err := http.Get(bblBinaryLocation)
		Expect(err).NotTo(HaveOccurred())

		f, err = ioutil.TempFile("", "")
		Expect(err).NotTo(HaveOccurred())

		_, err = io.Copy(f, resp.Body)
		Expect(err).NotTo(HaveOccurred())

		err = os.Chmod(f.Name(), 0700)
		Expect(err).NotTo(HaveOccurred())

		err = f.Close()
		Expect(err).NotTo(HaveOccurred())

		envName := "stack-migration-env"
		testName := os.Getenv("RUN_TEST")
		if testName != "" {
			envName = testName
		}
		bblStack = actors.NewBBL(configuration.StateFileDir, f.Name(), configuration, envName)
		bblTerraform = actors.NewBBL(configuration.StateFileDir, pathToBBL, configuration, envName)
		aws = actors.NewAWS(configuration)
		boshcli = actors.NewBOSHCLI()
		state = acceptance.NewState(configuration.StateFileDir)
	})

	AfterEach(func() {
		By("destroying with the old bbl", func() {
			session := bblStack.Destroy()
			Eventually(session, 10*time.Minute).Should(gexec.Exit())
		})

		By("destroying with the latest bbl", func() {
			session := bblTerraform.Destroy()
			Eventually(session, 10*time.Minute).Should(gexec.Exit())
		})

		err := os.Remove(f.Name())
		Expect(err).NotTo(HaveOccurred())
	})

	Describe("Up", func() {
		It("is able to bbl up idempotently with a director", func() {
			acceptance.SkipUnless("stack-migration-up")
			var (
				stackName       string
				directorAddress string
				caCertPath      string
			)

			By("bbl'ing up with cloudformation", func() {
				session := bblStack.Up("--iaas", "aws", "--name", bblStack.PredefinedEnvID())
				Eventually(session, 40*time.Minute).Should(gexec.Exit(0))
			})

			By("verifying the stack exists", func() {
				stackName = state.StackName()
				Expect(aws.StackExists(stackName)).To(BeTrue())
			})

			By("verifying the director exists", func() {
				directorAddress = bblStack.DirectorAddress()
				caCertPath = bblStack.SaveDirectorCA()

				exists, err := boshcli.DirectorExists(directorAddress, caCertPath)
				Expect(err).NotTo(HaveOccurred())
				Expect(exists).To(BeTrue())
			})

			By("migrating to terraform with latest bbl", func() {
				session := bblTerraform.Up()
				Eventually(session, 40*time.Minute).Should(gexec.Exit(0))
			})

			By("verifying the stack doesn't exists", func() {
				Expect(aws.StackExists(stackName)).To(BeFalse())
			})

			By("verifying the director still exists", func() {
				exists, err := boshcli.DirectorExists(directorAddress, caCertPath)
				Expect(err).NotTo(HaveOccurred())
				Expect(exists).To(BeTrue())
			})
		})
	})

	Describe("Create LBs", func() {
		It("is able to bbl create-lbs", func() {
			acceptance.SkipUnless("stack-migration-create-lbs")
			var (
				stackName string
				lbNames   []string
			)

			By("bbl'ing up with cloudformation", func() {
				session := bblStack.Up("--iaas", "aws", "--name", bblStack.PredefinedEnvID())
				Eventually(session, 40*time.Minute).Should(gexec.Exit(0))
			})

			By("verifying the stack exists", func() {
				stackName = state.StackName()
				Expect(aws.StackExists(stackName)).To(BeTrue())
			})

			By("verifying there are no LBs", func() {
				lbNames = aws.LoadBalancers(fmt.Sprintf("vpc-%s", bblStack.PredefinedEnvID()))
				Expect(lbNames).To(BeEmpty())
			})

			By("creating a concourse load balancer", func() {
				certPath, err := testhelpers.WriteContentsToTempFile(testhelpers.BBL_CERT)
				Expect(err).NotTo(HaveOccurred())

				chainPath, err := testhelpers.WriteContentsToTempFile(testhelpers.BBL_CHAIN)
				Expect(err).NotTo(HaveOccurred())

				keyPath, err := testhelpers.WriteContentsToTempFile(testhelpers.BBL_KEY)
				Expect(err).NotTo(HaveOccurred())

				session := bblTerraform.CreateLB("concourse", certPath, keyPath, chainPath)
				Eventually(session, 10*time.Minute).Should(gexec.Exit(0))
			})

			By("verifying that no stack exists", func() {
				Expect(aws.StackExists(stackName)).To(BeFalse())
			})

			By("checking that the LB was created", func() {
				vpcName := fmt.Sprintf("%s-vpc", bblStack.PredefinedEnvID())
				Expect(aws.LoadBalancers(vpcName)).To(HaveLen(1))
				Expect(aws.LoadBalancers(vpcName)).To(ConsistOf(
					MatchRegexp(".*-concourse-lb"),
				))
			})
		})

		It("deletes lbs from older bbl", func() {
			acceptance.SkipUnless("stack-migration-delete-lbs")
			var (
				stackName string
				lbNames   []string
			)

			By("bbl'ing up with cloudformation", func() {
				session := bblStack.Up("--iaas", "aws", "--name", bblStack.PredefinedEnvID())
				Eventually(session, 40*time.Minute).Should(gexec.Exit(0))
			})

			By("verifying the stack exists", func() {
				stackName = state.StackName()
				Expect(aws.StackExists(stackName)).To(BeTrue())
			})

			By("verifying there are no LBs", func() {
				lbNames = aws.LoadBalancers(fmt.Sprintf("vpc-%s", bblStack.PredefinedEnvID()))
				Expect(lbNames).To(BeEmpty())
			})

			By("creating cf lbs", func() {
				certPath, err := testhelpers.WriteContentsToTempFile(testhelpers.OTHER_BBL_CERT)
				Expect(err).NotTo(HaveOccurred())

				chainPath, err := testhelpers.WriteContentsToTempFile(testhelpers.OTHER_BBL_CHAIN)
				Expect(err).NotTo(HaveOccurred())

				keyPath, err := testhelpers.WriteContentsToTempFile(testhelpers.OTHER_BBL_KEY)
				Expect(err).NotTo(HaveOccurred())

				session := bblStack.CreateLB("cf", certPath, keyPath, chainPath)
				Eventually(session, 10*time.Minute).Should(gexec.Exit(0))
			})

			By("checking that the LB was created", func() {
				vpcName := fmt.Sprintf("vpc-%s", bblStack.PredefinedEnvID())
				Expect(aws.LoadBalancers(vpcName)).To(HaveLen(2))
				Expect(aws.LoadBalancers(vpcName)).To(ConsistOf(
					MatchRegexp("stack-.*-CFSSHPro-.*"),
					MatchRegexp("stack-.*-CFRouter-.*"),
				))
			})

			By("deleting the LBs", func() {
				session := bblTerraform.DeleteLBs()
				Eventually(session, 15*time.Minute).Should(gexec.Exit(0))
			})

			By("verifying that no stack exists", func() {
				Expect(aws.StackExists(stackName)).To(BeFalse())
			})

			By("confirming that the cf lbs do not exist", func() {
				vpcName := fmt.Sprintf("%s-vpc", bblStack.PredefinedEnvID())
				Expect(aws.LoadBalancers(vpcName)).To(BeEmpty())
			})
		})
	})
})
<|endoftext|>


<commit_before>package constants

const (
	SubPortNumber                = 6969
	DominatorPortNumber          = 6970
	ImageServerPortNumber        = 6971
	BasicFileGenServerPortNumber = 6972
	SimpleMdbServerPortNumber    = 6973
	ImageUnpackerPortNumber      = 6974
	ImaginatorPortNumber         = 6975
	HypervisorPortNumber         = 6976
	FleetManagerPortNumber       = 6977

	DefaultCpuPercent          = 50
	DefaultNetworkSpeedPercent = 10
	DefaultScanSpeedPercent    = 2

	AssignedOIDBase        = "1.3.6.1.4.1.9586.100.7"
	PermittedMethodListOID = AssignedOIDBase + ".1"
)

var RequiredPaths = map[string]rune{
	"/etc":        'd',
	"/etc/passwd": 'f',
	"/usr":        'd',
	"/usr/bin":    'd',
}

var ScanExcludeList = []string{
	"/home/.*",
	"/tmp/.*",
	"/var/log/.*",
	"/var/mail/.*",
	"/var/spool/.*",
	"/var/tmp/.*",
}
<commit_msg>Add lib/constants.GroupListOID.<commit_after>package constants

const (
	SubPortNumber                = 6969
	DominatorPortNumber          = 6970
	ImageServerPortNumber        = 6971
	BasicFileGenServerPortNumber = 6972
	SimpleMdbServerPortNumber    = 6973
	ImageUnpackerPortNumber      = 6974
	ImaginatorPortNumber         = 6975
	HypervisorPortNumber         = 6976
	FleetManagerPortNumber       = 6977

	DefaultCpuPercent          = 50
	DefaultNetworkSpeedPercent = 10
	DefaultScanSpeedPercent    = 2

	AssignedOIDBase        = "1.3.6.1.4.1.9586.100.7"
	PermittedMethodListOID = AssignedOIDBase + ".1"
	GroupListOID           = AssignedOIDBase + ".2"
)

var RequiredPaths = map[string]rune{
	"/etc":        'd',
	"/etc/passwd": 'f',
	"/usr":        'd',
	"/usr/bin":    'd',
}

var ScanExcludeList = []string{
	"/home/.*",
	"/tmp/.*",
	"/var/log/.*",
	"/var/mail/.*",
	"/var/spool/.*",
	"/var/tmp/.*",
}
<|endoftext|>