<commit_before>// Copyright 2015 Gary Burd. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package commands

import (
	"bytes"
	"go/scanner"
	"time"

	"nvim-go/context"
	"nvim-go/nvim"
	"nvim-go/nvim/profile"
	"nvim-go/nvim/quickfix"

	"github.com/garyburd/neovim-go/vim"
	"github.com/garyburd/neovim-go/vim/plugin"
	"golang.org/x/tools/imports"
)

var options = imports.Options{
	AllErrors: true,
	Comments:  true,
	TabIndent: true,
	TabWidth:  8,
}

func init() {
	plugin.HandleCommand("Gofmt", &plugin.CommandOptions{Eval: "expand('%:p:h')"}, Fmt)
}

// Fmt format to the current buffer source uses gofmt behavior.
func Fmt(v *vim.Vim, dir string) error {
	defer profile.Start(time.Now(), "GoFmt")
	var ctxt = context.Build{}
	defer ctxt.SetContext(dir)()

	var (
		b vim.Buffer
		w vim.Window
	)

	p := v.NewPipeline()
	p.CurrentBuffer(&b)
	p.CurrentWindow(&w)
	if err := p.Wait(); err != nil {
		return err
	}

	bufName, err := v.BufferName(b)
	if err != nil {
		return err
	}

	in, err := v.BufferLines(b, 0, -1, true)
	if err != nil {
		return err
	}

	buf, err := imports.Process("", bytes.Join(in, []byte{'\n'}), &options)
	if err != nil {
		var loclist []*quickfix.ErrorlistData

		if e, ok := err.(scanner.Error); ok {
			loclist = append(loclist, &quickfix.ErrorlistData{
				FileName: bufName,
				LNum:     e.Pos.Line,
				Col:      e.Pos.Column,
				Text:     e.Msg,
			})
		} else if el, ok := err.(scanner.ErrorList); ok {
			for _, e := range el {
				loclist = append(loclist, &quickfix.ErrorlistData{
					FileName: bufName,
					LNum:     e.Pos.Line,
					Col:      e.Pos.Column,
					Text:     e.Msg,
				})
			}
		}

		if err := quickfix.SetLoclist(v, loclist); err != nil {
			return nvim.Echomsg(v, "Gofmt:", err)
		}

		return quickfix.OpenLoclist(v, w, loclist, true)
	}

	quickfix.CloseLoclist(v)

	out := bytes.Split(bytes.TrimSuffix(buf, []byte{'\n'}), []byte{'\n'})

	return minUpdate(v, b, in, out)
}

func minUpdate(v *vim.Vim, b vim.Buffer, in [][]byte, out [][]byte) error {
	// Find matching head lines.
	n := len(out)
	if len(in) < len(out) {
		n = len(in)
	}
	head := 0
	for ; head < n; head++ {
		if !bytes.Equal(in[head], out[head]) {
			break
		}
	}

	// Nothing to do?
	if head == len(in) && head == len(out) {
		return nil
	}

	// Find matching tail lines.
	n -= head
	tail := 0
	for ; tail < n; tail++ {
		if !bytes.Equal(in[len(in)-tail-1], out[len(out)-tail-1]) {
			break
		}
	}

	// Update the buffer.
	start := head
	end := len(in) - tail
	repl := out[head : len(out)-tail]

	return v.SetBufferLines(b, start, end, true, repl)
}
<commit_msg>cmds/fmt: Add errors wrap<commit_after>// Copyright 2015 Gary Burd. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package commands

import (
	"bytes"
	"go/scanner"
	"time"

	"nvim-go/context"
	"nvim-go/nvim"
	"nvim-go/nvim/profile"
	"nvim-go/nvim/quickfix"

	"github.com/garyburd/neovim-go/vim"
	"github.com/garyburd/neovim-go/vim/plugin"
	"github.com/juju/errors"
	"golang.org/x/tools/imports"
)

var options = imports.Options{
	AllErrors: true,
	Comments:  true,
	TabIndent: true,
	TabWidth:  8,
}

func init() {
	plugin.HandleCommand("Gofmt", &plugin.CommandOptions{Eval: "expand('%:p:h')"}, Fmt)
}

// Fmt format to the current buffer source uses gofmt behavior.
func Fmt(v *vim.Vim, dir string) error {
	defer profile.Start(time.Now(), "GoFmt")
	var ctxt = context.Build{}
	defer ctxt.SetContext(dir)()

	var (
		b vim.Buffer
		w vim.Window
	)

	p := v.NewPipeline()
	p.CurrentBuffer(&b)
	p.CurrentWindow(&w)
	if err := p.Wait(); err != nil {
		return err
	}

	bufName, err := v.BufferName(b)
	if err != nil {
		return err
	}

	in, err := v.BufferLines(b, 0, -1, true)
	if err != nil {
		return err
	}

	buf, err := imports.Process("", bytes.Join(in, []byte{'\n'}), &options)
	if err != nil {
		var loclist []*quickfix.ErrorlistData

		if e, ok := err.(scanner.Error); ok {
			loclist = append(loclist, &quickfix.ErrorlistData{
				FileName: bufName,
				LNum:     e.Pos.Line,
				Col:      e.Pos.Column,
				Text:     e.Msg,
			})
		} else if el, ok := err.(scanner.ErrorList); ok {
			for _, e := range el {
				loclist = append(loclist, &quickfix.ErrorlistData{
					FileName: bufName,
					LNum:     e.Pos.Line,
					Col:      e.Pos.Column,
					Text:     e.Msg,
				})
			}
		}

		if err := quickfix.SetLoclist(v, loclist); err != nil {
			return nvim.Echomsg(v, "Gofmt:", err)
		}

		quickfix.OpenLoclist(v, w, loclist, true)
		return errors.Annotate(err, "GoFmt")
	}

	quickfix.CloseLoclist(v)

	out := bytes.Split(bytes.TrimSuffix(buf, []byte{'\n'}), []byte{'\n'})

	return minUpdate(v, b, in, out)
}

func minUpdate(v *vim.Vim, b vim.Buffer, in [][]byte, out [][]byte) error {
	// Find matching head lines.
	n := len(out)
	if len(in) < len(out) {
		n = len(in)
	}
	head := 0
	for ; head < n; head++ {
		if !bytes.Equal(in[head], out[head]) {
			break
		}
	}

	// Nothing to do?
	if head == len(in) && head == len(out) {
		return nil
	}

	// Find matching tail lines.
	n -= head
	tail := 0
	for ; tail < n; tail++ {
		if !bytes.Equal(in[len(in)-tail-1], out[len(out)-tail-1]) {
			break
		}
	}

	// Update the buffer.
	start := head
	end := len(in) - tail
	repl := out[head : len(out)-tail]

	return v.SetBufferLines(b, start, end, true, repl)
}
<|endoftext|>


<commit_before>package randomization

import (
	"net/http"
	"strings"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/user"
)

// deleteProjectStep1 gets the project name from the user.
func deleteProjectStep1(w http.ResponseWriter, r *http.Request) {

	if r.Method != "GET" {
		Serve404(w)
		return
	}

	ctx := appengine.NewContext(r)
	user := user.Current(ctx)

	_, projlist, err := getProjects(ctx, user.String(), false)
	if err != nil {
		msg := "A datastore error occured, your projects cannot be retrieved."
		log.Errorf(ctx, "Delete_project_step1: %v", err)
		rmsg := "Return to dashboard"
		messagePage(w, r, user, msg, rmsg, "/dashboard")
		return
	}

	if len(projlist) == 0 {
		msg := "You are not the owner of any projects.  A project can only be deleted by its owner."
		rmsg := "Return to dashboard"
		messagePage(w, r, user, msg, rmsg, "/dashboard")
		return
	}

	tvals := struct {
		User     string
		LoggedIn bool
		Proj     []*EncodedProjectView
	}{
		User:     user.String(),
		Proj:     formatEncodedProjects(projlist),
		LoggedIn: user != nil,
	}

	if err := tmpl.ExecuteTemplate(w, "delete_project_step1.html", tvals); err != nil {
		log.Errorf(ctx, "deleteProjectStep1: %v", err)
	}
}

// deleteProjectStep2 confirms that a project should be deleted.
func deleteProjectStep2(w http.ResponseWriter, r *http.Request) {

	if r.Method != "POST" {
		Serve404(w)
		return
	}

	ctx := appengine.NewContext(r)

	user := user.Current(ctx)

	if err := r.ParseForm(); err != nil {
		ServeError(ctx, w, err)
		return
	}

	pkey := r.FormValue("project_list")
	svec := strings.Split(pkey, "::")

	tvals := struct {
		User        string
		LoggedIn    bool
		ProjectName string
		Pkey        string
		Nokey       bool
	}{
		User:     user.String(),
		LoggedIn: user != nil,
		Pkey:     pkey,
		Nokey:    len(pkey) == 0,
	}

	if len(svec) >= 2 {
		tvals.ProjectName = svec[1]
	}

	if err := tmpl.ExecuteTemplate(w, "delete_project_step2.html", tvals); err != nil {
		log.Errorf(ctx, "deleteProjectStep2: %v", err)
	}
}

// deleteProjectStep3 deletes a project.
func deleteProjectStep3(w http.ResponseWriter, r *http.Request) {

	if r.Method != "POST" {
		Serve404(w)
		return
	}

	ctx := appengine.NewContext(r)
	user := user.Current(ctx)
	pkey := r.FormValue("Pkey")

	if err := r.ParseForm(); err != nil {
		log.Errorf(ctx, "deleteProjectStep3 [1]: %v", err)
		ServeError(ctx, w, err)
		return
	}

	// Delete the SharingByProject object, but first read the
	// users list from it so we can delete the project from their
	// SharingByUsers records.
	key := datastore.NewKey(ctx, "SharingByProject", pkey, 0, nil)
	var sbproj SharingByProject
	sharedWith := make([]string, 0)
	err := datastore.Get(ctx, key, &sbproj)
	if err == datastore.ErrNoSuchEntity {
		log.Errorf(ctx, "deleteProjectStep3 [2]: %v", err)
	} else if err != nil {
		log.Errorf(ctx, "deleteProjectStep3 [3] %v", err)
	} else {
		sharedWith = cleanSplit(sbproj.Users, ",")
		err = datastore.Delete(ctx, key)
		if err != nil {
			log.Errorf(ctx, "deleteProjectStep3 [4] %v", err)
		}
	}

	// Delete the project.
	key = datastore.NewKey(ctx, "EncodedProject", pkey, 0, nil)
	err = datastore.Delete(ctx, key)
	if err != nil {
		log.Errorf(ctx, "deleteProjectStep3 [5]: %v", err)
	}

	// Delete from each user's SharingByUser record.
	for _, user1 := range sharedWith {
		var sbuser SharingByUser
		key := datastore.NewKey(ctx, "SharingByUser", strings.ToLower(user1), 0, nil)
		err := datastore.Get(ctx, key, &sbuser)
		if err != nil {
			log.Errorf(ctx, "deleteProjectStep3 [6]: %v", err)
		}
		Projects := cleanSplit(sbuser.Projects, ",")

		// Get the unique project keys, except for pkey.
		mp := make(map[string]bool)
		for _, x := range Projects {
			if x != pkey {
				mp[x] = true
			}
		}
		var vec []string
		for k := range mp {
			vec = append(vec, k)
		}
		sbuser.Projects = strings.Join(vec, ",")

		_, err = datastore.Put(ctx, key, &sbuser)
		if err != nil {
			log.Errorf(ctx, "deleteProjectStep3 [7]: %v", err)
		}
	}

	tvals := struct {
		User     string
		LoggedIn bool
		Success  bool
	}{
		User:     user.String(),
		LoggedIn: err == nil,
		Success:  user != nil,
	}

	if err := tmpl.ExecuteTemplate(w, "delete_project_step3.html", tvals); err != nil {
		log.Errorf(ctx, "deleteProjectStep3 [9]: %v", err)
	}
}
<commit_msg>update<commit_after>package randomization

import (
	"net/http"
	"strings"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/user"
)

// deleteProjectStep1 gets the project name from the user.
func deleteProjectStep1(w http.ResponseWriter, r *http.Request) {

	if r.Method != "GET" {
		Serve404(w)
		return
	}

	ctx := appengine.NewContext(r)
	user := user.Current(ctx)

	_, projlist, err := getProjects(ctx, user.String(), false)
	if err != nil {
		msg := "A datastore error occured, your projects cannot be retrieved."
		log.Errorf(ctx, "Delete_project_step1: %v", err)
		rmsg := "Return to dashboard"
		messagePage(w, r, user, msg, rmsg, "/dashboard")
		return
	}

	if len(projlist) == 0 {
		msg := "You are not the owner of any projects.  A project can only be deleted by its owner."
		rmsg := "Return to dashboard"
		messagePage(w, r, user, msg, rmsg, "/dashboard")
		return
	}

	tvals := struct {
		User     string
		LoggedIn bool
		Proj     []*EncodedProjectView
	}{
		User:     user.String(),
		Proj:     formatEncodedProjects(projlist),
		LoggedIn: user != nil,
	}

	if err := tmpl.ExecuteTemplate(w, "delete_project_step1.html", tvals); err != nil {
		log.Errorf(ctx, "deleteProjectStep1: %v", err)
	}
}

// deleteProjectStep2 confirms that a project should be deleted.
func deleteProjectStep2(w http.ResponseWriter, r *http.Request) {

	if r.Method != "POST" {
		Serve404(w)
		return
	}

	ctx := appengine.NewContext(r)
	user := user.Current(ctx)

	if err := r.ParseForm(); err != nil {
		ServeError(ctx, w, err)
		return
	}

	pkey := r.FormValue("project_list")
	svec := strings.Split(pkey, "::")

	tvals := struct {
		User        string
		LoggedIn    bool
		ProjectName string
		Pkey        string
		Nokey       bool
	}{
		User:     user.String(),
		LoggedIn: user != nil,
		Pkey:     pkey,
		Nokey:    len(pkey) == 0,
	}

	if len(svec) >= 2 {
		tvals.ProjectName = svec[1]
	}

	if err := tmpl.ExecuteTemplate(w, "delete_project_step2.html", tvals); err != nil {
		log.Errorf(ctx, "deleteProjectStep2: %v", err)
	}
}

// deleteProjectStep3 deletes a project.
func deleteProjectStep3(w http.ResponseWriter, r *http.Request) {

	if r.Method != "POST" {
		Serve404(w)
		return
	}

	ctx := appengine.NewContext(r)
	user := user.Current(ctx)
	pkey := r.FormValue("Pkey")

	if !checkAccess(ctx, user, pkey, &w, r) {
		msg := "You do not have access to this project."
		rmsg := "Return"
		messagePage(w, r, user, msg, rmsg, "/")
		return
	}

	if err := r.ParseForm(); err != nil {
		log.Errorf(ctx, "deleteProjectStep3 [1]: %v", err)
		ServeError(ctx, w, err)
		return
	}

	// Delete the SharingByProject object, but first read the
	// users list from it so we can delete the project from their
	// SharingByUsers records.
	key := datastore.NewKey(ctx, "SharingByProject", pkey, 0, nil)
	var sbproj SharingByProject
	var sharedWith []string
	err := datastore.Get(ctx, key, &sbproj)
	if err == datastore.ErrNoSuchEntity {
		log.Errorf(ctx, "deleteProjectStep3 [2]: %v", err)
	} else if err != nil {
		log.Errorf(ctx, "deleteProjectStep3 [3] %v", err)
	} else {
		sharedWith = cleanSplit(sbproj.Users, ",")
		err = datastore.Delete(ctx, key)
		if err != nil {
			log.Errorf(ctx, "deleteProjectStep3 [4] %v", err)
		}
	}

	// Delete the project.
	key = datastore.NewKey(ctx, "EncodedProject", pkey, 0, nil)
	err = datastore.Delete(ctx, key)
	if err != nil {
		log.Errorf(ctx, "deleteProjectStep3 [5]: %v", err)
	}

	// Delete from each user's SharingByUser record.
	for _, user1 := range sharedWith {
		var sbuser SharingByUser
		key := datastore.NewKey(ctx, "SharingByUser", strings.ToLower(user1), 0, nil)
		err := datastore.Get(ctx, key, &sbuser)
		if err != nil {
			log.Errorf(ctx, "deleteProjectStep3 [6]: %v", err)
		}
		Projects := cleanSplit(sbuser.Projects, ",")

		// Get the unique project keys, except for pkey.
		mp := make(map[string]bool)
		for _, x := range Projects {
			if x != pkey {
				mp[x] = true
			}
		}
		var vec []string
		for k := range mp {
			vec = append(vec, k)
		}
		sbuser.Projects = strings.Join(vec, ",")

		_, err = datastore.Put(ctx, key, &sbuser)
		if err != nil {
			log.Errorf(ctx, "deleteProjectStep3 [7]: %v", err)
		}
	}

	tvals := struct {
		User     string
		LoggedIn bool
		Success  bool
	}{
		User:     user.String(),
		LoggedIn: err == nil,
		Success:  user != nil,
	}

	if err := tmpl.ExecuteTemplate(w, "delete_project_step3.html", tvals); err != nil {
		log.Errorf(ctx, "deleteProjectStep3 [9]: %v", err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os/exec"
	"sort"
	"time"

	compute "google.golang.org/api/compute/v1"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apis/extensions"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/wait"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// Before enabling this test you must make sure the associated project has
// enough quota. At the time of this writing GCE projects are allowed 3
// backend services by default. This test requires at least 5.

// This test exercises the GCE L7 loadbalancer controller cluster-addon. It
// will fail if the addon isn't running, or doesn't send traffic to the expected
// backend. Common failure modes include:
// * GCE L7 took too long to spin up
// * GCE L7 took too long to health check a backend
// * Repeated 404:
//   - L7 is sending traffic to the default backend of the addon.
//   - Backend is receiving /foo when it expects /bar.
// * Repeated 5xx:
//   - Out of quota (describe ing should show you if this is the case)
//   - Mismatched service/container port, or endpoints are dead.

var (
	appPrefix         = "foo-app-"
	pathPrefix        = "foo"
	testImage         = "gcr.io/google_containers/n-way-http:1.0"
	httpContainerPort = 8080

	expectedLBCreationTime    = 7 * time.Minute
	expectedLBHealthCheckTime = 7 * time.Minute

	// On average it takes ~6 minutes for a single backend to come online.
	// We *don't* expect this poll to consistently take 15 minutes for every
	// Ingress as GCE is creating/checking backends in parallel, but at the
	// same time, we're not testing GCE startup latency. So give it enough
	// time, and fail if the average is too high.
	lbPollTimeout  = 15 * time.Minute
	lbPollInterval = 30 * time.Second

	// One can scale this test by tweaking numApps and numIng, the former will
	// create more RCs/Services and add them to a single Ingress, while the latter
	// will create smaller, more fragmented Ingresses. The numbers 4, 2 are chosen
	// arbitrarity, we want to test more than a single Ingress, and it should have
	// more than 1 url endpoint going to a service.
	numApps = 4
	numIng  = 2
)

// timeSlice allows sorting of time.Duration
type timeSlice []time.Duration

func (p timeSlice) Len() int {
	return len(p)
}

func (p timeSlice) Less(i, j int) bool {
	return p[i] < p[j]
}

func (p timeSlice) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

// ruleByIndex returns an IngressRule for the given index.
func ruleByIndex(i int) extensions.IngressRule {
	return extensions.IngressRule{
		Host: fmt.Sprintf("foo%d.bar.com", i),
		IngressRuleValue: extensions.IngressRuleValue{
			HTTP: &extensions.HTTPIngressRuleValue{
				Paths: []extensions.HTTPIngressPath{
					{
						Path: fmt.Sprintf("/%v%d", pathPrefix, i),
						Backend: extensions.IngressBackend{
							ServiceName: fmt.Sprintf("%v%d", appPrefix, i),
							ServicePort: util.NewIntOrStringFromInt(httpContainerPort),
						},
					},
				},
			},
		},
	}
}

// createIngress creates an Ingress with num rules. Eg:
// start = 1 num = 2 will given you a single Ingress with 2 rules:
// Ingress {
//	 foo1.bar.com: /foo1
//	 foo2.bar.com: /foo2
// }
func createIngress(c *client.Client, ns string, start, num int) extensions.Ingress {
	ing := extensions.Ingress{
		ObjectMeta: api.ObjectMeta{
			Name:      fmt.Sprintf("%v%d", appPrefix, start),
			Namespace: ns,
		},
		Spec: extensions.IngressSpec{
			Backend: &extensions.IngressBackend{
				ServiceName: fmt.Sprintf("%v%d", appPrefix, start),
				ServicePort: util.NewIntOrStringFromInt(httpContainerPort),
			},
			Rules: []extensions.IngressRule{},
		},
	}
	for i := start; i < start+num; i++ {
		ing.Spec.Rules = append(ing.Spec.Rules, ruleByIndex(i))
	}
	Logf("Creating ingress %v", start)
	_, err := c.Extensions().Ingress(ns).Create(&ing)
	Expect(err).NotTo(HaveOccurred())
	return ing
}

// createApp will create a single RC and Svc. The Svc will match pods of the
// RC using the selector: 'name'=<name arg>
func createApp(c *client.Client, ns string, i int) {
	name := fmt.Sprintf("%v%d", appPrefix, i)
	l := map[string]string{}

	Logf("Creating svc %v", name)
	svc := svcByName(name, httpContainerPort)
	svc.Spec.Type = api.ServiceTypeNodePort
	_, err := c.Services(ns).Create(svc)
	Expect(err).NotTo(HaveOccurred())

	Logf("Creating rc %v", name)
	rc := rcByNamePort(name, 1, testImage, httpContainerPort, l)
	rc.Spec.Template.Spec.Containers[0].Args = []string{
		"--num=1",
		fmt.Sprintf("--start=%d", i),
		fmt.Sprintf("--prefix=%v", pathPrefix),
		fmt.Sprintf("--port=%d", httpContainerPort),
	}
	_, err = c.ReplicationControllers(ns).Create(rc)
	Expect(err).NotTo(HaveOccurred())
}

// gcloudUnmarshal unmarshals json output of gcloud into given out interface.
func gcloudUnmarshal(resource, regex string, out interface{}) {
	output, err := exec.Command("gcloud", "compute", resource, "list",
		fmt.Sprintf("--regex=%v", regex), "-q", "--format=json").CombinedOutput()
	if err != nil {
		Failf("Error unmarshalling gcloud output: %v", err)
	}
	if err := json.Unmarshal([]byte(output), out); err != nil {
		Failf("Error unmarshalling gcloud output: %v", err)
	}
}

func checkLeakedResources() error {
	msg := ""
	// Check all resources #16636.
	beList := []compute.BackendService{}
	gcloudUnmarshal("backend-services", "k8s-be-[0-9]+", &beList)
	if len(beList) != 0 {
		for _, b := range beList {
			msg += fmt.Sprintf("%v\n", b.Name)
		}
		return fmt.Errorf("Found backend services:\n%v", msg)
	}
	fwList := []compute.ForwardingRule{}
	gcloudUnmarshal("forwarding-rules", "k8s-fw-.*", &fwList)
	if len(fwList) != 0 {
		for _, f := range fwList {
			msg += fmt.Sprintf("%v\n", f.Name)
		}
		return fmt.Errorf("Found forwarding rules:\n%v", msg)
	}
	return nil
}

var _ = Describe("GCE L7 LoadBalancer Controller", func() {
	// These variables are initialized after framework's beforeEach.
	var ns string
	var client *client.Client
	var responseTimes, creationTimes []time.Duration

	framework := Framework{BaseName: "glbc"}

	BeforeEach(func() {
		// This test requires a GCE/GKE only cluster-addon
		SkipUnlessProviderIs("gce", "gke")
		framework.beforeEach()
		client = framework.Client
		ns = framework.Namespace.Name
		Expect(waitForRCPodsRunning(client, "kube-system", "glbc")).NotTo(HaveOccurred())
		Expect(checkLeakedResources()).NotTo(HaveOccurred())
		responseTimes = []time.Duration{}
		creationTimes = []time.Duration{}
	})

	AfterEach(func() {
		framework.afterEach()
		err := wait.Poll(lbPollInterval, lbPollTimeout, func() (bool, error) {
			if err := checkLeakedResources(); err != nil {
				Logf("Still waiting for glbc to cleanup: %v", err)
				return false, nil
			}
			return true, nil
		})
		Logf("Average creation time %+v, health check time %+v", creationTimes, responseTimes)
		if err != nil {
			Failf("Failed to cleanup GCE L7 resources.")
		}
		Logf("Successfully verified GCE L7 loadbalancer via Ingress.")
	})

	It("should create GCE L7 loadbalancers and verify Ingress", func() {
		// Create numApps apps, exposed via numIng Ingress each with 2 paths.
		// Eg with numApp=10, numIng=5:
		// apps: {foo-app-(0-10)}
		// ingress: {foo-app-(0, 2, 4, 6, 8)}
		// paths:
		//  ingress foo-app-0:
		//	  default1.bar.com
		//	  foo0.bar.com: /foo0
		//	  foo1.bar.com: /foo1
		if numApps < numIng {
			Failf("Need more apps than Ingress")
		}
		appsPerIngress := numApps / numIng
		By(fmt.Sprintf("Creating %d rcs + svc, and %d apps per Ingress", numApps, appsPerIngress))
		for appID := 0; appID < numApps; appID = appID + appsPerIngress {
			// Creates appsPerIngress apps, then creates one Ingress with paths to all the apps.
			for j := appID; j < appID+appsPerIngress; j++ {
				createApp(client, ns, j)
			}
			createIngress(client, ns, appID, appsPerIngress)
		}

		ings, err := client.Extensions().Ingress(ns).List(
			labels.Everything(), fields.Everything())
		Expect(err).NotTo(HaveOccurred())

		for _, ing := range ings.Items {
			// Wait for the loadbalancer IP.
			start := time.Now()
			address, err := waitForIngressAddress(client, ing.Namespace, ing.Name, lbPollTimeout)
			Expect(err).NotTo(HaveOccurred())
			By(fmt.Sprintf("Found address %v for ingress %v, took %v to come online",
				address, ing.Name, time.Since(start)))
			creationTimes = append(creationTimes, time.Since(start))

			// Check that all rules respond to a simple GET.
			for _, rules := range ing.Spec.Rules {
				// As of Kubernetes 1.1 we only support HTTP Ingress.
				if rules.IngressRuleValue.HTTP == nil {
					continue
				}
				for _, p := range rules.IngressRuleValue.HTTP.Paths {
					route := fmt.Sprintf("http://%v%v", address, p.Path)
					Logf("Testing route %v host %v with simple GET", route, rules.Host)

					GETStart := time.Now()
					var lastBody string
					pollErr := wait.Poll(lbPollInterval, lbPollTimeout, func() (bool, error) {
						var err error
						lastBody, err = simpleGET(http.DefaultClient, route, rules.Host)
						if err != nil {
							Logf("host %v path %v: %v", rules.Host, route, err)
							return false, nil
						}
						return true, nil
					})
					if pollErr != nil {
						Failf("Failed to execute a successful GET within %v, Last response body for %v, host %v:\n%v\n\n%v",
							lbPollTimeout, route, rules.Host, lastBody, pollErr)
					}
					rt := time.Since(GETStart)
					By(fmt.Sprintf("Route %v host %v took %v to respond", route, rules.Host, rt))
					responseTimes = append(responseTimes, rt)
				}
			}
		}
		// In most cases slow loadbalancer creation/startup translates directly to
		// GCE api sluggishness. However this might be because of something the
		// controller is doing, eg: maxing out QPS by repeated polling.
		sort.Sort(timeSlice(creationTimes))
		perc50 := creationTimes[len(creationTimes)/2]
		if perc50 > expectedLBCreationTime {
			Failf("Average creation time is too high: %+v", creationTimes)
		}
		sort.Sort(timeSlice(responseTimes))
		perc50 = responseTimes[len(responseTimes)/2]
		if perc50 > expectedLBHealthCheckTime {
			Failf("Average startup time is too high: %+v", responseTimes)
		}
	})
})
<commit_msg>Dump kubectl logs of Ingress controller at on e2e failure<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os/exec"
	"sort"
	"time"

	compute "google.golang.org/api/compute/v1"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/apis/extensions"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/labels"
	"k8s.io/kubernetes/pkg/util"
	"k8s.io/kubernetes/pkg/util/wait"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// Before enabling this test you must make sure the associated project has
// enough quota. At the time of this writing GCE projects are allowed 3
// backend services by default. This test requires at least 5.

// This test exercises the GCE L7 loadbalancer controller cluster-addon. It
// will fail if the addon isn't running, or doesn't send traffic to the expected
// backend. Common failure modes include:
// * GCE L7 took too long to spin up
// * GCE L7 took too long to health check a backend
// * Repeated 404:
//   - L7 is sending traffic to the default backend of the addon.
//   - Backend is receiving /foo when it expects /bar.
// * Repeated 5xx:
//   - Out of quota (describe ing should show you if this is the case)
//   - Mismatched service/container port, or endpoints are dead.

var (
	appPrefix         = "foo-app-"
	pathPrefix        = "foo"
	testImage         = "gcr.io/google_containers/n-way-http:1.0"
	httpContainerPort = 8080

	expectedLBCreationTime    = 7 * time.Minute
	expectedLBHealthCheckTime = 7 * time.Minute

	// Name of the loadbalancer controller within the cluster addon
	lbContainerName = "l7-lb-controller"

	// On average it takes ~6 minutes for a single backend to come online.
	// We *don't* expect this poll to consistently take 15 minutes for every
	// Ingress as GCE is creating/checking backends in parallel, but at the
	// same time, we're not testing GCE startup latency. So give it enough
	// time, and fail if the average is too high.
	lbPollTimeout  = 15 * time.Minute
	lbPollInterval = 30 * time.Second

	// One can scale this test by tweaking numApps and numIng, the former will
	// create more RCs/Services and add them to a single Ingress, while the latter
	// will create smaller, more fragmented Ingresses. The numbers 4, 2 are chosen
	// arbitrarity, we want to test more than a single Ingress, and it should have
	// more than 1 url endpoint going to a service.
	numApps = 4
	numIng  = 2
)

// timeSlice allows sorting of time.Duration
type timeSlice []time.Duration

func (p timeSlice) Len() int {
	return len(p)
}

func (p timeSlice) Less(i, j int) bool {
	return p[i] < p[j]
}

func (p timeSlice) Swap(i, j int) {
	p[i], p[j] = p[j], p[i]
}

// ruleByIndex returns an IngressRule for the given index.
func ruleByIndex(i int) extensions.IngressRule {
	return extensions.IngressRule{
		Host: fmt.Sprintf("foo%d.bar.com", i),
		IngressRuleValue: extensions.IngressRuleValue{
			HTTP: &extensions.HTTPIngressRuleValue{
				Paths: []extensions.HTTPIngressPath{
					{
						Path: fmt.Sprintf("/%v%d", pathPrefix, i),
						Backend: extensions.IngressBackend{
							ServiceName: fmt.Sprintf("%v%d", appPrefix, i),
							ServicePort: util.NewIntOrStringFromInt(httpContainerPort),
						},
					},
				},
			},
		},
	}
}

// createIngress creates an Ingress with num rules. Eg:
// start = 1 num = 2 will given you a single Ingress with 2 rules:
// Ingress {
//	 foo1.bar.com: /foo1
//	 foo2.bar.com: /foo2
// }
func createIngress(c *client.Client, ns string, start, num int) extensions.Ingress {
	ing := extensions.Ingress{
		ObjectMeta: api.ObjectMeta{
			Name:      fmt.Sprintf("%v%d", appPrefix, start),
			Namespace: ns,
		},
		Spec: extensions.IngressSpec{
			Backend: &extensions.IngressBackend{
				ServiceName: fmt.Sprintf("%v%d", appPrefix, start),
				ServicePort: util.NewIntOrStringFromInt(httpContainerPort),
			},
			Rules: []extensions.IngressRule{},
		},
	}
	for i := start; i < start+num; i++ {
		ing.Spec.Rules = append(ing.Spec.Rules, ruleByIndex(i))
	}
	Logf("Creating ingress %v", start)
	_, err := c.Extensions().Ingress(ns).Create(&ing)
	Expect(err).NotTo(HaveOccurred())
	return ing
}

// createApp will create a single RC and Svc. The Svc will match pods of the
// RC using the selector: 'name'=<name arg>
func createApp(c *client.Client, ns string, i int) {
	name := fmt.Sprintf("%v%d", appPrefix, i)
	l := map[string]string{}

	Logf("Creating svc %v", name)
	svc := svcByName(name, httpContainerPort)
	svc.Spec.Type = api.ServiceTypeNodePort
	_, err := c.Services(ns).Create(svc)
	Expect(err).NotTo(HaveOccurred())

	Logf("Creating rc %v", name)
	rc := rcByNamePort(name, 1, testImage, httpContainerPort, l)
	rc.Spec.Template.Spec.Containers[0].Args = []string{
		"--num=1",
		fmt.Sprintf("--start=%d", i),
		fmt.Sprintf("--prefix=%v", pathPrefix),
		fmt.Sprintf("--port=%d", httpContainerPort),
	}
	_, err = c.ReplicationControllers(ns).Create(rc)
	Expect(err).NotTo(HaveOccurred())
}

// gcloudUnmarshal unmarshals json output of gcloud into given out interface.
func gcloudUnmarshal(resource, regex string, out interface{}) {
	output, err := exec.Command("gcloud", "compute", resource, "list",
		fmt.Sprintf("--regex=%v", regex), "-q", "--format=json").CombinedOutput()
	if err != nil {
		Failf("Error unmarshalling gcloud output: %v", err)
	}
	if err := json.Unmarshal([]byte(output), out); err != nil {
		Failf("Error unmarshalling gcloud output: %v", err)
	}
}

func checkLeakedResources() error {
	msg := ""
	// Check all resources #16636.
	beList := []compute.BackendService{}
	gcloudUnmarshal("backend-services", "k8s-be-[0-9]+", &beList)
	if len(beList) != 0 {
		for _, b := range beList {
			msg += fmt.Sprintf("%v\n", b.Name)
		}
		return fmt.Errorf("Found backend services:\n%v", msg)
	}
	fwList := []compute.ForwardingRule{}
	gcloudUnmarshal("forwarding-rules", "k8s-fw-.*", &fwList)
	if len(fwList) != 0 {
		for _, f := range fwList {
			msg += fmt.Sprintf("%v\n", f.Name)
		}
		return fmt.Errorf("Found forwarding rules:\n%v", msg)
	}
	return nil
}

// run kubectl log on the L7 controller pod.
func kubectlLogLBController(c *client.Client) {
	selector := labels.SelectorFromSet(labels.Set(map[string]string{"name": "glbc"}))
	podList, err := c.Pods(api.NamespaceAll).List(selector, fields.Everything())
	if err != nil {
		Logf("Cannot log L7 controller output, error listing pods %v", err)
		return
	}
	if len(podList.Items) == 0 {
		Logf("Loadbalancer controller pod not found")
		return
	}
	for _, p := range podList.Items {
		Logf("\nLast 100 log lines of %v\n", p.Name)
		Logf(runKubectl("logs", p.Name, "--namespace=kube-system", "-c", lbContainerName, "--tail=100"))
	}
}

// dumpDebugAndFail dumps verbose debug output before failing.
func dumpDebugAndFail(err string, ns string, c *client.Client) {
	kubectlLogLBController(c)
	Logf("\nOutput of kubectl describe ing:\n")

	// TODO: runKubectl will hard fail if kubectl fails, swap it out for
	// something more befitting for a debug dumper.
	Logf(runKubectl("describe", "ing", fmt.Sprintf("--namespace=%v", ns)))
	Failf(err)
}

var _ = Describe("GCE L7 LoadBalancer Controller", func() {
	// These variables are initialized after framework's beforeEach.
	var ns string
	var client *client.Client
	var responseTimes, creationTimes []time.Duration

	framework := Framework{BaseName: "glbc"}

	BeforeEach(func() {
		// This test requires a GCE/GKE only cluster-addon
		SkipUnlessProviderIs("gce", "gke")
		framework.beforeEach()
		client = framework.Client
		ns = framework.Namespace.Name
		Expect(waitForRCPodsRunning(client, "kube-system", "glbc")).NotTo(HaveOccurred())
		Expect(checkLeakedResources()).NotTo(HaveOccurred())
		responseTimes = []time.Duration{}
		creationTimes = []time.Duration{}
	})

	AfterEach(func() {
		framework.afterEach()
		err := wait.Poll(lbPollInterval, lbPollTimeout, func() (bool, error) {
			if err := checkLeakedResources(); err != nil {
				Logf("Still waiting for glbc to cleanup: %v", err)
				return false, nil
			}
			return true, nil
		})
		Logf("Average creation time %+v, health check time %+v", creationTimes, responseTimes)
		if err != nil {
			Failf("Failed to cleanup GCE L7 resources.")
		}
		Logf("Successfully verified GCE L7 loadbalancer via Ingress.")
	})

	It("should create GCE L7 loadbalancers and verify Ingress", func() {
		// Create numApps apps, exposed via numIng Ingress each with 2 paths.
		// Eg with numApp=10, numIng=5:
		// apps: {foo-app-(0-10)}
		// ingress: {foo-app-(0, 2, 4, 6, 8)}
		// paths:
		//  ingress foo-app-0:
		//	  default1.bar.com
		//	  foo0.bar.com: /foo0
		//	  foo1.bar.com: /foo1
		if numApps < numIng {
			Failf("Need more apps than Ingress")
		}
		appsPerIngress := numApps / numIng
		By(fmt.Sprintf("Creating %d rcs + svc, and %d apps per Ingress", numApps, appsPerIngress))
		for appID := 0; appID < numApps; appID = appID + appsPerIngress {
			// Creates appsPerIngress apps, then creates one Ingress with paths to all the apps.
			for j := appID; j < appID+appsPerIngress; j++ {
				createApp(client, ns, j)
			}
			createIngress(client, ns, appID, appsPerIngress)
		}

		ings, err := client.Extensions().Ingress(ns).List(
			labels.Everything(), fields.Everything())
		Expect(err).NotTo(HaveOccurred())

		for _, ing := range ings.Items {
			// Wait for the loadbalancer IP.
			start := time.Now()
			address, err := waitForIngressAddress(client, ing.Namespace, ing.Name, lbPollTimeout)
			if err != nil {
				dumpDebugAndFail(fmt.Sprintf("Ingress failed to acquire an IP address within %v", lbPollTimeout), ns, client)
			}
			Expect(err).NotTo(HaveOccurred())
			By(fmt.Sprintf("Found address %v for ingress %v, took %v to come online",
				address, ing.Name, time.Since(start)))
			creationTimes = append(creationTimes, time.Since(start))

			// Check that all rules respond to a simple GET.
			for _, rules := range ing.Spec.Rules {
				// As of Kubernetes 1.1 we only support HTTP Ingress.
				if rules.IngressRuleValue.HTTP == nil {
					continue
				}
				for _, p := range rules.IngressRuleValue.HTTP.Paths {
					route := fmt.Sprintf("http://%v%v", address, p.Path)
					Logf("Testing route %v host %v with simple GET", route, rules.Host)

					GETStart := time.Now()
					var lastBody string
					pollErr := wait.Poll(lbPollInterval, lbPollTimeout, func() (bool, error) {
						var err error
						lastBody, err = simpleGET(http.DefaultClient, route, rules.Host)
						if err != nil {
							Logf("host %v path %v: %v", rules.Host, route, err)
							return false, nil
						}
						return true, nil
					})
					if pollErr != nil {
						dumpDebugAndFail(fmt.Sprintf("Failed to execute a successful GET within %v, Last response body for %v, host %v:\n%v\n\n%v",
							lbPollTimeout, route, rules.Host, lastBody, pollErr), ns, client)
					}
					rt := time.Since(GETStart)
					By(fmt.Sprintf("Route %v host %v took %v to respond", route, rules.Host, rt))
					responseTimes = append(responseTimes, rt)
				}
			}
		}
		// In most cases slow loadbalancer creation/startup translates directly to
		// GCE api sluggishness. However this might be because of something the
		// controller is doing, eg: maxing out QPS by repeated polling.
		sort.Sort(timeSlice(creationTimes))
		perc50 := creationTimes[len(creationTimes)/2]
		if perc50 > expectedLBCreationTime {
			dumpDebugAndFail(fmt.Sprintf("Average creation time is too high: %+v", creationTimes), ns, client)
		}
		sort.Sort(timeSlice(responseTimes))
		perc50 = responseTimes[len(responseTimes)/2]
		if perc50 > expectedLBHealthCheckTime {
			dumpDebugAndFail(fmt.Sprintf("Average startup time is too high: %+v", responseTimes), ns, client)
		}
	})
})
<|endoftext|>


<commit_before>// Copyright ©2015 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:generate ./genBlas.pl

// Ensure changes made to blas/cgo are reflected in blas/native where relevant.

/*
Package cgo provides bindings to a C BLAS library. This wrapper interface
panics when the input arguments are invalid as per the standard, for example
if a vector increment is zero. Please note that the treatment of NaN values
is not specified, and differs among the BLAS implementations.
github.com/gonum/blas/blas64 provides helpful wrapper functions to the BLAS
interface. The rest of this text describes the layout of the data for the input types.

Please note that in the function documentation, x[i] refers to the i^th element
of the vector, which will be different from the i^th element of the slice if
incX != 1.

Vector arguments are effectively strided slices. They have two input arguments,
a number of elements, n, and an increment, incX. The increment specifies the
distance between elements of the vector. The actual Go slice may be longer
than necessary.
The increment may be positive or negative, except in functions with only
a single vector argument where the increment may only be positive. If the increment
is negative, s[0] is the last element in the slice. Note that this is not the same
as counting backward from the end of the slice, as len(s) may be longer than
necessary. So, for example, if n = 5 and incX = 3, the elements of s are
	[0 * * 1 * * 2 * * 3 * * 4 * * * ...]
where ∗ elements are never accessed. If incX = -3, the same elements are
accessed, just in reverse order (4, 3, 2, 1, 0).

Dense matrices are specified by a number of rows, a number of columns, and a stride.
The stride specifies the number of entries in the slice between the first element
of successive rows. The stride must be at least as large as the number of columns
but may be longer.
	[a00 ... a0n a0* ... a1stride-1 a21 ... amn am* ... amstride-1]
Thus, dense[i*ld + j] refers to the {i, j}th element of the matrix.

Symmetric and triangular matrices (non-packed) are stored identically to Dense,
except that only elements in one triangle of the matrix are accessed.

Packed symmetric and packed triangular matrices are laid out with the entries
condensed such that all of the unreferenced elements are removed. So, the upper triangular
matrix
  [
    1  2  3
    0  4  5
    0  0  6
  ]
and the lower-triangular matrix
  [
    1  0  0
    2  3  0
    4  5  6
  ]
will both be compacted as [1 2 3 4 5 6]. The (i, j) element of the original
dense matrix can be found at element i*n - (i-1)*i/2 + j for upper triangular,
and at element i * (i+1) /2 + j for lower triangular.

Banded matrices are laid out in a compact format, constructed by removing the
zeros in the rows and aligning the diagonals. For example, the matrix
  [
    1  2  3  0  0  0
    4  5  6  7  0  0
    0  8  9 10 11  0
    0  0 12 13 14 15
    0  0  0 16 17 18
    0  0  0  0 19 20
  ]

implicitly becomes (∗ entries are never accessed)
  [
     *  1  2  3
     4  5  6  7
     8  9 10 11
    12 13 14 15
    16 17 18  *
    19 20  *  *
  ]
which is given to the BLAS routine as [∗ 1 2 3 4 ...].

See http://www.crest.iu.edu/research/mtl/reference/html/banded.html
for more information
*/
package cgo
<commit_msg>cgo: note that BLAS is underspecified<commit_after>// Copyright ©2015 The Gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:generate ./genBlas.pl

// Ensure changes made to blas/cgo are reflected in blas/native where relevant.

/*
Package cgo provides bindings to a C BLAS library. This wrapper interface
panics when the input arguments are invalid as per the standard, for example
if a vector increment is zero. Please note that the treatment of NaN values
is not specified, and differs among the BLAS implementations.
github.com/gonum/blas/blas64 provides helpful wrapper functions to the BLAS
interface. The rest of this text describes the layout of the data for the input types.

Please note that in the function documentation, x[i] refers to the i^th element
of the vector, which will be different from the i^th element of the slice if
incX != 1.

Vector arguments are effectively strided slices. They have two input arguments,
a number of elements, n, and an increment, incX. The increment specifies the
distance between elements of the vector. The actual Go slice may be longer
than necessary.
The increment may be positive or negative, except in functions with only
a single vector argument where the increment may only be positive. If the increment
is negative, s[0] is the last element in the slice. Note that this is not the same
as counting backward from the end of the slice, as len(s) may be longer than
necessary. So, for example, if n = 5 and incX = 3, the elements of s are
	[0 * * 1 * * 2 * * 3 * * 4 * * * ...]
where ∗ elements are never accessed. If incX = -3, the same elements are
accessed, just in reverse order (4, 3, 2, 1, 0).

Dense matrices are specified by a number of rows, a number of columns, and a stride.
The stride specifies the number of entries in the slice between the first element
of successive rows. The stride must be at least as large as the number of columns
but may be longer.
	[a00 ... a0n a0* ... a1stride-1 a21 ... amn am* ... amstride-1]
Thus, dense[i*ld + j] refers to the {i, j}th element of the matrix.

Symmetric and triangular matrices (non-packed) are stored identically to Dense,
except that only elements in one triangle of the matrix are accessed.

Packed symmetric and packed triangular matrices are laid out with the entries
condensed such that all of the unreferenced elements are removed. So, the upper triangular
matrix
  [
    1  2  3
    0  4  5
    0  0  6
  ]
and the lower-triangular matrix
  [
    1  0  0
    2  3  0
    4  5  6
  ]
will both be compacted as [1 2 3 4 5 6]. The (i, j) element of the original
dense matrix can be found at element i*n - (i-1)*i/2 + j for upper triangular,
and at element i * (i+1) /2 + j for lower triangular.

Banded matrices are laid out in a compact format, constructed by removing the
zeros in the rows and aligning the diagonals. For example, the matrix
  [
    1  2  3  0  0  0
    4  5  6  7  0  0
    0  8  9 10 11  0
    0  0 12 13 14 15
    0  0  0 16 17 18
    0  0  0  0 19 20
  ]

implicitly becomes (∗ entries are never accessed)
  [
     *  1  2  3
     4  5  6  7
     8  9 10 11
    12 13 14 15
    16 17 18  *
    19 20  *  *
  ]
which is given to the BLAS routine as [∗ 1 2 3 4 ...].

See http://www.crest.iu.edu/research/mtl/reference/html/banded.html
for more information

BUG(cgo): The cgo package is intrinsically dependent on the underlying C
implementation. The BLAS standard is silent on a number of behaviours, including
but not limited to how NaN values are treated. For this reason the result of
computations performed by the cgo BLAS package may disagree with the results
produced by native BLAS package. The cgo package is tested agains OpenBLAS; use
of other backing BLAS C libraries may cause result in test failure because of
this.
*/
package cgo
<|endoftext|>


<commit_before>package msgpack

import (
	"bytes"
	"io"
	"reflect"
	"testing"
)

type testDecStruct struct {
	IF  interface{}
	B   bool
	S   string
	I   int
	U   uint
	F64 float64
	SS  []string
	M   map[string]interface{}
}

type testDecEmptyStruct struct {
	B   bool   `empty:"true"`
	S   string `empty:"blank"`
	I   int    `empty:"1234"`
	I8  int8   `empty:"45"`
	I32 int32  `empty:"6789"`
}

type testDecArrayStruct struct {
	I int `msgpack:",array"`
	S string
}

func ptrInt(i int) *int {
	return &i
}

func TestDecode(t *testing.T) {
	t.Parallel()

	tests := map[string]struct {
		// arg is argument for Decode().
		arg func() interface{}
		// data is data to decode.
		data []interface{}
		// expected is the expected decoded value.
		expected interface{}
	}{
		"Int/Int64": {
			arg:      func() interface{} { return new(int) },
			data:     []interface{}{int64(1234)},
			expected: int(1234),
		},
		"Int/Float64": {
			arg:      func() interface{} { return new(int) },
			data:     []interface{}{float64(4321)},
			expected: int(4321),
		},
		"Int/Uint64": {
			arg:      func() interface{} { return new(int) },
			data:     []interface{}{uint64(5678)},
			expected: int(5678),
		},
		"Uint/Int64": {
			arg:      func() interface{} { return new(uint) },
			data:     []interface{}{int64(1234)},
			expected: uint(1234),
		},
		"Uint/Float64": {
			arg:      func() interface{} { return new(uint) },
			data:     []interface{}{float64(4321)},
			expected: uint(4321),
		},
		"Uint/Uint64": {
			arg:      func() interface{} { return new(uint) },
			data:     []interface{}{uint64(5678)},
			expected: uint(5678),
		},
		"Float64/Int64": {
			arg:      func() interface{} { return new(float64) },
			data:     []interface{}{int64(1234)},
			expected: float64(1234),
		},
		"Float64/Float64": {
			arg:      func() interface{} { return new(float64) },
			data:     []interface{}{float64(4321)},
			expected: float64(4321),
		},
		"Float64/Uint64": {
			arg:      func() interface{} { return new(float64) },
			data:     []interface{}{uint64(5678)},
			expected: float64(5678),
		},
		"Bool/True": {
			arg:      func() interface{} { return new(bool) },
			data:     []interface{}{true},
			expected: true,
		},
		"Bool/False": {
			arg:      func() interface{} { return new(bool) },
			data:     []interface{}{false},
			expected: false,
		},
		"String/String": {
			arg:      func() interface{} { return new(string) },
			data:     []interface{}{"hello"},
			expected: "hello",
		},
		"String/Bytes": {
			arg:      func() interface{} { return new(string) },
			data:     []interface{}{[]byte("world")},
			expected: "world",
		},
		"Bytes/String": {
			arg:      func() interface{} { return new([]byte) },
			data:     []interface{}{"hello"},
			expected: []byte("hello"),
		},
		"Bytes/Bytes": {
			arg:      func() interface{} { return new([]byte) },
			data:     []interface{}{[]byte("world")},
			expected: []byte("world"),
		},
		"Pointer/Int64": {
			arg:      func() interface{} { return new(*int) },
			data:     []interface{}{int64(-1)},
			expected: ptrInt(-1),
		},
		"Interface/Int64Pointer": {
			arg:      func() interface{} { return &testDecStruct{IF: ptrInt(1234)} },
			data:     []interface{}{mapLen(1), "IF", int64(5678)},
			expected: testDecStruct{IF: ptrInt(5678)},
		},
		"Interface/StringSlice": {
			arg:  func() interface{} { return &testDecStruct{IF: []string{"hello", "world"}} },
			data: []interface{}{mapLen(1), "IF", arrayLen(1), "foo"},
			expected: testDecStruct{
				IF: []string{"foo", ""},
			},
		},
		"StringSlice/ArrayLen/1": {
			arg:      func() interface{} { return []string{""} },
			data:     []interface{}{arrayLen(2), "foo", "bar"},
			expected: []string{"foo"},
		},
		"StringSlice/ArrayLen/2/ValueValue": {
			arg:      func() interface{} { return []string{"", ""} },
			data:     []interface{}{arrayLen(2), "foo", "bar"},
			expected: []string{"foo", "bar"},
		},
		"StringSlice/ArrayLen/2/ValueEmpty": {
			arg:      func() interface{} { return []string{"", "bar"} },
			data:     []interface{}{arrayLen(1), "foo"},
			expected: []string{"foo", ""},
		},
		"StringSlice/ArrayLen/Make/2": {
			arg:      func() interface{} { x := make([]string, 1); return &x },
			data:     []interface{}{arrayLen(2), "foo", "bar"},
			expected: []string{"foo", "bar"},
		},
		"StringSlice/ArrayLen/Make/3": {
			arg:      func() interface{} { x := make([]string, 3); return &x },
			data:     []interface{}{arrayLen(2), "foo", "bar"},
			expected: []string{"foo", "bar"},
		},
		"StringSlicePointer/ArrayLen/2": {
			arg:      func() interface{} { return new([]string) },
			data:     []interface{}{arrayLen(2), "foo", "bar"},
			expected: []string{"foo", "bar"},
		},
		"StringArray/ArrayLen/3/ValueValueEmpty": {
			arg:      func() interface{} { x := [...]string{"foo", "bar", "quux"}; return &x },
			data:     []interface{}{arrayLen(2), "hello", "world"},
			expected: [...]string{"hello", "world", ""},
		},
		"StringArray/ArrayLen/1/Value": {
			arg:      func() interface{} { x := [...]string{"foo"}; return &x },
			data:     []interface{}{arrayLen(2), "hello", "world"},
			expected: [...]string{"hello"},
		},
		"StructArray/Int64": {
			arg:      func() interface{} { return new(testDecArrayStruct) },
			data:     []interface{}{arrayLen(2), int64(22), "skidoo"},
			expected: testDecArrayStruct{I: 22, S: "skidoo"},
		},
		"Map/StringString": {
			arg:      func() interface{} { return make(map[string]string) },
			data:     []interface{}{mapLen(1), "foo", "bar"},
			expected: map[string]string{"foo": "bar"},
		},
		"MapPointer/StringString": {
			arg:      func() interface{} { return new(map[string]string) },
			data:     []interface{}{mapLen(1), "foo", "bar"},
			expected: map[string]string{"foo": "bar"},
		},
		"Interface/Extensions/ExtensionValue": {
			arg:      func() interface{} { return new(interface{}) },
			data:     []interface{}{extension{0, "hello"}},
			expected: extensionValue{kind: 0, data: []byte("hello")},
		},
		"Interface/Extensions/TestExtension": {
			arg:      func() interface{} { return new(interface{}) },
			data:     []interface{}{extension{1, "hello"}},
			expected: testExtension1{data: []byte("hello")},
		},
		"TestExtension/Extensions": {
			arg:      func() interface{} { return new(testExtension1) },
			data:     []interface{}{extension{1, "hello"}},
			expected: testExtension1{data: []byte("hello")},
		},
		"TestDecEmptyStruct/Empty/blank": {
			arg:      func() interface{} { return &testDecEmptyStruct{} },
			data:     []interface{}{mapLen(0)},
			expected: testDecEmptyStruct{B: true, S: "blank", I: 1234, I8: 45, I32: 6789},
		},
		"TestDecEmptyStruct/Empty/NotBlank": {
			arg:      func() interface{} { return &testDecEmptyStruct{} },
			data:     []interface{}{mapLen(1), "S", "not blank"},
			expected: testDecEmptyStruct{B: true, S: "not blank", I: 1234, I8: 45, I32: 6789},
		},
		// TODO(zchee): test errors like the following:
		// "Errors": {
		// 	arg:      func() interface{} { return &testDecStruct{I: 1234} },
		// 	data:     []interface{}{mapLen(1), "I", int64(5678)},
		// 	expected: testDecStruct{I: 1234},
		// },
	}
	for name, tt := range tests {
		tt := tt
		t.Run(name, func(t *testing.T) {
			t.Parallel()

			data, err := pack(tt.data...)
			if err != nil {
				t.Fatalf("pack(%+v) returned error %v", tt.data, err)
			}
			dec := NewDecoder(bytes.NewReader(data))
			buf, _ := dec.r.Peek(0)

			dec.SetExtensions(testExtensionMap)

			arg := tt.arg()
			if err := dec.Decode(arg); err != nil {
				t.Fatalf("decode(%+v, %T) returned error %v", tt.data, arg, err)
			}

			// scribble on bufio.Reader buffer to test that Decoder.Bytes() return value is copied
			buf = buf[:cap(buf)]
			for i := range buf {
				buf[i] = 0xff
			}

			rv := reflect.ValueOf(arg)
			if rv.Kind() == reflect.Ptr {
				rv = rv.Elem()
			}
			v := rv.Interface()
			if !reflect.DeepEqual(v, tt.expected) {
				t.Fatalf("decode(%+v, %T) returned %#v, want %#v", tt.data, arg, v, tt.expected)
			}

			// Decode should read to EOF.
			if _, err := dec.r.ReadByte(); err != io.EOF {
				t.Fatalf("decode(%+v, %T) did not read to EOF", tt.data, arg)
			}
		})
	}
}
<commit_msg>msgpack: add more testcases to TestDecode<commit_after>package msgpack

import (
	"bytes"
	"io"
	"reflect"
	"testing"
)

type testDecStruct struct {
	IF  interface{}
	B   bool
	S   string
	I   int
	U   uint
	F64 float64
	SS  []string
	M   map[string]interface{}
}

type testDecEmptyStruct struct {
	B   bool   `empty:"true"`
	S   string `empty:"blank"`
	I   int    `empty:"1234"`
	I8  int8   `empty:"45"`
	I32 int32  `empty:"6789"`
}

type testDecArrayStruct struct {
	I int `msgpack:",array"`
	S string
}

func ptrInt(i int) *int {
	return &i
}

func TestDecode(t *testing.T) {
	t.Parallel()

	tests := map[string]struct {
		// arg is argument for Decode().
		arg func() interface{}
		// data is data to decode.
		data []interface{}
		// expected is the expected decoded value.
		expected interface{}
	}{
		"Bool/Bool/True": {
			arg:      func() interface{} { return new(bool) },
			data:     []interface{}{true},
			expected: true,
		},
		"Bool/Bool/False": {
			arg:      func() interface{} { return new(bool) },
			data:     []interface{}{false},
			expected: false,
		},
		"Bool/Int/True": {
			arg:      func() interface{} { return new(bool) },
			data:     []interface{}{int64(1)},
			expected: true,
		},
		"Bool/Int/False": {
			arg:      func() interface{} { return new(bool) },
			data:     []interface{}{int64(0)},
			expected: false,
		},
		"Bool/Uint/True": {
			arg:      func() interface{} { return new(bool) },
			data:     []interface{}{uint64(1)},
			expected: true,
		},
		"Bool/Uint/False": {
			arg:      func() interface{} { return new(bool) },
			data:     []interface{}{uint64(0)},
			expected: false,
		},
		"Int/Int64": {
			arg:      func() interface{} { return new(int) },
			data:     []interface{}{int64(1234)},
			expected: int(1234),
		},
		"Int/Float64": {
			arg:      func() interface{} { return new(int) },
			data:     []interface{}{float64(4321)},
			expected: int(4321),
		},
		"Int/Uint64": {
			arg:      func() interface{} { return new(int) },
			data:     []interface{}{uint64(5678)},
			expected: int(5678),
		},
		"Uint/Int64": {
			arg:      func() interface{} { return new(uint) },
			data:     []interface{}{int64(1234)},
			expected: uint(1234),
		},
		"Uint/Float64": {
			arg:      func() interface{} { return new(uint) },
			data:     []interface{}{float64(4321)},
			expected: uint(4321),
		},
		"Uint/Uint64": {
			arg:      func() interface{} { return new(uint) },
			data:     []interface{}{uint64(5678)},
			expected: uint(5678),
		},
		"Float64/Int64": {
			arg:      func() interface{} { return new(float64) },
			data:     []interface{}{int64(1234)},
			expected: float64(1234),
		},
		"Float64/Float64": {
			arg:      func() interface{} { return new(float64) },
			data:     []interface{}{float64(4321)},
			expected: float64(4321),
		},
		"Float64/Uint64": {
			arg:      func() interface{} { return new(float64) },
			data:     []interface{}{uint64(5678)},
			expected: float64(5678),
		},
		"String/String": {
			arg:      func() interface{} { return new(string) },
			data:     []interface{}{"hello"},
			expected: "hello",
		},
		"String/Bytes": {
			arg:      func() interface{} { return new(string) },
			data:     []interface{}{[]byte("world")},
			expected: "world",
		},
		"Bytes/String": {
			arg:      func() interface{} { return new([]byte) },
			data:     []interface{}{"hello"},
			expected: []byte("hello"),
		},
		"Bytes/Bytes": {
			arg:      func() interface{} { return new([]byte) },
			data:     []interface{}{[]byte("world")},
			expected: []byte("world"),
		},
		"Bytes/Nil": {
			arg:      func() interface{} { return new([]byte) },
			data:     []interface{}{nil},
			expected: []byte(nil),
		},
		"Interface/Int64Pointer": {
			arg:  func() interface{} { return &testDecStruct{IF: ptrInt(1234)} },
			data: []interface{}{mapLen(1), "IF", int64(5678)},
			expected: testDecStruct{
				IF: ptrInt(5678),
			},
		},
		"Interface/StringSlice": {
			arg:  func() interface{} { return &testDecStruct{IF: []string{"hello", "world"}} },
			data: []interface{}{mapLen(1), "IF", arrayLen(1), "foo"},
			expected: testDecStruct{
				IF: []string{"foo", ""},
			},
		},
		"StringSlice/ArrayLen/1": {
			arg:      func() interface{} { return []string{""} },
			data:     []interface{}{arrayLen(2), "foo", "bar"},
			expected: []string{"foo"},
		},
		"StringSlice/ArrayLen/2/ValueValue": {
			arg:      func() interface{} { return []string{"", ""} },
			data:     []interface{}{arrayLen(2), "foo", "bar"},
			expected: []string{"foo", "bar"},
		},
		"StringSlice/ArrayLen/2/ValueEmpty": {
			arg:      func() interface{} { return []string{"", "bar"} },
			data:     []interface{}{arrayLen(1), "foo"},
			expected: []string{"foo", ""},
		},
		"StringSlice/ArrayLen/Make/2": {
			arg:      func() interface{} { x := make([]string, 1); return &x },
			data:     []interface{}{arrayLen(2), "foo", "bar"},
			expected: []string{"foo", "bar"},
		},
		"StringSlice/ArrayLen/Make/3": {
			arg:      func() interface{} { x := make([]string, 3); return &x },
			data:     []interface{}{arrayLen(2), "foo", "bar"},
			expected: []string{"foo", "bar"},
		},
		"StringSlicePointer/ArrayLen/2": {
			arg:      func() interface{} { return new([]string) },
			data:     []interface{}{arrayLen(2), "foo", "bar"},
			expected: []string{"foo", "bar"},
		},
		"StringArray/ArrayLen/3/ValueValueEmpty": {
			arg:      func() interface{} { x := [...]string{"foo", "bar", "quux"}; return &x },
			data:     []interface{}{arrayLen(2), "hello", "world"},
			expected: [...]string{"hello", "world", ""},
		},
		"StringArray/ArrayLen/1/Value": {
			arg:      func() interface{} { x := [...]string{"foo"}; return &x },
			data:     []interface{}{arrayLen(2), "hello", "world"},
			expected: [...]string{"hello"},
		},
		"StructArray/Int64": {
			arg:      func() interface{} { return new(testDecArrayStruct) },
			data:     []interface{}{arrayLen(2), int64(22), "skidoo"},
			expected: testDecArrayStruct{I: 22, S: "skidoo"},
		},
		"Map/StringString": {
			arg:      func() interface{} { return make(map[string]string) },
			data:     []interface{}{mapLen(1), "foo", "bar"},
			expected: map[string]string{"foo": "bar"},
		},
		"MapPointer/StringString": {
			arg:      func() interface{} { return new(map[string]string) },
			data:     []interface{}{mapLen(1), "foo", "bar"},
			expected: map[string]string{"foo": "bar"},
		},
		"Pointer/Int64": {
			arg:      func() interface{} { return new(*int) },
			data:     []interface{}{int64(-1)},
			expected: ptrInt(-1),
		},
		"Interface/Extensions/ExtensionValue": {
			arg:      func() interface{} { return new(interface{}) },
			data:     []interface{}{extension{0, "hello"}},
			expected: extensionValue{kind: 0, data: []byte("hello")},
		},
		"Interface/Extensions/TestExtension": {
			arg:      func() interface{} { return new(interface{}) },
			data:     []interface{}{extension{1, "hello"}},
			expected: testExtension1{data: []byte("hello")},
		},
		"TestExtension/Extensions": {
			arg:      func() interface{} { return new(testExtension1) },
			data:     []interface{}{extension{1, "hello"}},
			expected: testExtension1{data: []byte("hello")},
		},
		"TestDecEmptyStruct/Empty/blank": {
			arg:      func() interface{} { return &testDecEmptyStruct{} },
			data:     []interface{}{mapLen(0)},
			expected: testDecEmptyStruct{B: true, S: "blank", I: 1234, I8: 45, I32: 6789},
		},
		"TestDecEmptyStruct/Empty/NotBlank": {
			arg:      func() interface{} { return &testDecEmptyStruct{} },
			data:     []interface{}{mapLen(1), "S", "not blank"},
			expected: testDecEmptyStruct{B: true, S: "not blank", I: 1234, I8: 45, I32: 6789},
		},
		// TODO(zchee): test errors like the following:
		// "Errors": {
		// 	arg:      func() interface{} { return &testDecStruct{I: 1234} },
		// 	data:     []interface{}{mapLen(1), "I", int64(5678)},
		// 	expected: testDecStruct{I: 1234},
		// },
	}
	for name, tt := range tests {
		tt := tt
		t.Run(name, func(t *testing.T) {
			t.Parallel()

			data, err := pack(tt.data...)
			if err != nil {
				t.Fatalf("pack(%+v) returned error %v", tt.data, err)
			}
			dec := NewDecoder(bytes.NewReader(data))
			buf, _ := dec.r.Peek(0)

			dec.SetExtensions(testExtensionMap)

			arg := tt.arg()
			if err := dec.Decode(arg); err != nil {
				t.Fatalf("decode(%+v, %T) returned error %v", tt.data, arg, err)
			}

			// scribble on bufio.Reader buffer to test that Decoder.Bytes() return value is copied
			buf = buf[:cap(buf)]
			for i := range buf {
				buf[i] = 0xff
			}

			rv := reflect.ValueOf(arg)
			if rv.Kind() == reflect.Ptr {
				rv = rv.Elem()
			}
			v := rv.Interface()
			if !reflect.DeepEqual(v, tt.expected) {
				t.Fatalf("decode(%+v, %T) returned %#v, want %#v", tt.data, arg, v, tt.expected)
			}

			// Decode should read to EOF.
			if _, err := dec.r.ReadByte(); err != io.EOF {
				t.Fatalf("decode(%+v, %T) did not read to EOF", tt.data, arg)
			}
		})
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package stackdriver

import (
	"fmt"
	"sync"

	"github.com/go-kit/kit/log"
	"github.com/jkohen/prometheus/retrieval"
	"github.com/prometheus/prometheus/config"
)

type Storage struct {
	logger log.Logger
	cfg    *StackdriverConfig
	mtx    sync.RWMutex

	// For writes
	queues []*QueueManager
}

func NewStorage(logger log.Logger, cfg *StackdriverConfig) *Storage {
	return &Storage{
		logger: logger,
		cfg:    cfg,
	}
}

// Appender implements the retrieval.Appendable interface.
func (s *Storage) Appender() (retrieval.Appender, error) {
	return s, nil
}

// Add implements the retrieval.Appender interface.
func (s *Storage) Add(metricFamily *retrieval.MetricFamily) error {
	s.mtx.RLock()
	defer s.mtx.RUnlock()
	for _, q := range s.queues {
		q.Append(metricFamily)
	}
	return nil
}

// Close closes the storage and all its underlying resources.
func (s *Storage) Close() error {
	s.mtx.Lock()
	defer s.mtx.Unlock()

	for _, q := range s.queues {
		q.Stop()
	}

	return nil
}

// ApplyConfig updates the state as the new config requires.
func (s *Storage) ApplyConfig(conf *config.Config) error {
	// TODO(jkohen): try extracting this from the credentials
	var projectId string
	if value, ok := conf.GlobalConfig.ExternalLabels[ProjectIdLabel]; !ok {
		return fmt.Errorf(
			"the Stackdriver remote writer requires an external label '%s' in its configuration, and it must contain a project id or number",
			ProjectIdLabel)
	} else {
		projectId = fmt.Sprintf("projects/%v", value)
	}

	s.mtx.Lock()
	defer s.mtx.Unlock()

	// Update write queues

	newQueues := []*QueueManager{}
	// TODO: we should only stop & recreate queues which have changes,
	// as this can be quite disruptive.
	for i, rwConf := range conf.RemoteWriteConfigs {
		c, err := NewClient(i, &ClientConfig{
			Logger:    s.logger,
			ProjectId: projectId,
			URL:       rwConf.URL,
			Timeout:   rwConf.RemoteTimeout,
			Auth:      true,
		})
		if err != nil {
			return err
		}
		newQueues = append(newQueues, NewQueueManager(
			s.logger,
			rwConf.QueueConfig,
			conf.GlobalConfig.ExternalLabels,
			rwConf.WriteRelabelConfigs,
			c,
			s.cfg,
		))
	}

	for _, q := range s.queues {
		q.Stop()
	}

	s.queues = newQueues
	for _, q := range s.queues {
		q.Start()
	}

	return nil
}
<commit_msg>Added URL query param to disable auth, for load testing.<commit_after>/*
Copyright 2018 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package stackdriver

import (
	"fmt"
	"strconv"
	"sync"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"github.com/jkohen/prometheus/retrieval"
	"github.com/prometheus/prometheus/config"
)

type Storage struct {
	logger log.Logger
	cfg    *StackdriverConfig
	mtx    sync.RWMutex

	// For writes
	queues []*QueueManager
}

func NewStorage(logger log.Logger, cfg *StackdriverConfig) *Storage {
	return &Storage{
		logger: logger,
		cfg:    cfg,
	}
}

// Appender implements the retrieval.Appendable interface.
func (s *Storage) Appender() (retrieval.Appender, error) {
	return s, nil
}

// Add implements the retrieval.Appender interface.
func (s *Storage) Add(metricFamily *retrieval.MetricFamily) error {
	s.mtx.RLock()
	defer s.mtx.RUnlock()
	for _, q := range s.queues {
		q.Append(metricFamily)
	}
	return nil
}

// Close closes the storage and all its underlying resources.
func (s *Storage) Close() error {
	s.mtx.Lock()
	defer s.mtx.Unlock()

	for _, q := range s.queues {
		q.Stop()
	}

	return nil
}

// ApplyConfig updates the state as the new config requires.
func (s *Storage) ApplyConfig(conf *config.Config) error {
	// TODO(jkohen): try extracting this from the credentials
	var projectId string
	if value, ok := conf.GlobalConfig.ExternalLabels[ProjectIdLabel]; !ok {
		return fmt.Errorf(
			"the Stackdriver remote writer requires an external label '%s' in its configuration, and it must contain a project id or number",
			ProjectIdLabel)
	} else {
		projectId = fmt.Sprintf("projects/%v", value)
	}

	s.mtx.Lock()
	defer s.mtx.Unlock()

	// Update write queues

	newQueues := []*QueueManager{}
	// TODO: we should only stop & recreate queues which have changes,
	// as this can be quite disruptive.
	for i, rwConf := range conf.RemoteWriteConfigs {
		useAuth, err := strconv.ParseBool(rwConf.URL.Query().Get("auth"))
		if err != nil {
			useAuth = true // Default to auth enabled.
		}
		level.Info(s.logger).Log(
			"msg", "is auth enabled",
			"auth", useAuth,
			"url", rwConf.URL.String())
		c, err := NewClient(i, &ClientConfig{
			Logger:    s.logger,
			ProjectId: projectId,
			URL:       rwConf.URL,
			Timeout:   rwConf.RemoteTimeout,
			Auth:      useAuth,
		})
		if err != nil {
			return err
		}
		newQueues = append(newQueues, NewQueueManager(
			s.logger,
			rwConf.QueueConfig,
			conf.GlobalConfig.ExternalLabels,
			rwConf.WriteRelabelConfigs,
			c,
			s.cfg,
		))
	}

	for _, q := range s.queues {
		q.Stop()
	}

	s.queues = newQueues
	for _, q := range s.queues {
		q.Start()
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ast

import "fmt"

// A Visitor's Visit method is invoked for each node encountered by Walk.
// If the result visitor w is not nil, Walk visits each of the children
// of node with the visitor w, followed by a call of w.Visit(nil).
type Visitor interface {
	Visit(node interface{}) (w Visitor)
}

func walkIdent(v Visitor, x *Ident) {
	if x != nil {
		Walk(v, x)
	}
}

func walkCommentGroup(v Visitor, g *CommentGroup) {
	if g != nil {
		Walk(v, g)
	}
}

func walkBlockStmt(v Visitor, b *BlockStmt) {
	if b != nil {
		Walk(v, b)
	}
}

// Walk traverses an AST in depth-first order: If node != nil, it
// invokes v.Visit(node). If the visitor w returned by v.Visit(node) is
// not nil, Walk visits each of the children of node with the visitor w,
// followed by a call of w.Visit(nil).
//
// Walk may be called with any of the named ast node types. It also
// accepts arguments of type []*Field, []*Ident, []Expr, []Stmt and []Decl;
// the respective children are the slice elements.
//
func Walk(v Visitor, node interface{}) {
	if node == nil {
		return
	}
	if v = v.Visit(node); v == nil {
		return
	}

	// walk children
	// (the order of the cases matches the order
	// of the corresponding declaration in ast.go)
	switch n := node.(type) {
	// Comments and fields
	case *Comment:
		// nothing to do

	case *CommentGroup:
		for _, c := range n.List {
			Walk(v, c)
		}

	case *Field:
		walkCommentGroup(v, n.Doc)
		Walk(v, n.Names)
		Walk(v, n.Type)
		Walk(v, n.Tag)
		walkCommentGroup(v, n.Comment)

	case *FieldList:
		for _, f := range n.List {
			Walk(v, f)
		}

	// Expressions
	case *BadExpr, *Ident, *Ellipsis, *BasicLit:
		// nothing to do

	case *FuncLit:
		if n != nil {
			Walk(v, n.Type)
		}
		walkBlockStmt(v, n.Body)

	case *CompositeLit:
		Walk(v, n.Type)
		Walk(v, n.Elts)

	case *ParenExpr:
		Walk(v, n.X)

	case *SelectorExpr:
		Walk(v, n.X)
		walkIdent(v, n.Sel)

	case *IndexExpr:
		Walk(v, n.X)
		Walk(v, n.Index)

	case *SliceExpr:
		Walk(v, n.X)
		Walk(v, n.Index)
		Walk(v, n.End)

	case *TypeAssertExpr:
		Walk(v, n.X)
		Walk(v, n.Type)

	case *CallExpr:
		Walk(v, n.Fun)
		Walk(v, n.Args)

	case *StarExpr:
		Walk(v, n.X)

	case *UnaryExpr:
		Walk(v, n.X)

	case *BinaryExpr:
		Walk(v, n.X)
		Walk(v, n.Y)

	case *KeyValueExpr:
		Walk(v, n.Key)
		Walk(v, n.Value)

	// Types
	case *ArrayType:
		Walk(v, n.Len)
		Walk(v, n.Elt)

	case *StructType:
		Walk(v, n.Fields)

	case *FuncType:
		Walk(v, n.Params)
		if n.Results != nil {
			Walk(v, n.Results)
		}

	case *InterfaceType:
		Walk(v, n.Methods)

	case *MapType:
		Walk(v, n.Key)
		Walk(v, n.Value)

	case *ChanType:
		Walk(v, n.Value)

	// Statements
	case *BadStmt:
		// nothing to do

	case *DeclStmt:
		Walk(v, n.Decl)

	case *EmptyStmt:
		// nothing to do

	case *LabeledStmt:
		walkIdent(v, n.Label)
		Walk(v, n.Stmt)

	case *ExprStmt:
		Walk(v, n.X)

	case *IncDecStmt:
		Walk(v, n.X)

	case *AssignStmt:
		Walk(v, n.Lhs)
		Walk(v, n.Rhs)

	case *GoStmt:
		if n.Call != nil {
			Walk(v, n.Call)
		}

	case *DeferStmt:
		if n.Call != nil {
			Walk(v, n.Call)
		}

	case *ReturnStmt:
		Walk(v, n.Results)

	case *BranchStmt:
		walkIdent(v, n.Label)

	case *BlockStmt:
		Walk(v, n.List)

	case *IfStmt:
		Walk(v, n.Init)
		Walk(v, n.Cond)
		walkBlockStmt(v, n.Body)
		Walk(v, n.Else)

	case *CaseClause:
		Walk(v, n.Values)
		Walk(v, n.Body)

	case *SwitchStmt:
		Walk(v, n.Init)
		Walk(v, n.Tag)
		walkBlockStmt(v, n.Body)

	case *TypeCaseClause:
		Walk(v, n.Types)
		Walk(v, n.Body)

	case *TypeSwitchStmt:
		Walk(v, n.Init)
		Walk(v, n.Assign)
		walkBlockStmt(v, n.Body)

	case *CommClause:
		Walk(v, n.Lhs)
		Walk(v, n.Rhs)
		Walk(v, n.Body)

	case *SelectStmt:
		walkBlockStmt(v, n.Body)

	case *ForStmt:
		Walk(v, n.Init)
		Walk(v, n.Cond)
		Walk(v, n.Post)
		walkBlockStmt(v, n.Body)

	case *RangeStmt:
		Walk(v, n.Key)
		Walk(v, n.Value)
		Walk(v, n.X)
		walkBlockStmt(v, n.Body)

	// Declarations
	case *ImportSpec:
		walkCommentGroup(v, n.Doc)
		walkIdent(v, n.Name)
		Walk(v, n.Path)
		walkCommentGroup(v, n.Comment)

	case *ValueSpec:
		walkCommentGroup(v, n.Doc)
		Walk(v, n.Names)
		Walk(v, n.Type)
		Walk(v, n.Values)
		walkCommentGroup(v, n.Comment)

	case *TypeSpec:
		walkCommentGroup(v, n.Doc)
		walkIdent(v, n.Name)
		Walk(v, n.Type)
		walkCommentGroup(v, n.Comment)

	case *BadDecl:
		// nothing to do

	case *GenDecl:
		walkCommentGroup(v, n.Doc)
		for _, s := range n.Specs {
			Walk(v, s)
		}

	case *FuncDecl:
		walkCommentGroup(v, n.Doc)
		if n.Recv != nil {
			Walk(v, n.Recv)
		}
		walkIdent(v, n.Name)
		if n.Type != nil {
			Walk(v, n.Type)
		}
		walkBlockStmt(v, n.Body)

	// Files and packages
	case *File:
		walkCommentGroup(v, n.Doc)
		walkIdent(v, n.Name)
		Walk(v, n.Decls)
		for _, g := range n.Comments {
			Walk(v, g)
		}

	case *Package:
		for _, f := range n.Files {
			Walk(v, f)
		}

	case []*Ident:
		for _, x := range n {
			Walk(v, x)
		}

	case []Expr:
		for _, x := range n {
			Walk(v, x)
		}

	case []Stmt:
		for _, x := range n {
			Walk(v, x)
		}

	case []Decl:
		for _, x := range n {
			Walk(v, x)
		}

	default:
		fmt.Printf("ast.Walk: unexpected type %T", n)
		panic("ast.Walk")
	}

	v.Visit(nil)
}
<commit_msg>go/ast: add Inspect function for easy AST inspection w/o a visitor<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ast

import "fmt"

// A Visitor's Visit method is invoked for each node encountered by Walk.
// If the result visitor w is not nil, Walk visits each of the children
// of node with the visitor w, followed by a call of w.Visit(nil).
type Visitor interface {
	Visit(node interface{}) (w Visitor)
}

func walkIdent(v Visitor, x *Ident) {
	if x != nil {
		Walk(v, x)
	}
}

func walkCommentGroup(v Visitor, g *CommentGroup) {
	if g != nil {
		Walk(v, g)
	}
}

func walkBlockStmt(v Visitor, b *BlockStmt) {
	if b != nil {
		Walk(v, b)
	}
}

// Walk traverses an AST in depth-first order: If node != nil, it
// invokes v.Visit(node). If the visitor w returned by v.Visit(node) is
// not nil, Walk visits each of the children of node with the visitor w,
// followed by a call of w.Visit(nil).
//
// Walk may be called with any of the named ast node types. It also
// accepts arguments of type []*Field, []*Ident, []Expr, []Stmt and []Decl;
// the respective children are the slice elements.
//
func Walk(v Visitor, node interface{}) {
	if node == nil {
		return
	}
	if v = v.Visit(node); v == nil {
		return
	}

	// walk children
	// (the order of the cases matches the order
	// of the corresponding declaration in ast.go)
	switch n := node.(type) {
	// Comments and fields
	case *Comment:
		// nothing to do

	case *CommentGroup:
		for _, c := range n.List {
			Walk(v, c)
		}

	case *Field:
		walkCommentGroup(v, n.Doc)
		Walk(v, n.Names)
		Walk(v, n.Type)
		Walk(v, n.Tag)
		walkCommentGroup(v, n.Comment)

	case *FieldList:
		for _, f := range n.List {
			Walk(v, f)
		}

	// Expressions
	case *BadExpr, *Ident, *Ellipsis, *BasicLit:
		// nothing to do

	case *FuncLit:
		if n != nil {
			Walk(v, n.Type)
		}
		walkBlockStmt(v, n.Body)

	case *CompositeLit:
		Walk(v, n.Type)
		Walk(v, n.Elts)

	case *ParenExpr:
		Walk(v, n.X)

	case *SelectorExpr:
		Walk(v, n.X)
		walkIdent(v, n.Sel)

	case *IndexExpr:
		Walk(v, n.X)
		Walk(v, n.Index)

	case *SliceExpr:
		Walk(v, n.X)
		Walk(v, n.Index)
		Walk(v, n.End)

	case *TypeAssertExpr:
		Walk(v, n.X)
		Walk(v, n.Type)

	case *CallExpr:
		Walk(v, n.Fun)
		Walk(v, n.Args)

	case *StarExpr:
		Walk(v, n.X)

	case *UnaryExpr:
		Walk(v, n.X)

	case *BinaryExpr:
		Walk(v, n.X)
		Walk(v, n.Y)

	case *KeyValueExpr:
		Walk(v, n.Key)
		Walk(v, n.Value)

	// Types
	case *ArrayType:
		Walk(v, n.Len)
		Walk(v, n.Elt)

	case *StructType:
		Walk(v, n.Fields)

	case *FuncType:
		Walk(v, n.Params)
		if n.Results != nil {
			Walk(v, n.Results)
		}

	case *InterfaceType:
		Walk(v, n.Methods)

	case *MapType:
		Walk(v, n.Key)
		Walk(v, n.Value)

	case *ChanType:
		Walk(v, n.Value)

	// Statements
	case *BadStmt:
		// nothing to do

	case *DeclStmt:
		Walk(v, n.Decl)

	case *EmptyStmt:
		// nothing to do

	case *LabeledStmt:
		walkIdent(v, n.Label)
		Walk(v, n.Stmt)

	case *ExprStmt:
		Walk(v, n.X)

	case *IncDecStmt:
		Walk(v, n.X)

	case *AssignStmt:
		Walk(v, n.Lhs)
		Walk(v, n.Rhs)

	case *GoStmt:
		if n.Call != nil {
			Walk(v, n.Call)
		}

	case *DeferStmt:
		if n.Call != nil {
			Walk(v, n.Call)
		}

	case *ReturnStmt:
		Walk(v, n.Results)

	case *BranchStmt:
		walkIdent(v, n.Label)

	case *BlockStmt:
		Walk(v, n.List)

	case *IfStmt:
		Walk(v, n.Init)
		Walk(v, n.Cond)
		walkBlockStmt(v, n.Body)
		Walk(v, n.Else)

	case *CaseClause:
		Walk(v, n.Values)
		Walk(v, n.Body)

	case *SwitchStmt:
		Walk(v, n.Init)
		Walk(v, n.Tag)
		walkBlockStmt(v, n.Body)

	case *TypeCaseClause:
		Walk(v, n.Types)
		Walk(v, n.Body)

	case *TypeSwitchStmt:
		Walk(v, n.Init)
		Walk(v, n.Assign)
		walkBlockStmt(v, n.Body)

	case *CommClause:
		Walk(v, n.Lhs)
		Walk(v, n.Rhs)
		Walk(v, n.Body)

	case *SelectStmt:
		walkBlockStmt(v, n.Body)

	case *ForStmt:
		Walk(v, n.Init)
		Walk(v, n.Cond)
		Walk(v, n.Post)
		walkBlockStmt(v, n.Body)

	case *RangeStmt:
		Walk(v, n.Key)
		Walk(v, n.Value)
		Walk(v, n.X)
		walkBlockStmt(v, n.Body)

	// Declarations
	case *ImportSpec:
		walkCommentGroup(v, n.Doc)
		walkIdent(v, n.Name)
		Walk(v, n.Path)
		walkCommentGroup(v, n.Comment)

	case *ValueSpec:
		walkCommentGroup(v, n.Doc)
		Walk(v, n.Names)
		Walk(v, n.Type)
		Walk(v, n.Values)
		walkCommentGroup(v, n.Comment)

	case *TypeSpec:
		walkCommentGroup(v, n.Doc)
		walkIdent(v, n.Name)
		Walk(v, n.Type)
		walkCommentGroup(v, n.Comment)

	case *BadDecl:
		// nothing to do

	case *GenDecl:
		walkCommentGroup(v, n.Doc)
		for _, s := range n.Specs {
			Walk(v, s)
		}

	case *FuncDecl:
		walkCommentGroup(v, n.Doc)
		if n.Recv != nil {
			Walk(v, n.Recv)
		}
		walkIdent(v, n.Name)
		if n.Type != nil {
			Walk(v, n.Type)
		}
		walkBlockStmt(v, n.Body)

	// Files and packages
	case *File:
		walkCommentGroup(v, n.Doc)
		walkIdent(v, n.Name)
		Walk(v, n.Decls)
		for _, g := range n.Comments {
			Walk(v, g)
		}

	case *Package:
		for _, f := range n.Files {
			Walk(v, f)
		}

	case []*Ident:
		for _, x := range n {
			Walk(v, x)
		}

	case []Expr:
		for _, x := range n {
			Walk(v, x)
		}

	case []Stmt:
		for _, x := range n {
			Walk(v, x)
		}

	case []Decl:
		for _, x := range n {
			Walk(v, x)
		}

	default:
		fmt.Printf("ast.Walk: unexpected type %T", n)
		panic("ast.Walk")
	}

	v.Visit(nil)
}

type inspector func(node interface{}) bool

func (f inspector) Visit(node interface{}) Visitor {
	if node != nil && f(node) {
		return f
	}
	return nil
}

// Inspect traverses an AST in depth-first order: If node != nil, it
// invokes f(node). If f returns true, inspect invokes f for all the
// non-nil children of node, recursively.
//
func Inspect(ast interface{}, f func(node interface{}) bool) {
	Walk(inspector(f), ast)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"regexp"
	"strings"

	"github.com/kyma-project/test-infra/development/tools/pkg/common"
	"github.com/kyma-project/test-infra/development/tools/pkg/dnscollector"
	log "github.com/sirupsen/logrus"
	"golang.org/x/oauth2/google"
	compute "google.golang.org/api/compute/v1"
	dns "google.golang.org/api/dns/v1"
)

const defaultAddressRegexpList = "(remoteenvs-)?gkeint-(pr|commit)-.*,(remoteenvs-)?gke-upgrade-(pr|commit)-.*"
const minAgeInHours = 1
const minPatternLength = 5

var (
	project               = flag.String("project", "", "project id [required]")
	regions               = flag.String("regions", "", "comma-separted list of GCP regions [required]")
	dnsZone               = flag.String("dnsZone", "", "Name of the DNS Managed Zone [Required]")
	dryRun                = flag.Bool("dryRun", true, "Dry Run enabled, nothing is deleted")
	ageInHours            = flag.Int("ageInHours", 2, "IP Address age in hours. Addresses older than: now()-ageInHours are considered for removal.")
	addressNameRegexpList = flag.String("addressRegexpList", defaultAddressRegexpList, "Address name regexp list. Separate items with commas, spaces are trimmed. Matching addresses are considered for removal.")
)

func main() {
	flag.Parse()

	if *project == "" {
		fmt.Fprint(os.Stderr, "missing -project flag\n\n")
		flag.Usage()
		os.Exit(2)
	}

	if *dnsZone == "" {
		fmt.Fprint(os.Stderr, "missing -dnsZone flag\n\n")
		flag.Usage()
		os.Exit(2)
	}

	regionsList := splitPatterns(*regions)
	for _, region := range regionsList {
		if len(region) == 0 {
			fmt.Fprint(os.Stderr, "invalid region: \"\"\n\n")
			flag.Usage()
			os.Exit(2)
		}
	}

	patterns := splitPatterns(*addressNameRegexpList)
	regexpList := []*regexp.Regexp{}
	for _, pattern := range patterns {
		if len(pattern) < minPatternLength {
			fmt.Fprintf(os.Stderr, "invalid pattern: \"%s\". Value must not be shorter than %d characters.\n\n", pattern, minPatternLength)
			flag.Usage()
			os.Exit(2)
		}
		regexpList = append(regexpList, regexp.MustCompile(pattern))
	}

	if *ageInHours < minAgeInHours {
		fmt.Fprintf(os.Stderr, "invalid ageInHours. Value must not be smaller than %d\n\n", minAgeInHours)
		flag.Usage()
		os.Exit(2)
	}

	common.ShoutFirst("Running with arguments: project: \"%s\", regions: \"%s\", dnsZone: \"%s\", dryRun: %t, ageInHours: %d, addressRegexpList: %s", *project, quoteElems(regionsList), *dnsZone, *dryRun, *ageInHours, quoteElems(patterns))
	ctx := context.Background()

	computeConn, err := google.DefaultClient(ctx, compute.CloudPlatformScope)
	if err != nil {
		log.Fatalf("Could not get authenticated client: %v", err)
	}

	computeSvc, err := compute.New(computeConn)
	if err != nil {
		log.Fatalf("Could not initialize gke client for Compute API: %v", err)
	}

	dnsSvc, err := dns.New(computeConn)
	if err != nil {
		log.Fatalf("Could not initialize gke client for Compute API: %v", err)
	}

	computeAPI := &dnscollector.ComputeServiceWrapper{Context: ctx, Compute: computeSvc}
	dnsAPI := &dnscollector.DNSServiceWrapper{Context: ctx, DNS: dnsSvc}
	shouldRemoveFunc := dnscollector.DefaultIPAddressRemovalPredicate(regexpList, *ageInHours)

	cleaner := dnscollector.New(computeAPI, dnsAPI, shouldRemoveFunc)
	allSucceeded, err := cleaner.Run(*project, *dnsZone, regionsList, !(*dryRun))

	if err != nil {
		log.Fatalf("IP/DNS collector error: %v", err)
	}

	if !allSucceeded {
		log.Warn("Some operations failed.")
	}

	common.Shout("Finished")
}

func quoteElems(elems []string) string {

	res := "\"" + elems[0] + "\""
	for i := 1; i < len(elems); i++ {
		res = res + ",\"" + elems[i] + "\""
	}

	return "[" + res + "]"
}

func splitPatterns(commaSeparated string) []string {

	res := []string{}
	values := strings.Split(commaSeparated, ",")
	for _, pattern := range values {
		val := strings.Trim(pattern, " ")
		res = append(res, val)
	}

	return res
}
<commit_msg>Added load test and e2etest cluster to the cleaner (#806)<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"os"
	"regexp"
	"strings"

	"github.com/kyma-project/test-infra/development/tools/pkg/common"
	"github.com/kyma-project/test-infra/development/tools/pkg/dnscollector"
	log "github.com/sirupsen/logrus"
	"golang.org/x/oauth2/google"
	compute "google.golang.org/api/compute/v1"
	dns "google.golang.org/api/dns/v1"
)

const defaultAddressRegexpList = "(remoteenvs-)?gkeint-(pr|commit)-.*,(remoteenvs-)?gke-upgrade-(pr|commit)-.*,(remoteenvs-)?(load-test|e2etest)"
const minAgeInHours = 1
const minPatternLength = 5

var (
	project               = flag.String("project", "", "project id [required]")
	regions               = flag.String("regions", "", "comma-separted list of GCP regions [required]")
	dnsZone               = flag.String("dnsZone", "", "Name of the DNS Managed Zone [Required]")
	dryRun                = flag.Bool("dryRun", true, "Dry Run enabled, nothing is deleted")
	ageInHours            = flag.Int("ageInHours", 2, "IP Address age in hours. Addresses older than: now()-ageInHours are considered for removal.")
	addressNameRegexpList = flag.String("addressRegexpList", defaultAddressRegexpList, "Address name regexp list. Separate items with commas, spaces are trimmed. Matching addresses are considered for removal.")
)

func main() {
	flag.Parse()

	if *project == "" {
		fmt.Fprint(os.Stderr, "missing -project flag\n\n")
		flag.Usage()
		os.Exit(2)
	}

	if *dnsZone == "" {
		fmt.Fprint(os.Stderr, "missing -dnsZone flag\n\n")
		flag.Usage()
		os.Exit(2)
	}

	regionsList := splitPatterns(*regions)
	for _, region := range regionsList {
		if len(region) == 0 {
			fmt.Fprint(os.Stderr, "invalid region: \"\"\n\n")
			flag.Usage()
			os.Exit(2)
		}
	}

	patterns := splitPatterns(*addressNameRegexpList)
	regexpList := []*regexp.Regexp{}
	for _, pattern := range patterns {
		if len(pattern) < minPatternLength {
			fmt.Fprintf(os.Stderr, "invalid pattern: \"%s\". Value must not be shorter than %d characters.\n\n", pattern, minPatternLength)
			flag.Usage()
			os.Exit(2)
		}
		regexpList = append(regexpList, regexp.MustCompile(pattern))
	}

	if *ageInHours < minAgeInHours {
		fmt.Fprintf(os.Stderr, "invalid ageInHours. Value must not be smaller than %d\n\n", minAgeInHours)
		flag.Usage()
		os.Exit(2)
	}

	common.ShoutFirst("Running with arguments: project: \"%s\", regions: \"%s\", dnsZone: \"%s\", dryRun: %t, ageInHours: %d, addressRegexpList: %s", *project, quoteElems(regionsList), *dnsZone, *dryRun, *ageInHours, quoteElems(patterns))
	ctx := context.Background()

	computeConn, err := google.DefaultClient(ctx, compute.CloudPlatformScope)
	if err != nil {
		log.Fatalf("Could not get authenticated client: %v", err)
	}

	computeSvc, err := compute.New(computeConn)
	if err != nil {
		log.Fatalf("Could not initialize gke client for Compute API: %v", err)
	}

	dnsSvc, err := dns.New(computeConn)
	if err != nil {
		log.Fatalf("Could not initialize gke client for Compute API: %v", err)
	}

	computeAPI := &dnscollector.ComputeServiceWrapper{Context: ctx, Compute: computeSvc}
	dnsAPI := &dnscollector.DNSServiceWrapper{Context: ctx, DNS: dnsSvc}
	shouldRemoveFunc := dnscollector.DefaultIPAddressRemovalPredicate(regexpList, *ageInHours)

	cleaner := dnscollector.New(computeAPI, dnsAPI, shouldRemoveFunc)
	allSucceeded, err := cleaner.Run(*project, *dnsZone, regionsList, !(*dryRun))

	if err != nil {
		log.Fatalf("IP/DNS collector error: %v", err)
	}

	if !allSucceeded {
		log.Warn("Some operations failed.")
	}

	common.Shout("Finished")
}

func quoteElems(elems []string) string {

	res := "\"" + elems[0] + "\""
	for i := 1; i < len(elems); i++ {
		res = res + ",\"" + elems[i] + "\""
	}

	return "[" + res + "]"
}

func splitPatterns(commaSeparated string) []string {

	res := []string{}
	values := strings.Split(commaSeparated, ",")
	for _, pattern := range values {
		val := strings.Trim(pattern, " ")
		res = append(res, val)
	}

	return res
}
<|endoftext|>


<commit_before>package ui

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/config"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-ninja/model"
)

/*
type Thing struct {
	Name   string `json:"name"`
	ID     string `json:"id"`
	Device Device
}

type Device struct {
	Name     string `json:"name"`
	ID       string `json:"id"`
	IDType   string `json:"idType"`
	Guid     string `json:"guid"`
	Channels []Channel
}

type Channel struct {
	Protocol string `json:"protocol"`
	Name     string `json:"channel"`
	ID       string `json:"id"`
}*/

var sameRoomOnly = config.Bool(true, "homecloud.sameRoomOnly")

var conn *ninja.Connection
var tasks []*request
var thingModel *ninja.ServiceClient

var log = logger.GetLogger("ui")

type request struct {
	thingType string
	protocol  string
	filter    func(thing *model.Thing) bool
	cb        func([]*ninja.ServiceClient, error)
}

var foundLocation = make(chan bool)

var roomID *string

func runTasks() {

	// Find this sphere's location, if we care..
	if sameRoomOnly {
		for _, thing := range allThings {
			if thing.Type == "node" && thing.Device != nil && thing.Device.NaturalID == config.Serial() {
				if thing.Location != nil && (roomID == nil || *roomID != *thing.Location) {
					// Got it.
					log.Infof("Got this sphere's location: %s", thing.Location)
					roomID = thing.Location
					select {
					case foundLocation <- true:
					default:
					}

					break
				}
			}
		}
	}

	for _, task := range tasks {
		go func(t *request) {
			t.cb(getChannelServices(t.thingType, t.protocol, t.filter))
		}(task)
	}

}

var allThings []model.Thing

func fetchAll() error {

	var things []model.Thing

	err := thingModel.Call("fetchAll", []interface{}{}, &things, time.Second*20)
	//err = client.Call("fetch", "c7ac05e0-9999-4d93-bfe3-a0b4bb5e7e78", &thing)

	if err != nil {
		return fmt.Errorf("Failed to get things!: %s", err)
	}

	allThings = things
	runTasks()

	return nil
}

func startSearchTasks(c *ninja.Connection) {
	conn = c
	thingModel = conn.GetServiceClient("$home/services/ThingModel")

	dirty := false

	setDirty := func(params *json.RawMessage, topicKeys map[string]string) bool {
		log.Infof("Devices added/removed/updated. Marking dirty.")
		dirty = true
		return true
	}

	go func() {
		for {
			time.Sleep(time.Second * 30)
			setDirty(nil, nil)
		}
	}()

	go func() {
		for {
			err := fetchAll()
			if err == nil {
				break
			}

			log.Warningf("Failed to get fetch all things: %s", err)
			time.Sleep(time.Second * 3)
		}
	}()

	if sameRoomOnly {
		<-foundLocation
	}

	thingModel.OnEvent("created", setDirty)
	thingModel.OnEvent("updated", setDirty)
	thingModel.OnEvent("deleted", setDirty)

	go func() {
		time.Sleep(time.Second * 10)
		for {
			time.Sleep(time.Second * 5)
			if dirty {
				fetchAll()
				dirty = false
			}
		}
	}()
}

func getChannelServicesContinuous(thingType string, protocol string, filter func(thing *model.Thing) bool, cb func([]*ninja.ServiceClient, error)) {

	if filter == nil {
		filter = func(thing *model.Thing) bool {
			return roomID == nil || (thing.Location != nil && *thing.Location == *roomID)
		}
	}

	tasks = append(tasks, &request{thingType, protocol, filter, cb})

	cb(getChannelServices(thingType, protocol, filter))
}

func getChannelServices(thingType string, protocol string, filter func(thing *model.Thing) bool) ([]*ninja.ServiceClient, error) {

	//time.Sleep(time.Second * 3)

	var services []*ninja.ServiceClient

	for _, thing := range allThings {
		if thing.Type == thingType {

			// Handle more than one channel with same protocol
			channel := getChannel(&thing, protocol)
			if channel != nil {
				if filter(&thing) {
					services = append(services, conn.GetServiceClientFromAnnouncement(channel.ServiceAnnouncement))
				}
			}
		}
	}
	return services, nil
}

func getChannel(thing *model.Thing, protocol string) *model.Channel {

	if thing.Device == nil || thing.Device.Channels == nil {
		return nil
	}

	for _, channel := range *thing.Device.Channels {
		if channel.Protocol == protocol {
			if thing.Device == nil {
				//spew.Dump("NO device on thing!", thing)
				return nil
			}

			return channel
		}
	}

	return nil
}
<commit_msg>Undo testing code to enable nil roomId for sphere<commit_after>package ui

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/config"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-ninja/model"
)

/*
type Thing struct {
	Name   string `json:"name"`
	ID     string `json:"id"`
	Device Device
}

type Device struct {
	Name     string `json:"name"`
	ID       string `json:"id"`
	IDType   string `json:"idType"`
	Guid     string `json:"guid"`
	Channels []Channel
}

type Channel struct {
	Protocol string `json:"protocol"`
	Name     string `json:"channel"`
	ID       string `json:"id"`
}*/

var sameRoomOnly = config.Bool(true, "homecloud.sameRoomOnly")

var conn *ninja.Connection
var tasks []*request
var thingModel *ninja.ServiceClient

var log = logger.GetLogger("ui")

type request struct {
	thingType string
	protocol  string
	filter    func(thing *model.Thing) bool
	cb        func([]*ninja.ServiceClient, error)
}

var foundLocation = make(chan bool)

var roomID *string

func runTasks() {

	// Find this sphere's location, if we care..
	if sameRoomOnly {
		for _, thing := range allThings {
			if thing.Type == "node" && thing.Device != nil && thing.Device.NaturalID == config.Serial() {
				if thing.Location != nil && (roomID == nil || *roomID != *thing.Location) {
					// Got it.
					log.Infof("Got this sphere's location: %s", thing.Location)
					roomID = thing.Location
					select {
					case foundLocation <- true:
					default:
					}

					break
				}
			}
		}
	}

	for _, task := range tasks {
		go func(t *request) {
			t.cb(getChannelServices(t.thingType, t.protocol, t.filter))
		}(task)
	}

}

var allThings []model.Thing

func fetchAll() error {

	var things []model.Thing

	err := thingModel.Call("fetchAll", []interface{}{}, &things, time.Second*20)
	//err = client.Call("fetch", "c7ac05e0-9999-4d93-bfe3-a0b4bb5e7e78", &thing)

	if err != nil {
		return fmt.Errorf("Failed to get things!: %s", err)
	}

	allThings = things
	runTasks()

	return nil
}

func startSearchTasks(c *ninja.Connection) {
	conn = c
	thingModel = conn.GetServiceClient("$home/services/ThingModel")

	dirty := false

	setDirty := func(params *json.RawMessage, topicKeys map[string]string) bool {
		log.Infof("Devices added/removed/updated. Marking dirty.")
		dirty = true
		return true
	}

	go func() {
		for {
			time.Sleep(time.Second * 30)
			setDirty(nil, nil)
		}
	}()

	go func() {
		for {
			err := fetchAll()
			if err == nil {
				break
			}

			log.Warningf("Failed to get fetch all things: %s", err)
			time.Sleep(time.Second * 3)
		}
	}()

	if sameRoomOnly {
		<-foundLocation
	}

	thingModel.OnEvent("created", setDirty)
	thingModel.OnEvent("updated", setDirty)
	thingModel.OnEvent("deleted", setDirty)

	go func() {
		time.Sleep(time.Second * 10)
		for {
			time.Sleep(time.Second * 5)
			if dirty {
				fetchAll()
				dirty = false
			}
		}
	}()
}

func getChannelServicesContinuous(thingType string, protocol string, filter func(thing *model.Thing) bool, cb func([]*ninja.ServiceClient, error)) {

	if filter == nil {
		filter = func(thing *model.Thing) bool {
			return roomID != nil && (thing.Location != nil && *thing.Location == *roomID)
		}
	}

	tasks = append(tasks, &request{thingType, protocol, filter, cb})

	cb(getChannelServices(thingType, protocol, filter))
}

func getChannelServices(thingType string, protocol string, filter func(thing *model.Thing) bool) ([]*ninja.ServiceClient, error) {

	//time.Sleep(time.Second * 3)

	var services []*ninja.ServiceClient

	for _, thing := range allThings {
		if thing.Type == thingType {

			// Handle more than one channel with same protocol
			channel := getChannel(&thing, protocol)
			if channel != nil {
				if filter(&thing) {
					services = append(services, conn.GetServiceClientFromAnnouncement(channel.ServiceAnnouncement))
				}
			}
		}
	}
	return services, nil
}

func getChannel(thing *model.Thing, protocol string) *model.Channel {

	if thing.Device == nil || thing.Device.Channels == nil {
		return nil
	}

	for _, channel := range *thing.Device.Channels {
		if channel.Protocol == protocol {
			if thing.Device == nil {
				//spew.Dump("NO device on thing!", thing)
				return nil
			}

			return channel
		}
	}

	return nil
}
<|endoftext|>