<commit_before>// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// Package gvisor provides support for gVisor, user-space kernel, testing.
// See https://github.com/google/gvisor
package gvisor

import (
	"bytes"
	"fmt"
	"io"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/google/syzkaller/pkg/config"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/vm/vmimpl"
)

func init() {
	vmimpl.Register("gvisor", ctor, true)
}

type Config struct {
	Count     int    `json:"count"` // number of VMs to use
	RunscArgs string `json:"runsc_args"`
}

type Pool struct {
	env *vmimpl.Env
	cfg *Config
}

type instance struct {
	cfg      *Config
	image    string
	debug    bool
	rootDir  string
	imageDir string
	name     string
	port     int
	cmd      *exec.Cmd
	merger   *vmimpl.OutputMerger
}

func ctor(env *vmimpl.Env) (vmimpl.Pool, error) {
	cfg := &Config{
		Count: 1,
	}
	if err := config.LoadData(env.Config, cfg); err != nil {
		return nil, fmt.Errorf("failed to parse vm config: %v", err)
	}
	if cfg.Count < 1 || cfg.Count > 128 {
		return nil, fmt.Errorf("invalid config param count: %v, want [1, 128]", cfg.Count)
	}
	if env.Debug {
		cfg.Count = 1
	}
	if !osutil.IsExist(env.Image) {
		return nil, fmt.Errorf("image file %q does not exist", env.Image)
	}
	pool := &Pool{
		cfg: cfg,
		env: env,
	}
	return pool, nil
}

func (pool *Pool) Count() int {
	return pool.cfg.Count
}

func (pool *Pool) Create(workdir string, index int) (vmimpl.Instance, error) {
	rootDir := filepath.Clean(filepath.Join(workdir, "..", "gvisor_root"))
	imageDir := filepath.Join(workdir, "image")
	bundleDir := filepath.Join(workdir, "bundle")
	osutil.MkdirAll(rootDir)
	osutil.MkdirAll(bundleDir)
	osutil.MkdirAll(imageDir)

	caps := ""
	for _, c := range sandboxCaps {
		if caps != "" {
			caps += ", "
		}
		caps += "\"" + c + "\""
	}
	vmConfig := fmt.Sprintf(configTempl, imageDir, caps)
	if err := osutil.WriteFile(filepath.Join(bundleDir, "config.json"), []byte(vmConfig)); err != nil {
		return nil, err
	}
	bin, err := exec.LookPath(os.Args[0])
	if err != nil {
		return nil, fmt.Errorf("failed to lookup %v: %v", os.Args[0], err)
	}
	if err := osutil.CopyFile(bin, filepath.Join(imageDir, "init")); err != nil {
		return nil, err
	}

	rpipe, wpipe, err := osutil.LongPipe()
	if err != nil {
		return nil, err
	}
	var tee io.Writer
	if pool.env.Debug {
		tee = os.Stdout
	}
	merger := vmimpl.NewOutputMerger(tee)
	merger.Add("gvisor", rpipe)

	inst := &instance{
		cfg:      pool.cfg,
		image:    pool.env.Image,
		debug:    pool.env.Debug,
		rootDir:  rootDir,
		imageDir: imageDir,
		name:     fmt.Sprintf("%v-%v", pool.env.Name, index),
		merger:   merger,
	}

	// Kill the previous instance in case it's still running.
	osutil.Run(time.Minute, inst.runscCmd("delete", "-force", inst.name))
	time.Sleep(3 * time.Second)

	cmd := inst.runscCmd("run", "-bundle", bundleDir, inst.name)
	cmd.Stdout = wpipe
	cmd.Stderr = wpipe
	if err := cmd.Start(); err != nil {
		wpipe.Close()
		merger.Wait()
		return nil, err
	}
	inst.cmd = cmd
	wpipe.Close()

	if err := inst.waitBoot(); err != nil {
		inst.Close()
		return nil, err
	}
	return inst, nil
}

func (inst *instance) waitBoot() error {
	errorMsg := []byte("FATAL ERROR:")
	bootedMsg := []byte(initStartMsg)
	timeout := time.NewTimer(time.Minute)
	defer timeout.Stop()
	var output []byte
	for {
		select {
		case out := <-inst.merger.Output:
			output = append(output, out...)
			if pos := bytes.Index(output, errorMsg); pos != -1 {
				end := bytes.IndexByte(output[pos:], '\n')
				if end == -1 {
					end = len(output)
				} else {
					end += pos
				}
				return vmimpl.BootError{
					Title:  string(output[pos:end]),
					Output: output,
				}
			}
			if bytes.Contains(output, bootedMsg) {
				return nil
			}
		case err := <-inst.merger.Err:
			return vmimpl.BootError{
				Title:  fmt.Sprintf("runsc failed: %v", err),
				Output: output,
			}
		case <-timeout.C:
			return vmimpl.BootError{
				Title:  "init process did not start",
				Output: output,
			}
		}
	}
}

func (inst *instance) runscCmd(add ...string) *exec.Cmd {
	args := []string{
		"-root", inst.rootDir,
		"-watchdog-action=panic",
		"-panic-signal=12",
		"-network=none",
	}
	if inst.cfg.RunscArgs != "" {
		args = append(args, strings.Split(inst.cfg.RunscArgs, " ")...)
	}
	args = append(args, add...)
	cmd := osutil.Command(inst.image, args...)
	cmd.Env = []string{
		"GOTRACEBACK=all",
		"GORACE=halt_on_error=1",
	}
	return cmd
}

func (inst *instance) Close() {
	time.Sleep(3 * time.Second)
	osutil.Run(time.Minute, inst.runscCmd("delete", "-force", inst.name))
	inst.cmd.Process.Kill()
	inst.merger.Wait()
	inst.cmd.Wait()
	osutil.Run(time.Minute, inst.runscCmd("delete", "-force", inst.name))
	time.Sleep(3 * time.Second)
}

func (inst *instance) Forward(port int) (string, error) {
	if inst.port != 0 {
		return "", fmt.Errorf("forward port is already setup")
	}
	inst.port = port
	return "stdin", nil
}

func (inst *instance) Copy(hostSrc string) (string, error) {
	fname := filepath.Base(hostSrc)
	if err := osutil.CopyFile(hostSrc, filepath.Join(inst.imageDir, fname)); err != nil {
		return "", err
	}
	if err := os.Chmod(inst.imageDir, 0777); err != nil {
		return "", err
	}
	return filepath.Join("/", fname), nil
}

func (inst *instance) Run(timeout time.Duration, stop <-chan bool, command string) (
	<-chan []byte, <-chan error, error) {
	args := []string{"exec", "-user=0:0"}
	for _, c := range sandboxCaps {
		args = append(args, "-cap", c)
	}
	args = append(args, inst.name)
	args = append(args, strings.Split(command, " ")...)
	cmd := inst.runscCmd(args...)

	rpipe, wpipe, err := osutil.LongPipe()
	if err != nil {
		return nil, nil, err
	}
	defer wpipe.Close()
	inst.merger.Add("cmd", rpipe)
	cmd.Stdout = wpipe
	cmd.Stderr = wpipe

	guestSock, err := inst.guestProxy()
	if err != nil {
		return nil, nil, err
	}
	if guestSock != nil {
		defer guestSock.Close()
		cmd.Stdin = guestSock
	}

	if err := cmd.Start(); err != nil {
		return nil, nil, err
	}
	errc := make(chan error, 1)
	signal := func(err error) {
		select {
		case errc <- err:
		default:
		}
	}

	go func() {
		select {
		case <-time.After(timeout):
			signal(vmimpl.ErrTimeout)
		case <-stop:
			signal(vmimpl.ErrTimeout)
		case err := <-inst.merger.Err:
			cmd.Process.Kill()
			if cmdErr := cmd.Wait(); cmdErr == nil {
				// If the command exited successfully, we got EOF error from merger.
				// But in this case no error has happened and the EOF is expected.
				err = nil
			}
			signal(err)
			return
		}
		cmd.Process.Kill()
		cmd.Wait()
	}()
	return inst.merger.Output, errc, nil
}

func (inst *instance) guestProxy() (*os.File, error) {
	if inst.port == 0 {
		return nil, nil
	}
	// One does not simply let gvisor guest connect to host tcp port.
	// We create a unix socket, pass it to guest in stdin.
	// Guest will use it instead of dialing manager directly.
	// On host we connect to manager tcp port and proxy between the tcp and unix connections.
	socks, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)
	if err != nil {
		return nil, err
	}
	hostSock := os.NewFile(uintptr(socks[0]), "host unix proxy")
	guestSock := os.NewFile(uintptr(socks[1]), "guest unix proxy")
	conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%v", inst.port))
	if err != nil {
		conn.Close()
		hostSock.Close()
		guestSock.Close()
		return nil, err
	}
	go func() {
		io.Copy(hostSock, conn)
		hostSock.Close()
	}()
	go func() {
		io.Copy(conn, hostSock)
		conn.Close()
	}()
	return guestSock, nil
}

func (inst *instance) Diagnose() bool {
	osutil.Run(time.Minute, inst.runscCmd("debug", "-signal=12", inst.name))
	return true
}

func init() {
	if os.Getenv("SYZ_GVISOR_PROXY") != "" {
		fmt.Fprintf(os.Stderr, initStartMsg)
		select {}
	}
}

const initStartMsg = "SYZKALLER INIT STARTED\n"

const configTempl = `
{
	"root": {
		"path": "%[1]v",
		"readonly": true
	},
	"process":{
                "args": ["/init"],
                "cwd": "/tmp",
                "env": ["SYZ_GVISOR_PROXY=1"],
                "capabilities": {
                	"bounding": [%[2]v],
                	"effective": [%[2]v],
                	"inheritable": [%[2]v],
                	"permitted": [%[2]v],
                	"ambient": [%[2]v]
                }
	}
}
`

var sandboxCaps = []string{
	"CAP_CHOWN", "CAP_DAC_OVERRIDE", "CAP_DAC_READ_SEARCH", "CAP_FOWNER", "CAP_FSETID",
	"CAP_KILL", "CAP_SETGID", "CAP_SETUID", "CAP_SETPCAP", "CAP_LINUX_IMMUTABLE",
	"CAP_NET_BIND_SERVICE", "CAP_NET_BROADCAST", "CAP_NET_ADMIN", "CAP_NET_RAW",
	"CAP_IPC_LOCK", "CAP_IPC_OWNER", "CAP_SYS_MODULE", "CAP_SYS_RAWIO", "CAP_SYS_CHROOT",
	"CAP_SYS_PTRACE", "CAP_SYS_PACCT", "CAP_SYS_ADMIN", "CAP_SYS_BOOT", "CAP_SYS_NICE",
	"CAP_SYS_RESOURCE", "CAP_SYS_TIME", "CAP_SYS_TTY_CONFIG", "CAP_MKNOD", "CAP_LEASE",
	"CAP_AUDIT_WRITE", "CAP_AUDIT_CONTROL", "CAP_SETFCAP", "CAP_MAC_OVERRIDE", "CAP_MAC_ADMIN",
	"CAP_SYSLOG", "CAP_WAKE_ALARM", "CAP_BLOCK_SUSPEND", "CAP_AUDIT_READ",
}
<commit_msg>Pass -debug flag to runsc.<commit_after>// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// Package gvisor provides support for gVisor, user-space kernel, testing.
// See https://github.com/google/gvisor
package gvisor

import (
	"bytes"
	"fmt"
	"io"
	"net"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/google/syzkaller/pkg/config"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/vm/vmimpl"
)

func init() {
	vmimpl.Register("gvisor", ctor, true)
}

type Config struct {
	Count     int    `json:"count"` // number of VMs to use
	RunscArgs string `json:"runsc_args"`
}

type Pool struct {
	env *vmimpl.Env
	cfg *Config
}

type instance struct {
	cfg      *Config
	image    string
	debug    bool
	rootDir  string
	imageDir string
	name     string
	port     int
	cmd      *exec.Cmd
	merger   *vmimpl.OutputMerger
}

func ctor(env *vmimpl.Env) (vmimpl.Pool, error) {
	cfg := &Config{
		Count: 1,
	}
	if err := config.LoadData(env.Config, cfg); err != nil {
		return nil, fmt.Errorf("failed to parse vm config: %v", err)
	}
	if cfg.Count < 1 || cfg.Count > 128 {
		return nil, fmt.Errorf("invalid config param count: %v, want [1, 128]", cfg.Count)
	}
	if env.Debug {
		cfg.Count = 1
	}
	if !osutil.IsExist(env.Image) {
		return nil, fmt.Errorf("image file %q does not exist", env.Image)
	}
	pool := &Pool{
		cfg: cfg,
		env: env,
	}
	return pool, nil
}

func (pool *Pool) Count() int {
	return pool.cfg.Count
}

func (pool *Pool) Create(workdir string, index int) (vmimpl.Instance, error) {
	rootDir := filepath.Clean(filepath.Join(workdir, "..", "gvisor_root"))
	imageDir := filepath.Join(workdir, "image")
	bundleDir := filepath.Join(workdir, "bundle")
	osutil.MkdirAll(rootDir)
	osutil.MkdirAll(bundleDir)
	osutil.MkdirAll(imageDir)

	caps := ""
	for _, c := range sandboxCaps {
		if caps != "" {
			caps += ", "
		}
		caps += "\"" + c + "\""
	}
	vmConfig := fmt.Sprintf(configTempl, imageDir, caps)
	if err := osutil.WriteFile(filepath.Join(bundleDir, "config.json"), []byte(vmConfig)); err != nil {
		return nil, err
	}
	bin, err := exec.LookPath(os.Args[0])
	if err != nil {
		return nil, fmt.Errorf("failed to lookup %v: %v", os.Args[0], err)
	}
	if err := osutil.CopyFile(bin, filepath.Join(imageDir, "init")); err != nil {
		return nil, err
	}

	rpipe, wpipe, err := osutil.LongPipe()
	if err != nil {
		return nil, err
	}
	var tee io.Writer
	if pool.env.Debug {
		tee = os.Stdout
	}
	merger := vmimpl.NewOutputMerger(tee)
	merger.Add("gvisor", rpipe)

	inst := &instance{
		cfg:      pool.cfg,
		image:    pool.env.Image,
		debug:    pool.env.Debug,
		rootDir:  rootDir,
		imageDir: imageDir,
		name:     fmt.Sprintf("%v-%v", pool.env.Name, index),
		merger:   merger,
	}

	// Kill the previous instance in case it's still running.
	osutil.Run(time.Minute, inst.runscCmd("delete", "-force", inst.name))
	time.Sleep(3 * time.Second)

	cmd := inst.runscCmd("run", "-bundle", bundleDir, inst.name)
	cmd.Stdout = wpipe
	cmd.Stderr = wpipe
	if err := cmd.Start(); err != nil {
		wpipe.Close()
		merger.Wait()
		return nil, err
	}
	inst.cmd = cmd
	wpipe.Close()

	if err := inst.waitBoot(); err != nil {
		inst.Close()
		return nil, err
	}
	return inst, nil
}

func (inst *instance) waitBoot() error {
	errorMsg := []byte("FATAL ERROR:")
	bootedMsg := []byte(initStartMsg)
	timeout := time.NewTimer(time.Minute)
	defer timeout.Stop()
	var output []byte
	for {
		select {
		case out := <-inst.merger.Output:
			output = append(output, out...)
			if pos := bytes.Index(output, errorMsg); pos != -1 {
				end := bytes.IndexByte(output[pos:], '\n')
				if end == -1 {
					end = len(output)
				} else {
					end += pos
				}
				return vmimpl.BootError{
					Title:  string(output[pos:end]),
					Output: output,
				}
			}
			if bytes.Contains(output, bootedMsg) {
				return nil
			}
		case err := <-inst.merger.Err:
			return vmimpl.BootError{
				Title:  fmt.Sprintf("runsc failed: %v", err),
				Output: output,
			}
		case <-timeout.C:
			return vmimpl.BootError{
				Title:  "init process did not start",
				Output: output,
			}
		}
	}
}

func (inst *instance) runscCmd(add ...string) *exec.Cmd {
	args := []string{
		"-root", inst.rootDir,
		"-watchdog-action=panic",
		"-panic-signal=12",
		"-network=none",
		"-debug",
	}
	if inst.cfg.RunscArgs != "" {
		args = append(args, strings.Split(inst.cfg.RunscArgs, " ")...)
	}
	args = append(args, add...)
	cmd := osutil.Command(inst.image, args...)
	cmd.Env = []string{
		"GOTRACEBACK=all",
		"GORACE=halt_on_error=1",
	}
	return cmd
}

func (inst *instance) Close() {
	time.Sleep(3 * time.Second)
	osutil.Run(time.Minute, inst.runscCmd("delete", "-force", inst.name))
	inst.cmd.Process.Kill()
	inst.merger.Wait()
	inst.cmd.Wait()
	osutil.Run(time.Minute, inst.runscCmd("delete", "-force", inst.name))
	time.Sleep(3 * time.Second)
}

func (inst *instance) Forward(port int) (string, error) {
	if inst.port != 0 {
		return "", fmt.Errorf("forward port is already setup")
	}
	inst.port = port
	return "stdin", nil
}

func (inst *instance) Copy(hostSrc string) (string, error) {
	fname := filepath.Base(hostSrc)
	if err := osutil.CopyFile(hostSrc, filepath.Join(inst.imageDir, fname)); err != nil {
		return "", err
	}
	if err := os.Chmod(inst.imageDir, 0777); err != nil {
		return "", err
	}
	return filepath.Join("/", fname), nil
}

func (inst *instance) Run(timeout time.Duration, stop <-chan bool, command string) (
	<-chan []byte, <-chan error, error) {
	args := []string{"exec", "-user=0:0"}
	for _, c := range sandboxCaps {
		args = append(args, "-cap", c)
	}
	args = append(args, inst.name)
	args = append(args, strings.Split(command, " ")...)
	cmd := inst.runscCmd(args...)

	rpipe, wpipe, err := osutil.LongPipe()
	if err != nil {
		return nil, nil, err
	}
	defer wpipe.Close()
	inst.merger.Add("cmd", rpipe)
	cmd.Stdout = wpipe
	cmd.Stderr = wpipe

	guestSock, err := inst.guestProxy()
	if err != nil {
		return nil, nil, err
	}
	if guestSock != nil {
		defer guestSock.Close()
		cmd.Stdin = guestSock
	}

	if err := cmd.Start(); err != nil {
		return nil, nil, err
	}
	errc := make(chan error, 1)
	signal := func(err error) {
		select {
		case errc <- err:
		default:
		}
	}

	go func() {
		select {
		case <-time.After(timeout):
			signal(vmimpl.ErrTimeout)
		case <-stop:
			signal(vmimpl.ErrTimeout)
		case err := <-inst.merger.Err:
			cmd.Process.Kill()
			if cmdErr := cmd.Wait(); cmdErr == nil {
				// If the command exited successfully, we got EOF error from merger.
				// But in this case no error has happened and the EOF is expected.
				err = nil
			}
			signal(err)
			return
		}
		cmd.Process.Kill()
		cmd.Wait()
	}()
	return inst.merger.Output, errc, nil
}

func (inst *instance) guestProxy() (*os.File, error) {
	if inst.port == 0 {
		return nil, nil
	}
	// One does not simply let gvisor guest connect to host tcp port.
	// We create a unix socket, pass it to guest in stdin.
	// Guest will use it instead of dialing manager directly.
	// On host we connect to manager tcp port and proxy between the tcp and unix connections.
	socks, err := syscall.Socketpair(syscall.AF_UNIX, syscall.SOCK_STREAM, 0)
	if err != nil {
		return nil, err
	}
	hostSock := os.NewFile(uintptr(socks[0]), "host unix proxy")
	guestSock := os.NewFile(uintptr(socks[1]), "guest unix proxy")
	conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%v", inst.port))
	if err != nil {
		conn.Close()
		hostSock.Close()
		guestSock.Close()
		return nil, err
	}
	go func() {
		io.Copy(hostSock, conn)
		hostSock.Close()
	}()
	go func() {
		io.Copy(conn, hostSock)
		conn.Close()
	}()
	return guestSock, nil
}

func (inst *instance) Diagnose() bool {
	osutil.Run(time.Minute, inst.runscCmd("debug", "-signal=12", inst.name))
	return true
}

func init() {
	if os.Getenv("SYZ_GVISOR_PROXY") != "" {
		fmt.Fprintf(os.Stderr, initStartMsg)
		select {}
	}
}

const initStartMsg = "SYZKALLER INIT STARTED\n"

const configTempl = `
{
	"root": {
		"path": "%[1]v",
		"readonly": true
	},
	"process":{
                "args": ["/init"],
                "cwd": "/tmp",
                "env": ["SYZ_GVISOR_PROXY=1"],
                "capabilities": {
                	"bounding": [%[2]v],
                	"effective": [%[2]v],
                	"inheritable": [%[2]v],
                	"permitted": [%[2]v],
                	"ambient": [%[2]v]
                }
	}
}
`

var sandboxCaps = []string{
	"CAP_CHOWN", "CAP_DAC_OVERRIDE", "CAP_DAC_READ_SEARCH", "CAP_FOWNER", "CAP_FSETID",
	"CAP_KILL", "CAP_SETGID", "CAP_SETUID", "CAP_SETPCAP", "CAP_LINUX_IMMUTABLE",
	"CAP_NET_BIND_SERVICE", "CAP_NET_BROADCAST", "CAP_NET_ADMIN", "CAP_NET_RAW",
	"CAP_IPC_LOCK", "CAP_IPC_OWNER", "CAP_SYS_MODULE", "CAP_SYS_RAWIO", "CAP_SYS_CHROOT",
	"CAP_SYS_PTRACE", "CAP_SYS_PACCT", "CAP_SYS_ADMIN", "CAP_SYS_BOOT", "CAP_SYS_NICE",
	"CAP_SYS_RESOURCE", "CAP_SYS_TIME", "CAP_SYS_TTY_CONFIG", "CAP_MKNOD", "CAP_LEASE",
	"CAP_AUDIT_WRITE", "CAP_AUDIT_CONTROL", "CAP_SETFCAP", "CAP_MAC_OVERRIDE", "CAP_MAC_ADMIN",
	"CAP_SYSLOG", "CAP_WAKE_ALARM", "CAP_BLOCK_SUSPEND", "CAP_AUDIT_READ",
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package translator

import (
	"fmt"
	"strings"
	"time"

	stackdriver "google.golang.org/api/monitoring/v3"
	apierr "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/klog"
	"k8s.io/metrics/pkg/apis/custom_metrics"
	"k8s.io/metrics/pkg/apis/external_metrics"
	"sigs.k8s.io/custom-metrics-apiserver/pkg/provider"
)

const (
	PodSchemaKey  = "pods"  // PodSchemaKey is the schema key for pod metrics
	NodeSchemaKey = "nodes" // NodeSchemaKey is the schema key for node metrics
)

// GetRespForSingleObject returns translates Stackdriver response to a Custom Metric associated with
// a single object.
func (t *Translator) GetRespForSingleObject(response *stackdriver.ListTimeSeriesResponse, groupResource schema.GroupResource, metricName string, metricSelector labels.Selector, namespace string, name string) (*custom_metrics.MetricValue, error) {
	values, err := t.getMetricValuesFromResponse(groupResource, response, metricName)
	if err != nil {
		return nil, err
	}
	if len(values) == 0 {
		return nil, provider.NewMetricNotFoundForError(groupResource, metricName, name)
	}
	if len(values) > 1 {
		return nil, apierr.NewInternalError(fmt.Errorf("Expected exactly one value for resource %q in namespace %q, but received %v values", name, namespace, len(values)))
	}
	// Since len(values) = 1, this loop will execute only once.
	for _, value := range values {
		metricValue, err := t.metricFor(value, groupResource, namespace, name, metricName, metricSelector)
		if err != nil {
			return nil, err
		}
		return metricValue, nil
	}
	// This code is unreacheable
	return nil, apierr.NewInternalError(fmt.Errorf("Illegal state"))
}

// GetRespForMultipleObjects translates Stackdriver response to a Custom Metric associated
// with multiple pods.
func (t *Translator) GetRespForMultipleObjects(response *stackdriver.ListTimeSeriesResponse, list []metav1.ObjectMeta, groupResource schema.GroupResource, metricName string, metricSelector labels.Selector) ([]custom_metrics.MetricValue, error) {
	values, err := t.getMetricValuesFromResponse(groupResource, response, metricName)
	if err != nil {
		return nil, err
	}
	return t.metricsFor(values, groupResource, metricName, metricSelector, list)
}

// GetRespForExternalMetric translates Stackdriver response to list of External Metrics
func (t *Translator) GetRespForExternalMetric(response *stackdriver.ListTimeSeriesResponse, metricName string) ([]external_metrics.ExternalMetricValue, error) {
	metricValues := []external_metrics.ExternalMetricValue{}
	for _, series := range response.TimeSeries {
		if len(series.Points) <= 0 {
			// This shouldn't happen with correct query to Stackdriver
			return nil, apierr.NewInternalError(fmt.Errorf("Empty time series returned from Stackdriver"))
		}
		// Points in a time series are returned in reverse time order
		point := series.Points[0]
		endTime, err := time.Parse(time.RFC3339, point.Interval.EndTime)
		if err != nil {
			return nil, apierr.NewInternalError(fmt.Errorf("Timeseries from Stackdriver has incorrect end time: %s", point.Interval.EndTime))
		}
		metricValue := external_metrics.ExternalMetricValue{
			Timestamp:    metav1.NewTime(endTime),
			MetricName:   metricName,
			MetricLabels: t.getMetricLabels(series),
		}
		value := *point.Value
		switch {
		case value.Int64Value != nil:
			metricValue.Value = *resource.NewQuantity(*value.Int64Value, resource.DecimalSI)
		case value.DoubleValue != nil:
			metricValue.Value = *resource.NewMilliQuantity(int64(*value.DoubleValue*1000), resource.DecimalSI)
		default:
			return nil, apierr.NewBadRequest(fmt.Sprintf("Expected metric of type DoubleValue or Int64Value, but received TypedValue: %v", value))
		}
		metricValues = append(metricValues, metricValue)
	}
	return metricValues, nil
}

// GetMetricsFromSDDescriptorsResp returns an array of MetricInfo for all metric descriptors
// returned by Stackdriver API that satisfy the requirements:
// - valueType is "INT64" or "DOUBLE"
// - metric name doesn't contain "/" character after "custom.googleapis.com/" prefix
func (t *Translator) GetMetricsFromSDDescriptorsResp(response *stackdriver.ListMetricDescriptorsResponse) []provider.CustomMetricInfo {
	metrics := []provider.CustomMetricInfo{}
	for _, descriptor := range response.MetricDescriptors {
		if descriptor.ValueType == "INT64" || descriptor.ValueType == "DOUBLE" {
			metrics = append(metrics, provider.CustomMetricInfo{
				GroupResource: schema.GroupResource{Group: "", Resource: "*"},
				Metric:        escapeMetric(descriptor.Type),
				Namespaced:    true,
			})
		}
	}
	return metrics
}

// CheckMetricUniquenessForPod checks if each pod has at most one container with given metric
func (t *Translator) CheckMetricUniquenessForPod(response *stackdriver.ListTimeSeriesResponse, metricName string) error {
	metricContainer := make(map[string]string)
	for _, series := range response.TimeSeries {
		name, err := t.metricKey(series, PodSchemaKey)
		if err != nil {
			return err
		}

		containerName, ok := series.Resource.Labels["container_name"]
		if !ok {
			return apierr.NewInternalError(fmt.Errorf("container_name is missing"))
		}
		metricContainerName, ok := metricContainer[name]
		if !ok {
			metricContainer[name] = containerName
		} else {
			if metricContainerName != containerName {
				return apierr.NewBadRequest(fmt.Sprintf("Only one container in pod can have specific metric. Containers %x %x have the same metric %x in pod %x", metricContainerName, containerName, metricName, name))
			}
		}
	}
	return nil
}

func (t *Translator) getMetricValuesFromResponse(groupResource schema.GroupResource, response *stackdriver.ListTimeSeriesResponse, metricName string) (map[string]resource.Quantity, error) {
	metricValues := make(map[string]resource.Quantity)
	for _, series := range response.TimeSeries {
		if len(series.Points) <= 0 {
			// This shouldn't happen with correct query to Stackdriver
			return nil, apierr.NewInternalError(fmt.Errorf("Empty time series returned from Stackdriver"))
		}
		// Points in a time series are returned in reverse time order
		point := *series.Points[0]
		value := point.Value
		name, err := t.metricKey(series, groupResource.String())
		if err != nil {
			return nil, err
		}

		currentQuantity, ok := metricValues[name]
		if !ok {
			currentQuantity = *resource.NewQuantity(0, resource.DecimalSI)
		}
		switch {
		case value.Int64Value != nil:
			currentQuantity.Add(*resource.NewQuantity(*value.Int64Value, resource.DecimalSI))
			metricValues[name] = currentQuantity
		case value.DoubleValue != nil:
			currentQuantity.Add(*resource.NewMilliQuantity(int64(*value.DoubleValue*1000), resource.DecimalSI))
			metricValues[name] = currentQuantity
		default:
			return nil, apierr.NewBadRequest(fmt.Sprintf("Expected metric of type DoubleValue or Int64Value, but received TypedValue: %v", value))
		}
	}
	return metricValues, nil
}

func (t *Translator) metricFor(value resource.Quantity, groupResource schema.GroupResource, namespace string, name string, metricName string, metricSelector labels.Selector) (*custom_metrics.MetricValue, error) {
	kind, err := t.mapper.KindFor(groupResource.WithVersion(""))
	if err != nil {
		return nil, err
	}
	var metricLabelSelector *metav1.LabelSelector
	if !metricSelector.Empty() {
		metricLabelSelector, err = metav1.ParseToLabelSelector(metricSelector.String())
		if err != nil {
			return nil, err
		}
	}

	return &custom_metrics.MetricValue{
		DescribedObject: custom_metrics.ObjectReference{
			APIVersion: groupResource.Group + "/" + runtime.APIVersionInternal,
			Kind:       kind.Kind,
			Name:       name,
			Namespace:  namespace,
		},
		Metric: custom_metrics.MetricIdentifier{
			Name:     metricName,
			Selector: metricLabelSelector,
		},
		// TODO(kawych): metric timestamp should be retrieved from Stackdriver response instead
		Timestamp: metav1.Time{t.clock.Now()},
		Value:     value,
	}, nil
}

func (t *Translator) metricsFor(values map[string]resource.Quantity, groupResource schema.GroupResource, metricName string, metricSelector labels.Selector, list []metav1.ObjectMeta) ([]custom_metrics.MetricValue, error) {
	res := make([]custom_metrics.MetricValue, 0)

	for _, item := range list {
		if _, ok := values[t.resourceKey(item)]; !ok {
			klog.V(4).Infof("Metric '%s' not found for pod '%s'", metricName, item.GetName())
			continue
		}
		value, err := t.metricFor(values[t.resourceKey(item)], groupResource, item.GetNamespace(), item.GetName(), metricName, metricSelector)
		if err != nil {
			return nil, err
		}
		res = append(res, *value)
	}
	return res, nil
}

func (t *Translator) resourceKey(object metav1.ObjectMeta) string {
	if t.useNewResourceModel {
		return object.GetNamespace() + ":" + object.GetName()
	}
	return fmt.Sprintf("%s", object.GetUID())
}

func (t *Translator) metricKey(timeSeries *stackdriver.TimeSeries, resourceSchema string) (string, error) {
	if t.useNewResourceModel {
		switch timeSeries.Resource.Type {
		case "k8s_pod":
			return timeSeries.Resource.Labels["namespace_name"] + ":" + timeSeries.Resource.Labels["pod_name"], nil
		case "k8s_container":
			// The same key as pod, because only one container in pod can provide specific metric. Uniqueness is checked in CheckMetricUniquenessForPod.
			return timeSeries.Resource.Labels["namespace_name"] + ":" + timeSeries.Resource.Labels["pod_name"], nil
		case "k8s_node":
			return ":" + timeSeries.Resource.Labels["node_name"], nil
		case "prometheus_target":
			if resourceSchema == "nodes" {
				// use node label if exists
				if nodeName, ok := timeSeries.Resource.Labels["node"]; ok {
					return ":" + nodeName, nil
				}
				// for prometheus_target metric, its instance usually consist of node id and the port who emited the metric.
				// splitting with ":" and take the first part is to ignore the port part.
				return ":" + strings.Split(timeSeries.Resource.Labels["instance"], ":")[0], nil
			}
			return timeSeries.Resource.Labels["namespace"] + ":" + timeSeries.Metric.Labels["pod"], nil
		default:
			klog.Errorf("Expected resource type as one of [\"k8s_pod\", \"k8s_container\", \"k8s_node\", \"prometheus_target\"], but received %s", timeSeries.Resource.Type)
		}
	} else {
		return timeSeries.Resource.Labels["pod_id"], nil
	}
	return "", apierr.NewInternalError(fmt.Errorf("Stackdriver returned incorrect resource type %q", timeSeries.Resource.Type))
}

func escapeMetric(metricName string) string {
	return strings.Replace(metricName, "/", "|", -1)
}
<commit_msg>update prometheus node metrics to use node lable from metric section<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

		http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package translator

import (
	"fmt"
	"strings"
	"time"

	stackdriver "google.golang.org/api/monitoring/v3"
	apierr "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/klog"
	"k8s.io/metrics/pkg/apis/custom_metrics"
	"k8s.io/metrics/pkg/apis/external_metrics"
	"sigs.k8s.io/custom-metrics-apiserver/pkg/provider"
)

const (
	PodSchemaKey  = "pods"  // PodSchemaKey is the schema key for pod metrics
	NodeSchemaKey = "nodes" // NodeSchemaKey is the schema key for node metrics
)

// GetRespForSingleObject returns translates Stackdriver response to a Custom Metric associated with
// a single object.
func (t *Translator) GetRespForSingleObject(response *stackdriver.ListTimeSeriesResponse, groupResource schema.GroupResource, metricName string, metricSelector labels.Selector, namespace string, name string) (*custom_metrics.MetricValue, error) {
	values, err := t.getMetricValuesFromResponse(groupResource, response, metricName)
	if err != nil {
		return nil, err
	}
	if len(values) == 0 {
		return nil, provider.NewMetricNotFoundForError(groupResource, metricName, name)
	}
	if len(values) > 1 {
		return nil, apierr.NewInternalError(fmt.Errorf("Expected exactly one value for resource %q in namespace %q, but received %v values", name, namespace, len(values)))
	}
	// Since len(values) = 1, this loop will execute only once.
	for _, value := range values {
		metricValue, err := t.metricFor(value, groupResource, namespace, name, metricName, metricSelector)
		if err != nil {
			return nil, err
		}
		return metricValue, nil
	}
	// This code is unreacheable
	return nil, apierr.NewInternalError(fmt.Errorf("Illegal state"))
}

// GetRespForMultipleObjects translates Stackdriver response to a Custom Metric associated
// with multiple pods.
func (t *Translator) GetRespForMultipleObjects(response *stackdriver.ListTimeSeriesResponse, list []metav1.ObjectMeta, groupResource schema.GroupResource, metricName string, metricSelector labels.Selector) ([]custom_metrics.MetricValue, error) {
	values, err := t.getMetricValuesFromResponse(groupResource, response, metricName)
	if err != nil {
		return nil, err
	}
	return t.metricsFor(values, groupResource, metricName, metricSelector, list)
}

// GetRespForExternalMetric translates Stackdriver response to list of External Metrics
func (t *Translator) GetRespForExternalMetric(response *stackdriver.ListTimeSeriesResponse, metricName string) ([]external_metrics.ExternalMetricValue, error) {
	metricValues := []external_metrics.ExternalMetricValue{}
	for _, series := range response.TimeSeries {
		if len(series.Points) <= 0 {
			// This shouldn't happen with correct query to Stackdriver
			return nil, apierr.NewInternalError(fmt.Errorf("Empty time series returned from Stackdriver"))
		}
		// Points in a time series are returned in reverse time order
		point := series.Points[0]
		endTime, err := time.Parse(time.RFC3339, point.Interval.EndTime)
		if err != nil {
			return nil, apierr.NewInternalError(fmt.Errorf("Timeseries from Stackdriver has incorrect end time: %s", point.Interval.EndTime))
		}
		metricValue := external_metrics.ExternalMetricValue{
			Timestamp:    metav1.NewTime(endTime),
			MetricName:   metricName,
			MetricLabels: t.getMetricLabels(series),
		}
		value := *point.Value
		switch {
		case value.Int64Value != nil:
			metricValue.Value = *resource.NewQuantity(*value.Int64Value, resource.DecimalSI)
		case value.DoubleValue != nil:
			metricValue.Value = *resource.NewMilliQuantity(int64(*value.DoubleValue*1000), resource.DecimalSI)
		default:
			return nil, apierr.NewBadRequest(fmt.Sprintf("Expected metric of type DoubleValue or Int64Value, but received TypedValue: %v", value))
		}
		metricValues = append(metricValues, metricValue)
	}
	return metricValues, nil
}

// GetMetricsFromSDDescriptorsResp returns an array of MetricInfo for all metric descriptors
// returned by Stackdriver API that satisfy the requirements:
// - valueType is "INT64" or "DOUBLE"
// - metric name doesn't contain "/" character after "custom.googleapis.com/" prefix
func (t *Translator) GetMetricsFromSDDescriptorsResp(response *stackdriver.ListMetricDescriptorsResponse) []provider.CustomMetricInfo {
	metrics := []provider.CustomMetricInfo{}
	for _, descriptor := range response.MetricDescriptors {
		if descriptor.ValueType == "INT64" || descriptor.ValueType == "DOUBLE" {
			metrics = append(metrics, provider.CustomMetricInfo{
				GroupResource: schema.GroupResource{Group: "", Resource: "*"},
				Metric:        escapeMetric(descriptor.Type),
				Namespaced:    true,
			})
		}
	}
	return metrics
}

// CheckMetricUniquenessForPod checks if each pod has at most one container with given metric
func (t *Translator) CheckMetricUniquenessForPod(response *stackdriver.ListTimeSeriesResponse, metricName string) error {
	metricContainer := make(map[string]string)
	for _, series := range response.TimeSeries {
		name, err := t.metricKey(series, PodSchemaKey)
		if err != nil {
			return err
		}

		containerName, ok := series.Resource.Labels["container_name"]
		if !ok {
			return apierr.NewInternalError(fmt.Errorf("container_name is missing"))
		}
		metricContainerName, ok := metricContainer[name]
		if !ok {
			metricContainer[name] = containerName
		} else {
			if metricContainerName != containerName {
				return apierr.NewBadRequest(fmt.Sprintf("Only one container in pod can have specific metric. Containers %x %x have the same metric %x in pod %x", metricContainerName, containerName, metricName, name))
			}
		}
	}
	return nil
}

func (t *Translator) getMetricValuesFromResponse(groupResource schema.GroupResource, response *stackdriver.ListTimeSeriesResponse, metricName string) (map[string]resource.Quantity, error) {
	metricValues := make(map[string]resource.Quantity)
	for _, series := range response.TimeSeries {
		if len(series.Points) <= 0 {
			// This shouldn't happen with correct query to Stackdriver
			return nil, apierr.NewInternalError(fmt.Errorf("Empty time series returned from Stackdriver"))
		}
		// Points in a time series are returned in reverse time order
		point := *series.Points[0]
		value := point.Value
		name, err := t.metricKey(series, groupResource.String())
		if err != nil {
			return nil, err
		}

		currentQuantity, ok := metricValues[name]
		if !ok {
			currentQuantity = *resource.NewQuantity(0, resource.DecimalSI)
		}
		switch {
		case value.Int64Value != nil:
			currentQuantity.Add(*resource.NewQuantity(*value.Int64Value, resource.DecimalSI))
			metricValues[name] = currentQuantity
		case value.DoubleValue != nil:
			currentQuantity.Add(*resource.NewMilliQuantity(int64(*value.DoubleValue*1000), resource.DecimalSI))
			metricValues[name] = currentQuantity
		default:
			return nil, apierr.NewBadRequest(fmt.Sprintf("Expected metric of type DoubleValue or Int64Value, but received TypedValue: %v", value))
		}
	}
	return metricValues, nil
}

func (t *Translator) metricFor(value resource.Quantity, groupResource schema.GroupResource, namespace string, name string, metricName string, metricSelector labels.Selector) (*custom_metrics.MetricValue, error) {
	kind, err := t.mapper.KindFor(groupResource.WithVersion(""))
	if err != nil {
		return nil, err
	}
	var metricLabelSelector *metav1.LabelSelector
	if !metricSelector.Empty() {
		metricLabelSelector, err = metav1.ParseToLabelSelector(metricSelector.String())
		if err != nil {
			return nil, err
		}
	}

	return &custom_metrics.MetricValue{
		DescribedObject: custom_metrics.ObjectReference{
			APIVersion: groupResource.Group + "/" + runtime.APIVersionInternal,
			Kind:       kind.Kind,
			Name:       name,
			Namespace:  namespace,
		},
		Metric: custom_metrics.MetricIdentifier{
			Name:     metricName,
			Selector: metricLabelSelector,
		},
		// TODO(kawych): metric timestamp should be retrieved from Stackdriver response instead
		Timestamp: metav1.Time{t.clock.Now()},
		Value:     value,
	}, nil
}

func (t *Translator) metricsFor(values map[string]resource.Quantity, groupResource schema.GroupResource, metricName string, metricSelector labels.Selector, list []metav1.ObjectMeta) ([]custom_metrics.MetricValue, error) {
	res := make([]custom_metrics.MetricValue, 0)

	for _, item := range list {
		if _, ok := values[t.resourceKey(item)]; !ok {
			klog.V(4).Infof("Metric '%s' not found for pod '%s'", metricName, item.GetName())
			continue
		}
		value, err := t.metricFor(values[t.resourceKey(item)], groupResource, item.GetNamespace(), item.GetName(), metricName, metricSelector)
		if err != nil {
			return nil, err
		}
		res = append(res, *value)
	}
	return res, nil
}

func (t *Translator) resourceKey(object metav1.ObjectMeta) string {
	if t.useNewResourceModel {
		return object.GetNamespace() + ":" + object.GetName()
	}
	return fmt.Sprintf("%s", object.GetUID())
}

func (t *Translator) metricKey(timeSeries *stackdriver.TimeSeries, resourceSchema string) (string, error) {
	if t.useNewResourceModel {
		switch timeSeries.Resource.Type {
		case "k8s_pod":
			return timeSeries.Resource.Labels["namespace_name"] + ":" + timeSeries.Resource.Labels["pod_name"], nil
		case "k8s_container":
			// The same key as pod, because only one container in pod can provide specific metric. Uniqueness is checked in CheckMetricUniquenessForPod.
			return timeSeries.Resource.Labels["namespace_name"] + ":" + timeSeries.Resource.Labels["pod_name"], nil
		case "k8s_node":
			return ":" + timeSeries.Resource.Labels["node_name"], nil
		case "prometheus_target":
			if resourceSchema == "nodes" {
				// use node label if exists
				if nodeName, ok := timeSeries.Metric.Labels["node"]; ok {
					return ":" + nodeName, nil
				}
				// for prometheus_target metric, its instance usually consist of node id and the port who emited the metric.
				// splitting with ":" and take the first part is to ignore the port part.
				return ":" + strings.Split(timeSeries.Resource.Labels["instance"], ":")[0], nil
			}
			return timeSeries.Resource.Labels["namespace"] + ":" + timeSeries.Metric.Labels["pod"], nil
		default:
			klog.Errorf("Expected resource type as one of [\"k8s_pod\", \"k8s_container\", \"k8s_node\", \"prometheus_target\"], but received %s", timeSeries.Resource.Type)
		}
	} else {
		return timeSeries.Resource.Labels["pod_id"], nil
	}
	return "", apierr.NewInternalError(fmt.Errorf("Stackdriver returned incorrect resource type %q", timeSeries.Resource.Type))
}

func escapeMetric(metricName string) string {
	return strings.Replace(metricName, "/", "|", -1)
}
<|endoftext|>


<commit_before>// Copyright 2022 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package kexec

// Params communicates boot information to
// the purgatory, and possibly the kernel.
type Params struct {
	Entry  uint64
	Params uint64
	_      [5]uint64
}
<commit_msg>kexec: delete unused code<commit_after><|endoftext|>


<commit_before>package kubelego

import (
	"testing"

	"github.com/jetstack/kube-lego/pkg/kubelego_const"

	"github.com/stretchr/testify/assert"
	k8sApi "k8s.io/kubernetes/pkg/api"
	"sort"
)

type mockTls struct {
	SecretName  string
	IngressName string
	Namespace   string
	hosts       []string
}

func (m *mockTls) Hosts() []string {
	return m.hosts
}

func (m *mockTls) SecretMetadata() *k8sApi.ObjectMeta {
	return &k8sApi.ObjectMeta{
		Name:      m.SecretName,
		Namespace: m.Namespace,
	}
}

func (m *mockTls) IngressMetadata() *k8sApi.ObjectMeta {
	return &k8sApi.ObjectMeta{
		Name:      m.IngressName,
		Namespace: m.Namespace,
	}
}

func (m *mockTls) Process() error {
	// process a lot
	return nil
}

func getTlsExample() []kubelego.Tls {
	return []kubelego.Tls{
		&mockTls{
			SecretName:  "secret1",
			IngressName: "ingress1",
			Namespace:   "namespace1",
			hosts:       []string{"domain1"},
		},
		&mockTls{
			SecretName:  "secret2",
			IngressName: "ingress2",
			Namespace:   "namespace1",
			hosts:       []string{"domain2"},
		},
		&mockTls{
			SecretName:  "secret1",
			IngressName: "ingress3",
			Namespace:   "namespace2",
			hosts:       []string{"domain3", "domain4"},
		},
		&mockTls{
			SecretName:  "secret1",
			IngressName: "ingress4",
			Namespace:   "namespace1",
			hosts:       []string{"domain1"},
		},
	}
}

func TestKubeLego_TlsIgnoreDuplicatedSecrets(t *testing.T) {
	k := KubeLego{}
	input := getTlsExample()
	output := k.TlsIgnoreDuplicatedSecrets(input)
	assert.EqualValues(t, 2, len(output))
}

func TestKubeLego_TlsAggregateHosts(t *testing.T) {
	k := KubeLego{}
	input := getTlsExample()
	output := k.TlsAggregateHosts(input)
	expected := []string{"domain1", "domain2", "domain3", "domain4"}
	sort.Strings(output)
	sort.Strings(expected)
	assert.EqualValues(
		t,
		expected,
		output,
	)
}
<commit_msg>Fix tests<commit_after>package kubelego

import (
	"testing"
	"time"

	"github.com/jetstack/kube-lego/pkg/kubelego_const"

	"github.com/stretchr/testify/assert"
	k8sApi "k8s.io/kubernetes/pkg/api"
	"sort"
)

type mockTls struct {
	SecretName  string
	IngressName string
	Namespace   string
	hosts       []string
}

func (m *mockTls) Hosts() []string {
	return m.hosts
}

func (m *mockTls) SecretMetadata() *k8sApi.ObjectMeta {
	return &k8sApi.ObjectMeta{
		Name:      m.SecretName,
		Namespace: m.Namespace,
	}
}

func (m *mockTls) IngressMetadata() *k8sApi.ObjectMeta {
	return &k8sApi.ObjectMeta{
		Name:      m.IngressName,
		Namespace: m.Namespace,
	}
}

func (m *mockTls) Process(mV time.Duration) error {
	// process a lot
	return nil
}

func getTlsExample() []kubelego.Tls {
	return []kubelego.Tls{
		&mockTls{
			SecretName:  "secret1",
			IngressName: "ingress1",
			Namespace:   "namespace1",
			hosts:       []string{"domain1"},
		},
		&mockTls{
			SecretName:  "secret2",
			IngressName: "ingress2",
			Namespace:   "namespace1",
			hosts:       []string{"domain2"},
		},
		&mockTls{
			SecretName:  "secret1",
			IngressName: "ingress3",
			Namespace:   "namespace2",
			hosts:       []string{"domain3", "domain4"},
		},
		&mockTls{
			SecretName:  "secret1",
			IngressName: "ingress4",
			Namespace:   "namespace1",
			hosts:       []string{"domain1"},
		},
	}
}

func TestKubeLego_TlsIgnoreDuplicatedSecrets(t *testing.T) {
	k := KubeLego{}
	input := getTlsExample()
	output := k.TlsIgnoreDuplicatedSecrets(input)
	assert.EqualValues(t, 2, len(output))
}

func TestKubeLego_TlsAggregateHosts(t *testing.T) {
	k := KubeLego{}
	input := getTlsExample()
	output := k.TlsAggregateHosts(input)
	expected := []string{"domain1", "domain2", "domain3", "domain4"}
	sort.Strings(output)
	sort.Strings(expected)
	assert.EqualValues(
		t,
		expected,
		output,
	)
}
<|endoftext|>


<commit_before>package opentracing

import (
	"context"
	"io"
	"net/http"
	"time"

	"github.com/hellofresh/gcloud-opentracing"
	"github.com/hellofresh/janus/pkg/config"
	"github.com/hellofresh/janus/pkg/opentracing/appdash"
	"github.com/opentracing/opentracing-go"
	log "github.com/sirupsen/logrus"
	jaeger "github.com/uber/jaeger-client-go"
	jaegercfg "github.com/uber/jaeger-client-go/config"
	"github.com/uber/jaeger-lib/metrics"
)

type noopCloser struct{}

func (n noopCloser) Close() error { return nil }

// Build a tracer based on the configuration provided
func Build(config config.Tracing) (opentracing.Tracer, io.Closer, error) {
	switch config.Tracer {
	case "gcloud":
		log.Debug("Using google cloud platform (stackdriver trace) as tracing system")
		tracer, err := buildGCloud(config.GoogleCloudTracing)
		return tracer, noopCloser{}, err
	case "appdash":
		tracer, err := buildAppdash(config.AppdashTracing)
		return tracer, noopCloser{}, err
	case "jaeger":
		return buildJaeger(config.JaegerTracing)
	default:
		log.Debug("No tracer selected")
		return &opentracing.NoopTracer{}, noopCloser{}, nil
	}
}

// FromContext creates a span from a context that contains a parent span
func FromContext(ctx context.Context, name string) opentracing.Span {
	span, _ := opentracing.StartSpanFromContext(ctx, name)
	return span
}

// ToContext sets a span to a context
func ToContext(r *http.Request, span opentracing.Span) *http.Request {
	return r.WithContext(opentracing.ContextWithSpan(r.Context(), span))
}

func buildGCloud(config config.GoogleCloudTracing) (opentracing.Tracer, error) {
	tracer, err := gcloudtracer.NewTracer(
		context.Background(),
		gcloudtracer.WithLogger(log.StandardLogger()),
		gcloudtracer.WithProject(config.ProjectID),
		gcloudtracer.WithJWTCredentials(gcloudtracer.JWTCredentials{
			Email:        config.Email,
			PrivateKey:   []byte(config.PrivateKey),
			PrivateKeyID: config.PrivateKeyID,
		}),
	)
	if err != nil {
		return nil, err
	}

	return tracer, nil
}

func buildAppdash(config config.AppdashTracing) (opentracing.Tracer, error) {
	server := appdash.NewServer(config.DSN, config.URL)

	appdashFields := log.WithFields(log.Fields{
		"appdash_dsn":    config.DSN,
		"appdash_web_ui": config.URL,
	})

	if config.URL != "" {
		appdashFields.Debug("Using local appdash server as tracing system")
		err := server.Listen()
		if err != nil {
			return nil, err
		}
	} else {
		appdashFields.Debug("Using remote appdash server as tracing system")
	}

	return server.GetTracer(), nil
}

func buildJaeger(c config.JaegerTracing) (opentracing.Tracer, io.Closer, error) {
	cfg := jaegercfg.Configuration{
		Sampler: &jaegercfg.SamplerConfig{
			Type:  jaeger.SamplerTypeConst,
			Param: 1,
		},
		Reporter: &jaegercfg.ReporterConfig{
			LogSpans:            true,
			BufferFlushInterval: 1 * time.Second,
			LocalAgentHostPort:  c.DSN,
		},
	}

	return cfg.New(
		"janus",
		jaegercfg.Logger(jaegerLoggerAdapter{log.StandardLogger()}),
		jaegercfg.Metrics(metrics.NullFactory),
	)
}
<commit_msg>No more appdash<commit_after>package opentracing

import (
	"context"
	"io"
	"net/http"
	"time"

	"github.com/hellofresh/gcloud-opentracing"
	"github.com/hellofresh/janus/pkg/config"
	"github.com/opentracing/opentracing-go"
	log "github.com/sirupsen/logrus"
	jaeger "github.com/uber/jaeger-client-go"
	jaegercfg "github.com/uber/jaeger-client-go/config"
	"github.com/uber/jaeger-lib/metrics"
)

type noopCloser struct{}

func (n noopCloser) Close() error { return nil }

// Build a tracer based on the configuration provided
func Build(config config.Tracing) (opentracing.Tracer, io.Closer, error) {
	switch config.Tracer {
	case "gcloud":
		log.Debug("Using google cloud platform (stackdriver trace) as tracing system")
		tracer, err := buildGCloud(config.GoogleCloudTracing)
		return tracer, noopCloser{}, err
	case "jaeger":
		return buildJaeger(config.JaegerTracing)
	default:
		log.Debug("No tracer selected")
		return &opentracing.NoopTracer{}, noopCloser{}, nil
	}
}

// FromContext creates a span from a context that contains a parent span
func FromContext(ctx context.Context, name string) opentracing.Span {
	span, _ := opentracing.StartSpanFromContext(ctx, name)
	return span
}

// ToContext sets a span to a context
func ToContext(r *http.Request, span opentracing.Span) *http.Request {
	return r.WithContext(opentracing.ContextWithSpan(r.Context(), span))
}

func buildGCloud(config config.GoogleCloudTracing) (opentracing.Tracer, error) {
	tracer, err := gcloudtracer.NewTracer(
		context.Background(),
		gcloudtracer.WithLogger(log.StandardLogger()),
		gcloudtracer.WithProject(config.ProjectID),
		gcloudtracer.WithJWTCredentials(gcloudtracer.JWTCredentials{
			Email:        config.Email,
			PrivateKey:   []byte(config.PrivateKey),
			PrivateKeyID: config.PrivateKeyID,
		}),
	)
	if err != nil {
		return nil, err
	}

	return tracer, nil
}

func buildJaeger(c config.JaegerTracing) (opentracing.Tracer, io.Closer, error) {
	cfg := jaegercfg.Configuration{
		Sampler: &jaegercfg.SamplerConfig{
			Type:  jaeger.SamplerTypeConst,
			Param: 1,
		},
		Reporter: &jaegercfg.ReporterConfig{
			LogSpans:            true,
			BufferFlushInterval: 1 * time.Second,
			LocalAgentHostPort:  c.DSN,
		},
	}

	return cfg.New(
		c.ServiceName,
		jaegercfg.Logger(jaegerLoggerAdapter{log.StandardLogger()}),
		jaegercfg.Metrics(metrics.NullFactory),
	)
}
<|endoftext|>


<commit_before>// +build openbsd

package protect

import "golang.org/x/sys/unix"

// ProtectEnabled lets us know if we have protection or not
var ProtectEnabled = true

// Pledge on OpenBSD lets us "promise" to only run a subset of
// system calls: http://man.openbsd.org/pledge
func Pledge(s string) error {
	return unix.Pledge(s, "")
}
<commit_msg>Update plegde call on OpenBSD. (#1015)<commit_after>// +build openbsd

package protect

import "golang.org/x/sys/unix"

// ProtectEnabled lets us know if we have protection or not
var ProtectEnabled = true

// Pledge on OpenBSD lets us "promise" to only run a subset of
// system calls: http://man.openbsd.org/pledge
func Pledge(s string) error {
	return unix.PledgePromises(s)
}
<|endoftext|>


<commit_before>package yamlconfig

import (
	"fmt"
	"github.com/denkhaus/tcgl/applog"
	"github.com/globocom/config"
	"os"
	"os/user"
	"path"
	"strconv"
	"time"
)

type loadDefFn func(conf *Config)

type Config struct {
	defaults       map[string]interface{}
	configFileName string
}

func NewConfig(fileName string) *Config {
	config := &Config{
		configFileName: fileName,
		defaults:       make(map[string]interface{}),
	}

	return config
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
//
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) ThrowKeyPanic(key string) {
	panic(fmt.Sprintf("config error: key %s not available", key))
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
//
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) ThrowConversionPanic(src interface{}, err error) {
	panic(fmt.Sprintf("config error: cannot convert %v :: error : %s", src, err.Error()))
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
//
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) stringSlice2IntSlice(values []string) []int {

	if values != nil {
		res := make([]int, len(values))
		for n, src := range values {
			if v, err := strconv.Atoi(src); err == nil {
				res[n] = v
			} else {
				c.ThrowConversionPanic(src, err)
			}
		}
		return res
	}

	return nil
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
// GetInt
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) GetInt(key string) int {

	value, err := config.GetInt(key)
	if err != nil {
		if value, ok := c.defaults[key]; ok {
			return value.(int)
		}
		c.ThrowKeyPanic(key)
	}

	return value
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
// GetIntList
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) GetIntList(key string) []int {

	if value, err := config.GetList(key); err == nil {
		return c.stringSlice2IntSlice(value)
	} else {
		if val, ok := c.defaults[key]; ok {
			return val.([]int)
		} else {
			c.ThrowKeyPanic(key)
		}
	}

	return nil
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
// GetString
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) GetString(key string) string {

	value, err := config.GetString(key)
	if err != nil {
		if val, ok := c.defaults[key]; ok {
			return val.(string)
		} else {
			c.ThrowKeyPanic(key)
		}
	}

	return value
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
// GetStringList
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) GetStringList(key string) []string {

	value, err := config.GetList(key)
	if err != nil {
		if value, ok := c.defaults[key]; ok {
			return value.([]string)
		} else {
			c.ThrowKeyPanic(key)
		}
	}

	return value
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
// GetDuration
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) GetDuration(key string) time.Duration {

	value, err := config.GetDuration(key)
	if err != nil {
		if val, ok := c.defaults[key]; ok {
			return val.(time.Duration)
		} else {
			c.ThrowKeyPanic(key)
		}
	}

	return value
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
//
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) SetDefault(key string, value interface{}) {
	c.defaults[key] = value
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
//
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) writeDefConfigFile(filePath string) error {

	for key, value := range c.defaults {
		config.Set(key, value)
	}

	applog.Infof("Create new config file at %s", filePath)
	return config.WriteConfigFile(filePath, 0644)
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
// If filePath is defined and the file exists, Load gets the configuration from there, otherwise
// it looks for the filename, specified in NewConfig in the current users home directory. If no file can be found
// the function creates a file with the users default values.
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) Load(loadDefaults loadDefFn, filePath string, watchConfig bool) error {

	if len(filePath) == 0 {
		usr, err := user.Current()
		if err != nil {
			return err
		}

		filePath = path.Join(usr.HomeDir, c.configFileName)
	}

	loadDefaults(c)
	if _, err := os.Stat(filePath); err == nil {
		if watchConfig {
			if err := config.ReadAndWatchConfigFile(filePath); err != nil {
				return err
			}
		} else {
			if err := config.ReadConfigFile(filePath); err != nil {
				return err
			}
		}
	} else {
		if err = c.writeDefConfigFile(filePath); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>progress<commit_after>package yamlconfig

import (
	"fmt"
	"github.com/denkhaus/tcgl/applog"
	"github.com/globocom/config"
	"os"
	"os/user"
	"path"
	"strconv"
	"time"
)

type loadDefFn func(conf *Config)

type Config struct {
	defaults       map[string]interface{}
	configFileName string
}

func NewConfig(fileName string) *Config {
	config := &Config{
		configFileName: fileName,
		defaults:       make(map[string]interface{}),
	}

	return config
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
//
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) ThrowKeyPanic(key string) {
	panic(fmt.Sprintf("config error: key %s not available", key))
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
//
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) ThrowConversionPanic(src interface{}, err error) {
	panic(fmt.Sprintf("config error: cannot convert %v :: error : %s", src, err.Error()))
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
//
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) stringSlice2IntSlice(values []string) []int {

	if values != nil {
		res := make([]int, len(values))
		for n, src := range values {
			if v, err := strconv.Atoi(src); err == nil {
				res[n] = v
			} else {
				c.ThrowConversionPanic(src, err)
			}
		}
		return res
	}

	return nil
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
// GetInt
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) GetObject(key string) interface{} {

	value, err := config.Get(key)
	if err != nil {
		if value, ok := c.defaults[key]; ok {
			return value
		}
		c.ThrowKeyPanic(key)
	}

	return value
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
// GetFloat
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) GetFloat(key string) float64 {
	obj := c.GetObject(key)
	return obj.(float64)
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
// GetInt
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) GetInt(key string) int {
	value, err := config.GetInt(key)
	if err != nil {
		if value, ok := c.defaults[key]; ok {
			return value.(int)
		}
		c.ThrowKeyPanic(key)
	}

	return value
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
// GetIntList
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) GetIntList(key string) []int {
	if value, err := config.GetList(key); err == nil {
		return c.stringSlice2IntSlice(value)
	} else {
		if val, ok := c.defaults[key]; ok {
			return val.([]int)
		} else {
			c.ThrowKeyPanic(key)
		}
	}

	return nil
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
// GetString
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) GetString(key string) string {
	value, err := config.GetString(key)
	if err != nil {
		if val, ok := c.defaults[key]; ok {
			return val.(string)
		} else {
			c.ThrowKeyPanic(key)
		}
	}

	return value
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
// GetBool
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) GetBool(key string) bool {
	value, err := config.GetBool(key)
	if err != nil {
		if val, ok := c.defaults[key]; ok {
			return val.(bool)
		} else {
			c.ThrowKeyPanic(key)
		}
	}

	return value
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
// GetStringList
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) GetStringList(key string) []string {
	value, err := config.GetList(key)
	if err != nil {
		if value, ok := c.defaults[key]; ok {
			return value.([]string)
		} else {
			c.ThrowKeyPanic(key)
		}
	}

	return value
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
// GetDuration
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) GetDuration(key string) time.Duration {
	value, err := config.GetDuration(key)
	if err != nil {
		if val, ok := c.defaults[key]; ok {
			return val.(time.Duration)
		} else {
			c.ThrowKeyPanic(key)
		}
	}

	return value
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
//
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) SetDefault(key string, value interface{}) {
	c.defaults[key] = value
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
//
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) writeDefConfigFile(filePath string) error {

	for key, value := range c.defaults {
		config.Set(key, value)
	}

	applog.Infof("Create new config file at %s", filePath)
	return config.WriteConfigFile(filePath, 0644)
}

///////////////////////////////////////////////////////////////////////////////////////////////////////
// If filePath is defined and the file exists, Load gets the configuration from there, otherwise
// it looks for the filename, specified in NewConfig in the current users home directory. If no file can be found
// the function creates a file with the users default values.
///////////////////////////////////////////////////////////////////////////////////////////////////////
func (c *Config) Load(loadDefaults loadDefFn, filePath string, watchConfig bool) error {

	if len(filePath) == 0 {
		usr, err := user.Current()
		if err != nil {
			return err
		}

		filePath = path.Join(usr.HomeDir, c.configFileName)
	}

	loadDefaults(c)
	if _, err := os.Stat(filePath); err == nil {
		if watchConfig {
			if err := config.ReadAndWatchConfigFile(filePath); err != nil {
				return err
			}
		} else {
			if err := config.ReadConfigFile(filePath); err != nil {
				return err
			}
		}
	} else {
		if err = c.writeDefConfigFile(filePath); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// trybot_updater is an application that updates a repo's buildbucket.config file.
package main

import (
	"bytes"
	"context"
	"flag"
	"sort"
	"strings"
	"text/template"
	"time"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/gerrit"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/gitiles"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/task_scheduler/go/specs"
)

const (
	// The format of this file is that of a gerrit extension config (not a proto).
	// The buildbucket extension parses the config like this:
	// https://chromium.googlesource.com/infra/gerrit-plugins/buildbucket/+/refs/heads/master/src/main/java/com/googlesource/chromium/plugins/buildbucket/GetConfig.java
	bbCfgFileName = "buildbucket.config"
	// Branch buildbucket.config lies in. Hopefully this will change one day, see b/38258213.
	bbCfgBranch = "refs/meta/config"
	// Template used to create buildbucket.config content.
	bbCfgTemplate = `
{{- range .EmptyBuckets}}[bucket "{{.}}"]{{end}}
[bucket "{{.BucketName}}"]
{{- range .Jobs}}
	builder = {{.}}
{{- end}}
`
)

var (
	// Flags.
	repoUrl       = flag.String("repo_url", common.REPO_SKIA, "Repo that needs buildbucket.config updated from it's tasks.json file.")
	bucketName    = flag.String("bucket_name", "luci.skia.skia.primary", "Name of the bucket to update in buildbucket.config.")
	emptyBuckets  = common.NewMultiStringFlag("empty_bucket", nil, "Empty buckets to specify in buildbucket.config. Eg: luci.chromium.try. See skbug.com/9639 for why these buckets are empty.")
	pollingPeriod = flag.Duration("polling_period", 10*time.Minute, "How often to poll tasks.json.")
	submit        = flag.Bool("submit", false, "If set, automatically submit the Gerrit change to update buildbucket.config")
	local         = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
	promPort      = flag.String("prom_port", ":20000", "Metrics service address (e.g., ':20000')")

	bbCfgTemplateParsed = template.Must(template.New("buildbucket_config").Parse(bbCfgTemplate))
)

// getBuildbucketCfgFromJobs reads tasks.json from the specified repository and returns
// contents of what the new buildbucket.config file should be.
func getBuildbucketCfgFromJobs(ctx context.Context, repo *gitiles.Repo) (string, error) {
	// Read tasks.json from the specified repository.
	tasksContents, err := repo.ReadFileAtRef(ctx, specs.TASKS_CFG_FILE, git.DefaultBranch)
	if err != nil {
		return "", skerr.Fmt("Could not read %s: %s", specs.TASKS_CFG_FILE, err)
	}
	tasksCfg, err := specs.ParseTasksCfg(string(tasksContents))
	if err != nil {
		return "", skerr.Fmt("Could not parse %s: %s", specs.TASKS_CFG_FILE, err)
	}

	// Create a sorted slice of jobs.
	jobs := make([]string, 0, len(tasksCfg.Jobs))
	for j := range tasksCfg.Jobs {
		jobs = append(jobs, j)
	}
	sort.Strings(jobs)

	// Use jobs to create content of buildbucket.config.
	bbCfg := new(bytes.Buffer)
	if err := bbCfgTemplateParsed.Execute(bbCfg, struct {
		EmptyBuckets []string
		BucketName   string
		Jobs         []string
	}{
		EmptyBuckets: *emptyBuckets,
		BucketName:   *bucketName,
		Jobs:         jobs,
	}); err != nil {
		return "", skerr.Fmt("Failed to execute bbCfg template: %s", err)
	}
	return bbCfg.String(), nil
}

// getCurrentBuildbucketCfg returns the current contents of buildbucket.config for the
// specified repository.
func getCurrentBuildbucketCfg(ctx context.Context, repo *gitiles.Repo) (string, error) {
	contents, err := repo.ReadFileAtRef(ctx, bbCfgFileName, bbCfgBranch)
	if err != nil {
		return "", skerr.Fmt("Could not read %s: %s", bbCfgFileName, err)
	}
	return string(contents), nil
}

// updateBuildbucketCfg creates a Gerrit CL to update buildbucket.config. If submit flag is true then that CL
// is automatically self-approved and submitted.
func updateBuildbucketCfg(ctx context.Context, g *gerrit.Gerrit, repo *gitiles.Repo, cfgContents string) error {
	commitMsg := "Update buildbucket.config"
	repoSplit := strings.Split(*repoUrl, "/")
	project := strings.TrimSuffix(repoSplit[len(repoSplit)-1], ".git")
	baseCommitInfo, err := repo.Details(ctx, bbCfgBranch)
	if err != nil {
		return skerr.Fmt("Could not get details of %s: %s", bbCfgBranch, err)
	}
	baseCommit := baseCommitInfo.Hash
	ci, err := gerrit.CreateAndEditChange(ctx, g, project, bbCfgBranch, commitMsg, baseCommit, func(ctx context.Context, g gerrit.GerritInterface, ci *gerrit.ChangeInfo) error {
		if err := g.EditFile(ctx, ci, bbCfgFileName, cfgContents); err != nil {
			return skerr.Fmt("Could not edit %s: %s", bbCfgFileName, err)
		}
		return nil
	})
	if err != nil {
		return skerr.Fmt("Could not create Gerrit change: %s", err)
	}
	sklog.Infof("Uploaded change https://skia-review.googlesource.com/c/%s/+/%d", project, ci.Issue)

	if *submit {
		// TODO(rmistry): Change reviewer to be the trooper after verifying that things work.
		reviewers := []string{"rmistry@google.com"}
		if err := g.SetReview(ctx, ci, "", gerrit.CONFIG_CHROMIUM.SelfApproveLabels, reviewers); err != nil {
			return abandonGerritChange(ctx, g, ci, err)
		}
		if err := g.Submit(ctx, ci); err != nil {
			return abandonGerritChange(ctx, g, ci, err)
		}
		sklog.Infof("Submitted change https://skia-review.googlesource.com/c/%s/+/%d", project, ci.Issue)
	}

	return nil
}

// abandonGerritChange abandons the specified CL and returns the specified err after abandoning.
// If abandoning fails then that error is wrapped with the specified err.
func abandonGerritChange(ctx context.Context, g *gerrit.Gerrit, issue *gerrit.ChangeInfo, err error) error {
	if abandonErr := g.Abandon(ctx, issue, ""); abandonErr != nil {
		return skerr.Wrapf(err, "failed to abandon CL with %s", abandonErr)
	}
	return err
}

func main() {
	common.InitWithMust("trybot_updater", common.PrometheusOpt(promPort))
	defer sklog.Flush()
	ctx := context.Background()

	// OAuth2.0 TokenSource.
	ts, err := auth.NewDefaultTokenSource(false, auth.SCOPE_USERINFO_EMAIL, auth.SCOPE_GERRIT)
	if err != nil {
		sklog.Fatal(err)
	}
	// Authenticated HTTP client.
	httpClient := httputils.DefaultClientConfig().WithTokenSource(ts).With2xxOnly().Client()

	// Instantiate Gerrit.
	gUrl := strings.Split(*repoUrl, ".googlesource.com")[0] + "-review.googlesource.com"
	g, err := gerrit.NewGerrit(gUrl, httpClient)
	if err != nil {
		sklog.Fatal(err)
	}

	// Instantiate Gitiles using the specified repo URL.
	repo := gitiles.NewRepo(*repoUrl, httpClient)

	// TODO(rmistry): Use pubsub instead of polling.
	go util.RepeatCtx(ctx, *pollingPeriod, func(ctx context.Context) {
		existingCfg, err := getCurrentBuildbucketCfg(ctx, repo)
		if err != nil {
			sklog.Errorf("Could not get contents of buildbucket.config from %s: %s", repo, err)
		}

		newCfg, err := getBuildbucketCfgFromJobs(ctx, repo)
		if err != nil {
			sklog.Errorf("Could not get list of jobs from %s: %s", repo, err)
		}

		// Only update buildbucket.config if the config is different.
		if newCfg != existingCfg {
			if err := updateBuildbucketCfg(ctx, g, repo, newCfg); err != nil {
				sklog.Errorf("Could not update buildbucket.config: %s", err)
				sklog.Info("Sleep for 10 mins since there was a error with Gerrit to give it time to recover.")
				time.Sleep(10 * time.Minute)
			}
		} else {
			sklog.Info("Config has not changed")
		}
	})

	select {}
}
<commit_msg>[trybot-updater] Return after errors<commit_after>// trybot_updater is an application that updates a repo's buildbucket.config file.
package main

import (
	"bytes"
	"context"
	"flag"
	"sort"
	"strings"
	"text/template"
	"time"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/common"
	"go.skia.org/infra/go/gerrit"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/gitiles"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/util"
	"go.skia.org/infra/task_scheduler/go/specs"
)

const (
	// The format of this file is that of a gerrit extension config (not a proto).
	// The buildbucket extension parses the config like this:
	// https://chromium.googlesource.com/infra/gerrit-plugins/buildbucket/+/refs/heads/master/src/main/java/com/googlesource/chromium/plugins/buildbucket/GetConfig.java
	bbCfgFileName = "buildbucket.config"
	// Branch buildbucket.config lies in. Hopefully this will change one day, see b/38258213.
	bbCfgBranch = "refs/meta/config"
	// Template used to create buildbucket.config content.
	bbCfgTemplate = `
{{- range .EmptyBuckets}}[bucket "{{.}}"]{{end}}
[bucket "{{.BucketName}}"]
{{- range .Jobs}}
	builder = {{.}}
{{- end}}
`
)

var (
	// Flags.
	repoUrl       = flag.String("repo_url", common.REPO_SKIA, "Repo that needs buildbucket.config updated from it's tasks.json file.")
	bucketName    = flag.String("bucket_name", "luci.skia.skia.primary", "Name of the bucket to update in buildbucket.config.")
	emptyBuckets  = common.NewMultiStringFlag("empty_bucket", nil, "Empty buckets to specify in buildbucket.config. Eg: luci.chromium.try. See skbug.com/9639 for why these buckets are empty.")
	pollingPeriod = flag.Duration("polling_period", 10*time.Minute, "How often to poll tasks.json.")
	submit        = flag.Bool("submit", false, "If set, automatically submit the Gerrit change to update buildbucket.config")
	local         = flag.Bool("local", false, "Running locally if true. As opposed to in production.")
	promPort      = flag.String("prom_port", ":20000", "Metrics service address (e.g., ':20000')")

	bbCfgTemplateParsed = template.Must(template.New("buildbucket_config").Parse(bbCfgTemplate))
)

// getBuildbucketCfgFromJobs reads tasks.json from the specified repository and returns
// contents of what the new buildbucket.config file should be.
func getBuildbucketCfgFromJobs(ctx context.Context, repo *gitiles.Repo) (string, error) {
	// Read tasks.json from the specified repository.
	tasksContents, err := repo.ReadFileAtRef(ctx, specs.TASKS_CFG_FILE, git.DefaultBranch)
	if err != nil {
		return "", skerr.Fmt("Could not read %s: %s", specs.TASKS_CFG_FILE, err)
	}
	tasksCfg, err := specs.ParseTasksCfg(string(tasksContents))
	if err != nil {
		return "", skerr.Fmt("Could not parse %s: %s", specs.TASKS_CFG_FILE, err)
	}

	// Create a sorted slice of jobs.
	jobs := make([]string, 0, len(tasksCfg.Jobs))
	for j := range tasksCfg.Jobs {
		jobs = append(jobs, j)
	}
	sort.Strings(jobs)

	// Use jobs to create content of buildbucket.config.
	bbCfg := new(bytes.Buffer)
	if err := bbCfgTemplateParsed.Execute(bbCfg, struct {
		EmptyBuckets []string
		BucketName   string
		Jobs         []string
	}{
		EmptyBuckets: *emptyBuckets,
		BucketName:   *bucketName,
		Jobs:         jobs,
	}); err != nil {
		return "", skerr.Fmt("Failed to execute bbCfg template: %s", err)
	}
	return bbCfg.String(), nil
}

// getCurrentBuildbucketCfg returns the current contents of buildbucket.config for the
// specified repository.
func getCurrentBuildbucketCfg(ctx context.Context, repo *gitiles.Repo) (string, error) {
	contents, err := repo.ReadFileAtRef(ctx, bbCfgFileName, bbCfgBranch)
	if err != nil {
		return "", skerr.Fmt("Could not read %s: %s", bbCfgFileName, err)
	}
	return string(contents), nil
}

// updateBuildbucketCfg creates a Gerrit CL to update buildbucket.config. If submit flag is true then that CL
// is automatically self-approved and submitted.
func updateBuildbucketCfg(ctx context.Context, g *gerrit.Gerrit, repo *gitiles.Repo, cfgContents string) error {
	commitMsg := "Update buildbucket.config"
	repoSplit := strings.Split(*repoUrl, "/")
	project := strings.TrimSuffix(repoSplit[len(repoSplit)-1], ".git")
	baseCommitInfo, err := repo.Details(ctx, bbCfgBranch)
	if err != nil {
		return skerr.Fmt("Could not get details of %s: %s", bbCfgBranch, err)
	}
	baseCommit := baseCommitInfo.Hash
	ci, err := gerrit.CreateAndEditChange(ctx, g, project, bbCfgBranch, commitMsg, baseCommit, func(ctx context.Context, g gerrit.GerritInterface, ci *gerrit.ChangeInfo) error {
		if err := g.EditFile(ctx, ci, bbCfgFileName, cfgContents); err != nil {
			return skerr.Fmt("Could not edit %s: %s", bbCfgFileName, err)
		}
		return nil
	})
	if err != nil {
		return skerr.Fmt("Could not create Gerrit change: %s", err)
	}
	sklog.Infof("Uploaded change https://skia-review.googlesource.com/c/%s/+/%d", project, ci.Issue)

	if *submit {
		// TODO(rmistry): Change reviewer to be the trooper after verifying that things work.
		reviewers := []string{"rmistry@google.com"}
		if err := g.SetReview(ctx, ci, "", gerrit.CONFIG_CHROMIUM.SelfApproveLabels, reviewers); err != nil {
			return abandonGerritChange(ctx, g, ci, err)
		}
		if err := g.Submit(ctx, ci); err != nil {
			return abandonGerritChange(ctx, g, ci, err)
		}
		sklog.Infof("Submitted change https://skia-review.googlesource.com/c/%s/+/%d", project, ci.Issue)
	}

	return nil
}

// abandonGerritChange abandons the specified CL and returns the specified err after abandoning.
// If abandoning fails then that error is wrapped with the specified err.
func abandonGerritChange(ctx context.Context, g *gerrit.Gerrit, issue *gerrit.ChangeInfo, err error) error {
	if abandonErr := g.Abandon(ctx, issue, ""); abandonErr != nil {
		return skerr.Wrapf(err, "failed to abandon CL with %s", abandonErr)
	}
	return err
}

func main() {
	common.InitWithMust("trybot_updater", common.PrometheusOpt(promPort))
	defer sklog.Flush()
	ctx := context.Background()

	// OAuth2.0 TokenSource.
	ts, err := auth.NewDefaultTokenSource(false, auth.SCOPE_USERINFO_EMAIL, auth.SCOPE_GERRIT)
	if err != nil {
		sklog.Fatal(err)
	}
	// Authenticated HTTP client.
	httpClient := httputils.DefaultClientConfig().WithTokenSource(ts).With2xxOnly().Client()

	// Instantiate Gerrit.
	gUrl := strings.Split(*repoUrl, ".googlesource.com")[0] + "-review.googlesource.com"
	g, err := gerrit.NewGerrit(gUrl, httpClient)
	if err != nil {
		sklog.Fatal(err)
	}

	// Instantiate Gitiles using the specified repo URL.
	repo := gitiles.NewRepo(*repoUrl, httpClient)

	// TODO(rmistry): Use pubsub instead of polling.
	util.RepeatCtx(ctx, *pollingPeriod, func(ctx context.Context) {
		existingCfg, err := getCurrentBuildbucketCfg(ctx, repo)
		if err != nil {
			sklog.Errorf("Could not get contents of buildbucket.config from %s: %s", repo, err)
			return
		}

		newCfg, err := getBuildbucketCfgFromJobs(ctx, repo)
		if err != nil {
			sklog.Errorf("Could not get list of jobs from %s: %s", repo, err)
			return
		}

		// Only update buildbucket.config if the config is different.
		if newCfg != existingCfg {
			if err := updateBuildbucketCfg(ctx, g, repo, newCfg); err != nil {
				sklog.Errorf("Could not update buildbucket.config: %s", err)
				sklog.Info("Sleep for 10 mins since there was a error with Gerrit to give it time to recover.")
				time.Sleep(10 * time.Minute)
			}
		} else {
			sklog.Info("Config has not changed")
		}
	})
}
<|endoftext|>


<commit_before>package tb

import (
	"testing"
	"time"
)

func TestNewBucket(t *testing.T) {
	t.Parallel()

	b := NewBucket(10, 0)
	b.Take(10)
	time.Sleep(100 * time.Millisecond)
	if w, g := int64(0), b.Take(1); w != g {
		t.Fatal("Expected no filling when freq < 1/c")
	}
}

func TestBucket_Take_single(t *testing.T) {
	t.Parallel()

	b := NewBucket(10, -1)
	defer b.Close()

	ex := [...]int64{5, 5, 1, 1, 5, 4, 1, 0}
	for i := 0; i < len(ex)-1; i += 2 {
		if got, want := b.Take(ex[i]), ex[i+1]; got != want {
			t.Errorf("Want: %d, Got: %d", want, got)
		}
	}
}

func TestBucket_Put_single(t *testing.T) {
	t.Parallel()

	b := NewBucket(10, 0)
	defer b.Close()

	b.Take(10)

	ex := [...]int64{5, 5, 10, 5, 15, 0}
	for i := 0; i < len(ex)-1; i += 2 {
		if got, want := b.Put(ex[i]), ex[i+1]; got != want {
			t.Errorf("Want: %d, Got: %d", want, got)
		}
	}
}

func TestBucket_Take_multi(t *testing.T) {
	t.Parallel()

	b := NewBucket(10, -1)
	defer b.Close()

	exs := [2][]int64{{4, 4, 2, 2}, {2, 2, 1, 1}}
	for i := 0; i < 2; i++ {
		go func(i int) {
			for j := 0; j < len(exs[i])-1; j += 2 {
				if got, want := b.Take(exs[i][j]), exs[i][j+1]; got != want {
					t.Errorf("Want: %d, Got: %d", want, got)
				}
			}
		}(i)
	}
}

func TestBucket_Put_multi(t *testing.T) {
	t.Parallel()

	b := NewBucket(10, 0)
	defer b.Close()

	b.Take(10)

	exs := [2][]int64{{4, 4, 2, 2}, {2, 2, 1, 1}}
	for i := 0; i < 2; i++ {
		go func(i int) {
			for j := 0; j < len(exs[i])-1; j += 2 {
				if got, want := b.Put(exs[i][j]), exs[i][j+1]; got != want {
					t.Errorf("Want: %d, Got: %d", want, got)
				}
			}
		}(i)
	}
}

func TestBucket_Take_throughput(t *testing.T) {
	t.Parallel()
	if testing.Short() {
		t.Skip("Skipping test in short mode")
	}

	b := NewBucket(1000, -1)
	defer b.Close()

	b.Take(1000)

	var (
		out   int64
		began = time.Now()
	)
	for out < 1000 {
		out += b.Take(1000 - out)
	}

	ended := time.Since(began)
	if int(ended.Seconds()) != 1 {
		t.Errorf("Want 1000 tokens to take 1s. Got: %d", int(ended.Seconds()))
	}
}

func BenchmarkBucket_Take_sequential(b *testing.B) {
	bucket := NewBucket(int64(b.N), -1)
	defer bucket.Close()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		bucket.Take(8)
	}
}

func BenchmarkBucket_Put_sequential(b *testing.B) {
	bucket := NewBucket(int64(b.N), 0)
	defer bucket.Close()

	bucket.Take(int64(b.N))

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		bucket.Put(8)
	}
}
func TestBucket_Close(t *testing.T) {
	b := NewBucket(10000, -1)
	b.Close()
	b.Take(10000)
	time.Sleep(10 * time.Millisecond)
	if want, got := int64(0), b.Take(1); want != got {
		t.Errorf("Want: %d Got: %d", want, got)
	}
}
<commit_msg>Update tsenart/tb<commit_after>package tb

import (
	"runtime"
	"testing"
	"time"
)

func TestNewBucket(t *testing.T) {
	t.Parallel()

	b := NewBucket(10, 0)
	b.Take(10)
	time.Sleep(100 * time.Millisecond)
	if w, g := int64(0), b.Take(1); w != g {
		t.Fatal("Expected no filling when freq < 1/c")
	}
}

func TestBucket_Take_single(t *testing.T) {
	t.Parallel()

	b := NewBucket(10, -1)
	defer b.Close()

	ex := [...]int64{5, 5, 1, 1, 5, 4, 1, 0}
	for i := 0; i < len(ex)-1; i += 2 {
		if got, want := b.Take(ex[i]), ex[i+1]; got != want {
			t.Errorf("Want: %d, Got: %d", want, got)
		}
	}
}

func TestBucket_Put_single(t *testing.T) {
	t.Parallel()

	b := NewBucket(10, 0)
	defer b.Close()

	b.Take(10)

	ex := [...]int64{5, 5, 10, 5, 15, 0}
	for i := 0; i < len(ex)-1; i += 2 {
		if got, want := b.Put(ex[i]), ex[i+1]; got != want {
			t.Errorf("Want: %d, Got: %d", want, got)
		}
	}
}

func TestBucket_Take_multi(t *testing.T) {
	t.Parallel()

	b := NewBucket(10, -1)
	defer b.Close()

	exs := [2][]int64{{4, 4, 2, 2}, {2, 2, 1, 1}}
	for i := 0; i < 2; i++ {
		go func(i int) {
			for j := 0; j < len(exs[i])-1; j += 2 {
				if got, want := b.Take(exs[i][j]), exs[i][j+1]; got != want {
					t.Errorf("Want: %d, Got: %d", want, got)
				}
			}
		}(i)
	}
}

func TestBucket_Put_multi(t *testing.T) {
	t.Parallel()

	b := NewBucket(10, 0)
	defer b.Close()

	b.Take(10)

	exs := [2][]int64{{4, 4, 2, 2}, {2, 2, 1, 1}}
	for i := 0; i < 2; i++ {
		go func(i int) {
			for j := 0; j < len(exs[i])-1; j += 2 {
				if got, want := b.Put(exs[i][j]), exs[i][j+1]; got != want {
					t.Errorf("Want: %d, Got: %d", want, got)
				}
			}
		}(i)
	}
}

func TestBucket_Take_throughput(t *testing.T) {
	t.Parallel()

	if testing.Short() {
		t.Skip("Skipping test in short mode")
	}

	runtime.GOMAXPROCS(2)

	b := NewBucket(1000, -1)
	defer b.Close()

	b.Take(1000)

	var (
		out   int64
		began = time.Now()
	)
	for out < 1000 {
		out += b.Take(1000 - out)
	}

	ended := time.Since(began)
	if int(ended.Seconds()) != 1 {
		t.Errorf("Want 1000 tokens to take 1s. Got: %d", int(ended.Seconds()))
	}
}

func BenchmarkBucket_Take_sequential(b *testing.B) {
	bucket := NewBucket(int64(b.N), -1)
	defer bucket.Close()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		bucket.Take(8)
	}
}

func BenchmarkBucket_Put_sequential(b *testing.B) {
	bucket := NewBucket(int64(b.N), 0)
	defer bucket.Close()

	bucket.Take(int64(b.N))

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		bucket.Put(8)
	}
}
func TestBucket_Close(t *testing.T) {
	b := NewBucket(10000, -1)
	b.Close()
	b.Take(10000)
	time.Sleep(10 * time.Millisecond)
	if want, got := int64(0), b.Take(1); want != got {
		t.Errorf("Want: %d Got: %d", want, got)
	}
}
<|endoftext|>