<commit_before>package redis

import (
	"bufio"
	"errors"
	"fmt"
	"net"
	"net/url"
	"strconv"
	"strings"
	"sync"

	"github.com/influxdb/telegraf/plugins"
)

type Redis struct {
	Servers []string
}

var sampleConfig = `
	# An array of URI to gather stats about. Specify an ip or hostname
	# with optional port add password. ie redis://localhost, redis://10.10.3.33:18832,
	# 10.0.0.1:10000, etc.
	#
	# If no servers are specified, then localhost is used as the host.
	servers = ["localhost"]
`

func (r *Redis) SampleConfig() string {
	return sampleConfig
}

func (r *Redis) Description() string {
	return "Read metrics from one or many redis servers"
}

var Tracking = map[string]string{
	"uptime_in_seconds":           "uptime",
	"connected_clients":           "clients",
	"used_memory":                 "used_memory",
	"used_memory_rss":             "used_memory_rss",
	"used_memory_peak":            "used_memory_peak",
	"used_memory_lua":             "used_memory_lua",
	"rdb_changes_since_last_save": "rdb_changes_since_last_save",
	"total_connections_received":  "total_connections_received",
	"total_commands_processed":    "total_commands_processed",
	"instantaneous_ops_per_sec":   "instantaneous_ops_per_sec",
	"sync_full":                   "sync_full",
	"sync_partial_ok":             "sync_partial_ok",
	"sync_partial_err":            "sync_partial_err",
	"expired_keys":                "expired_keys",
	"evicted_keys":                "evicted_keys",
	"keyspace_hits":               "keyspace_hits",
	"keyspace_misses":             "keyspace_misses",
	"pubsub_channels":             "pubsub_channels",
	"pubsub_patterns":             "pubsub_patterns",
	"latest_fork_usec":            "latest_fork_usec",
	"connected_slaves":            "connected_slaves",
	"master_repl_offset":          "master_repl_offset",
	"repl_backlog_active":         "repl_backlog_active",
	"repl_backlog_size":           "repl_backlog_size",
	"repl_backlog_histlen":        "repl_backlog_histlen",
	"mem_fragmentation_ratio":     "mem_fragmentation_ratio",
	"used_cpu_sys":                "used_cpu_sys",
	"used_cpu_user":               "used_cpu_user",
	"used_cpu_sys_children":       "used_cpu_sys_children",
	"used_cpu_user_children":      "used_cpu_user_children",
}

var ErrProtocolError = errors.New("redis protocol error")

// Reads stats from all configured servers accumulates stats.
// Returns one of the errors encountered while gather stats (if any).
func (r *Redis) Gather(acc plugins.Accumulator) error {
	if len(r.Servers) == 0 {
		url := &url.URL{
			Host: ":6379",
		}
		r.gatherServer(url, acc)
		return nil
	}

	var wg sync.WaitGroup

	var outerr error

	for _, serv := range r.Servers {
		u, err := url.Parse(serv)
		if err != nil {
			return fmt.Errorf("Unable to parse to address '%s': %s", serv, err)
		} else if u.Scheme == "" {
			// fallback to simple string based address (i.e. "10.0.0.1:10000")
			u.Scheme = "tcp"
			u.Host = serv
			u.Path = ""
		}
		wg.Add(1)
		go func(serv string) {
			defer wg.Done()
			outerr = r.gatherServer(u, acc)
		}(serv)
	}

	wg.Wait()

	return outerr
}

const defaultPort = "6379"

func (r *Redis) gatherServer(addr *url.URL, acc plugins.Accumulator) error {
	_, _, err := net.SplitHostPort(addr.Host)
	if err != nil {
		addr.Host = addr.Host + ":" + defaultPort
	}

	c, err := net.Dial("tcp", addr.Host)
	if err != nil {
		return fmt.Errorf("Unable to connect to redis server '%s': %s", addr.Host, err)
	}
	defer c.Close()

	if addr.User != nil {
		pwd, set := addr.User.Password()
		if set && pwd != "" {
			c.Write([]byte(fmt.Sprintf("AUTH %s\r\n", pwd)))

			rdr := bufio.NewReader(c)

			line, err := rdr.ReadString('\n')
			if err != nil {
				return err
			}
			if line[0] != '+' {
				return fmt.Errorf("%s", strings.TrimSpace(line)[1:])
			}
		}
	}

	c.Write([]byte("info\r\n"))

	rdr := bufio.NewReader(c)

	line, err := rdr.ReadString('\n')
	if err != nil {
		return err
	}

	if line[0] != '$' {
		return fmt.Errorf("bad line start: %s", ErrProtocolError)
	}

	line = strings.TrimSpace(line)

	szStr := line[1:]

	sz, err := strconv.Atoi(szStr)
	if err != nil {
		return fmt.Errorf("bad size string <<%s>>: %s", szStr, ErrProtocolError)
	}

	var read int

	for read < sz {
		line, err := rdr.ReadString('\n')
		if err != nil {
			return err
		}

		read += len(line)

		if len(line) == 1 || line[0] == '#' {
			continue
		}

		parts := strings.SplitN(line, ":", 2)

		name := string(parts[0])

		metric, ok := Tracking[name]
		if !ok {
			continue
		}

		_, rPort, err := net.SplitHostPort(addr.Host)
		if err != nil {
			rPort = defaultPort
		}
		tags := map[string]string{"host": addr.String(), "port": rPort}

		val := strings.TrimSpace(parts[1])

		ival, err := strconv.ParseUint(val, 10, 64)
		if err == nil {
			acc.Add(metric, ival, tags)
			continue
		}

		fval, err := strconv.ParseFloat(val, 64)
		if err != nil {
			return err
		}

		acc.Add(metric, fval, tags)
	}

	return nil
}

func init() {
	plugins.Add("redis", func() plugins.Plugin {
		return &Redis{}
	})
}
<commit_msg>Redis plugin, add key metrics and simplify parsing<commit_after>package redis

import (
	"bufio"
	"errors"
	"fmt"
	"net"
	"net/url"
	// "strconv"
	"strings"
	"sync"

	"github.com/influxdb/telegraf/plugins"
)

type Redis struct {
	Servers []string
}

var sampleConfig = `
	# An array of URI to gather stats about. Specify an ip or hostname
	# with optional port add password. ie redis://localhost, redis://10.10.3.33:18832,
	# 10.0.0.1:10000, etc.
	#
	# If no servers are specified, then localhost is used as the host.
	servers = ["localhost"]
`

func (r *Redis) SampleConfig() string {
	return sampleConfig
}

func (r *Redis) Description() string {
	return "Read metrics from one or many redis servers"
}

var Tracking = map[string]string{
	"uptime_in_seconds":           "uptime",
	"connected_clients":           "clients",
	"used_memory":                 "used_memory",
	"used_memory_rss":             "used_memory_rss",
	"used_memory_peak":            "used_memory_peak",
	"used_memory_lua":             "used_memory_lua",
	"rdb_changes_since_last_save": "rdb_changes_since_last_save",
	"total_connections_received":  "total_connections_received",
	"total_commands_processed":    "total_commands_processed",
	"instantaneous_ops_per_sec":   "instantaneous_ops_per_sec",
	"sync_full":                   "sync_full",
	"sync_partial_ok":             "sync_partial_ok",
	"sync_partial_err":            "sync_partial_err",
	"expired_keys":                "expired_keys",
	"evicted_keys":                "evicted_keys",
	"keyspace_hits":               "keyspace_hits",
	"keyspace_misses":             "keyspace_misses",
	"pubsub_channels":             "pubsub_channels",
	"pubsub_patterns":             "pubsub_patterns",
	"latest_fork_usec":            "latest_fork_usec",
	"connected_slaves":            "connected_slaves",
	"master_repl_offset":          "master_repl_offset",
	"repl_backlog_active":         "repl_backlog_active",
	"repl_backlog_size":           "repl_backlog_size",
	"repl_backlog_histlen":        "repl_backlog_histlen",
	"mem_fragmentation_ratio":     "mem_fragmentation_ratio",
	"used_cpu_sys":                "used_cpu_sys",
	"used_cpu_user":               "used_cpu_user",
	"used_cpu_sys_children":       "used_cpu_sys_children",
	"used_cpu_user_children":      "used_cpu_user_children",
}

var ErrProtocolError = errors.New("redis protocol error")

// Reads stats from all configured servers accumulates stats.
// Returns one of the errors encountered while gather stats (if any).
func (r *Redis) Gather(acc plugins.Accumulator) error {
	if len(r.Servers) == 0 {
		url := &url.URL{
			Host: ":6379",
		}
		r.gatherServer(url, acc)
		return nil
	}

	var wg sync.WaitGroup

	var outerr error

	for _, serv := range r.Servers {
		u, err := url.Parse(serv)
		if err != nil {
			return fmt.Errorf("Unable to parse to address '%s': %s", serv, err)
		} else if u.Scheme == "" {
			// fallback to simple string based address (i.e. "10.0.0.1:10000")
			u.Scheme = "tcp"
			u.Host = serv
			u.Path = ""
		}
		wg.Add(1)
		go func(serv string) {
			defer wg.Done()
			outerr = r.gatherServer(u, acc)
		}(serv)
	}

	wg.Wait()

	return outerr
}

const defaultPort = "6379"

func (r *Redis) gatherServer(addr *url.URL, acc plugins.Accumulator) error {
	_, _, err := net.SplitHostPort(addr.Host)
	if err != nil {
		addr.Host = addr.Host + ":" + defaultPort
	}

	c, err := net.Dial("tcp", addr.Host)
	if err != nil {
		return fmt.Errorf("Unable to connect to redis server '%s': %s", addr.Host, err)
	}
	defer c.Close()

	if addr.User != nil {
		pwd, set := addr.User.Password()
		if set && pwd != "" {
			c.Write([]byte(fmt.Sprintf("AUTH %s\r\n", pwd)))

			rdr := bufio.NewReader(c)

			line, err := rdr.ReadString('\n')
			if err != nil {
				return err
			}
			if line[0] != '+' {
				return fmt.Errorf("%s", strings.TrimSpace(line)[1:])
			}
		}
	}

	c.Write([]byte("info\r\n"))

	rdr := bufio.NewReader(c)
	scanner := bufio.NewScanner(rdr)
	for scanner.Scan() {
		fmt.Println(scanner.Text())
	}
	if err := scanner.Err(); err != nil {
		fmt.Println("reading standard input:", err)
	}

	// line, err := rdr.ReadString('\n')
	// if err != nil {
	// 	return err
	// }

	// if line[0] != '$' {
	// 	return fmt.Errorf("bad line start: %s", ErrProtocolError)
	// }

	// line = strings.TrimSpace(line)

	// szStr := line[0:]

	// sz, err := strconv.Atoi(szStr)
	// if err != nil {
	// 	return fmt.Errorf("bad size string <<%s>>: %s", szStr, ErrProtocolError)
	// }

	// var read int

	// for read < sz {
	// 	line, err := rdr.ReadString('\n')
	// 	fmt.Printf(line)
	// 	if err != nil {
	// 		return err
	// 	}

	// 	read += len(line)
	// 	if len(line) == 1 || line[0] == '#' {
	// 		continue
	// 	}

	// 	_, rPort, err := net.SplitHostPort(addr.Host)
	// 	if err != nil {
	// 		rPort = defaultPort
	// 	}
	// 	tags := map[string]string{"host": addr.String(), "port": rPort}

	// 	parts := strings.SplitN(line, ":", 2)
	// 	if len(parts) < 2 {
	// 		continue
	// 	}
	// 	name := string(parts[0])
	// 	metric, ok := Tracking[name]
	// 	if !ok {
	// 		// See if this is the keyspace line
	// 		if strings.Contains(string(parts[1]), "keys=") {
	// 			tags["database"] = name
	// 			acc.Add("foo", 999, tags)
	// 		}
	// 		continue
	// 	}

	// 	val := strings.TrimSpace(parts[1])
	// 	ival, err := strconv.ParseUint(val, 10, 64)
	// 	if err == nil {
	// 		acc.Add(metric, ival, tags)
	// 		continue
	// 	}

	// 	fval, err := strconv.ParseFloat(val, 64)
	// 	if err != nil {
	// 		return err
	// 	}

	// 	acc.Add(metric, fval, tags)
	// }

	return nil
}

func init() {
	plugins.Add("redis", func() plugins.Plugin {
		return &Redis{}
	})
}
<|endoftext|>


<commit_before>package memo

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"strings"
	"sync"
	"testing"
	"time"
)

// BPS is emulation download speed for url
const BPS = 10

type slowReader struct {
	delay time.Duration
	r     io.Reader
}

func (sr slowReader) Read(p []byte) (int, error) {
	time.Sleep(sr.delay)
	return sr.r.Read(p[:1])
}

func newReader(r io.Reader, bps int) io.Reader {
	delay := time.Second / time.Duration(bps)
	return slowReader{r: r, delay: delay}
}

func httpGetBodyMock(str string) (interface{}, error) {
	s := strings.NewReader(str)
	r := newReader(s, BPS)
	return ioutil.ReadAll(r)
}

var HTTPGetBody = httpGetBodyMock

func incomingURLs() <-chan string {
	ch := make(chan string)
	go func() {
		for _, url := range []string{
			"https://golang.org",
			"https://godoc.org",
			"https://play.golang.org",
			"http://gopl.io",
			"https://golang.org",
			"https://godoc.org",
			"https://play.golang.org",
			"http://gopl.io",
		} {
			ch <- url
		}
		close(ch)
	}()
	return ch
}

type M interface {
	Get(key string) (interface{}, error)
}

func Sequential(t *testing.T, m M) {
	for url := range incomingURLs() {
		start := time.Now()
		value, err := m.Get(url)
		if err != nil {
			log.Print(err)
			continue
		}
		fmt.Printf("%s, %s, %d bytes\n",
			url, time.Since(start), len(value.([]byte)))
	}
}

func Concurrent(t *testing.T, m M) {
	var n sync.WaitGroup
	for url := range incomingURLs() {
		n.Add(1)
		go func(url string) {
			defer n.Done()
			start := time.Now()
			value, err := m.Get(url)
			if err != nil {
				log.Print(err)
				return
			}
			fmt.Printf("%s, %s, %d bytes\n",
				url, time.Since(start), len(value.([]byte)))
		}(url)
	}
	n.Wait()
}

func TestSequential(t *testing.T) {
	m := New(httpGetBodyMock)
	Sequential(t, m)
}

func TestConcurrent(t *testing.T) {
	m := New(httpGetBodyMock)
	Concurrent(t, m)
}
<commit_msg>[9.3] add original httpGetBody to tests<commit_after>package memo

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
	"sync"
	"testing"
	"time"
)

// BPS is emulation download speed for url
const BPS = 10

type slowReader struct {
	delay time.Duration
	r     io.Reader
}

func (sr slowReader) Read(p []byte) (int, error) {
	time.Sleep(sr.delay)
	return sr.r.Read(p[:1])
}

func newReader(r io.Reader, bps int) io.Reader {
	delay := time.Second / time.Duration(bps)
	return slowReader{r: r, delay: delay}
}

func httpGetBodyMock(str string) (interface{}, error) {
	s := strings.NewReader(str)
	r := newReader(s, BPS)
	return ioutil.ReadAll(r)
}

func httpGetBody(url string) (interface{}, error) {
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	return ioutil.ReadAll(resp.Body)
}

var HTTPGetBody = httpGetBodyMock

func incomingURLs() <-chan string {
	ch := make(chan string)
	go func() {
		for _, url := range []string{
			"https://golang.org",
			"https://godoc.org",
			"https://play.golang.org",
			"http://gopl.io",
			"https://golang.org",
			"https://godoc.org",
			"https://play.golang.org",
			"http://gopl.io",
		} {
			ch <- url
		}
		close(ch)
	}()
	return ch
}

type M interface {
	Get(key string) (interface{}, error)
}

func Sequential(t *testing.T, m M) {
	for url := range incomingURLs() {
		start := time.Now()
		value, err := m.Get(url)
		if err != nil {
			log.Print(err)
			continue
		}
		fmt.Printf("%s, %s, %d bytes\n",
			url, time.Since(start), len(value.([]byte)))
	}
}

func Concurrent(t *testing.T, m M) {
	var n sync.WaitGroup
	for url := range incomingURLs() {
		n.Add(1)
		go func(url string) {
			defer n.Done()
			start := time.Now()
			value, err := m.Get(url)
			if err != nil {
				log.Print(err)
				return
			}
			fmt.Printf("%s, %s, %d bytes\n",
				url, time.Since(start), len(value.([]byte)))
		}(url)
	}
	n.Wait()
}

func TestSequential(t *testing.T) {
	m := New(httpGetBodyMock)
	Sequential(t, m)
}

func TestConcurrent(t *testing.T) {
	m := New(httpGetBodyMock)
	Concurrent(t, m)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fake

import (
	"fmt"
	"io"
	"path/filepath"
	"strings"

	clientgentypes "k8s.io/code-generator/cmd/client-gen/types"
	"k8s.io/gengo/generator"
	"k8s.io/gengo/namer"
	"k8s.io/gengo/types"
)

// genClientset generates a package for a clientset.
type genClientset struct {
	generator.DefaultGen
	groups               []clientgentypes.GroupVersions
	groupGoNames         map[clientgentypes.GroupVersion]string
	fakeClientsetPackage string
	outputPackage        string
	imports              namer.ImportTracker
	clientsetGenerated   bool
	// the import path of the generated real clientset.
	realClientsetPackage string
}

var _ generator.Generator = &genClientset{}

func (g *genClientset) Namers(c *generator.Context) namer.NameSystems {
	return namer.NameSystems{
		"raw": namer.NewRawNamer(g.outputPackage, g.imports),
	}
}

// We only want to call GenerateType() once.
func (g *genClientset) Filter(c *generator.Context, t *types.Type) bool {
	ret := !g.clientsetGenerated
	g.clientsetGenerated = true
	return ret
}

func (g *genClientset) Imports(c *generator.Context) (imports []string) {
	imports = append(imports, g.imports.ImportLines()...)
	for _, group := range g.groups {
		for _, version := range group.Versions {
			groupClientPackage := filepath.Join(g.fakeClientsetPackage, "typed", group.PackageName, version.NonEmpty())
			fakeGroupClientPackage := filepath.Join(groupClientPackage, "fake")

			groupAlias := strings.ToLower(g.groupGoNames[clientgentypes.GroupVersion{Group: group.Group, Version: version.Version}])
			imports = append(imports, strings.ToLower(fmt.Sprintf("%s%s \"%s\"", groupAlias, version.NonEmpty(), groupClientPackage)))
			imports = append(imports, strings.ToLower(fmt.Sprintf("fake%s%s \"%s\"", groupAlias, version.NonEmpty(), fakeGroupClientPackage)))
		}
	}
	// the package that has the clientset Interface
	imports = append(imports, fmt.Sprintf("clientset \"%s\"", g.realClientsetPackage))
	// imports for the code in commonTemplate
	imports = append(imports,
		"k8s.io/client-go/testing",
		"k8s.io/client-go/discovery",
		"fakediscovery \"k8s.io/client-go/discovery/fake\"",
		"k8s.io/apimachinery/pkg/runtime",
		"k8s.io/apimachinery/pkg/watch",
	)

	return
}

func (g *genClientset) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error {
	// TODO: We actually don't need any type information to generate the clientset,
	// perhaps we can adapt the go2ild framework to this kind of usage.
	sw := generator.NewSnippetWriter(w, c, "$", "$")

	allGroups := clientgentypes.ToGroupVersionInfo(g.groups, g.groupGoNames)

	sw.Do(common, nil)
	sw.Do(checkImpl, nil)

	for _, group := range allGroups {
		m := map[string]interface{}{
			"group":        group.Group,
			"version":      group.Version,
			"PackageAlias": group.PackageAlias,
			"GroupGoName":  group.GroupGoName,
			"Version":      namer.IC(group.Version.String()),
		}

		sw.Do(clientsetInterfaceImplTemplate, m)
		// don't generated the default method if generating internalversion clientset
		if group.IsDefaultVersion && group.Version != "" {
			sw.Do(clientsetInterfaceDefaultVersionImpl, m)
		}
	}

	return sw.Error()
}

// This part of code is version-independent, unchanging.
var common = `
// NewSimpleClientset returns a clientset that will respond with the provided objects.
// It's backed by a very simple object tracker that processes creates, updates and deletions as-is,
// without applying any validations and/or defaults. It shouldn't be considered a replacement
// for a real clientset and is mostly useful in simple unit tests.
func NewSimpleClientset(objects ...runtime.Object) *Clientset {
	o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder())
	for _, obj := range objects {
		if err := o.Add(obj); err != nil {
			panic(err)
		}
	}

	fakePtr := testing.Fake{}
	fakePtr.AddReactor("*", "*", testing.ObjectReaction(o))
	fakePtr.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) {
		gvr := action.GetResource()
		ns := action.GetNamespace()
		watch, err := o.Watch(gvr, ns)
		if err != nil {
			return false, nil, err
		}
		return true, watch, nil
	})

	return &Clientset{fakePtr, &fakediscovery.FakeDiscovery{Fake: &fakePtr}}
}

// Clientset implements clientset.Interface. Meant to be embedded into a
// struct to get a default implementation. This makes faking out just the method
// you want to test easier.
type Clientset struct {
	testing.Fake
	discovery *fakediscovery.FakeDiscovery
}

func (c *Clientset) Discovery() discovery.DiscoveryInterface {
	return c.discovery
}
`

var checkImpl = `
var _ clientset.Interface = &Clientset{}
`

var clientsetInterfaceImplTemplate = `
// $.GroupGoName$$.Version$ retrieves the $.GroupGoName$$.Version$Client
func (c *Clientset) $.GroupGoName$$.Version$() $.PackageAlias$.$.GroupGoName$$.Version$Interface {
	return &fake$.PackageAlias$.Fake$.GroupGoName$$.Version${Fake: &c.Fake}
}
`

var clientsetInterfaceDefaultVersionImpl = `
// $.GroupGoName$ retrieves the $.GroupGoName$$.Version$Client
func (c *Clientset) $.GroupGoName$() $.PackageAlias$.$.GroupGoName$$.Version$Interface {
	return &fake$.PackageAlias$.Fake$.GroupGoName$$.Version${Fake: &c.Fake}
}
`
<commit_msg>generate code that passes go vet<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package fake

import (
	"fmt"
	"io"
	"path/filepath"
	"strings"

	clientgentypes "k8s.io/code-generator/cmd/client-gen/types"
	"k8s.io/gengo/generator"
	"k8s.io/gengo/namer"
	"k8s.io/gengo/types"
)

// genClientset generates a package for a clientset.
type genClientset struct {
	generator.DefaultGen
	groups               []clientgentypes.GroupVersions
	groupGoNames         map[clientgentypes.GroupVersion]string
	fakeClientsetPackage string
	outputPackage        string
	imports              namer.ImportTracker
	clientsetGenerated   bool
	// the import path of the generated real clientset.
	realClientsetPackage string
}

var _ generator.Generator = &genClientset{}

func (g *genClientset) Namers(c *generator.Context) namer.NameSystems {
	return namer.NameSystems{
		"raw": namer.NewRawNamer(g.outputPackage, g.imports),
	}
}

// We only want to call GenerateType() once.
func (g *genClientset) Filter(c *generator.Context, t *types.Type) bool {
	ret := !g.clientsetGenerated
	g.clientsetGenerated = true
	return ret
}

func (g *genClientset) Imports(c *generator.Context) (imports []string) {
	imports = append(imports, g.imports.ImportLines()...)
	for _, group := range g.groups {
		for _, version := range group.Versions {
			groupClientPackage := filepath.Join(g.fakeClientsetPackage, "typed", group.PackageName, version.NonEmpty())
			fakeGroupClientPackage := filepath.Join(groupClientPackage, "fake")

			groupAlias := strings.ToLower(g.groupGoNames[clientgentypes.GroupVersion{Group: group.Group, Version: version.Version}])
			imports = append(imports, strings.ToLower(fmt.Sprintf("%s%s \"%s\"", groupAlias, version.NonEmpty(), groupClientPackage)))
			imports = append(imports, strings.ToLower(fmt.Sprintf("fake%s%s \"%s\"", groupAlias, version.NonEmpty(), fakeGroupClientPackage)))
		}
	}
	// the package that has the clientset Interface
	imports = append(imports, fmt.Sprintf("clientset \"%s\"", g.realClientsetPackage))
	// imports for the code in commonTemplate
	imports = append(imports,
		"k8s.io/client-go/testing",
		"k8s.io/client-go/discovery",
		"fakediscovery \"k8s.io/client-go/discovery/fake\"",
		"k8s.io/apimachinery/pkg/runtime",
		"k8s.io/apimachinery/pkg/watch",
	)

	return
}

func (g *genClientset) GenerateType(c *generator.Context, t *types.Type, w io.Writer) error {
	// TODO: We actually don't need any type information to generate the clientset,
	// perhaps we can adapt the go2ild framework to this kind of usage.
	sw := generator.NewSnippetWriter(w, c, "$", "$")

	allGroups := clientgentypes.ToGroupVersionInfo(g.groups, g.groupGoNames)

	sw.Do(common, nil)
	sw.Do(checkImpl, nil)

	for _, group := range allGroups {
		m := map[string]interface{}{
			"group":        group.Group,
			"version":      group.Version,
			"PackageAlias": group.PackageAlias,
			"GroupGoName":  group.GroupGoName,
			"Version":      namer.IC(group.Version.String()),
		}

		sw.Do(clientsetInterfaceImplTemplate, m)
		// don't generated the default method if generating internalversion clientset
		if group.IsDefaultVersion && group.Version != "" {
			sw.Do(clientsetInterfaceDefaultVersionImpl, m)
		}
	}

	return sw.Error()
}

// This part of code is version-independent, unchanging.
var common = `
// NewSimpleClientset returns a clientset that will respond with the provided objects.
// It's backed by a very simple object tracker that processes creates, updates and deletions as-is,
// without applying any validations and/or defaults. It shouldn't be considered a replacement
// for a real clientset and is mostly useful in simple unit tests.
func NewSimpleClientset(objects ...runtime.Object) *Clientset {
	o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder())
	for _, obj := range objects {
		if err := o.Add(obj); err != nil {
			panic(err)
		}
	}

	cs := &Clientset{}
	cs.discovery = &fakediscovery.FakeDiscovery{Fake: &cs.Fake}
	cs.AddReactor("*", "*", testing.ObjectReaction(o))
	cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) {
		gvr := action.GetResource()
		ns := action.GetNamespace()
		watch, err := o.Watch(gvr, ns)
		if err != nil {
			return false, nil, err
		}
		return true, watch, nil
	})

	return cs
}

// Clientset implements clientset.Interface. Meant to be embedded into a
// struct to get a default implementation. This makes faking out just the method
// you want to test easier.
type Clientset struct {
	testing.Fake
	discovery *fakediscovery.FakeDiscovery
}

func (c *Clientset) Discovery() discovery.DiscoveryInterface {
	return c.discovery
}
`

var checkImpl = `
var _ clientset.Interface = &Clientset{}
`

var clientsetInterfaceImplTemplate = `
// $.GroupGoName$$.Version$ retrieves the $.GroupGoName$$.Version$Client
func (c *Clientset) $.GroupGoName$$.Version$() $.PackageAlias$.$.GroupGoName$$.Version$Interface {
	return &fake$.PackageAlias$.Fake$.GroupGoName$$.Version${Fake: &c.Fake}
}
`

var clientsetInterfaceDefaultVersionImpl = `
// $.GroupGoName$ retrieves the $.GroupGoName$$.Version$Client
func (c *Clientset) $.GroupGoName$() $.PackageAlias$.$.GroupGoName$$.Version$Interface {
	return &fake$.PackageAlias$.Fake$.GroupGoName$$.Version${Fake: &c.Fake}
}
`
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import "unsafe"

const (
	_AT_PLATFORM = 15 //  introduced in at least 2.6.11

	_HWCAP_VFP   = 1 << 6  // introduced in at least 2.6.11
	_HWCAP_VFPv3 = 1 << 13 // introduced in 2.6.30
)

var randomNumber uint32
var armArch uint8 = 6 // we default to ARMv6
var hwcap uint32      // set by setup_auxv

func checkgoarm() {
	if goarm > 5 && hwcap&_HWCAP_VFP == 0 {
		print("runtime: this CPU has no floating point hardware, so it cannot run\n")
		print("this GOARM=", goarm, " binary. Recompile using GOARM=5.\n")
		exit(1)
	}
	if goarm > 6 && hwcap&_HWCAP_VFPv3 == 0 {
		print("runtime: this CPU has no VFPv3 floating point hardware, so it cannot run\n")
		print("this GOARM=", goarm, " binary. Recompile using GOARM=5.\n")
		exit(1)
	}
}

func archauxv(tag, val uintptr) {
	switch tag {
	case _AT_RANDOM:
		// sysargs filled in startupRandomData, but that
		// pointer may not be word aligned, so we must treat
		// it as a byte array.
		randomNumber = uint32(startupRandomData[4]) | uint32(startupRandomData[5])<<8 |
			uint32(startupRandomData[6])<<16 | uint32(startupRandomData[7])<<24

	case _AT_PLATFORM: // v5l, v6l, v7l
		t := *(*uint8)(unsafe.Pointer(val + 1))
		if '5' <= t && t <= '7' {
			armArch = t - '0'
		}

	case _AT_HWCAP: // CPU capability bit flags
		hwcap = uint32(val)
	}
}

//go:nosplit
func cputicks() int64 {
	// Currently cputicks() is used in blocking profiler and to seed fastrand().
	// nanotime() is a poor approximation of CPU ticks that is enough for the profiler.
	// randomNumber provides better seeding of fastrand.
	return nanotime() + int64(randomNumber)
}
<commit_msg>runtime: skip floating point hardware check on Android<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import "unsafe"

const (
	_AT_PLATFORM = 15 //  introduced in at least 2.6.11

	_HWCAP_VFP   = 1 << 6  // introduced in at least 2.6.11
	_HWCAP_VFPv3 = 1 << 13 // introduced in 2.6.30
)

var randomNumber uint32
var armArch uint8 = 6 // we default to ARMv6
var hwcap uint32      // set by setup_auxv

func checkgoarm() {
	// On Android, /proc/self/auxv might be unreadable and hwcap won't
	// reflect the CPU capabilities. Assume that every Android arm device
	// has the necessary floating point hardware available.
	if GOOS == "android" {
		return
	}
	if goarm > 5 && hwcap&_HWCAP_VFP == 0 {
		print("runtime: this CPU has no floating point hardware, so it cannot run\n")
		print("this GOARM=", goarm, " binary. Recompile using GOARM=5.\n")
		exit(1)
	}
	if goarm > 6 && hwcap&_HWCAP_VFPv3 == 0 {
		print("runtime: this CPU has no VFPv3 floating point hardware, so it cannot run\n")
		print("this GOARM=", goarm, " binary. Recompile using GOARM=5.\n")
		exit(1)
	}
}

func archauxv(tag, val uintptr) {
	switch tag {
	case _AT_RANDOM:
		// sysargs filled in startupRandomData, but that
		// pointer may not be word aligned, so we must treat
		// it as a byte array.
		randomNumber = uint32(startupRandomData[4]) | uint32(startupRandomData[5])<<8 |
			uint32(startupRandomData[6])<<16 | uint32(startupRandomData[7])<<24

	case _AT_PLATFORM: // v5l, v6l, v7l
		t := *(*uint8)(unsafe.Pointer(val + 1))
		if '5' <= t && t <= '7' {
			armArch = t - '0'
		}

	case _AT_HWCAP: // CPU capability bit flags
		hwcap = uint32(val)
	}
}

//go:nosplit
func cputicks() int64 {
	// Currently cputicks() is used in blocking profiler and to seed fastrand().
	// nanotime() is a poor approximation of CPU ticks that is enough for the profiler.
	// randomNumber provides better seeding of fastrand.
	return nanotime() + int64(randomNumber)
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

type sigTabT struct {
	flags int
	name  string
}

// Incoming notes are compared against this table using strncmp, so the
// order matters: longer patterns must appear before their prefixes.
// There are _SIG constants in os2_plan9.go for the table index of some
// of these.
//
// If you add entries to this table, you must respect the prefix ordering
// and also update the constant values is os2_plan9.go.
var sigtable = [...]sigTabT{
	// Traps that we cannot be recovered.
	{_SigThrow, "sys: trap: debug exception"},
	{_SigThrow, "sys: trap: invalid opcode"},

	// We can recover from some memory errors in runtime·sigpanic.
	{_SigPanic, "sys: trap: fault read addr"},  // SIGRFAULT
	{_SigPanic, "sys: trap: fault write addr"}, // SIGWFAULT

	// We can also recover from math errors.
	{_SigPanic, "sys: trap: divide error"}, // SIGINTDIV
	{_SigPanic, "sys: fp:"},                // SIGFLOAT

	// All other traps are normally handled as if they were marked SigThrow.
	// We mark them SigPanic here so that debug.SetPanicOnFault will work.
	{_SigPanic, "sys: trap:"}, // SIGTRAP

	// Writes to a closed pipe can be handled if desired, otherwise they're ignored.
	{_SigNotify, "sys: write on closed pipe"},

	// Other system notes are more serious and cannot be recovered.
	{_SigThrow, "sys:"},

	// Issued to all other procs when calling runtime·exit.
	{_SigGoExit, "go: exit "},

	// Kill is sent by external programs to cause an exit.
	{_SigKill, "kill"},

	// Interrupts can be handled if desired, otherwise they cause an exit.
	{_SigNotify + _SigKill, "interrupt"},
	{_SigNotify + _SigKill, "hangup"},

	// Alarms can be handled if desired, otherwise they're ignored.
	{_SigNotify, "alarm"},
}
<commit_msg>runtime: handle abort note on Plan 9<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

type sigTabT struct {
	flags int
	name  string
}

// Incoming notes are compared against this table using strncmp, so the
// order matters: longer patterns must appear before their prefixes.
// There are _SIG constants in os2_plan9.go for the table index of some
// of these.
//
// If you add entries to this table, you must respect the prefix ordering
// and also update the constant values is os2_plan9.go.
var sigtable = [...]sigTabT{
	// Traps that we cannot be recovered.
	{_SigThrow, "sys: trap: debug exception"},
	{_SigThrow, "sys: trap: invalid opcode"},

	// We can recover from some memory errors in runtime·sigpanic.
	{_SigPanic, "sys: trap: fault read addr"},  // SIGRFAULT
	{_SigPanic, "sys: trap: fault write addr"}, // SIGWFAULT

	// We can also recover from math errors.
	{_SigPanic, "sys: trap: divide error"}, // SIGINTDIV
	{_SigPanic, "sys: fp:"},                // SIGFLOAT

	// All other traps are normally handled as if they were marked SigThrow.
	// We mark them SigPanic here so that debug.SetPanicOnFault will work.
	{_SigPanic, "sys: trap:"}, // SIGTRAP

	// Writes to a closed pipe can be handled if desired, otherwise they're ignored.
	{_SigNotify, "sys: write on closed pipe"},

	// Other system notes are more serious and cannot be recovered.
	{_SigThrow, "sys:"},

	// Issued to all other procs when calling runtime·exit.
	{_SigGoExit, "go: exit "},

	// Kill is sent by external programs to cause an exit.
	{_SigKill, "kill"},

	// Interrupts can be handled if desired, otherwise they cause an exit.
	{_SigNotify + _SigKill, "interrupt"},
	{_SigNotify + _SigKill, "hangup"},

	// Alarms can be handled if desired, otherwise they're ignored.
	{_SigNotify, "alarm"},

	// Aborts can be handled if desired, otherwise they cause a stack trace.
	{_SigNotify + _SigThrow, "abort"},
}
<|endoftext|>


<commit_before>package s3

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"fmt"
	"io"
	"net/http"
	"strings"
	"text/template"

	"github.com/gogo/protobuf/types"
	"github.com/gorilla/mux"
	log "github.com/sirupsen/logrus"

	"github.com/pachyderm/pachyderm/src/client"
)

const locationResponse = `<?xml version="1.0" encoding="UTF-8"?>
<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">PACHYDERM</LocationConstraint>`

const listBucketSource = `<?xml version="1.0" encoding="UTF-8"?>
<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
    <Owner>
    	<ID>000000000000000000000000000000</ID>
    	<DisplayName>pachyderm</DisplayName>
    </Owner>
    <Buckets>
        {{ range . }}
            <Bucket>
                <Name>{{ .Repo.Name }}</Name>
                <CreationDate>{{ formatTime .Created }}</CreationDate>
            </Bucket>
        {{ end }}
    </Buckets>
</ListAllMyBucketsResult>`

func writeBadRequest(w http.ResponseWriter, err error) {
	http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
}

func writeMaybeNotFound(w http.ResponseWriter, r *http.Request, err error) {
	if strings.Contains(err.Error(), "not found") {
		http.NotFound(w, r)
	} else {
		writeServerError(w, err)
	}
}

func writeServerError(w http.ResponseWriter, err error) {
	http.Error(w, fmt.Sprintf("%v", err), http.StatusInternalServerError)
}

type handler struct {
	pc                  *client.APIClient
	listBucketsTemplate *template.Template
}

func newHandler(pc *client.APIClient) handler {
	funcMap := template.FuncMap{
		"formatTime": func(timestamp *types.Timestamp) string {
			return timestamp.String()
		},
	}

	listBucketsTemplate := template.Must(template.New("list-buckets").
		Funcs(funcMap).
		Parse(listBucketSource))

	return handler{
		pc:                  pc,
		listBucketsTemplate: listBucketsTemplate,
	}
}

func (h handler) ping(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusOK)
}

func (h handler) root(w http.ResponseWriter, r *http.Request) {
	buckets, err := h.pc.ListRepo()
	if err != nil {
		writeServerError(w, err)
		return
	}

	if err = h.listBucketsTemplate.Execute(w, buckets); err != nil {
		writeServerError(w, err)
		return
	}

	w.Header().Set("Content-Type", "application/xml")
}

func (h handler) repo(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	repo := vars["repo"]

	if r.Method == http.MethodGet {
		if err := r.ParseForm(); err != nil {
			writeBadRequest(w, err)
			return
		}

		if _, err := h.pc.InspectRepo(repo); err != nil {
			writeMaybeNotFound(w, r, err)
			return
		}

		if _, ok := r.Form["location"]; ok {
			w.Header().Set("Content-Type", "application/xml")
			w.Write([]byte(locationResponse))
		} else {
			w.WriteHeader(http.StatusOK)
		}
	} else if r.Method == http.MethodPut {
		err := h.pc.CreateRepo(repo)

		if err != nil {
			writeServerError(w, err)
		} else {
			w.WriteHeader(http.StatusOK)
		}
	} else if r.Method == http.MethodDelete {
		err := h.pc.DeleteRepo(repo, false)

		if err != nil {
			writeMaybeNotFound(w, r, err)
		} else {
			w.WriteHeader(http.StatusNoContent)
		}
	}
}

func (h handler) getObject(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	repo := vars["repo"]
	branch := vars["branch"]
	file := vars["file"]

	fileInfo, err := h.pc.InspectFile(repo, branch, file)
	if err != nil {
		if strings.Contains(err.Error(), "has no head") {
			// occurs if the branch exists, but there's no head commit
			http.NotFound(w, r)
		} else {
			writeMaybeNotFound(w, r, err)
		}

		return
	}

	timestamp, err := types.TimestampFromProto(fileInfo.Committed)
	if err != nil {
		writeServerError(w, err)
		return
	}

	reader, err := h.pc.GetFileReadSeeker(repo, branch, file)
	if err != nil {
		writeServerError(w, err)
		return
	}

	http.ServeContent(w, r, "", timestamp, reader)
}

func (h handler) putObject(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	repo := vars["repo"]
	branch := vars["branch"]
	file := vars["file"]

	expectedHash := r.Header.Get("Content-MD5")

	if expectedHash != "" {
		expectedHashBytes, err := base64.StdEncoding.DecodeString(expectedHash)

		if err != nil {
			writeBadRequest(w, fmt.Errorf("could not decode `Content-MD5`, as it is not base64-encoded"))
			return
		}

		h.putObjectVerifying(w, r, repo, branch, file, expectedHashBytes)
		return
	}

	h.putObjectUnverified(w, r, repo, branch, file)
}

func (h handler) putObjectVerifying(w http.ResponseWriter, r *http.Request, repo, branch, file string, expectedHash []byte) {
	hasher := md5.New()
	reader := io.TeeReader(r.Body, hasher)

	_, err := h.pc.PutFileOverwrite(repo, branch, file, reader, 0)
	if err != nil {
		// the error may be because the repo or branch does not exist -
		// double-check that by inspecting the branch, so we can serve a 404
		// instead
		_, inspectError := h.pc.InspectBranch(repo, branch)
		writeMaybeNotFound(w, r, inspectError)
		return
	}

	actualHash := hasher.Sum(nil)

	if !bytes.Equal(expectedHash, actualHash) {
		err = fmt.Errorf("content checksums differ; expected=%x, actual=%x", expectedHash, actualHash)
		writeBadRequest(w, err)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (h handler) putObjectUnverified(w http.ResponseWriter, r *http.Request, repo, branch, file string) {
	_, err := h.pc.PutFileOverwrite(repo, branch, file, r.Body, 0)
	if err != nil {
		// the error may be because the repo or branch does not exist -
		// double-check that by inspecting the branch, so we can serve a 404
		// instead
		_, inspectError := h.pc.InspectBranch(repo, branch)
		writeMaybeNotFound(w, r, inspectError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

// Server runs an HTTP server with an S3-like API for PFS. This allows you to
// use s3 clients to acccess PFS contents.
//
// This returns an `http.Server` instance. It is the responsibility of the
// caller to start the server. This also makes it possible for the caller to
// enable graceful shutdown if desired; see the `http` package for details.
//
// Bucket names correspond to repo names, and files are accessible via the s3
// key pattern "<branch>/<filepath>". For example, to get the file "a/b/c.txt"
// on the "foo" repo's "master" branch, you'd making an s3 get request with
// bucket = "foo", key = "master/a/b/c.txt".
//
// Note: in s3, bucket names are constrained by IETF RFC 1123, (and its
// predecessor RFC 952) but pachyderm's repo naming constraints are slightly
// more liberal. If the s3 client does any kind of bucket name validation
// (this includes minio), repos whose names do not comply with RFC 1123 will
// not be accessible.
//
// Note: In `s3cmd`, you must set the access key and secret key, even though
// this API will ignore them - otherwise, you'll get an opaque config error:
// https://github.com/s3tools/s3cmd/issues/845#issuecomment-464885959
func Server(pc *client.APIClient, port uint16) *http.Server {
	handler := newHandler(pc)

	// repo validation regex is the same as minio
	router := mux.NewRouter()
	router.HandleFunc(`/`, handler.root).Methods("GET", "HEAD")
	router.HandleFunc(`/{repo:[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]}/`, handler.repo).Methods("DELETE", "GET", "PUT", "HEAD")
	router.HandleFunc(`/{repo:[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]}/{branch}/{file:.+}`, handler.getObject).Methods("GET", "HEAD")
	router.HandleFunc(`/{repo:[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]}/{branch}/{file:.+}`, handler.putObject).Methods("PUT", "HEAD")
	router.HandleFunc(`/_ping`, handler.ping).Methods("GET", "HEAD")

	// Note: error log is not customized on this `http.Server`, which means
	// it'll default to using the stdlib logger and produce log messages that
	// don't look like the ones produced elsewhere, and aren't configured
	// properly. In testing, this didn't seem to be a big deal because it's
	// rather hard to trigger it anyways, but if we find a reliable way to
	// create error logs, it might be worthwhile to fix.
	return &http.Server{
		Addr: fmt.Sprintf(":%d", port),
		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			log.Debugf("s3 gateway request: %s %s", r.Method, r.RequestURI)
			router.ServeHTTP(w, r)
		}),
	}
}
<commit_msg>Break up repo endpoint<commit_after>package s3

import (
	"bytes"
	"crypto/md5"
	"encoding/base64"
	"fmt"
	"io"
	"net/http"
	"strings"
	"text/template"

	"github.com/gogo/protobuf/types"
	"github.com/gorilla/mux"
	log "github.com/sirupsen/logrus"

	"github.com/pachyderm/pachyderm/src/client"
)

const locationResponse = `<?xml version="1.0" encoding="UTF-8"?>
<LocationConstraint xmlns="http://s3.amazonaws.com/doc/2006-03-01/">PACHYDERM</LocationConstraint>`

const listBucketSource = `<?xml version="1.0" encoding="UTF-8"?>
<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
    <Owner>
    	<ID>000000000000000000000000000000</ID>
    	<DisplayName>pachyderm</DisplayName>
    </Owner>
    <Buckets>
        {{ range . }}
            <Bucket>
                <Name>{{ .Repo.Name }}</Name>
                <CreationDate>{{ formatTime .Created }}</CreationDate>
            </Bucket>
        {{ end }}
    </Buckets>
</ListAllMyBucketsResult>`

func writeBadRequest(w http.ResponseWriter, err error) {
	http.Error(w, fmt.Sprintf("%v", err), http.StatusBadRequest)
}

func writeMaybeNotFound(w http.ResponseWriter, r *http.Request, err error) {
	if strings.Contains(err.Error(), "not found") {
		http.NotFound(w, r)
	} else {
		writeServerError(w, err)
	}
}

func writeServerError(w http.ResponseWriter, err error) {
	http.Error(w, fmt.Sprintf("%v", err), http.StatusInternalServerError)
}

type handler struct {
	pc                  *client.APIClient
	listBucketsTemplate *template.Template
}

func newHandler(pc *client.APIClient) handler {
	funcMap := template.FuncMap{
		"formatTime": func(timestamp *types.Timestamp) string {
			return timestamp.String()
		},
	}

	listBucketsTemplate := template.Must(template.New("list-buckets").
		Funcs(funcMap).
		Parse(listBucketSource))

	return handler{
		pc:                  pc,
		listBucketsTemplate: listBucketsTemplate,
	}
}

func (h handler) ping(w http.ResponseWriter, r *http.Request) {
	w.WriteHeader(http.StatusOK)
}

func (h handler) root(w http.ResponseWriter, r *http.Request) {
	buckets, err := h.pc.ListRepo()
	if err != nil {
		writeServerError(w, err)
		return
	}

	if err = h.listBucketsTemplate.Execute(w, buckets); err != nil {
		writeServerError(w, err)
		return
	}

	w.Header().Set("Content-Type", "application/xml")
}

func (h handler) getRepo(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	repo := vars["repo"]

	if err := r.ParseForm(); err != nil {
		writeBadRequest(w, err)
		return
	}

	if _, err := h.pc.InspectRepo(repo); err != nil {
		writeMaybeNotFound(w, r, err)
		return
	}

	if _, ok := r.Form["location"]; ok {
		w.Header().Set("Content-Type", "application/xml")
		w.Write([]byte(locationResponse))
	} else {
		w.WriteHeader(http.StatusOK)
	}
}

func (h handler) putRepo(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	repo := vars["repo"]

	err := h.pc.CreateRepo(repo)

	if err != nil {
		writeServerError(w, err)
	} else {
		w.WriteHeader(http.StatusOK)
	}
}

func (h handler) deleteRepo(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	repo := vars["repo"]

	err := h.pc.DeleteRepo(repo, false)

	if err != nil {
		writeMaybeNotFound(w, r, err)
	} else {
		w.WriteHeader(http.StatusNoContent)
	}
}

func (h handler) getObject(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	repo := vars["repo"]
	branch := vars["branch"]
	file := vars["file"]

	fileInfo, err := h.pc.InspectFile(repo, branch, file)
	if err != nil {
		if strings.Contains(err.Error(), "has no head") {
			// occurs if the branch exists, but there's no head commit
			http.NotFound(w, r)
		} else {
			writeMaybeNotFound(w, r, err)
		}

		return
	}

	timestamp, err := types.TimestampFromProto(fileInfo.Committed)
	if err != nil {
		writeServerError(w, err)
		return
	}

	reader, err := h.pc.GetFileReadSeeker(repo, branch, file)
	if err != nil {
		writeServerError(w, err)
		return
	}

	http.ServeContent(w, r, "", timestamp, reader)
}

func (h handler) putObject(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	repo := vars["repo"]
	branch := vars["branch"]
	file := vars["file"]

	expectedHash := r.Header.Get("Content-MD5")

	if expectedHash != "" {
		expectedHashBytes, err := base64.StdEncoding.DecodeString(expectedHash)

		if err != nil {
			writeBadRequest(w, fmt.Errorf("could not decode `Content-MD5`, as it is not base64-encoded"))
			return
		}

		h.putObjectVerifying(w, r, repo, branch, file, expectedHashBytes)
		return
	}

	h.putObjectUnverified(w, r, repo, branch, file)
}

func (h handler) putObjectVerifying(w http.ResponseWriter, r *http.Request, repo, branch, file string, expectedHash []byte) {
	hasher := md5.New()
	reader := io.TeeReader(r.Body, hasher)

	_, err := h.pc.PutFileOverwrite(repo, branch, file, reader, 0)
	if err != nil {
		// the error may be because the repo or branch does not exist -
		// double-check that by inspecting the branch, so we can serve a 404
		// instead
		_, inspectError := h.pc.InspectBranch(repo, branch)
		writeMaybeNotFound(w, r, inspectError)
		return
	}

	actualHash := hasher.Sum(nil)

	if !bytes.Equal(expectedHash, actualHash) {
		err = fmt.Errorf("content checksums differ; expected=%x, actual=%x", expectedHash, actualHash)
		writeBadRequest(w, err)
		return
	}

	w.WriteHeader(http.StatusOK)
}

func (h handler) putObjectUnverified(w http.ResponseWriter, r *http.Request, repo, branch, file string) {
	_, err := h.pc.PutFileOverwrite(repo, branch, file, r.Body, 0)
	if err != nil {
		// the error may be because the repo or branch does not exist -
		// double-check that by inspecting the branch, so we can serve a 404
		// instead
		_, inspectError := h.pc.InspectBranch(repo, branch)
		writeMaybeNotFound(w, r, inspectError)
		return
	}

	w.WriteHeader(http.StatusOK)
}

// Server runs an HTTP server with an S3-like API for PFS. This allows you to
// use s3 clients to acccess PFS contents.
//
// This returns an `http.Server` instance. It is the responsibility of the
// caller to start the server. This also makes it possible for the caller to
// enable graceful shutdown if desired; see the `http` package for details.
//
// Bucket names correspond to repo names, and files are accessible via the s3
// key pattern "<branch>/<filepath>". For example, to get the file "a/b/c.txt"
// on the "foo" repo's "master" branch, you'd making an s3 get request with
// bucket = "foo", key = "master/a/b/c.txt".
//
// Note: in s3, bucket names are constrained by IETF RFC 1123, (and its
// predecessor RFC 952) but pachyderm's repo naming constraints are slightly
// more liberal. If the s3 client does any kind of bucket name validation
// (this includes minio), repos whose names do not comply with RFC 1123 will
// not be accessible.
//
// Note: In `s3cmd`, you must set the access key and secret key, even though
// this API will ignore them - otherwise, you'll get an opaque config error:
// https://github.com/s3tools/s3cmd/issues/845#issuecomment-464885959
func Server(pc *client.APIClient, port uint16) *http.Server {
	handler := newHandler(pc)

	// repo validation regex is the same as minio
	router := mux.NewRouter()
	router.HandleFunc(`/`, handler.root).Methods("GET", "HEAD")
	router.HandleFunc(`/{repo:[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]}/`, handler.getRepo).Methods("GET", "HEAD")
	router.HandleFunc(`/{repo:[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]}/`, handler.putRepo).Methods("PUT")
	router.HandleFunc(`/{repo:[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]}/`, handler.deleteRepo).Methods("DELETE")
	router.HandleFunc(`/{repo:[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]}/{branch}/{file:.+}`, handler.getObject).Methods("GET", "HEAD")
	router.HandleFunc(`/{repo:[a-z0-9][a-z0-9\.\-]{1,61}[a-z0-9]}/{branch}/{file:.+}`, handler.putObject).Methods("PUT")
	router.HandleFunc(`/_ping`, handler.ping).Methods("GET", "HEAD")

	// Note: error log is not customized on this `http.Server`, which means
	// it'll default to using the stdlib logger and produce log messages that
	// don't look like the ones produced elsewhere, and aren't configured
	// properly. In testing, this didn't seem to be a big deal because it's
	// rather hard to trigger it anyways, but if we find a reliable way to
	// create error logs, it might be worthwhile to fix.
	return &http.Server{
		Addr: fmt.Sprintf(":%d", port),
		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			log.Debugf("s3 gateway request: %s %s", r.Method, r.RequestURI)
			router.ServeHTTP(w, r)
		}),
	}
}
<|endoftext|>


<commit_before>// +build linux

package endpoint

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"syscall"

	"github.com/armon/go-metrics"
	log "github.com/sirupsen/logrus"
	"github.com/weaveworks/common/fs"
	"github.com/weaveworks/scope/probe/endpoint/procspy"
	"github.com/weaveworks/scope/probe/host"
	"github.com/weaveworks/scope/probe/process"
	"github.com/weaveworks/tcptracer-bpf/pkg/tracer"
)

// An ebpfConnection represents a TCP connection
type ebpfConnection struct {
	tuple            fourTuple
	networkNamespace uint32
	incoming         bool
	pid              int
}

// EbpfTracker contains the sets of open and closed TCP connections.
// Closed connections are kept in the `closedConnections` slice for one iteration of `walkConnections`.
type EbpfTracker struct {
	sync.Mutex
	tracer          *tracer.Tracer
	ready           bool
	stopping        bool
	dead            bool
	lastTimestampV4 uint64

	// debugBPF specifies if EbpfTracker must be started in debug mode. This
	// allows to easily debug issues like:
	// https://github.com/weaveworks/scope/issues/2650
	//
	// Scope could be started this way:
	//   $ sudo WEAVESCOPE_DOCKER_ARGS="-e SCOPE_DEBUG_BPF=1" ./scope launch
	//
	// Then, EbpfTracker could be tricked into restarting with:
	//   $ echo stop | sudo tee /proc/$(pidof scope-probe)/root/var/run/scope/debug-bpf
	debugBPF bool

	openConnections   map[fourTuple]ebpfConnection
	closedConnections []ebpfConnection
	closedDuringInit  map[fourTuple]struct{}
}

// releaseRegex should match all possible variations of a common Linux
// version string:
//   - 4.1
//   - 4.22-foo
//   - 4.1.2-foo
//   - 4.1.2-33.44+bar
//   - etc.
// For example, on a Ubuntu system the vendor specific release part
// (after the first `-`) could look like:
// '<ABI number>.<upload number>-<flavour>' or
// '<ABI number>-<flavour>'
// See https://wiki.ubuntu.com/Kernel/FAQ
var releaseRegex = regexp.MustCompile(`^(\d+)\.(\d+)\.?(\d*)-?(\d*)(.*)$`)

func isKernelSupported() error {
	release, version, err := host.GetKernelReleaseAndVersion()
	if err != nil {
		return err
	}

	releaseParts := releaseRegex.FindStringSubmatch(release)
	if len(releaseParts) != 6 {
		return fmt.Errorf("got invalid release version %q (expected format '4.4[.2-1]')", release)
	}

	major, err := strconv.Atoi(releaseParts[1])
	if err != nil {
		return err
	}

	minor, err := strconv.Atoi(releaseParts[2])
	if err != nil {
		return err
	}

	if major > 4 {
		return nil
	}

	if major < 4 || minor < 4 {
		return fmt.Errorf("got kernel %s but need kernel >=4.4", release)
	}

	if strings.Contains(version, "Ubuntu") {
		// Check for specific Ubuntu kernel versions with
		// known issues.

		abiNumber, err := strconv.Atoi(releaseParts[4])
		if err != nil {
			// By now we know it's at least kernel 4.4 and
			// not "119-ish", so allow it.
			return nil
		}
		if major == 4 && minor == 4 && abiNumber >= 119 && abiNumber < 127 {
			// https://github.com/weaveworks/scope/issues/3131
			// https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1763454
			return fmt.Errorf("got Ubuntu kernel %s with known bug", release)
		}
	}

	return nil
}

func newEbpfTracker() (*EbpfTracker, error) {
	if err := isKernelSupported(); err != nil {
		return nil, fmt.Errorf("kernel not supported: %v", err)
	}

	var debugBPF bool
	if os.Getenv("SCOPE_DEBUG_BPF") != "" {
		log.Infof("ebpf tracker started in debug mode")
		debugBPF = true
	}

	tracer.TimestampOffset = 200000 // Delay events by 0.2ms to avoid out-of-order reporting
	tracker := &EbpfTracker{
		debugBPF: debugBPF,
	}
	if err := tracker.restart(); err != nil {
		return nil, err
	}

	return tracker, nil
}

// TCPEventV4 handles IPv4 TCP events from the eBPF tracer
func (t *EbpfTracker) TCPEventV4(e tracer.TcpV4) {
	if t.debugBPF {
		debugBPFFile := "/var/run/scope/debug-bpf"
		b, err := ioutil.ReadFile("/var/run/scope/debug-bpf")
		if err == nil && strings.TrimSpace(string(b[:])) == "stop" {
			os.Remove(debugBPFFile)
			log.Warnf("ebpf tracker stopped as requested by user")
			t.stop()
			return
		}
	}

	if t.lastTimestampV4 > e.Timestamp {
		// A kernel bug can cause the timestamps to be wrong (e.g. on Ubuntu with Linux 4.4.0-47.68)
		// Upgrading the kernel will fix the problem. For further info see:
		// https://github.com/iovisor/bcc/issues/790#issuecomment-263704235
		// https://github.com/weaveworks/scope/issues/2334
		log.Errorf("tcp tracer received event with timestamp %v even though the last timestamp was %v. Stopping the eBPF tracker.", e.Timestamp, t.lastTimestampV4)
		t.stop()
		metrics.IncrCounterWithLabels([]string{"ebpf", "errors"}, 1, []metrics.Label{
			{Name: "kind", Value: "timestamp-out-of-order"},
		})
		return
	}

	t.lastTimestampV4 = e.Timestamp

	if e.Type == tracer.EventFdInstall {
		t.handleFdInstall(e.Type, int(e.Pid), int(e.Fd))
	} else {
		tuple := makeFourTuple(e.SAddr, e.DAddr, e.SPort, e.DPort)
		t.handleConnection(e.Type, tuple, int(e.Pid), e.NetNS)
	}
}

// TCPEventV6 handles IPv6 TCP events from the eBPF tracer. This is
// currently a no-op.
func (t *EbpfTracker) TCPEventV6(e tracer.TcpV6) {
	// TODO: IPv6 not supported in Scope
}

// LostV4 handles IPv4 TCP event misses from the eBPF tracer.
func (t *EbpfTracker) LostV4(count uint64) {
	log.Errorf("tcp tracer lost %d events. Stopping the eBPF tracker", count)
	metrics.IncrCounterWithLabels([]string{"ebpf", "errors"}, 1, []metrics.Label{
		{Name: "kind", Value: "lost-events"},
	})
	t.stop()
}

// LostV6 handles IPv4 TCP event misses from the eBPF tracer. This is
// currently a no-op.
func (t *EbpfTracker) LostV6(count uint64) {
	// TODO: IPv6 not supported in Scope
}

func tupleFromPidFd(pid int, fd int) (tuple fourTuple, netns uint32, ok bool) {
	// read /proc/$pid/ns/net
	//
	// probe/endpoint/procspy/proc_linux.go supports Linux < 3.8 but we
	// don't need that here since ebpf-enabled kernels will be > 3.8
	netnsIno, err := procspy.ReadNetnsFromPID(pid)
	if err != nil {
		log.Debugf("netns proc file for pid %d disappeared before we could read it: %v", pid, err)
		return fourTuple{}, 0, false
	}

	// find /proc/$pid/fd/$fd's ino
	fdFilename := fmt.Sprintf("/proc/%d/fd/%d", pid, fd)
	var statFdFile syscall.Stat_t
	if err := fs.Stat(fdFilename, &statFdFile); err != nil {
		log.Debugf("proc file %q disappeared before we could read it", fdFilename)
		return fourTuple{}, 0, false
	}

	if statFdFile.Mode&syscall.S_IFMT != syscall.S_IFSOCK {
		log.Errorf("file %q is not a socket", fdFilename)
		return fourTuple{}, 0, false
	}
	ino := statFdFile.Ino

	// read both /proc/pid/net/{tcp,tcp6}
	buf := bytes.NewBuffer(make([]byte, 0, 5000))
	if _, err := procspy.ReadTCPFiles(pid, buf); err != nil {
		log.Debugf("TCP proc file for pid %d disappeared before we could read it: %v", pid, err)
		return fourTuple{}, 0, false
	}

	// find /proc/$pid/fd/$fd's ino in /proc/pid/net/tcp
	pn := procspy.NewProcNet(buf.Bytes())
	for {
		n := pn.Next()
		if n == nil {
			log.Debugf("connection for proc file %q not found. buf=%q", fdFilename, buf.String())
			break
		}
		if n.Inode == ino {
			tuple := makeFourTuple(n.LocalAddress, n.RemoteAddress, n.LocalPort, n.RemotePort)
			return tuple, netnsIno, true
		}
	}

	return fourTuple{}, 0, false
}

func (t *EbpfTracker) handleFdInstall(ev tracer.EventType, pid int, fd int) {
	if !process.IsProcInAccept("/proc", strconv.Itoa(pid)) {
		t.tracer.RemoveFdInstallWatcher(uint32(pid))
	}
	tuple, netns, ok := tupleFromPidFd(pid, fd)
	log.Debugf("EbpfTracker: got fd-install event: pid=%d fd=%d -> tuple=%s netns=%v ok=%v", pid, fd, tuple, netns, ok)
	if !ok {
		return
	}

	t.Lock()
	defer t.Unlock()

	t.openConnections[tuple] = ebpfConnection{
		incoming:         true,
		tuple:            tuple,
		pid:              pid,
		networkNamespace: netns,
	}
}

func (t *EbpfTracker) handleConnection(ev tracer.EventType, tuple fourTuple, pid int, networkNamespace uint32) {
	t.Lock()
	defer t.Unlock()

	log.Debugf("handleConnection(%v, [%v:%v --> %v:%v], pid=%v, netNS=%v)",
		ev, tuple.fromAddr, tuple.fromPort, tuple.toAddr, tuple.toPort, pid, networkNamespace)

	switch ev {
	case tracer.EventConnect:
		t.openConnections[tuple] = ebpfConnection{
			incoming:         false,
			tuple:            tuple,
			pid:              pid,
			networkNamespace: networkNamespace,
		}
	case tracer.EventAccept:
		t.openConnections[tuple] = ebpfConnection{
			incoming:         true,
			tuple:            tuple,
			pid:              pid,
			networkNamespace: networkNamespace,
		}
	case tracer.EventClose:
		if !t.ready {
			t.closedDuringInit[tuple] = struct{}{}
		}
		if deadConn, ok := t.openConnections[tuple]; ok {
			delete(t.openConnections, tuple)
			t.closedConnections = append(t.closedConnections, deadConn)
		} else {
			log.Debugf("EbpfTracker: unmatched close event: %s pid=%d netns=%v", tuple, pid, networkNamespace)
		}
	default:
		log.Debugf("EbpfTracker: unknown event: %s (%d)", ev, ev)
	}
}

// walkConnections calls f with all open connections and connections that have come and gone
// since the last call to walkConnections
func (t *EbpfTracker) walkConnections(f func(ebpfConnection)) {
	t.Lock()
	defer t.Unlock()

	for _, connection := range t.openConnections {
		f(connection)
	}
	for _, connection := range t.closedConnections {
		f(connection)
	}
	t.closedConnections = t.closedConnections[:0]
}

func (t *EbpfTracker) feedInitialConnections(conns procspy.ConnIter, seenTuples map[string]fourTuple, processesWaitingInAccept []int, hostNodeID string) {
	t.Lock()
	for conn := conns.Next(); conn != nil; conn = conns.Next() {
		if conn.Proc.PID == 0 {
			continue // no point in tracking a connection which we can't associate to a process
		}
		tuple, namespaceID, incoming := connectionTuple(conn, seenTuples)
		if _, ok := t.closedDuringInit[tuple]; !ok {
			if _, ok := t.openConnections[tuple]; !ok {
				log.Debugf("initialConnection([%v], in=%v, pid=%v, netNS=%v)",
					tuple, incoming, conn.Proc.PID, namespaceID)
				t.openConnections[tuple] = ebpfConnection{
					incoming:         incoming,
					tuple:            tuple,
					pid:              int(conn.Proc.PID),
					networkNamespace: namespaceID,
				}
			}
		}
	}
	t.closedDuringInit = nil
	t.ready = true
	t.Unlock()

	for _, p := range processesWaitingInAccept {
		t.tracer.AddFdInstallWatcher(uint32(p))
		log.Debugf("EbpfTracker: install fd-install watcher: pid=%d", p)
	}
}

func (t *EbpfTracker) isDead() bool {
	t.Lock()
	defer t.Unlock()
	return t.dead
}

func (t *EbpfTracker) stop() {
	t.Lock()
	alreadyDead := t.dead || t.stopping
	t.stopping = true
	t.Unlock()

	// Do not call tracer.Stop() in this thread, otherwise tracer.Stop() will
	// deadlock waiting for this thread to pick up the next event.
	go func() {
		if !alreadyDead && t.tracer != nil {
			t.tracer.Stop()
			t.tracer = nil
		}

		// Only advertise the tracer as dead after the tracer is fully stopped so that
		// restart() is not called in parallel in another thread.
		t.Lock()
		t.stopping = false
		t.dead = true
		t.Unlock()
	}()
}

func (t *EbpfTracker) restart() error {
	t.Lock()
	defer t.Unlock()

	t.dead = false
	t.ready = false

	t.openConnections = map[fourTuple]ebpfConnection{}
	t.closedDuringInit = map[fourTuple]struct{}{}
	t.closedConnections = []ebpfConnection{}

	tracer, err := tracer.NewTracer(t)
	if err != nil {
		return err
	}

	t.tracer = tracer
	tracer.Start()

	return nil
}
<commit_msg>comment: add explanatory comment on handleFdInstall()<commit_after>// +build linux

package endpoint

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"syscall"

	"github.com/armon/go-metrics"
	log "github.com/sirupsen/logrus"
	"github.com/weaveworks/common/fs"
	"github.com/weaveworks/scope/probe/endpoint/procspy"
	"github.com/weaveworks/scope/probe/host"
	"github.com/weaveworks/scope/probe/process"
	"github.com/weaveworks/tcptracer-bpf/pkg/tracer"
)

// An ebpfConnection represents a TCP connection
type ebpfConnection struct {
	tuple            fourTuple
	networkNamespace uint32
	incoming         bool
	pid              int
}

// EbpfTracker contains the sets of open and closed TCP connections.
// Closed connections are kept in the `closedConnections` slice for one iteration of `walkConnections`.
type EbpfTracker struct {
	sync.Mutex
	tracer          *tracer.Tracer
	ready           bool
	stopping        bool
	dead            bool
	lastTimestampV4 uint64

	// debugBPF specifies if EbpfTracker must be started in debug mode. This
	// allows to easily debug issues like:
	// https://github.com/weaveworks/scope/issues/2650
	//
	// Scope could be started this way:
	//   $ sudo WEAVESCOPE_DOCKER_ARGS="-e SCOPE_DEBUG_BPF=1" ./scope launch
	//
	// Then, EbpfTracker could be tricked into restarting with:
	//   $ echo stop | sudo tee /proc/$(pidof scope-probe)/root/var/run/scope/debug-bpf
	debugBPF bool

	openConnections   map[fourTuple]ebpfConnection
	closedConnections []ebpfConnection
	closedDuringInit  map[fourTuple]struct{}
}

// releaseRegex should match all possible variations of a common Linux
// version string:
//   - 4.1
//   - 4.22-foo
//   - 4.1.2-foo
//   - 4.1.2-33.44+bar
//   - etc.
// For example, on a Ubuntu system the vendor specific release part
// (after the first `-`) could look like:
// '<ABI number>.<upload number>-<flavour>' or
// '<ABI number>-<flavour>'
// See https://wiki.ubuntu.com/Kernel/FAQ
var releaseRegex = regexp.MustCompile(`^(\d+)\.(\d+)\.?(\d*)-?(\d*)(.*)$`)

func isKernelSupported() error {
	release, version, err := host.GetKernelReleaseAndVersion()
	if err != nil {
		return err
	}

	releaseParts := releaseRegex.FindStringSubmatch(release)
	if len(releaseParts) != 6 {
		return fmt.Errorf("got invalid release version %q (expected format '4.4[.2-1]')", release)
	}

	major, err := strconv.Atoi(releaseParts[1])
	if err != nil {
		return err
	}

	minor, err := strconv.Atoi(releaseParts[2])
	if err != nil {
		return err
	}

	if major > 4 {
		return nil
	}

	if major < 4 || minor < 4 {
		return fmt.Errorf("got kernel %s but need kernel >=4.4", release)
	}

	if strings.Contains(version, "Ubuntu") {
		// Check for specific Ubuntu kernel versions with
		// known issues.

		abiNumber, err := strconv.Atoi(releaseParts[4])
		if err != nil {
			// By now we know it's at least kernel 4.4 and
			// not "119-ish", so allow it.
			return nil
		}
		if major == 4 && minor == 4 && abiNumber >= 119 && abiNumber < 127 {
			// https://github.com/weaveworks/scope/issues/3131
			// https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1763454
			return fmt.Errorf("got Ubuntu kernel %s with known bug", release)
		}
	}

	return nil
}

func newEbpfTracker() (*EbpfTracker, error) {
	if err := isKernelSupported(); err != nil {
		return nil, fmt.Errorf("kernel not supported: %v", err)
	}

	var debugBPF bool
	if os.Getenv("SCOPE_DEBUG_BPF") != "" {
		log.Infof("ebpf tracker started in debug mode")
		debugBPF = true
	}

	tracer.TimestampOffset = 200000 // Delay events by 0.2ms to avoid out-of-order reporting
	tracker := &EbpfTracker{
		debugBPF: debugBPF,
	}
	if err := tracker.restart(); err != nil {
		return nil, err
	}

	return tracker, nil
}

// TCPEventV4 handles IPv4 TCP events from the eBPF tracer
func (t *EbpfTracker) TCPEventV4(e tracer.TcpV4) {
	if t.debugBPF {
		debugBPFFile := "/var/run/scope/debug-bpf"
		b, err := ioutil.ReadFile("/var/run/scope/debug-bpf")
		if err == nil && strings.TrimSpace(string(b[:])) == "stop" {
			os.Remove(debugBPFFile)
			log.Warnf("ebpf tracker stopped as requested by user")
			t.stop()
			return
		}
	}

	if t.lastTimestampV4 > e.Timestamp {
		// A kernel bug can cause the timestamps to be wrong (e.g. on Ubuntu with Linux 4.4.0-47.68)
		// Upgrading the kernel will fix the problem. For further info see:
		// https://github.com/iovisor/bcc/issues/790#issuecomment-263704235
		// https://github.com/weaveworks/scope/issues/2334
		log.Errorf("tcp tracer received event with timestamp %v even though the last timestamp was %v. Stopping the eBPF tracker.", e.Timestamp, t.lastTimestampV4)
		t.stop()
		metrics.IncrCounterWithLabels([]string{"ebpf", "errors"}, 1, []metrics.Label{
			{Name: "kind", Value: "timestamp-out-of-order"},
		})
		return
	}

	t.lastTimestampV4 = e.Timestamp

	if e.Type == tracer.EventFdInstall {
		t.handleFdInstall(e.Type, int(e.Pid), int(e.Fd))
	} else {
		tuple := makeFourTuple(e.SAddr, e.DAddr, e.SPort, e.DPort)
		t.handleConnection(e.Type, tuple, int(e.Pid), e.NetNS)
	}
}

// TCPEventV6 handles IPv6 TCP events from the eBPF tracer. This is
// currently a no-op.
func (t *EbpfTracker) TCPEventV6(e tracer.TcpV6) {
	// TODO: IPv6 not supported in Scope
}

// LostV4 handles IPv4 TCP event misses from the eBPF tracer.
func (t *EbpfTracker) LostV4(count uint64) {
	log.Errorf("tcp tracer lost %d events. Stopping the eBPF tracker", count)
	metrics.IncrCounterWithLabels([]string{"ebpf", "errors"}, 1, []metrics.Label{
		{Name: "kind", Value: "lost-events"},
	})
	t.stop()
}

// LostV6 handles IPv4 TCP event misses from the eBPF tracer. This is
// currently a no-op.
func (t *EbpfTracker) LostV6(count uint64) {
	// TODO: IPv6 not supported in Scope
}

func tupleFromPidFd(pid int, fd int) (tuple fourTuple, netns uint32, ok bool) {
	// read /proc/$pid/ns/net
	//
	// probe/endpoint/procspy/proc_linux.go supports Linux < 3.8 but we
	// don't need that here since ebpf-enabled kernels will be > 3.8
	netnsIno, err := procspy.ReadNetnsFromPID(pid)
	if err != nil {
		log.Debugf("netns proc file for pid %d disappeared before we could read it: %v", pid, err)
		return fourTuple{}, 0, false
	}

	// find /proc/$pid/fd/$fd's ino
	fdFilename := fmt.Sprintf("/proc/%d/fd/%d", pid, fd)
	var statFdFile syscall.Stat_t
	if err := fs.Stat(fdFilename, &statFdFile); err != nil {
		log.Debugf("proc file %q disappeared before we could read it", fdFilename)
		return fourTuple{}, 0, false
	}

	if statFdFile.Mode&syscall.S_IFMT != syscall.S_IFSOCK {
		log.Errorf("file %q is not a socket", fdFilename)
		return fourTuple{}, 0, false
	}
	ino := statFdFile.Ino

	// read both /proc/pid/net/{tcp,tcp6}
	buf := bytes.NewBuffer(make([]byte, 0, 5000))
	if _, err := procspy.ReadTCPFiles(pid, buf); err != nil {
		log.Debugf("TCP proc file for pid %d disappeared before we could read it: %v", pid, err)
		return fourTuple{}, 0, false
	}

	// find /proc/$pid/fd/$fd's ino in /proc/pid/net/tcp
	pn := procspy.NewProcNet(buf.Bytes())
	for {
		n := pn.Next()
		if n == nil {
			log.Debugf("connection for proc file %q not found. buf=%q", fdFilename, buf.String())
			break
		}
		if n.Inode == ino {
			tuple := makeFourTuple(n.LocalAddress, n.RemoteAddress, n.LocalPort, n.RemotePort)
			return tuple, netnsIno, true
		}
	}

	return fourTuple{}, 0, false
}

// this callback exists to close a hole whereby we don't get a kprobe
// for tcp_accept if accept was called before the probe started.
// It's fairly safe to assume all such connections are incoming, but not 100%
func (t *EbpfTracker) handleFdInstall(ev tracer.EventType, pid int, fd int) {
	if !process.IsProcInAccept("/proc", strconv.Itoa(pid)) {
		t.tracer.RemoveFdInstallWatcher(uint32(pid))
	}
	tuple, netns, ok := tupleFromPidFd(pid, fd)
	log.Debugf("EbpfTracker: got fd-install event: pid=%d fd=%d -> tuple=%s netns=%v ok=%v", pid, fd, tuple, netns, ok)
	if !ok {
		return
	}

	t.Lock()
	defer t.Unlock()

	t.openConnections[tuple] = ebpfConnection{
		incoming:         true,
		tuple:            tuple,
		pid:              pid,
		networkNamespace: netns,
	}
}

func (t *EbpfTracker) handleConnection(ev tracer.EventType, tuple fourTuple, pid int, networkNamespace uint32) {
	t.Lock()
	defer t.Unlock()

	log.Debugf("handleConnection(%v, [%v:%v --> %v:%v], pid=%v, netNS=%v)",
		ev, tuple.fromAddr, tuple.fromPort, tuple.toAddr, tuple.toPort, pid, networkNamespace)

	switch ev {
	case tracer.EventConnect:
		t.openConnections[tuple] = ebpfConnection{
			incoming:         false,
			tuple:            tuple,
			pid:              pid,
			networkNamespace: networkNamespace,
		}
	case tracer.EventAccept:
		t.openConnections[tuple] = ebpfConnection{
			incoming:         true,
			tuple:            tuple,
			pid:              pid,
			networkNamespace: networkNamespace,
		}
	case tracer.EventClose:
		if !t.ready {
			t.closedDuringInit[tuple] = struct{}{}
		}
		if deadConn, ok := t.openConnections[tuple]; ok {
			delete(t.openConnections, tuple)
			t.closedConnections = append(t.closedConnections, deadConn)
		} else {
			log.Debugf("EbpfTracker: unmatched close event: %s pid=%d netns=%v", tuple, pid, networkNamespace)
		}
	default:
		log.Debugf("EbpfTracker: unknown event: %s (%d)", ev, ev)
	}
}

// walkConnections calls f with all open connections and connections that have come and gone
// since the last call to walkConnections
func (t *EbpfTracker) walkConnections(f func(ebpfConnection)) {
	t.Lock()
	defer t.Unlock()

	for _, connection := range t.openConnections {
		f(connection)
	}
	for _, connection := range t.closedConnections {
		f(connection)
	}
	t.closedConnections = t.closedConnections[:0]
}

func (t *EbpfTracker) feedInitialConnections(conns procspy.ConnIter, seenTuples map[string]fourTuple, processesWaitingInAccept []int, hostNodeID string) {
	t.Lock()
	for conn := conns.Next(); conn != nil; conn = conns.Next() {
		if conn.Proc.PID == 0 {
			continue // no point in tracking a connection which we can't associate to a process
		}
		tuple, namespaceID, incoming := connectionTuple(conn, seenTuples)
		if _, ok := t.closedDuringInit[tuple]; !ok {
			if _, ok := t.openConnections[tuple]; !ok {
				log.Debugf("initialConnection([%v], in=%v, pid=%v, netNS=%v)",
					tuple, incoming, conn.Proc.PID, namespaceID)
				t.openConnections[tuple] = ebpfConnection{
					incoming:         incoming,
					tuple:            tuple,
					pid:              int(conn.Proc.PID),
					networkNamespace: namespaceID,
				}
			}
		}
	}
	t.closedDuringInit = nil
	t.ready = true
	t.Unlock()

	for _, p := range processesWaitingInAccept {
		t.tracer.AddFdInstallWatcher(uint32(p))
		log.Debugf("EbpfTracker: install fd-install watcher: pid=%d", p)
	}
}

func (t *EbpfTracker) isDead() bool {
	t.Lock()
	defer t.Unlock()
	return t.dead
}

func (t *EbpfTracker) stop() {
	t.Lock()
	alreadyDead := t.dead || t.stopping
	t.stopping = true
	t.Unlock()

	// Do not call tracer.Stop() in this thread, otherwise tracer.Stop() will
	// deadlock waiting for this thread to pick up the next event.
	go func() {
		if !alreadyDead && t.tracer != nil {
			t.tracer.Stop()
			t.tracer = nil
		}

		// Only advertise the tracer as dead after the tracer is fully stopped so that
		// restart() is not called in parallel in another thread.
		t.Lock()
		t.stopping = false
		t.dead = true
		t.Unlock()
	}()
}

func (t *EbpfTracker) restart() error {
	t.Lock()
	defer t.Unlock()

	t.dead = false
	t.ready = false

	t.openConnections = map[fourTuple]ebpfConnection{}
	t.closedDuringInit = map[fourTuple]struct{}{}
	t.closedConnections = []ebpfConnection{}

	tracer, err := tracer.NewTracer(t)
	if err != nil {
		return err
	}

	t.tracer = tracer
	tracer.Start()

	return nil
}
<|endoftext|>


<commit_before>//  Copyright 2019 Google Inc. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package packages

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"strings"

	"github.com/GoogleCloudPlatform/osconfig/osinfo"
	"github.com/GoogleCloudPlatform/osconfig/util"
)

var (
	dpkg      string
	dpkgquery string
	aptGet    string

	dpkgInstallArgs   = []string{"--install"}
	dpkgQueryArgs     = []string{"-W", "-f", "${Package} ${Architecture} ${Version}\n"}
	aptGetInstallArgs = []string{"install", "-y"}
	aptGetRemoveArgs  = []string{"remove", "-y"}
	aptGetUpdateArgs  = []string{"update"}

	aptGetUpgradeCmd     = "upgrade"
	aptGetFullUpgradeCmd = "full-upgrade"
	aptGetDistUpgradeCmd = "dist-upgrade"
	aptGetUpgradableArgs = []string{"--just-print", "-qq"}
)

func init() {
	if runtime.GOOS != "windows" {
		dpkg = "/usr/bin/dpkg"
		dpkgquery = "/usr/bin/dpkg-query"
		aptGet = "/usr/bin/apt-get"
	}
	AptExists = util.Exists(aptGet)
	DpkgExists = util.Exists(dpkg)
	DpkgQueryExists = util.Exists(dpkgquery)
}

// AptUpgradeType is the apt upgrade type.
type AptUpgradeType int

const (
	// AptGetUpgrade specifies apt-get upgrade should be run.
	AptGetUpgrade AptUpgradeType = iota
	// AptGetDistUpgrade specifies apt-get dist-upgrade should be run.
	AptGetDistUpgrade
	// AptGetFullUpgrade specifies apt-get full-upgrade should be run.
	AptGetFullUpgrade
)

type aptGetUpgradeOpts struct {
	upgradeType AptUpgradeType
	showNew     bool
}

// AptGetUpgradeOption is an option for apt-get upgrade.
type AptGetUpgradeOption func(*aptGetUpgradeOpts)

// AptGetUpgradeType returns a AptGetUpgradeOption that specifies upgrade type.
func AptGetUpgradeType(upgradeType AptUpgradeType) AptGetUpgradeOption {
	return func(args *aptGetUpgradeOpts) {
		args.upgradeType = upgradeType
	}
}

// AptGetUpgradeShowNew returns a AptGetUpgradeOption that indicates whether 'new' packages should be returned.
func AptGetUpgradeShowNew(showNew bool) AptGetUpgradeOption {
	return func(args *aptGetUpgradeOpts) {
		args.showNew = showNew
	}
}

// InstallAptPackages installs apt packages.
func InstallAptPackages(pkgs []string) error {
	args := append(aptGetInstallArgs, pkgs...)
	install := exec.Command(aptGet, args...)
	install.Env = append(os.Environ(),
		"DEBIAN_FRONTEND=noninteractive",
	)
	out, err := run(install)
	DebugLogger.Printf("apt-get %q output:\n%s", args, strings.ReplaceAll(string(out), "\n", "\n "))
	if err != nil {
		err = fmt.Errorf("error running apt-get with args %q: %v, stdout: %s", args, err, out)
	}
	return err
}

// RemoveAptPackages removes apt packages.
func RemoveAptPackages(pkgs []string) error {
	args := append(aptGetRemoveArgs, pkgs...)
	remove := exec.Command(aptGet, args...)
	remove.Env = append(os.Environ(),
		"DEBIAN_FRONTEND=noninteractive",
	)
	out, err := run(remove)
	DebugLogger.Printf("apt-get %q output:\n%s", args, strings.ReplaceAll(string(out), "\n", "\n "))
	if err != nil {
		err = fmt.Errorf("error running apt-get with args %q: %v, stdout: %s", args, err, out)
	}
	return err
}

func parseAptUpdates(data []byte, showNew bool) []PkgInfo {
	/*
		Inst libldap-common [2.4.45+dfsg-1ubuntu1.2] (2.4.45+dfsg-1ubuntu1.3 Ubuntu:18.04/bionic-updates, Ubuntu:18.04/bionic-security [all])
		Inst firmware-linux-free (3.4 Debian:9.9/stable [all]) []
		Inst google-cloud-sdk [245.0.0-0] (246.0.0-0 cloud-sdk-stretch:cloud-sdk-stretch [all])
		Inst linux-image-4.9.0-9-amd64 (4.9.168-1+deb9u2 Debian-Security:9/stable [amd64])
		Inst linux-image-amd64 [4.9+80+deb9u6] (4.9+80+deb9u7 Debian:9.9/stable [amd64])
		Conf firmware-linux-free (3.4 Debian:9.9/stable [all])
		Conf google-cloud-sdk (246.0.0-0 cloud-sdk-stretch:cloud-sdk-stretch [all])
		Conf linux-image-4.9.0-9-amd64 (4.9.168-1+deb9u2 Debian-Security:9/stable [amd64])
		Conf linux-image-amd64 (4.9+80+deb9u7 Debian:9.9/stable [amd64])
	*/

	lines := bytes.Split(bytes.TrimSpace(data), []byte("\n"))

	var pkgs []PkgInfo
	for _, ln := range lines {
		pkg := bytes.Fields(ln)
		if len(pkg) < 5 || string(pkg[0]) != "Inst" {
			continue
		}
		// Inst google-cloud-sdk [245.0.0-0] (246.0.0-0 cloud-sdk-stretch:cloud-sdk-stretch [all])
		pkg = pkg[1:] // ==> google-cloud-sdk [245.0.0-0] (246.0.0-0 cloud-sdk-stretch:cloud-sdk-stretch [all])
		if bytes.HasPrefix(pkg[1], []byte("[")) {
			pkg = append(pkg[:1], pkg[2:]...) // ==> google-cloud-sdk (246.0.0-0 cloud-sdk-stretch:cloud-sdk-stretch [all])
		} else if !showNew {
			// This is a newly installed package and not an upgrade, ignore if showNew is false.
			continue
		}
		// Drop trailing "[]" if they exist.
		if bytes.Contains(pkg[len(pkg)-1], []byte("[]")) {
			pkg = pkg[:len(pkg)-1]
		}
		if !bytes.HasPrefix(pkg[1], []byte("(")) || !bytes.HasSuffix(pkg[len(pkg)-1], []byte(")")) {
			continue
		}
		ver := bytes.Trim(pkg[1], "(")             // (246.0.0-0 => 246.0.0-0
		arch := bytes.Trim(pkg[len(pkg)-1], "[])") // [all]) => all
		pkgs = append(pkgs, PkgInfo{Name: string(pkg[0]), Arch: osinfo.Architecture(string(arch)), Version: string(ver)})
	}
	return pkgs
}

// AptUpdates returns all the packages that will be installed when running
// apt-get [dist-|full-]upgrade.
func AptUpdates(opts ...AptGetUpgradeOption) ([]PkgInfo, error) {
	aptOpts := &aptGetUpgradeOpts{
		upgradeType: AptGetUpgrade,
		showNew:     false,
	}

	for _, opt := range opts {
		opt(aptOpts)
	}

	args := aptGetUpgradableArgs
	switch aptOpts.upgradeType {
	case AptGetUpgrade:
		args = append(aptGetUpgradableArgs, aptGetUpgradeCmd)
	case AptGetDistUpgrade:
		args = append(aptGetUpgradableArgs, aptGetDistUpgradeCmd)
	case AptGetFullUpgrade:
		args = append(aptGetUpgradableArgs, aptGetFullUpgradeCmd)
	default:
		return nil, fmt.Errorf("unknown upgrade type: %q", aptOpts.upgradeType)
	}

	if out, err := run(exec.Command(aptGet, aptGetUpdateArgs...)); err != nil {
		return nil, fmt.Errorf("error running apt-get with args %q: %v, stdout: %s", args, err, out)
	}

	out, err := run(exec.Command(aptGet, args...))
	DebugLogger.Printf("apt-get %q output:\n%s", args, strings.ReplaceAll(string(out), "\n", "\n "))
	if err != nil {
		return nil, fmt.Errorf("error running apt-get with args %q: %v, stdout: %s", args, err, out)
	}

	return parseAptUpdates(out, aptOpts.showNew), nil
}

func parseInstalledDebpackages(data []byte) []PkgInfo {
	/*
	   foo amd64 1.2.3-4
	   bar noarch 1.2.3-4
	   ...
	*/
	lines := strings.Split(strings.TrimSpace(string(data)), "\n")

	var pkgs []PkgInfo
	for _, ln := range lines {
		pkg := strings.Fields(ln)
		if len(pkg) != 3 {
			continue
		}

		pkgs = append(pkgs, PkgInfo{Name: pkg[0], Arch: osinfo.Architecture(pkg[1]), Version: pkg[2]})
	}
	return pkgs
}

// InstalledDebPackages queries for all installed deb packages.
func InstalledDebPackages() ([]PkgInfo, error) {
	out, err := run(exec.Command(dpkgquery, dpkgQueryArgs...))
	DebugLogger.Printf("dpkgquery %q output:\n%s", dpkgQueryArgs, strings.ReplaceAll(string(out), "\n", "\n "))
	if err != nil {
		return nil, fmt.Errorf("error running dpkgquery with args %q: %v, stdout: %s", dpkgQueryArgs, err, out)
	}
	return parseInstalledDebpackages(out), nil
}

// DpkgInstall installs a deb package.
func DpkgInstall(path string) error {
	args := append(dpkgInstallArgs, path)
	out, err := run(exec.Command(dpkg, args...))
	DebugLogger.Printf("dpkg %q output:\n%s", args, strings.ReplaceAll(string(out), "\n", "\n "))
	if err != nil {
		err = fmt.Errorf("error running dpkg with args %q: %v, stdout: %s", args, err, out)
	}
	return err
}
<commit_msg>Run 'dpkg --configure -a' when needed. (#174)<commit_after>//  Copyright 2019 Google Inc. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package packages

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"strings"

	"github.com/GoogleCloudPlatform/osconfig/osinfo"
	"github.com/GoogleCloudPlatform/osconfig/util"
)

var (
	dpkg      string
	dpkgquery string
	aptGet    string

	dpkgInstallArgs   = []string{"--install"}
	dpkgQueryArgs     = []string{"-W", "-f", "${Package} ${Architecture} ${Version}\n"}
	dpkgRepairArgs    = []string{"--configure", "-a"}
	aptGetInstallArgs = []string{"install", "-y"}
	aptGetRemoveArgs  = []string{"remove", "-y"}
	aptGetUpdateArgs  = []string{"update"}

	aptGetUpgradeCmd     = "upgrade"
	aptGetFullUpgradeCmd = "full-upgrade"
	aptGetDistUpgradeCmd = "dist-upgrade"
	aptGetUpgradableArgs = []string{"--just-print", "-qq"}
)

func init() {
	if runtime.GOOS != "windows" {
		dpkg = "/usr/bin/dpkg"
		dpkgquery = "/usr/bin/dpkg-query"
		aptGet = "/usr/bin/apt-get"
	}
	AptExists = util.Exists(aptGet)
	DpkgExists = util.Exists(dpkg)
	DpkgQueryExists = util.Exists(dpkgquery)
}

// AptUpgradeType is the apt upgrade type.
type AptUpgradeType int

const (
	// AptGetUpgrade specifies apt-get upgrade should be run.
	AptGetUpgrade AptUpgradeType = iota
	// AptGetDistUpgrade specifies apt-get dist-upgrade should be run.
	AptGetDistUpgrade
	// AptGetFullUpgrade specifies apt-get full-upgrade should be run.
	AptGetFullUpgrade
)

type aptGetUpgradeOpts struct {
	upgradeType AptUpgradeType
	showNew     bool
}

// AptGetUpgradeOption is an option for apt-get upgrade.
type AptGetUpgradeOption func(*aptGetUpgradeOpts)

// AptGetUpgradeType returns a AptGetUpgradeOption that specifies upgrade type.
func AptGetUpgradeType(upgradeType AptUpgradeType) AptGetUpgradeOption {
	return func(args *aptGetUpgradeOpts) {
		args.upgradeType = upgradeType
	}
}

// AptGetUpgradeShowNew returns a AptGetUpgradeOption that indicates whether 'new' packages should be returned.
func AptGetUpgradeShowNew(showNew bool) AptGetUpgradeOption {
	return func(args *aptGetUpgradeOpts) {
		args.showNew = showNew
	}
}

func dpkgRepair(out []byte) bool {
	// Error code 100 may occur for non repairable errors, just check the output.
	if !bytes.Contains(out, []byte("dpkg --configure -a")) {
		return false
	}
	DebugLogger.Printf("apt-get error, attempting dpkg repair.")
	// Ignore error here, just log and rerun apt-get.
	out, _ = run(exec.Command(dpkg, dpkgRepairArgs...))
	DebugLogger.Printf("dpkg %q output:\n%s", dpkgRepairArgs, strings.ReplaceAll(string(out), "\n", "\n "))

	return true
}

// InstallAptPackages installs apt packages.
func InstallAptPackages(pkgs []string) error {
	args := append(aptGetInstallArgs, pkgs...)
	install := exec.Command(aptGet, args...)
	install.Env = append(os.Environ(),
		"DEBIAN_FRONTEND=noninteractive",
	)
	out, err := run(install)
	DebugLogger.Printf("apt-get %q output:\n%s", args, strings.ReplaceAll(string(out), "\n", "\n "))
	if err != nil {
		if dpkgRepair(out) {
			out, err = run(install)
			DebugLogger.Printf("apt-get %q output:\n%s", args, strings.ReplaceAll(string(out), "\n", "\n "))
		}
	}

	if err != nil {
		err = fmt.Errorf("error running apt-get with args %q: %v, stdout: %s", args, err, out)
	}
	return err
}

// RemoveAptPackages removes apt packages.
func RemoveAptPackages(pkgs []string) error {
	args := append(aptGetRemoveArgs, pkgs...)
	remove := exec.Command(aptGet, args...)
	remove.Env = append(os.Environ(),
		"DEBIAN_FRONTEND=noninteractive",
	)
	out, err := run(remove)
	if err != nil {
		if dpkgRepair(out) {
			out, err = run(remove)
			DebugLogger.Printf("apt-get %q output:\n%s", args, strings.ReplaceAll(string(out), "\n", "\n "))
		}
	}

	DebugLogger.Printf("apt-get %q output:\n%s", args, strings.ReplaceAll(string(out), "\n", "\n "))
	if err != nil {
		err = fmt.Errorf("error running apt-get with args %q: %v, stdout: %s", args, err, out)
	}
	return err
}

func parseAptUpdates(data []byte, showNew bool) []PkgInfo {
	/*
		Inst libldap-common [2.4.45+dfsg-1ubuntu1.2] (2.4.45+dfsg-1ubuntu1.3 Ubuntu:18.04/bionic-updates, Ubuntu:18.04/bionic-security [all])
		Inst firmware-linux-free (3.4 Debian:9.9/stable [all]) []
		Inst google-cloud-sdk [245.0.0-0] (246.0.0-0 cloud-sdk-stretch:cloud-sdk-stretch [all])
		Inst linux-image-4.9.0-9-amd64 (4.9.168-1+deb9u2 Debian-Security:9/stable [amd64])
		Inst linux-image-amd64 [4.9+80+deb9u6] (4.9+80+deb9u7 Debian:9.9/stable [amd64])
		Conf firmware-linux-free (3.4 Debian:9.9/stable [all])
		Conf google-cloud-sdk (246.0.0-0 cloud-sdk-stretch:cloud-sdk-stretch [all])
		Conf linux-image-4.9.0-9-amd64 (4.9.168-1+deb9u2 Debian-Security:9/stable [amd64])
		Conf linux-image-amd64 (4.9+80+deb9u7 Debian:9.9/stable [amd64])
	*/

	lines := bytes.Split(bytes.TrimSpace(data), []byte("\n"))

	var pkgs []PkgInfo
	for _, ln := range lines {
		pkg := bytes.Fields(ln)
		if len(pkg) < 5 || string(pkg[0]) != "Inst" {
			continue
		}
		// Inst google-cloud-sdk [245.0.0-0] (246.0.0-0 cloud-sdk-stretch:cloud-sdk-stretch [all])
		pkg = pkg[1:] // ==> google-cloud-sdk [245.0.0-0] (246.0.0-0 cloud-sdk-stretch:cloud-sdk-stretch [all])
		if bytes.HasPrefix(pkg[1], []byte("[")) {
			pkg = append(pkg[:1], pkg[2:]...) // ==> google-cloud-sdk (246.0.0-0 cloud-sdk-stretch:cloud-sdk-stretch [all])
		} else if !showNew {
			// This is a newly installed package and not an upgrade, ignore if showNew is false.
			continue
		}
		// Drop trailing "[]" if they exist.
		if bytes.Contains(pkg[len(pkg)-1], []byte("[]")) {
			pkg = pkg[:len(pkg)-1]
		}
		if !bytes.HasPrefix(pkg[1], []byte("(")) || !bytes.HasSuffix(pkg[len(pkg)-1], []byte(")")) {
			continue
		}
		ver := bytes.Trim(pkg[1], "(")             // (246.0.0-0 => 246.0.0-0
		arch := bytes.Trim(pkg[len(pkg)-1], "[])") // [all]) => all
		pkgs = append(pkgs, PkgInfo{Name: string(pkg[0]), Arch: osinfo.Architecture(string(arch)), Version: string(ver)})
	}
	return pkgs
}

// AptUpdates returns all the packages that will be installed when running
// apt-get [dist-|full-]upgrade.
func AptUpdates(opts ...AptGetUpgradeOption) ([]PkgInfo, error) {
	aptOpts := &aptGetUpgradeOpts{
		upgradeType: AptGetUpgrade,
		showNew:     false,
	}

	for _, opt := range opts {
		opt(aptOpts)
	}

	args := aptGetUpgradableArgs
	switch aptOpts.upgradeType {
	case AptGetUpgrade:
		args = append(aptGetUpgradableArgs, aptGetUpgradeCmd)
	case AptGetDistUpgrade:
		args = append(aptGetUpgradableArgs, aptGetDistUpgradeCmd)
	case AptGetFullUpgrade:
		args = append(aptGetUpgradableArgs, aptGetFullUpgradeCmd)
	default:
		return nil, fmt.Errorf("unknown upgrade type: %q", aptOpts.upgradeType)
	}

	if out, err := run(exec.Command(aptGet, aptGetUpdateArgs...)); err != nil {
		return nil, fmt.Errorf("error running apt-get with args %q: %v, stdout: %s", aptGetUpdateArgs, err, out)
	}

	out, err := run(exec.Command(aptGet, args...))
	DebugLogger.Printf("apt-get %q output:\n%s", args, strings.ReplaceAll(string(out), "\n", "\n "))
	if err != nil {
		return nil, fmt.Errorf("error running apt-get with args %q: %v, stdout: %s", args, err, out)
	}

	return parseAptUpdates(out, aptOpts.showNew), nil
}

func parseInstalledDebpackages(data []byte) []PkgInfo {
	/*
	   foo amd64 1.2.3-4
	   bar noarch 1.2.3-4
	   ...
	*/
	lines := strings.Split(strings.TrimSpace(string(data)), "\n")

	var pkgs []PkgInfo
	for _, ln := range lines {
		pkg := strings.Fields(ln)
		if len(pkg) != 3 {
			continue
		}

		pkgs = append(pkgs, PkgInfo{Name: pkg[0], Arch: osinfo.Architecture(pkg[1]), Version: pkg[2]})
	}
	return pkgs
}

// InstalledDebPackages queries for all installed deb packages.
func InstalledDebPackages() ([]PkgInfo, error) {
	out, err := run(exec.Command(dpkgquery, dpkgQueryArgs...))
	DebugLogger.Printf("dpkgquery %q output:\n%s", dpkgQueryArgs, strings.ReplaceAll(string(out), "\n", "\n "))
	if err != nil {
		return nil, fmt.Errorf("error running dpkgquery with args %q: %v, stdout: %s", dpkgQueryArgs, err, out)
	}
	return parseInstalledDebpackages(out), nil
}

// DpkgInstall installs a deb package.
func DpkgInstall(path string) error {
	args := append(dpkgInstallArgs, path)
	out, err := run(exec.Command(dpkg, args...))
	DebugLogger.Printf("dpkg %q output:\n%s", args, strings.ReplaceAll(string(out), "\n", "\n "))
	if err != nil {
		err = fmt.Errorf("error running dpkg with args %q: %v, stdout: %s", args, err, out)
	}
	return err
}
<|endoftext|>