<commit_before>package command

import (
	"flag"
	"fmt"
	"net"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/funkygao/Go-Redis"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/gofmt"
	log "github.com/funkygao/log4go"
	"github.com/funkygao/termui"
	"github.com/pmylund/sortutil"
	"github.com/ryanuber/columnize"
)

type Redis struct {
	Ui  cli.Ui
	Cmd string

	mu       sync.Mutex
	topInfos []redisTopInfo
}

func (this *Redis) Run(args []string) (exitCode int) {
	var (
		zone   string
		add    string
		list   bool
		byHost bool
		del    string
		top    bool
		ping   bool
	)
	cmdFlags := flag.NewFlagSet("redis", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&add, "add", "", "")
	cmdFlags.BoolVar(&list, "list", true, "")
	cmdFlags.BoolVar(&byHost, "host", false, "")
	cmdFlags.BoolVar(&top, "top", false, "")
	cmdFlags.BoolVar(&ping, "ping", false, "")
	cmdFlags.StringVar(&del, "del", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	if top || ping {
		list = false
	}

	if add != "" {
		host, port, err := net.SplitHostPort(add)
		swallow(err)

		nport, err := strconv.Atoi(port)
		swallow(err)
		zkzone.AddRedis(host, nport)
	} else if del != "" {
		host, port, err := net.SplitHostPort(del)
		swallow(err)

		nport, err := strconv.Atoi(port)
		swallow(err)
		zkzone.DelRedis(host, nport)
	} else {
		if top {
			this.runTop(zkzone)
		} else if ping {
			this.runPing(zkzone)
		} else if list {
			machineMap := make(map[string]struct{})
			var machines []string
			hostPorts := zkzone.AllRedis()
			sort.Strings(hostPorts)
			for _, hp := range hostPorts {
				host, port, _ := net.SplitHostPort(hp)
				ips, _ := net.LookupIP(host)
				if _, present := machineMap[ips[0].String()]; !present {
					machineMap[ips[0].String()] = struct{}{}

					machines = append(machines, ips[0].String())
				}
				if !byHost {
					this.Ui.Output(fmt.Sprintf("%35s %s", host, port))
				}

			}

			if byHost {
				sort.Strings(machines)
				for _, ip := range machines {
					this.Ui.Output(fmt.Sprintf("%20s", ip))
				}
			}

			this.Ui.Output(fmt.Sprintf("Total instances:%d machines:%d", len(hostPorts), len(machines)))
		}
	}

	return
}

type redisTopInfo struct {
	host                       string
	port                       int
	dbsize, ops, rx, tx, conns int64
	t0                         time.Time
	latency                    time.Duration
}

func (this *Redis) runTop(zkzone *zk.ZkZone) {
	termui.Init()
	limit := termui.TermHeight() - 3
	termui.Close()
	this.topInfos = make([]redisTopInfo, 0, 100)
	for {
		var wg sync.WaitGroup
		this.topInfos = this.topInfos[:0]
		for _, hostPort := range zkzone.AllRedis() {
			host, port, err := net.SplitHostPort(hostPort)
			if err != nil {
				log.Error("invalid redis instance: %s", hostPort)
				continue
			}

			nport, err := strconv.Atoi(port)
			if err != nil || nport < 0 {
				log.Error("invalid redis instance: %s", hostPort)
				continue
			}

			wg.Add(1)
			go this.updateRedisInfo(&wg, host, nport)
		}
		wg.Wait()
		refreshScreen()

		sortutil.DescByField(this.topInfos, "ops")
		lines := []string{"Host|Port|dbsize|conns|ops|rx/bps|tx/bps"}

		for i := 0; i < min(limit, len(this.topInfos)); i++ {
			info := this.topInfos[i]
			lines = append(lines, fmt.Sprintf("%s|%d|%s|%s|%s|%s|%s",
				info.host, info.port,
				gofmt.Comma(info.dbsize), gofmt.Comma(info.conns), gofmt.Comma(info.ops),
				gofmt.ByteSize(info.rx*1024/8), gofmt.ByteSize(info.tx*1024/8)))
		}

		this.Ui.Output(columnize.SimpleFormat(lines))

		time.Sleep(time.Second * 5)
	}
}

func (this *Redis) updateRedisInfo(wg *sync.WaitGroup, host string, port int) {
	defer wg.Done()

	spec := redis.DefaultSpec().Host(host).Port(port)
	client, err := redis.NewSynchClientWithSpec(spec)
	if err != nil {
		return
	}
	defer client.Quit()

	infoMap, err := client.Info()
	if err != nil {
		return
	}

	dbSize, _ := client.Dbsize()
	conns, _ := strconv.ParseInt(infoMap["connected_clients"], 10, 64)
	ops, _ := strconv.ParseInt(infoMap["instantaneous_ops_per_sec"], 10, 64)
	rxKbps, _ := strconv.ParseFloat(infoMap["instantaneous_input_kbps"], 64)
	txKbps, _ := strconv.ParseFloat(infoMap["instantaneous_output_kbps"], 64)

	this.mu.Lock()
	this.topInfos = append(this.topInfos, redisTopInfo{
		host:   host,
		port:   port,
		dbsize: dbSize,
		ops:    ops,
		rx:     int64(rxKbps),
		tx:     int64(txKbps),
		conns:  conns,
	})
	this.mu.Unlock()
}

func (this *Redis) runPing(zkzone *zk.ZkZone) {
	var wg sync.WaitGroup
	allRedis := zkzone.AllRedis()
	this.topInfos = make([]redisTopInfo, 0, len(allRedis))

	for _, hostPort := range allRedis {
		host, port, err := net.SplitHostPort(hostPort)
		if err != nil {
			this.Ui.Error(hostPort)
			continue
		}

		nport, err := strconv.Atoi(port)
		if err != nil || nport < 0 {
			this.Ui.Error(hostPort)
			continue
		}

		wg.Add(1)
		go func(wg *sync.WaitGroup, host string, port int) {
			defer wg.Done()

			t0 := time.Now()

			spec := redis.DefaultSpec().Host(host).Port(port)
			client, err := redis.NewSynchClientWithSpec(spec)
			if err != nil {
				this.Ui.Error(fmt.Sprintf("[%s:%d] %v", host, port, err))
				return
			}
			defer client.Quit()

			if err := client.Ping(); err != nil {
				this.Ui.Error(fmt.Sprintf("[%s:%d] %v", host, port, err))
				return
			}

			latency := time.Since(t0)

			this.mu.Lock()
			this.topInfos = append(this.topInfos, redisTopInfo{
				host:    host,
				port:    port,
				t0:      t0,
				latency: latency,
			})
			this.mu.Unlock()
		}(&wg, host, nport)
	}
	wg.Wait()

	latency := metrics.NewRegisteredHistogram("redis.latency", metrics.DefaultRegistry, metrics.NewExpDecaySample(1028, 0.015))

	sortutil.AscByField(this.topInfos, "latency")
	lines := []string{"Host|Port|At|latency"}
	for _, info := range this.topInfos {
		latency.Update(info.latency.Nanoseconds() / 1e6)

		lines = append(lines, fmt.Sprintf("%s|%d|%s|%s",
			info.host, info.port, info.t0, info.latency))
	}
	this.Ui.Output(columnize.SimpleFormat(lines))

	// summary
	ps := latency.Percentiles([]float64{0.90, 0.95, 0.99, 0.999})
	this.Ui.Info(fmt.Sprintf("N:%d Min:%dms Max:%dms Mean:%.1fms 90%%:%.1fms 95%%:%.1fms 99%%:%.1fms",
		latency.Count(), latency.Min(), latency.Max(), latency.Mean(), ps[0], ps[1], ps[2]))
}

func (*Redis) Synopsis() string {
	return "Monitor redis instances"
}

func (this *Redis) Help() string {
	help := fmt.Sprintf(`
Usage: %s redis [options]

    %s

    -z zone

    -list

    -top
      Monitor all redis instances ops

    -ping
      Ping all redis instances

    -host
      Work with -list, print host instead of redis instance

    -add host:port

    -del host:port

`, this.Cmd, this.Synopsis())
	return strings.TrimSpace(help)
}
<commit_msg>display redis topology<commit_after>package command

import (
	"flag"
	"fmt"
	"net"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/funkygao/Go-Redis"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/gofmt"
	log "github.com/funkygao/log4go"
	"github.com/funkygao/termui"
	"github.com/pmylund/sortutil"
	"github.com/ryanuber/columnize"
)

type Redis struct {
	Ui  cli.Ui
	Cmd string

	mu       sync.Mutex
	topInfos []redisTopInfo
}

func (this *Redis) Run(args []string) (exitCode int) {
	var (
		zone   string
		add    string
		list   bool
		byHost int
		del    string
		top    bool
		ping   bool
	)
	cmdFlags := flag.NewFlagSet("redis", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&add, "add", "", "")
	cmdFlags.BoolVar(&list, "list", true, "")
	cmdFlags.IntVar(&byHost, "host", 0, "")
	cmdFlags.BoolVar(&top, "top", false, "")
	cmdFlags.BoolVar(&ping, "ping", false, "")
	cmdFlags.StringVar(&del, "del", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	if top || ping {
		list = false
	}

	if add != "" {
		host, port, err := net.SplitHostPort(add)
		swallow(err)

		nport, err := strconv.Atoi(port)
		swallow(err)
		zkzone.AddRedis(host, nport)
	} else if del != "" {
		host, port, err := net.SplitHostPort(del)
		swallow(err)

		nport, err := strconv.Atoi(port)
		swallow(err)
		zkzone.DelRedis(host, nport)
	} else {
		if top {
			this.runTop(zkzone)
		} else if ping {
			this.runPing(zkzone)
		} else if list {
			machineMap := make(map[string]struct{})
			machinePortMap := make(map[string][]string)
			var machines []string
			hostPorts := zkzone.AllRedis()
			sort.Strings(hostPorts)
			for _, hp := range hostPorts {
				host, port, _ := net.SplitHostPort(hp)
				ips, _ := net.LookupIP(host)
				ip := ips[0].String()
				if _, present := machineMap[ip]; !present {
					machineMap[ip] = struct{}{}
					machinePortMap[ip] = make([]string, 0)

					machines = append(machines, ip)
				}

				if byHost == 0 {
					this.Ui.Output(fmt.Sprintf("%35s %s", host, port))
				} else {
					machinePortMap[ip] = append(machinePortMap[ip], port)
				}

			}

			if byHost > 0 {
				sort.Strings(machines)
				for _, ip := range machines {
					sort.Strings(machinePortMap[ip])
					this.Ui.Info(fmt.Sprintf("%20s %2d ports", ip, len(machinePortMap[ip])))
					if byHost > 1 {
						this.Ui.Output(fmt.Sprintf("%+v", machinePortMap[ip]))
					}

				}
			}

			this.Ui.Output(fmt.Sprintf("Total instances:%d machines:%d", len(hostPorts), len(machines)))
		}
	}

	return
}

type redisTopInfo struct {
	host                       string
	port                       int
	dbsize, ops, rx, tx, conns int64
	t0                         time.Time
	latency                    time.Duration
}

func (this *Redis) runTop(zkzone *zk.ZkZone) {
	termui.Init()
	limit := termui.TermHeight() - 3
	termui.Close()
	this.topInfos = make([]redisTopInfo, 0, 100)
	for {
		var wg sync.WaitGroup
		this.topInfos = this.topInfos[:0]
		for _, hostPort := range zkzone.AllRedis() {
			host, port, err := net.SplitHostPort(hostPort)
			if err != nil {
				log.Error("invalid redis instance: %s", hostPort)
				continue
			}

			nport, err := strconv.Atoi(port)
			if err != nil || nport < 0 {
				log.Error("invalid redis instance: %s", hostPort)
				continue
			}

			wg.Add(1)
			go this.updateRedisInfo(&wg, host, nport)
		}
		wg.Wait()
		refreshScreen()

		sortutil.DescByField(this.topInfos, "ops")
		lines := []string{"Host|Port|dbsize|conns|ops|rx/bps|tx/bps"}

		for i := 0; i < min(limit, len(this.topInfos)); i++ {
			info := this.topInfos[i]
			lines = append(lines, fmt.Sprintf("%s|%d|%s|%s|%s|%s|%s",
				info.host, info.port,
				gofmt.Comma(info.dbsize), gofmt.Comma(info.conns), gofmt.Comma(info.ops),
				gofmt.ByteSize(info.rx*1024/8), gofmt.ByteSize(info.tx*1024/8)))
		}

		this.Ui.Output(columnize.SimpleFormat(lines))

		time.Sleep(time.Second * 5)
	}
}

func (this *Redis) updateRedisInfo(wg *sync.WaitGroup, host string, port int) {
	defer wg.Done()

	spec := redis.DefaultSpec().Host(host).Port(port)
	client, err := redis.NewSynchClientWithSpec(spec)
	if err != nil {
		return
	}
	defer client.Quit()

	infoMap, err := client.Info()
	if err != nil {
		return
	}

	dbSize, _ := client.Dbsize()
	conns, _ := strconv.ParseInt(infoMap["connected_clients"], 10, 64)
	ops, _ := strconv.ParseInt(infoMap["instantaneous_ops_per_sec"], 10, 64)
	rxKbps, _ := strconv.ParseFloat(infoMap["instantaneous_input_kbps"], 64)
	txKbps, _ := strconv.ParseFloat(infoMap["instantaneous_output_kbps"], 64)

	this.mu.Lock()
	this.topInfos = append(this.topInfos, redisTopInfo{
		host:   host,
		port:   port,
		dbsize: dbSize,
		ops:    ops,
		rx:     int64(rxKbps),
		tx:     int64(txKbps),
		conns:  conns,
	})
	this.mu.Unlock()
}

func (this *Redis) runPing(zkzone *zk.ZkZone) {
	var wg sync.WaitGroup
	allRedis := zkzone.AllRedis()
	this.topInfos = make([]redisTopInfo, 0, len(allRedis))

	for _, hostPort := range allRedis {
		host, port, err := net.SplitHostPort(hostPort)
		if err != nil {
			this.Ui.Error(hostPort)
			continue
		}

		nport, err := strconv.Atoi(port)
		if err != nil || nport < 0 {
			this.Ui.Error(hostPort)
			continue
		}

		wg.Add(1)
		go func(wg *sync.WaitGroup, host string, port int) {
			defer wg.Done()

			t0 := time.Now()

			spec := redis.DefaultSpec().Host(host).Port(port)
			client, err := redis.NewSynchClientWithSpec(spec)
			if err != nil {
				this.Ui.Error(fmt.Sprintf("[%s:%d] %v", host, port, err))
				return
			}
			defer client.Quit()

			if err := client.Ping(); err != nil {
				this.Ui.Error(fmt.Sprintf("[%s:%d] %v", host, port, err))
				return
			}

			latency := time.Since(t0)

			this.mu.Lock()
			this.topInfos = append(this.topInfos, redisTopInfo{
				host:    host,
				port:    port,
				t0:      t0,
				latency: latency,
			})
			this.mu.Unlock()
		}(&wg, host, nport)
	}
	wg.Wait()

	latency := metrics.NewRegisteredHistogram("redis.latency", metrics.DefaultRegistry, metrics.NewExpDecaySample(1028, 0.015))

	sortutil.AscByField(this.topInfos, "latency")
	lines := []string{"Host|Port|At|latency"}
	for _, info := range this.topInfos {
		latency.Update(info.latency.Nanoseconds() / 1e6)

		lines = append(lines, fmt.Sprintf("%s|%d|%s|%s",
			info.host, info.port, info.t0, info.latency))
	}
	this.Ui.Output(columnize.SimpleFormat(lines))

	// summary
	ps := latency.Percentiles([]float64{0.90, 0.95, 0.99, 0.999})
	this.Ui.Info(fmt.Sprintf("N:%d Min:%dms Max:%dms Mean:%.1fms 90%%:%.1fms 95%%:%.1fms 99%%:%.1fms",
		latency.Count(), latency.Min(), latency.Max(), latency.Mean(), ps[0], ps[1], ps[2]))
}

func (*Redis) Synopsis() string {
	return "Monitor redis instances"
}

func (this *Redis) Help() string {
	help := fmt.Sprintf(`
Usage: %s redis [options]

    %s

    -z zone

    -list

    -host 1|2
      Work with -list, print host instead of redis instance
      1: only display host
      2: 0 + port info

    -top
      Monitor all redis instances ops

    -ping
      Ping all redis instances

    -add host:port

    -del host:port

`, this.Cmd, this.Synopsis())
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main_test

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"testing"
	"time"
)

var godocTests = []struct {
	args      []string
	matches   []string // regular expressions
	dontmatch []string // regular expressions
}{
	{
		args: []string{"fmt"},
		matches: []string{
			`import "fmt"`,
			`Package fmt implements formatted I/O`,
		},
	},
	{
		args: []string{"io", "WriteString"},
		matches: []string{
			`func WriteString\(`,
			`WriteString writes the contents of the string s to w`,
		},
	},
	{
		args: []string{"nonexistingpkg"},
		matches: []string{
			`no such file or directory|does not exist|cannot find the file`,
		},
	},
	{
		args: []string{"fmt", "NonexistentSymbol"},
		matches: []string{
			`No match found\.`,
		},
	},
	{
		args: []string{"-src", "syscall", "Open"},
		matches: []string{
			`func Open\(`,
		},
		dontmatch: []string{
			`No match found\.`,
		},
	},
}

// buildGodoc builds the godoc executable.
// It returns its path, and a cleanup function.
//
// TODO(adonovan): opt: do this at most once, and do the cleanup
// exactly once.  How though?  There's no atexit.
func buildGodoc(t *testing.T) (bin string, cleanup func()) {
	tmp, err := ioutil.TempDir("", "godoc-regtest-")
	if err != nil {
		t.Fatal(err)
	}

	bin = filepath.Join(tmp, "godoc")
	if runtime.GOOS == "windows" {
		bin += ".exe"
	}
	cmd := exec.Command("go", "build", "-o", bin)
	if err := cmd.Run(); err != nil {
		t.Fatalf("Building godoc: %v", err)
	}

	return bin, func() { os.RemoveAll(tmp) }
}

// Basic regression test for godoc command-line tool.
func TestCLI(t *testing.T) {
	bin, cleanup := buildGodoc(t)
	defer cleanup()
	for _, test := range godocTests {
		cmd := exec.Command(bin, test.args...)
		cmd.Args[0] = "godoc"
		out, err := cmd.CombinedOutput()
		if err != nil {
			t.Errorf("Running with args %#v: %v", test.args, err)
			continue
		}
		for _, pat := range test.matches {
			re := regexp.MustCompile(pat)
			if !re.Match(out) {
				t.Errorf("godoc %v =\n%s\nwanted /%v/", strings.Join(test.args, " "), out, pat)
			}
		}
		for _, pat := range test.dontmatch {
			re := regexp.MustCompile(pat)
			if re.Match(out) {
				t.Errorf("godoc %v =\n%s\ndid not want /%v/", strings.Join(test.args, " "), out, pat)
			}
		}
	}
}

func serverAddress(t *testing.T) string {
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		ln, err = net.Listen("tcp6", "[::1]:0")
	}
	if err != nil {
		t.Fatal(err)
	}
	defer ln.Close()
	return ln.Addr().String()
}

func waitForServer(t *testing.T, address string) {
	// Poll every 50ms for a total of 5s.
	for i := 0; i < 100; i++ {
		time.Sleep(50 * time.Millisecond)
		conn, err := net.Dial("tcp", address)
		if err != nil {
			continue
		}
		conn.Close()
		return
	}
	t.Fatalf("Server %q failed to respond in 5 seconds", address)
}

// Basic integration test for godoc HTTP interface.
func TestWeb(t *testing.T) {
	bin, cleanup := buildGodoc(t)
	defer cleanup()
	addr := serverAddress(t)
	cmd := exec.Command(bin, fmt.Sprintf("-http=%s", addr))
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	cmd.Args[0] = "godoc"
	if err := cmd.Start(); err != nil {
		t.Fatalf("failed to start godoc: %s", err)
	}
	defer cmd.Process.Kill()
	waitForServer(t, addr)
	tests := []struct {
		path      string
		match     []string
		dontmatch []string
	}{
		{
			path:  "/",
			match: []string{"Go is an open source programming language"},
		},
		{
			path:  "/pkg/fmt/",
			match: []string{"Package fmt implements formatted I/O"},
		},
		{
			path:  "/src/fmt/",
			match: []string{"scan_test.go"},
		},
		{
			path:  "/src/fmt/print.go",
			match: []string{"// Println formats using"},
		},
		{
			path: "/pkg",
			match: []string{
				"Standard library",
				"Package fmt implements formatted I/O",
			},
			dontmatch: []string{
				"internal/syscall",
				"cmd/gc",
			},
		},
		{
			path: "/pkg/?m=all",
			match: []string{
				"Standard library",
				"Package fmt implements formatted I/O",
				"internal/syscall",
			},
			dontmatch: []string{
				"cmd/gc",
			},
		},
	}
	for _, test := range tests {
		url := fmt.Sprintf("http://%s%s", addr, test.path)
		resp, err := http.Get(url)
		if err != nil {
			t.Errorf("GET %s failed: %s", url, err)
			continue
		}
		body, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			t.Errorf("GET %s: failed to read body: %s (response: %v)", url, err, resp)
		}
		isErr := false
		for _, substr := range test.match {
			if !bytes.Contains(body, []byte(substr)) {
				t.Errorf("GET %s: wanted substring %q in body", url, substr)
				isErr = true
			}
		}
		for _, substr := range test.dontmatch {
			if bytes.Contains(body, []byte(substr)) {
				t.Errorf("GET %s: didn't want substring %q in body", url, substr)
				isErr = true
			}
		}
		if isErr {
			t.Errorf("GET %s: got:\n%s", url, body)
		}
	}
}

// Basic integration test for godoc -analysis=type (via HTTP interface).
func TestTypeAnalysis(t *testing.T) {
	// Write a fake GOROOT/GOPATH.
	tmpdir, err := ioutil.TempDir("", "godoc-analysis")
	if err != nil {
		t.Fatalf("ioutil.TempDir failed: %s", err)
	}
	defer os.RemoveAll(tmpdir)
	for _, f := range []struct{ file, content string }{
		{"goroot/src/lib/lib.go", `
package lib
type T struct{}
const C = 3
var V T
func (T) F() int { return C }
`},
		{"gopath/src/app/main.go", `
package main
import "lib"
func main() { print(lib.V) }
`},
	} {
		file := filepath.Join(tmpdir, f.file)
		if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil {
			t.Fatalf("MkdirAll(%s) failed: %s", filepath.Dir(file), err)
		}
		if err := ioutil.WriteFile(file, []byte(f.content), 0644); err != nil {
			t.Fatal(err)
		}
	}

	// Start the server.
	bin, cleanup := buildGodoc(t)
	defer cleanup()
	addr := serverAddress(t)
	cmd := exec.Command(bin, fmt.Sprintf("-http=%s", addr), "-analysis=type")
	cmd.Env = append(cmd.Env, fmt.Sprintf("GOROOT=%s", filepath.Join(tmpdir, "goroot")))
	cmd.Env = append(cmd.Env, fmt.Sprintf("GOPATH=%s", filepath.Join(tmpdir, "gopath")))
	for _, e := range os.Environ() {
		if strings.HasPrefix(e, "GOROOT=") || strings.HasPrefix(e, "GOPATH=") {
			continue
		}
		cmd.Env = append(cmd.Env, e)
	}
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	cmd.Args[0] = "godoc"
	if err := cmd.Start(); err != nil {
		t.Fatalf("failed to start godoc: %s", err)
	}
	defer cmd.Process.Kill()
	waitForServer(t, addr)

	t0 := time.Now()

	// Make an HTTP request and check for a regular expression match.
	// The patterns are very crude checks that basic type information
	// has been annotated onto the source view.
tryagain:
	for _, test := range []struct{ url, pattern string }{
		{"/src/lib/lib.go", "L2.*package .*Package docs for lib.*/lib"},
		{"/src/lib/lib.go", "L3.*type .*type info for T.*struct"},
		{"/src/lib/lib.go", "L5.*var V .*type T struct"},
		{"/src/lib/lib.go", "L6.*func .*type T struct.*T.*return .*const C untyped int.*C"},

		{"/src/app/main.go", "L2.*package .*Package docs for app"},
		{"/src/app/main.go", "L3.*import .*Package docs for lib.*lib"},
		{"/src/app/main.go", "L4.*func main.*package lib.*lib.*var lib.V lib.T.*V"},
	} {
		url := fmt.Sprintf("http://%s%s", addr, test.url)
		resp, err := http.Get(url)
		if err != nil {
			t.Errorf("GET %s failed: %s", url, err)
			continue
		}
		body, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			t.Errorf("GET %s: failed to read body: %s (response: %v)", url, err, resp)
			continue
		}

		if !bytes.Contains(body, []byte("Static analysis features")) {
			// Type analysis results usually become available within
			// ~4ms after godoc startup (for this input on my machine).
			if elapsed := time.Since(t0); elapsed > 500*time.Millisecond {
				t.Fatalf("type analysis results still unavailable after %s", elapsed)
			}
			time.Sleep(10 * time.Millisecond)
			goto tryagain
		}

		match, err := regexp.Match(test.pattern, body)
		if err != nil {
			t.Errorf("regexp.Match(%q) failed: %s", test.pattern, err)
			continue
		}
		if !match {
			// This is a really ugly failure message.
			t.Errorf("GET %s: body doesn't match %q, got:\n%s",
				url, test.pattern, string(body))
		}
	}
}
<commit_msg>go.tools/cmd/godoc: wait for analysis to complete in TestTypeAnalysis<commit_after>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main_test

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"testing"
	"time"
)

var godocTests = []struct {
	args      []string
	matches   []string // regular expressions
	dontmatch []string // regular expressions
}{
	{
		args: []string{"fmt"},
		matches: []string{
			`import "fmt"`,
			`Package fmt implements formatted I/O`,
		},
	},
	{
		args: []string{"io", "WriteString"},
		matches: []string{
			`func WriteString\(`,
			`WriteString writes the contents of the string s to w`,
		},
	},
	{
		args: []string{"nonexistingpkg"},
		matches: []string{
			`no such file or directory|does not exist|cannot find the file`,
		},
	},
	{
		args: []string{"fmt", "NonexistentSymbol"},
		matches: []string{
			`No match found\.`,
		},
	},
	{
		args: []string{"-src", "syscall", "Open"},
		matches: []string{
			`func Open\(`,
		},
		dontmatch: []string{
			`No match found\.`,
		},
	},
}

// buildGodoc builds the godoc executable.
// It returns its path, and a cleanup function.
//
// TODO(adonovan): opt: do this at most once, and do the cleanup
// exactly once.  How though?  There's no atexit.
func buildGodoc(t *testing.T) (bin string, cleanup func()) {
	tmp, err := ioutil.TempDir("", "godoc-regtest-")
	if err != nil {
		t.Fatal(err)
	}

	bin = filepath.Join(tmp, "godoc")
	if runtime.GOOS == "windows" {
		bin += ".exe"
	}
	cmd := exec.Command("go", "build", "-o", bin)
	if err := cmd.Run(); err != nil {
		t.Fatalf("Building godoc: %v", err)
	}

	return bin, func() { os.RemoveAll(tmp) }
}

// Basic regression test for godoc command-line tool.
func TestCLI(t *testing.T) {
	bin, cleanup := buildGodoc(t)
	defer cleanup()
	for _, test := range godocTests {
		cmd := exec.Command(bin, test.args...)
		cmd.Args[0] = "godoc"
		out, err := cmd.CombinedOutput()
		if err != nil {
			t.Errorf("Running with args %#v: %v", test.args, err)
			continue
		}
		for _, pat := range test.matches {
			re := regexp.MustCompile(pat)
			if !re.Match(out) {
				t.Errorf("godoc %v =\n%s\nwanted /%v/", strings.Join(test.args, " "), out, pat)
			}
		}
		for _, pat := range test.dontmatch {
			re := regexp.MustCompile(pat)
			if re.Match(out) {
				t.Errorf("godoc %v =\n%s\ndid not want /%v/", strings.Join(test.args, " "), out, pat)
			}
		}
	}
}

func serverAddress(t *testing.T) string {
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		ln, err = net.Listen("tcp6", "[::1]:0")
	}
	if err != nil {
		t.Fatal(err)
	}
	defer ln.Close()
	return ln.Addr().String()
}

func waitForServer(t *testing.T, address string) {
	// Poll every 50ms for a total of 5s.
	for i := 0; i < 100; i++ {
		time.Sleep(50 * time.Millisecond)
		conn, err := net.Dial("tcp", address)
		if err != nil {
			continue
		}
		conn.Close()
		return
	}
	t.Fatalf("Server %q failed to respond in 5 seconds", address)
}

// Basic integration test for godoc HTTP interface.
func TestWeb(t *testing.T) {
	bin, cleanup := buildGodoc(t)
	defer cleanup()
	addr := serverAddress(t)
	cmd := exec.Command(bin, fmt.Sprintf("-http=%s", addr))
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	cmd.Args[0] = "godoc"
	if err := cmd.Start(); err != nil {
		t.Fatalf("failed to start godoc: %s", err)
	}
	defer cmd.Process.Kill()
	waitForServer(t, addr)
	tests := []struct {
		path      string
		match     []string
		dontmatch []string
	}{
		{
			path:  "/",
			match: []string{"Go is an open source programming language"},
		},
		{
			path:  "/pkg/fmt/",
			match: []string{"Package fmt implements formatted I/O"},
		},
		{
			path:  "/src/fmt/",
			match: []string{"scan_test.go"},
		},
		{
			path:  "/src/fmt/print.go",
			match: []string{"// Println formats using"},
		},
		{
			path: "/pkg",
			match: []string{
				"Standard library",
				"Package fmt implements formatted I/O",
			},
			dontmatch: []string{
				"internal/syscall",
				"cmd/gc",
			},
		},
		{
			path: "/pkg/?m=all",
			match: []string{
				"Standard library",
				"Package fmt implements formatted I/O",
				"internal/syscall",
			},
			dontmatch: []string{
				"cmd/gc",
			},
		},
	}
	for _, test := range tests {
		url := fmt.Sprintf("http://%s%s", addr, test.path)
		resp, err := http.Get(url)
		if err != nil {
			t.Errorf("GET %s failed: %s", url, err)
			continue
		}
		body, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			t.Errorf("GET %s: failed to read body: %s (response: %v)", url, err, resp)
		}
		isErr := false
		for _, substr := range test.match {
			if !bytes.Contains(body, []byte(substr)) {
				t.Errorf("GET %s: wanted substring %q in body", url, substr)
				isErr = true
			}
		}
		for _, substr := range test.dontmatch {
			if bytes.Contains(body, []byte(substr)) {
				t.Errorf("GET %s: didn't want substring %q in body", url, substr)
				isErr = true
			}
		}
		if isErr {
			t.Errorf("GET %s: got:\n%s", url, body)
		}
	}
}

// Basic integration test for godoc -analysis=type (via HTTP interface).
func TestTypeAnalysis(t *testing.T) {
	// Write a fake GOROOT/GOPATH.
	tmpdir, err := ioutil.TempDir("", "godoc-analysis")
	if err != nil {
		t.Fatalf("ioutil.TempDir failed: %s", err)
	}
	defer os.RemoveAll(tmpdir)
	for _, f := range []struct{ file, content string }{
		{"goroot/src/lib/lib.go", `
package lib
type T struct{}
const C = 3
var V T
func (T) F() int { return C }
`},
		{"gopath/src/app/main.go", `
package main
import "lib"
func main() { print(lib.V) }
`},
	} {
		file := filepath.Join(tmpdir, f.file)
		if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil {
			t.Fatalf("MkdirAll(%s) failed: %s", filepath.Dir(file), err)
		}
		if err := ioutil.WriteFile(file, []byte(f.content), 0644); err != nil {
			t.Fatal(err)
		}
	}

	// Start the server.
	bin, cleanup := buildGodoc(t)
	defer cleanup()
	addr := serverAddress(t)
	cmd := exec.Command(bin, fmt.Sprintf("-http=%s", addr), "-analysis=type")
	cmd.Env = append(cmd.Env, fmt.Sprintf("GOROOT=%s", filepath.Join(tmpdir, "goroot")))
	cmd.Env = append(cmd.Env, fmt.Sprintf("GOPATH=%s", filepath.Join(tmpdir, "gopath")))
	for _, e := range os.Environ() {
		if strings.HasPrefix(e, "GOROOT=") || strings.HasPrefix(e, "GOPATH=") {
			continue
		}
		cmd.Env = append(cmd.Env, e)
	}
	cmd.Stdout = os.Stderr
	stderr, err := cmd.StderrPipe()
	if err != nil {
		t.Fatal(err)
	}
	cmd.Args[0] = "godoc"
	if err := cmd.Start(); err != nil {
		t.Fatalf("failed to start godoc: %s", err)
	}
	defer cmd.Process.Kill()
	waitForServer(t, addr)

	// Wait for type analysis to complete.
	reader := bufio.NewReader(stderr)
	for {
		s, err := reader.ReadString('\n')
		if err != nil {
			t.Fatal(err)
		}
		fmt.Fprint(os.Stderr, s)
		if strings.Contains(s, "Type analysis complete.") {
			break
		}
	}
	go io.Copy(os.Stderr, reader)

	t0 := time.Now()

	// Make an HTTP request and check for a regular expression match.
	// The patterns are very crude checks that basic type information
	// has been annotated onto the source view.
tryagain:
	for _, test := range []struct{ url, pattern string }{
		{"/src/lib/lib.go", "L2.*package .*Package docs for lib.*/lib"},
		{"/src/lib/lib.go", "L3.*type .*type info for T.*struct"},
		{"/src/lib/lib.go", "L5.*var V .*type T struct"},
		{"/src/lib/lib.go", "L6.*func .*type T struct.*T.*return .*const C untyped int.*C"},

		{"/src/app/main.go", "L2.*package .*Package docs for app"},
		{"/src/app/main.go", "L3.*import .*Package docs for lib.*lib"},
		{"/src/app/main.go", "L4.*func main.*package lib.*lib.*var lib.V lib.T.*V"},
	} {
		url := fmt.Sprintf("http://%s%s", addr, test.url)
		resp, err := http.Get(url)
		if err != nil {
			t.Errorf("GET %s failed: %s", url, err)
			continue
		}
		body, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			t.Errorf("GET %s: failed to read body: %s (response: %v)", url, err, resp)
			continue
		}

		if !bytes.Contains(body, []byte("Static analysis features")) {
			// Type analysis results usually become available within
			// ~4ms after godoc startup (for this input on my machine).
			if elapsed := time.Since(t0); elapsed > 500*time.Millisecond {
				t.Fatalf("type analysis results still unavailable after %s", elapsed)
			}
			time.Sleep(10 * time.Millisecond)
			goto tryagain
		}

		match, err := regexp.Match(test.pattern, body)
		if err != nil {
			t.Errorf("regexp.Match(%q) failed: %s", test.pattern, err)
			continue
		}
		if !match {
			// This is a really ugly failure message.
			t.Errorf("GET %s: body doesn't match %q, got:\n%s",
				url, test.pattern, string(body))
		}
	}
}
<|endoftext|>


<commit_before>
package main

import (
	"os"
)

const (
	// Default value.
	MaxRabbits	= 30

	// Chance to ascend deeper (closer to /). The weight
	// has to be fair, because ascending is very limited
	// and you only have one option.
	AscendChance	= 0.35

	// Chance to move twice instead of once.
	TwoStepChance	= 0.50
)

type directoryForest struct {
	// List of rabbits and their locations. Only one
	// rabbit per location.
	rabbits		map[string]Rabbit
	// Number of rabbits currently in the forest.
	numRabbits	uint
	// Max number of rabbits allowed in the forest. Configurable.
	maxRabbits	uint
}

func newDirectoryForest() directoryForest {
	return directoryForest{
		make(map[string]Rabbit), 0, MaxRabbits,
	}
}

// For a location to exist, the directory must exist.
func (f *directoryForest) LocationExists(loc string) bool {
	fi, err := os.Stat(loc)
	if err != nil {
		return false
	}
	return fi.IsDir()
}

// Returns a location near the passed location. Nearby is
// found by a small number of random directory changes. Will
// not be the same directory, unless it has to (can't ascend
// or descend).
//
// XXX: Currently this does not check if a rabbit already
// exists at the new location. So we just lose rabbits
// if one encounters another.
func (f *directoryForest) NearbyLocation(loc string) string {
	steps := 1
	if chance(TwoStepChance) {
		steps = 2
	}

	for i := 0; i < steps; i++ {
		// Can't move.
		if !canAscend(loc) && !canDescend(loc) {
			return loc
		} else if chance(AscendChance) && canAscend(loc) {
			loc = ascend(loc)
		} else {
			loc = randDescension(loc)
		}
	}

	return loc

}

// A random faraway location. Rabbits typically start here
// and run here when they're fleeing.
func (f *directoryForest) FarawayLocation() string {
	loc := baseLocation()

	steps := 1
	if chance(TwoStepChance) {
		steps = 2
	}

	for i := 0; i < steps; i++ {
		if canDescend(loc) {
			loc = randDescension(loc)
		}
	}

	return loc
}<commit_msg>Added trying again when landing on a location that's the same.<commit_after>
package main

import (
	"os"
)

const (
	// Default value.
	MaxRabbits	= 30

	// Chance to ascend deeper (closer to /). The weight
	// has to be fair, because ascending is very limited
	// and you only have one option.
	AscendChance	= 0.35

	// Chance to move twice instead of once.
	TwoStepChance	= 0.50
)

type directoryForest struct {
	// List of rabbits and their locations. Only one
	// rabbit per location.
	rabbits		map[string]Rabbit
	// Number of rabbits currently in the forest.
	numRabbits	uint
	// Max number of rabbits allowed in the forest. Configurable.
	maxRabbits	uint
}

func newDirectoryForest() directoryForest {
	return directoryForest{
		make(map[string]Rabbit), 0, MaxRabbits,
	}
}

// For a location to exist, the directory must exist.
func (f *directoryForest) LocationExists(loc string) bool {
	fi, err := os.Stat(loc)
	if err != nil {
		return false
	}
	return fi.IsDir()
}

// Returns a location near the passed location. Nearby is
// found by a small number of random directory changes. Will
// not be the same directory, unless it has to (can't ascend
// or descend).
//
// XXX: Currently this does not check if a rabbit already
// exists at the new location. So we just lose rabbits
// if one encounters another.
func (f *directoryForest) NearbyLocation(loc string) string {
	newloc := loc

	steps := 1
	if chance(TwoStepChance) {
		steps = 2
	}

tryagain:
	for i := 0; i < steps; i++ {
		// Can't move.
		if !canAscend(newloc) && !canDescend(newloc) {
			return newloc
		} else if chance(AscendChance) && canAscend(newloc) {
			newloc = ascend(newloc)
		} else {
			newloc = randDescension(newloc)
		}
	}

	if newloc == loc && steps == 2 {
		// Guaranteed to not be the same because you must
		// step twice to get to the same destination.
		steps = 1
		goto tryagain
	}

	return newloc

}

// A random faraway location. Rabbits typically start here
// and run here when they're fleeing.
func (f *directoryForest) FarawayLocation() string {
	loc := baseLocation()

	steps := 1
	if chance(TwoStepChance) {
		steps = 2
	}

	for i := 0; i < steps; i++ {
		if canDescend(loc) {
			loc = randDescension(loc)
		}
	}

	return loc
}<|endoftext|>


<commit_before>package chuper

import (
	"fmt"
	"net/http"
	"net/url"
	"time"

	"github.com/PuerkitoBio/fetchbot"
	"github.com/PuerkitoBio/goquery"
)

const (
	DefaultCrawlDelay      = 5 * time.Second
	DefaultCrawlPoliteness = false
)

var (
	DefaultHTTPClient = http.DefaultClient

	DefaultCache = NewMemoryCache()

	DefaultErrorHandler = fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		fmt.Printf("chuper - %s - error: %s %s - %s\n", time.Now().Format(time.RFC3339), ctx.Cmd.Method(), ctx.Cmd.URL(), err)
	})

	DefaultLogHandlerFunc = func(ctx *fetchbot.Context, res *http.Response, err error) {
		if err == nil {
			fmt.Printf("chuper - %s - info: [%d] %s %s - %s\n", time.Now().Format(time.RFC3339), res.StatusCode, ctx.Cmd.Method(), ctx.Cmd.URL(), res.Header.Get("Content-Type"))
		}
	}
)

type Crawler struct {
	CrawlDelay      time.Duration
	CrawlDuration   time.Duration
	CrawlPoliteness bool
	HTTPClient      fetchbot.Doer
	Cache           Cache
	ErrorHandler    fetchbot.Handler
	LogHandlerFunc  func(ctx *fetchbot.Context, res *http.Response, err error)

	mux *fetchbot.Mux
	f   *fetchbot.Fetcher
	q   *fetchbot.Queue
}

// New returns an initialized Crawler.
func New() *Crawler {
	return &Crawler{
		CrawlDelay:      DefaultCrawlDelay,
		CrawlPoliteness: DefaultCrawlPoliteness,
		HTTPClient:      DefaultHTTPClient,
		Cache:           DefaultCache,
		ErrorHandler:    DefaultErrorHandler,
		LogHandlerFunc:  DefaultLogHandlerFunc,
		mux:             fetchbot.NewMux(),
	}
}

func (c *Crawler) Start() *fetchbot.Queue {
	c.mux.HandleErrors(c.ErrorHandler)
	l := newLogHandler(c.mux, c.LogHandlerFunc)

	f := fetchbot.New(l)
	f.CrawlDelay = c.CrawlDelay
	f.DisablePoliteness = !c.CrawlPoliteness
	f.HttpClient = c.HTTPClient

	c.f = f
	c.q = c.f.Start()

	if c.CrawlDuration > 0 {
		go func() {
			t := time.After(c.CrawlDuration)
			<-t
			c.q.Close()
		}()
	}

	return c.q
}

func (c *Crawler) Block() {
	c.q.Block()
}

func (c *Crawler) Enqueue(method string, rawURL ...string) error {
	for _, u := range rawURL {
		ok := true
		if c.mustCache() {
			ok, _ = c.Cache.SetNX(u, true)
		}
		if ok {
			if _, err := c.q.SendString(method, u); err != nil {
				return err
			}
		}
	}
	return nil
}

func (c *Crawler) EnqueueWithSource(method string, URL string, sourceURL string) (bool, error) {
	ok := true
	if c.mustCache() {
		ok, _ = c.Cache.SetNX(URL, true)
	}
	if ok {
		u, err := url.Parse(URL)
		if err != nil {
			return ok, err
		}
		s, err := url.Parse(sourceURL)
		if err != nil {
			return ok, err
		}
		cmd := Cmd{&fetchbot.Cmd{U: u, M: "GET"}, s}
		err = c.q.Send(cmd)
		return ok, err
	}
	return ok, nil
}

type ResponseCriteria struct {
	Method      string
	ContentType string
	Status      int
	MinStatus   int
	MaxStatus   int
	Path        string
	Host        string
}

func (c *Crawler) Match(r *ResponseCriteria) *fetchbot.ResponseMatcher {
	m := c.mux.Response()

	if r.Method != "" {
		m.Method(r.Method)
	}

	if r.ContentType != "" {
		m.ContentType(r.ContentType)
	}

	if r.Status != 0 {
		m.Status(r.Status)
	} else {
		if r.MinStatus != 0 && r.MaxStatus != 0 {
			m.StatusRange(r.MinStatus, r.MaxStatus)
		} else {
			if r.MinStatus != 0 {
				m.Status(r.MinStatus)
			}
			if r.MaxStatus != 0 {
				m.Status(r.MaxStatus)
			}
		}
	}

	if r.Path != "" {
		m.Path(r.Path)
	}

	if r.Host != "" {
		m.Host(r.Host)
	}

	return m
}

func (c *Crawler) Register(rc *ResponseCriteria, procs ...Processor) {
	m := c.Match(rc)
	h := newDocHandler(c.Cache, procs...)
	m.Handler(h)
}

func (c *Crawler) mustCache() bool {
	if c.Cache == nil {
		return false
	}
	return true
}

func newLogHandler(wrapped fetchbot.Handler, f func(ctx *fetchbot.Context, res *http.Response, err error)) fetchbot.Handler {
	return fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		f(ctx, res, err)
		wrapped.Handle(ctx, res, err)
	})
}

func newDocHandler(cache Cache, procs ...Processor) fetchbot.Handler {
	return fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		context := &Context{ctx, cache}
		doc, err := goquery.NewDocumentFromResponse(res)
		if err != nil {
			fmt.Printf("chuper - %s - error: %s %s - %s\n", time.Now().Format(time.RFC3339), ctx.Cmd.Method(), ctx.Cmd.URL(), err)
			return
		}
		for _, p := range procs {
			ok := p.Process(context, doc)
			if !ok {
				return
			}
		}
	})
}
<commit_msg>Add Finish method to Crawler<commit_after>package chuper

import (
	"fmt"
	"net/http"
	"net/url"
	"time"

	"github.com/PuerkitoBio/fetchbot"
	"github.com/PuerkitoBio/goquery"
)

const (
	DefaultCrawlDelay      = 5 * time.Second
	DefaultCrawlPoliteness = false
)

var (
	DefaultHTTPClient = http.DefaultClient

	DefaultCache = NewMemoryCache()

	DefaultErrorHandler = fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		fmt.Printf("chuper - %s - error: %s %s - %s\n", time.Now().Format(time.RFC3339), ctx.Cmd.Method(), ctx.Cmd.URL(), err)
	})

	DefaultLogHandlerFunc = func(ctx *fetchbot.Context, res *http.Response, err error) {
		if err == nil {
			fmt.Printf("chuper - %s - info: [%d] %s %s - %s\n", time.Now().Format(time.RFC3339), res.StatusCode, ctx.Cmd.Method(), ctx.Cmd.URL(), res.Header.Get("Content-Type"))
		}
	}
)

type Crawler struct {
	CrawlDelay      time.Duration
	CrawlDuration   time.Duration
	CrawlPoliteness bool
	HTTPClient      fetchbot.Doer
	Cache           Cache
	ErrorHandler    fetchbot.Handler
	LogHandlerFunc  func(ctx *fetchbot.Context, res *http.Response, err error)

	mux *fetchbot.Mux
	f   *fetchbot.Fetcher
	q   *fetchbot.Queue
}

// New returns an initialized Crawler.
func New() *Crawler {
	return &Crawler{
		CrawlDelay:      DefaultCrawlDelay,
		CrawlPoliteness: DefaultCrawlPoliteness,
		HTTPClient:      DefaultHTTPClient,
		Cache:           DefaultCache,
		ErrorHandler:    DefaultErrorHandler,
		LogHandlerFunc:  DefaultLogHandlerFunc,
		mux:             fetchbot.NewMux(),
	}
}

func (c *Crawler) Start() *fetchbot.Queue {
	c.mux.HandleErrors(c.ErrorHandler)
	l := newLogHandler(c.mux, c.LogHandlerFunc)

	f := fetchbot.New(l)
	f.CrawlDelay = c.CrawlDelay
	f.DisablePoliteness = !c.CrawlPoliteness
	f.HttpClient = c.HTTPClient

	c.f = f
	c.q = c.f.Start()

	if c.CrawlDuration > 0 {
		go func() {
			t := time.After(c.CrawlDuration)
			<-t
			c.q.Close()
		}()
	}

	return c.q
}

func (c *Crawler) Block() {
	c.q.Block()
}

func (c *Crawler) Finish() {
	c.q.Close()
}

func (c *Crawler) Enqueue(method string, rawURL ...string) error {
	for _, u := range rawURL {
		ok := true
		if c.mustCache() {
			ok, _ = c.Cache.SetNX(u, true)
		}
		if ok {
			if _, err := c.q.SendString(method, u); err != nil {
				return err
			}
		}
	}
	return nil
}

func (c *Crawler) EnqueueWithSource(method string, URL string, sourceURL string) (bool, error) {
	ok := true
	if c.mustCache() {
		ok, _ = c.Cache.SetNX(URL, true)
	}
	if ok {
		u, err := url.Parse(URL)
		if err != nil {
			return ok, err
		}
		s, err := url.Parse(sourceURL)
		if err != nil {
			return ok, err
		}
		cmd := Cmd{&fetchbot.Cmd{U: u, M: "GET"}, s}
		err = c.q.Send(cmd)
		return ok, err
	}
	return ok, nil
}

type ResponseCriteria struct {
	Method      string
	ContentType string
	Status      int
	MinStatus   int
	MaxStatus   int
	Path        string
	Host        string
}

func (c *Crawler) Match(r *ResponseCriteria) *fetchbot.ResponseMatcher {
	m := c.mux.Response()

	if r.Method != "" {
		m.Method(r.Method)
	}

	if r.ContentType != "" {
		m.ContentType(r.ContentType)
	}

	if r.Status != 0 {
		m.Status(r.Status)
	} else {
		if r.MinStatus != 0 && r.MaxStatus != 0 {
			m.StatusRange(r.MinStatus, r.MaxStatus)
		} else {
			if r.MinStatus != 0 {
				m.Status(r.MinStatus)
			}
			if r.MaxStatus != 0 {
				m.Status(r.MaxStatus)
			}
		}
	}

	if r.Path != "" {
		m.Path(r.Path)
	}

	if r.Host != "" {
		m.Host(r.Host)
	}

	return m
}

func (c *Crawler) Register(rc *ResponseCriteria, procs ...Processor) {
	m := c.Match(rc)
	h := newDocHandler(c.Cache, procs...)
	m.Handler(h)
}

func (c *Crawler) mustCache() bool {
	if c.Cache == nil {
		return false
	}
	return true
}

func newLogHandler(wrapped fetchbot.Handler, f func(ctx *fetchbot.Context, res *http.Response, err error)) fetchbot.Handler {
	return fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		f(ctx, res, err)
		wrapped.Handle(ctx, res, err)
	})
}

func newDocHandler(cache Cache, procs ...Processor) fetchbot.Handler {
	return fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		context := &Context{ctx, cache}
		doc, err := goquery.NewDocumentFromResponse(res)
		if err != nil {
			fmt.Printf("chuper - %s - error: %s %s - %s\n", time.Now().Format(time.RFC3339), ctx.Cmd.Method(), ctx.Cmd.URL(), err)
			return
		}
		for _, p := range procs {
			ok := p.Process(context, doc)
			if !ok {
				return
			}
		}
	})
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"os"
	"path/filepath"
	"strings"
	"testing"

	"k8s.io/enhancements/pkg/kepval/keps"
	"k8s.io/enhancements/pkg/kepval/prrs"
)

const (
	kepsDir     = "keps"
	prrsDir     = "keps/prod-readiness"
	kepMetadata = "kep.yaml"
)

// This is the actual validation check of all keps in this repo
func TestValidation(t *testing.T) {
	// Find all the keps
	files := []string{}
	err := filepath.Walk(
		filepath.Join("..", "..", kepsDir),
		func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if info.IsDir() {
				return nil
			}

			dir := filepath.Dir(path)
			// true if the file is a symlink
			if info.Mode()&os.ModeSymlink != 0 {
				// assume symlink from old KEP location to new
				newLocation, err := os.Readlink(path)
				if err != nil {
					return err
				}
				files = append(files, filepath.Join(dir, filepath.Dir(newLocation), kepMetadata))
				return nil
			}
			if ignore(dir, info.Name()) {
				return nil
			}
			files = append(files, path)
			return nil
		},
	)
	// This indicates a problem walking the filepath, not a validation error.
	if err != nil {
		t.Fatal(err)
	}

	if len(files) == 0 {
		t.Fatal("must find more than 0 keps")
	}

	kepParser := &keps.Parser{}
	prrParser := &prrs.Parser{}
	prrsDir := filepath.Join("..", "..", prrsDir)

	for _, filename := range files {
		t.Run(filename, func(t *testing.T) {
			kepFile, err := os.Open(filename)
			if err != nil {
				t.Fatalf("could not open file %s: %v\n", filename, err)
			}
			defer kepFile.Close()

			kep := kepParser.Parse(kepFile)
			if kep.Error != nil {
				t.Errorf("%v has an error: %v", filename, kep.Error)
			}

			requiredPRRApproval := len(kep.Number) > 0 && kep.LatestMilestone >= "v1.21"
			if !requiredPRRApproval {
				return
			}

			var stageMilestone string
			switch kep.Stage {
			case "alpha":
				stageMilestone = kep.Milestone.Alpha
			case "beta":
				stageMilestone = kep.Milestone.Beta
			case "stable":
				stageMilestone = kep.Milestone.Stable
			}

			prrFilename := kep.Number + ".yaml"
			prrFilename = filepath.Join(prrsDir, kep.OwningSIG, prrFilename)
			prrFile, err := os.Open(prrFilename)
			if os.IsNotExist(err) {
				t.Errorf("PRR approval is required to target milestone %v (stage %v)", stageMilestone, kep.Stage)
				t.Errorf("For more details about PRR approval see: https://github.com/kubernetes/community/blob/master/sig-architecture/production-readiness.md")
				t.Errorf("To get PRR approval modify appropriately file %s and have this approved by PRR team", prrFilename)
				return
			}
			if err != nil {
				t.Fatalf("could not open file %s: %v\n", prrFilename, err)
			}
			prr := prrParser.Parse(prrFile)
			if prr.Error != nil {
				t.Errorf("PRR approval file %v has an error: %v", prrFilename, prr.Error)
				return
			}

			var stagePRRApprover string
			switch kep.Stage {
			case "alpha":
				stagePRRApprover = prr.Alpha.Approver
			case "beta":
				stagePRRApprover = prr.Beta.Approver
			case "stable":
				stagePRRApprover = prr.Stable.Approver
			}
			if len(stageMilestone) > 0 && stageMilestone >= "v1.21" {
				// PRR approval is needed.
				if len(stagePRRApprover) == 0 {
					t.Errorf("PRR approval is required to target milestone %v (stage %v)", stageMilestone, kep.Stage)
					t.Errorf("For more details about PRR approval see: https://github.com/kubernetes/community/blob/master/sig-architecture/production-readiness.md")
					t.Errorf("To get PRR approval modify appropriately file %s and have this approved by PRR team", prrFilename)
				}
			}
		})
	}
}

// ignore certain files in the keps/ subdirectory
func ignore(dir, name string) bool {
	if dir == "../../keps/NNNN-kep-template" {
		return true // ignore the template directory because its metadata file does not use a valid sig name
	}

	if name == kepMetadata {
		return false // always check metadata files
	}

	if !strings.HasSuffix(name, "md") {
		return true
	}

	if name == "0023-documentation-for-images.md" ||
		name == "0004-cloud-provider-template.md" ||
		name == "YYYYMMDD-kep-template.md" ||
		name == "README.md" ||
		name == "kep-faq.md" {
		return true
	}

	return false
}
<commit_msg>Disable enforcement of PRR requirements<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"os"
	"path/filepath"
	"strings"
	"testing"

	"k8s.io/enhancements/pkg/kepval/keps"
	"k8s.io/enhancements/pkg/kepval/prrs"
)

const (
	kepsDir     = "keps"
	prrsDir     = "keps/prod-readiness"
	kepMetadata = "kep.yaml"
)

// This is the actual validation check of all keps in this repo
func TestValidation(t *testing.T) {
	// Find all the keps
	files := []string{}
	err := filepath.Walk(
		filepath.Join("..", "..", kepsDir),
		func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if info.IsDir() {
				return nil
			}

			dir := filepath.Dir(path)
			// true if the file is a symlink
			if info.Mode()&os.ModeSymlink != 0 {
				// assume symlink from old KEP location to new
				newLocation, err := os.Readlink(path)
				if err != nil {
					return err
				}
				files = append(files, filepath.Join(dir, filepath.Dir(newLocation), kepMetadata))
				return nil
			}
			if ignore(dir, info.Name()) {
				return nil
			}
			files = append(files, path)
			return nil
		},
	)
	// This indicates a problem walking the filepath, not a validation error.
	if err != nil {
		t.Fatal(err)
	}

	if len(files) == 0 {
		t.Fatal("must find more than 0 keps")
	}

	kepParser := &keps.Parser{}
	prrParser := &prrs.Parser{}
	prrsDir := filepath.Join("..", "..", prrsDir)

	for _, filename := range files {
		t.Run(filename, func(t *testing.T) {
			kepFile, err := os.Open(filename)
			if err != nil {
				t.Fatalf("could not open file %s: %v\n", filename, err)
			}
			defer kepFile.Close()

			kep := kepParser.Parse(kepFile)
			if kep.Error != nil {
				t.Errorf("%v has an error: %v", filename, kep.Error)
			}

			// temporarily disable PRR enforcement per https://github.com/kubernetes/enhancements/issues/2239
			// so setting this to something always false
			requiredPRRApproval := len(kep.Number) < 0
			//requiredPRRApproval := len(kep.Number) > 0 && kep.LatestMilestone >= "v1.21"
			if !requiredPRRApproval {
				return
			}

			var stageMilestone string
			switch kep.Stage {
			case "alpha":
				stageMilestone = kep.Milestone.Alpha
			case "beta":
				stageMilestone = kep.Milestone.Beta
			case "stable":
				stageMilestone = kep.Milestone.Stable
			}

			prrFilename := kep.Number + ".yaml"
			prrFilename = filepath.Join(prrsDir, kep.OwningSIG, prrFilename)
			prrFile, err := os.Open(prrFilename)
			if os.IsNotExist(err) {
				t.Errorf("PRR approval is required to target milestone %v (stage %v)", stageMilestone, kep.Stage)
				t.Errorf("For more details about PRR approval see: https://github.com/kubernetes/community/blob/master/sig-architecture/production-readiness.md")
				t.Errorf("To get PRR approval modify appropriately file %s and have this approved by PRR team", prrFilename)
				return
			}
			if err != nil {
				t.Fatalf("could not open file %s: %v\n", prrFilename, err)
			}
			prr := prrParser.Parse(prrFile)
			if prr.Error != nil {
				t.Errorf("PRR approval file %v has an error: %v", prrFilename, prr.Error)
				return
			}

			var stagePRRApprover string
			switch kep.Stage {
			case "alpha":
				stagePRRApprover = prr.Alpha.Approver
			case "beta":
				stagePRRApprover = prr.Beta.Approver
			case "stable":
				stagePRRApprover = prr.Stable.Approver
			}
			if len(stageMilestone) > 0 && stageMilestone >= "v1.21" {
				// PRR approval is needed.
				if len(stagePRRApprover) == 0 {
					t.Errorf("PRR approval is required to target milestone %v (stage %v)", stageMilestone, kep.Stage)
					t.Errorf("For more details about PRR approval see: https://github.com/kubernetes/community/blob/master/sig-architecture/production-readiness.md")
					t.Errorf("To get PRR approval modify appropriately file %s and have this approved by PRR team", prrFilename)
				}
			}
		})
	}
}

// ignore certain files in the keps/ subdirectory
func ignore(dir, name string) bool {
	if dir == "../../keps/NNNN-kep-template" {
		return true // ignore the template directory because its metadata file does not use a valid sig name
	}

	if name == kepMetadata {
		return false // always check metadata files
	}

	if !strings.HasSuffix(name, "md") {
		return true
	}

	if name == "0023-documentation-for-images.md" ||
		name == "0004-cloud-provider-template.md" ||
		name == "YYYYMMDD-kep-template.md" ||
		name == "README.md" ||
		name == "kep-faq.md" {
		return true
	}

	return false
}
<|endoftext|>


<commit_before>package main

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestFeatures(t *testing.T) {
	assert := assert.New(t)

	assert.False(*faceDetection)
	assert.False(*landmarkDetection)
	assert.False(*logoDetection)
	assert.False(*labelDetection)
	assert.False(*textDetection)
	assert.False(*safeSearchDetection)
	assert.False(*imageProperties)

	f := features()
	if assert.Equal(1, len(f)) {
		feature := f[0]
		assert.EqualValues(0, feature.MaxResults)
		assert.Equal("FACE_DETECTION", feature.Type)
	}

	*landmarkDetection = true
	f = features()
	if assert.Equal(1, len(f)) {
		feature := f[0]
		assert.EqualValues(0, feature.MaxResults)
		assert.Equal("LANDMARK_DETECTION", feature.Type)
	}

	*logoDetection = true
	f = features()
	if assert.Equal(2, len(f)) {
		feature := f[0]
		assert.EqualValues(0, feature.MaxResults)
		assert.Equal("LANDMARK_DETECTION", feature.Type)
		feature = f[1]
		assert.EqualValues(0, feature.MaxResults)
		assert.Equal("LOGO_DETECTION", feature.Type)
	}
}
<commit_msg>Fix main test<commit_after>package main

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestFeatures(t *testing.T) {
	assert := assert.New(t)

	assert.False(*faceDetection)
	assert.False(*landmarkDetection)
	assert.False(*logoDetection)
	assert.False(*labelDetection)
	assert.False(*textDetection)
	assert.False(*safeSearchDetection)
	assert.False(*imageProperties)

	f := features()
	if assert.Equal(1, len(f)) {
		feature := f[0]
		assert.EqualValues(0, feature.MaxResults)
		assert.Equal("LABEL_DETECTION", feature.Type)
	}

	*landmarkDetection = true
	f = features()
	if assert.Equal(1, len(f)) {
		feature := f[0]
		assert.EqualValues(0, feature.MaxResults)
		assert.Equal("LANDMARK_DETECTION", feature.Type)
	}

	*logoDetection = true
	f = features()
	if assert.Equal(2, len(f)) {
		feature := f[0]
		assert.EqualValues(0, feature.MaxResults)
		assert.Equal("LANDMARK_DETECTION", feature.Type)
		feature = f[1]
		assert.EqualValues(0, feature.MaxResults)
		assert.Equal("LOGO_DETECTION", feature.Type)
	}
}
<|endoftext|>


<commit_before>//
// csvcols - is a command line that takes each argument in order and outputs a line in CSV format.
// It can also take a delimiter and line of text splitting it into a CSV formatted set of columns.
//
// @author R. S. Doiel, <rsdoiel@gmail.com>
//
// Copyright (c) 2017, R. S. Doiel
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice, this
//   list of conditions and the following disclaimer.
//
// * Redistributions in binary form must reproduce the above copyright notice,
//   this list of conditions and the following disclaimer in the documentation
//   and/or other materials provided with the distribution.
//
// * Neither the name of findfile nor the names of its
//   contributors may be used to endorse or promote products derived from
//   this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"strings"

	// My packages
	"github.com/rsdoiel/cli"
	"github.com/rsdoiel/shelltools"
)

var (
	usage = `USAGE: %s [OPTIONS] ARGS_AS_COLS`

	description = `
SYNOPSIS

%s converts a set of command line args into columns output in CSV format.
`

	examples = `
EXAMPLES

Simple usage of building a CSV file one row at a time.

    %s one two three > 3col.csv
    %s 1 2 3 >> 3col.csv
    cat 3col.csv

Example parsing a pipe delimited string into a CSV line

    %s -d "|" "one|two|three" > 3col.csv
    %s -delimiter "|" "1|2|3" >> 3col.csv
    cat 3col.csv
`

	// Basic Options
	showHelp    bool
	showLicense bool
	showVersion bool

	// App Options
	delimiter string
)

func init() {
	// Basic Options
	flag.BoolVar(&showHelp, "h", false, "display help")
	flag.BoolVar(&showHelp, "help", false, "display help")
	flag.BoolVar(&showLicense, "l", false, "display license")
	flag.BoolVar(&showLicense, "license", false, "display license")
	flag.BoolVar(&showVersion, "v", false, "display version")
	flag.BoolVar(&showVersion, "version", false, "display version")

	// App Options
	flag.StringVar(&delimiter, "d", "", "set delimiter for conversion")
	flag.StringVar(&delimiter, "delimiter", "", "set delimiter for conversion")
}

func main() {
	appName := path.Base(os.Args[0])
	flag.Parse()
	args := flag.Args()

	// Configuration and command line interation
	cfg := cli.New(appName, "csvcols", fmt.Sprintf(shelltools.LicenseText, appName, shelltools.Version), shelltools.Version)
	cfg.UsageText = fmt.Sprintf(usage, appName)
	cfg.DescriptionText = fmt.Sprintf(description, appName)
	cfg.OptionsText = "OPTIONS\n"
	cfg.ExampleText = fmt.Sprintf(examples, appName, appName, appName, appName)

	if showHelp == true {
		fmt.Println(cfg.Usage())
		os.Exit(0)
	}

	if showLicense == true {
		fmt.Println(cfg.License())
		os.Exit(0)
	}

	if showVersion == true {
		fmt.Println(cfg.Version())
		os.Exit(0)
	}

	if len(delimiter) > 0 && len(args) == 1 {
		args = strings.Split(args[0], delimiter)
	}

	out := csv.NewWriter(os.Stdout)
	if err := out.Write(args); err != nil {
		log.Fatalf("error wrint args as csv, %s", err)
	}
	out.Flush()
	if err := out.Error(); err != nil {
		log.Fatal(err)
	}

}
<commit_msg>Quick Save<commit_after>//
// csvcols - is a command line that takes each argument in order and outputs a line in CSV format.
// It can also take a delimiter and line of text splitting it into a CSV formatted set of columns.
//
// @author R. S. Doiel, <rsdoiel@gmail.com>
//
// Copyright (c) 2017, R. S. Doiel
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice, this
//   list of conditions and the following disclaimer.
//
// * Redistributions in binary form must reproduce the above copyright notice,
//   this list of conditions and the following disclaimer in the documentation
//   and/or other materials provided with the distribution.
//
// * Neither the name of findfile nor the names of its
//   contributors may be used to endorse or promote products derived from
//   this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
// FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
// DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
// SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
// CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
// OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"strings"

	// My packages
	"github.com/rsdoiel/cli"
	"github.com/rsdoiel/shelltools"
)

var (
	usage = `USAGE: %s [OPTIONS] ARGS_AS_COLS`

	description = `
SYNOPSIS

%s converts a set of command line args into columns output in CSV format.
`

	examples = `
EXAMPLES

Simple usage of building a CSV file one row at a time.

    %s one two three > 3col.csv
    %s 1 2 3 >> 3col.csv
    cat 3col.csv

Example parsing a pipe delimited string into a CSV line

    %s -d "|" "one|two|three" > 3col.csv
    %s -delimiter "|" "1|2|3" >> 3col.csv
    cat 3col.csv
`

	// Basic Options
	showHelp    bool
	showLicense bool
	showVersion bool

	// App Options
	delimiter string
)

func init() {
	// Basic Options
	flag.BoolVar(&showHelp, "h", false, "display help")
	flag.BoolVar(&showHelp, "help", false, "display help")
	flag.BoolVar(&showLicense, "l", false, "display license")
	flag.BoolVar(&showLicense, "license", false, "display license")
	flag.BoolVar(&showVersion, "v", false, "display version")
	flag.BoolVar(&showVersion, "version", false, "display version")

	// App Options
	flag.StringVar(&delimiter, "d", "", "set delimiter for conversion")
	flag.StringVar(&delimiter, "delimiter", "", "set delimiter for conversion")
}

func main() {
	appName := path.Base(os.Args[0])
	flag.Parse()
	args := flag.Args()

	// Configuration and command line interation
	cfg := cli.New(appName, "csvcols", fmt.Sprintf(shelltools.LicenseText, appName, shelltools.Version), shelltools.Version)
	cfg.UsageText = fmt.Sprintf(usage, appName)
	cfg.DescriptionText = fmt.Sprintf(description, appName)
	cfg.OptionsText = "OPTIONS\n"
	cfg.ExampleText = fmt.Sprintf(examples, appName, appName, appName, appName)

	if showHelp == true {
		fmt.Println(cfg.Usage())
		os.Exit(0)
	}

	if showLicense == true {
		fmt.Println(cfg.License())
		os.Exit(0)
	}

	if showVersion == true {
		fmt.Println(cfg.Version())
		os.Exit(0)
	}

	if len(delimiter) > 0 && len(args) == 1 {
		args = strings.Split(args[0], delimiter)
	}

	// Clean up fields removing outer quotes if necessary
	fields := []string{}
	for _, val := range args {
		if strings.HasPrefix(val, "\"") && strings.HasSuffix(val, "\"") {
			val = strings.TrimPrefix(strings.TrimSuffix(val, "\""), "\"")
		}
		fields = append(fields, strings.TrimSpace(val))
	}

	out := csv.NewWriter(os.Stdout)
	if err := out.Write(fields); err != nil {
		log.Fatalf("error wrint args as csv, %s", err)
	}
	out.Flush()
	if err := out.Error(); err != nil {
		log.Fatal(err)
	}

}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package fs

import (
	"container/list"
	"errors"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
)

const dirSeparator = '/'

// Implementation detail, do not touch.
//
// How long we cache the most recent listing for a particular directory from
// GCS before regarding it as stale.
//
// Intended to paper over performance issues caused by quick follow-up calls;
// for example when the fuse VFS performs a readdir followed quickly by a
// lookup for each child. The drawback is that this increases the time before a
// write by a foreign machine within a recently-listed directory will be seen
// locally.
//
// TODO(jacobsa): Set this according to real-world performance issues when the
// kernel does e.g. ReadDir followed by LookUp. Can probably be set quite
// small.
//
// TODO(jacobsa): Can this be moved to a decorator implementation of gcs.Bucket
// instead of living here?
var ListingCacheTTL = 10 * time.Second

// Implementation detail, do not touch.
//
// How long we remember that we took some action on the contents of a directory
// (linking or unlinking), and pretend the action is reflected in the listing
// even if it is not.
//
// Intended to paper over the fact that GCS doesn't offer list-your-own-writes
// consistency: it may be an arbitrarily long time before you see the creation
// or deletion of an object in a subsequent listing, and even if you see it in
// one listing you may not in the next. The drawback is that modifications to
// recently-modified directories by foreign machines will not be reflected
// locally for awhile.
//
// TODO(jacobsa): Set this according to information about listing staleness
// distributions from the GCS team.
//
// TODO(jacobsa): Can this be moved to a decorator implementation of gcs.Bucket
// instead of living here?
var ChildActionMemoryTTL = 5 * time.Minute

// See the childModifications field of dir.
type childModification struct {
	time time.Time
	name string

	// INVARIANT: nil or of type *file or *dir.
	node fusefs.Node
}

// A "directory" in GCS, defined by an object name prefix.
//
// For example, if the bucket contains objects "foo/bar" and "foo/baz", this
// implicitly defines the directory "foo/". No matter what the contents of the
// bucket, there is an implicit root directory "".
type dir struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	logger *log.Logger
	clock  timeutil.Clock
	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// INVARIANT: objectPrefix is "" (representing the root directory) or ends
	// with dirSeparator.
	objectPrefix string

	/////////////////////////
	// Mutable state
	/////////////////////////

	// TODO(jacobsa): Make sure to initialize this correctly.
	fixmemu syncutil.InvariantMutex

	// Our current best understanding of the contents of the directory in GCS,
	// formed by listing the bucket and then patching according to child addition
	// and removal records at the time, and patched since then by subsequent
	// additions and removals. May be nil if no listing has happened or the
	// listing expired (see below).
	//
	// The time after which this should be generated anew from a new listing is
	// also stored. This is set to the time at which the listing completed plus
	// ListingCacheTTL.
	//
	// INVARIANT: All nodes are of type *dir or *file.
	// INVARIANT: All nodes are indexed by names that agree with node contents.
	contents           map[string]fusefs.Node // GUARDED_BY(mu)
	contentsExpiration time.Time              // GUARDED_BY(mu)

	// A collection of children that have recently been added or removed locally
	// and the time at which it happened, ordered by the sequence in which it
	// happened. Elements M with M.node == nil are removals; all others are
	// additions.
	//
	// For a record M in this list with M's age less than ChildActionMemoryTTL,
	// any listing from the bucket should be augmented by pretending M just
	// happened.
	//
	// TODO(jacobsa): Make sure to test link followed by unlink, and unlink
	// followed by link.
	//
	// TODO(jacobsa): Make sure to test that these expire eventually, i.e. that
	// foreign overwrites, deletes, and recreates are reflected eventually.
	//
	// INVARIANT: All elements are of type childModification.
	// INVARIANT: Contains no duplicate names.
	// INVARIANT: For each M with M.node == nil, contents does not contain M.name.
	// INVARIANT: For each M with M.node != nil,
	//              contents == nil || contents[M.name] == M.node.
	childModifications list.List // GUARDED_BY(mu)

	// An index of childModifications by name.
	//
	// INVARIANT: For all names N in the map, the indexed modification has name N.
	// INVARIANT: Contains exactly the set of names in childModifications.
	childModificationsIndex map[string]*list.Element // GUARDED_BY(mu)
}

// Make sure dir implements the interfaces we think it does.
var (
	_ fusefs.Node               = &dir{}
	_ fusefs.NodeCreater        = &dir{}
	_ fusefs.NodeMknoder        = &dir{}
	_ fusefs.NodeStringLookuper = &dir{}

	_ fusefs.Handle             = &dir{}
	_ fusefs.HandleReadDirAller = &dir{}
)

func newDir(
	logger *log.Logger,
	clock timeutil.Clock,
	bucket gcs.Bucket,
	objectPrefix string) *dir {
	return &dir{
		logger:       logger,
		clock:        clock,
		bucket:       bucket,
		objectPrefix: objectPrefix,
	}
}

// Ensure that d.contents is fresh and usable. Must be called before using
// d.contents.
//
// TODO(jacobsa): If contents hasn't expired, return immediately. Otherwise
// list parasitically while holding the lock (why not, we can make this more
// subtle later if we must) and modify the result appropriately.
//
// EXCLUSIVE_LOCKS_REQUIRED(d.mu)
func (d *dir) ensureContents(ctx context.Context) error

func (d *dir) Attr() fuse.Attr {
	return fuse.Attr{
		// TODO(jacobsa): Reflect that we allow writes now. Make sure to test.
		// TODO(jacobsa): Expose ACLs from GCS?
		Mode: os.ModeDir | 0500,
	}
}

func (d *dir) ReadDirAll(ctx context.Context) (ents []fuse.Dirent, err error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	d.logger.Printf("ReadDirAll: [%s]/%s", d.bucket.Name(), d.objectPrefix)

	// Ensure that we can use d.contents.
	if err := d.ensureContents(ctx); err != nil {
		err = fmt.Errorf("d.ensureContents: %v", err)
		return
	}

	// Read out entries from the contents map.
	for name, node := range d.contents {
		ent := fuse.Dirent{
			Name: name,
		}

		if _, ok := node.(*dir); ok {
			ent.Type = fuse.DT_Dir
		} else {
			ent.Type = fuse.DT_File
		}

		ents = append(ents, ent)
	}

	return
}

func (d *dir) Lookup(ctx context.Context, name string) (fusefs.Node, error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	d.logger.Printf("Lookup: ([%s]/%s) %s", d.bucket.Name(), d.objectPrefix, name)

	// Ensure that we can use d.contents.
	if err := d.ensureContents(ctx); err != nil {
		err = fmt.Errorf("d.ensureContents: %v", err)
		return
	}

	// Find the object within the map.
	if n, ok := d.contents[name]; ok {
		return n, nil
	}

	return nil, fuse.ENOENT
}

func (d *dir) Create(
	ctx context.Context,
	req *fuse.CreateRequest,
	resp *fuse.CreateResponse) (
	fusefs.Node,
	fusefs.Handle,
	error) {
	// Tell fuse to use Mknod followed by Open, rather than re-implementing much
	// of Open here.
	return nil, nil, fuse.ENOSYS
}

func (d *dir) Mknod(
	ctx context.Context,
	req *fuse.MknodRequest) (fusefs.Node, error) {
	return nil, errors.New("TODO(jacobsa): Support Mknod.")
}
<commit_msg>Fixed a comment (LookUp -> Lookup).<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package fs

import (
	"container/list"
	"errors"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"

	"bazil.org/fuse"
	fusefs "bazil.org/fuse/fs"
)

const dirSeparator = '/'

// Implementation detail, do not touch.
//
// How long we cache the most recent listing for a particular directory from
// GCS before regarding it as stale.
//
// Intended to paper over performance issues caused by quick follow-up calls;
// for example when the fuse VFS performs a readdir followed quickly by a
// lookup for each child. The drawback is that this increases the time before a
// write by a foreign machine within a recently-listed directory will be seen
// locally.
//
// TODO(jacobsa): Set this according to real-world performance issues when the
// kernel does e.g. ReadDir followed by Lookup. Can probably be set quite
// small.
//
// TODO(jacobsa): Can this be moved to a decorator implementation of gcs.Bucket
// instead of living here?
var ListingCacheTTL = 10 * time.Second

// Implementation detail, do not touch.
//
// How long we remember that we took some action on the contents of a directory
// (linking or unlinking), and pretend the action is reflected in the listing
// even if it is not.
//
// Intended to paper over the fact that GCS doesn't offer list-your-own-writes
// consistency: it may be an arbitrarily long time before you see the creation
// or deletion of an object in a subsequent listing, and even if you see it in
// one listing you may not in the next. The drawback is that modifications to
// recently-modified directories by foreign machines will not be reflected
// locally for awhile.
//
// TODO(jacobsa): Set this according to information about listing staleness
// distributions from the GCS team.
//
// TODO(jacobsa): Can this be moved to a decorator implementation of gcs.Bucket
// instead of living here?
var ChildActionMemoryTTL = 5 * time.Minute

// See the childModifications field of dir.
type childModification struct {
	time time.Time
	name string

	// INVARIANT: nil or of type *file or *dir.
	node fusefs.Node
}

// A "directory" in GCS, defined by an object name prefix.
//
// For example, if the bucket contains objects "foo/bar" and "foo/baz", this
// implicitly defines the directory "foo/". No matter what the contents of the
// bucket, there is an implicit root directory "".
type dir struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	logger *log.Logger
	clock  timeutil.Clock
	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// INVARIANT: objectPrefix is "" (representing the root directory) or ends
	// with dirSeparator.
	objectPrefix string

	/////////////////////////
	// Mutable state
	/////////////////////////

	// TODO(jacobsa): Make sure to initialize this correctly.
	fixmemu syncutil.InvariantMutex

	// Our current best understanding of the contents of the directory in GCS,
	// formed by listing the bucket and then patching according to child addition
	// and removal records at the time, and patched since then by subsequent
	// additions and removals. May be nil if no listing has happened or the
	// listing expired (see below).
	//
	// The time after which this should be generated anew from a new listing is
	// also stored. This is set to the time at which the listing completed plus
	// ListingCacheTTL.
	//
	// INVARIANT: All nodes are of type *dir or *file.
	// INVARIANT: All nodes are indexed by names that agree with node contents.
	contents           map[string]fusefs.Node // GUARDED_BY(mu)
	contentsExpiration time.Time              // GUARDED_BY(mu)

	// A collection of children that have recently been added or removed locally
	// and the time at which it happened, ordered by the sequence in which it
	// happened. Elements M with M.node == nil are removals; all others are
	// additions.
	//
	// For a record M in this list with M's age less than ChildActionMemoryTTL,
	// any listing from the bucket should be augmented by pretending M just
	// happened.
	//
	// TODO(jacobsa): Make sure to test link followed by unlink, and unlink
	// followed by link.
	//
	// TODO(jacobsa): Make sure to test that these expire eventually, i.e. that
	// foreign overwrites, deletes, and recreates are reflected eventually.
	//
	// INVARIANT: All elements are of type childModification.
	// INVARIANT: Contains no duplicate names.
	// INVARIANT: For each M with M.node == nil, contents does not contain M.name.
	// INVARIANT: For each M with M.node != nil,
	//              contents == nil || contents[M.name] == M.node.
	childModifications list.List // GUARDED_BY(mu)

	// An index of childModifications by name.
	//
	// INVARIANT: For all names N in the map, the indexed modification has name N.
	// INVARIANT: Contains exactly the set of names in childModifications.
	childModificationsIndex map[string]*list.Element // GUARDED_BY(mu)
}

// Make sure dir implements the interfaces we think it does.
var (
	_ fusefs.Node               = &dir{}
	_ fusefs.NodeCreater        = &dir{}
	_ fusefs.NodeMknoder        = &dir{}
	_ fusefs.NodeStringLookuper = &dir{}

	_ fusefs.Handle             = &dir{}
	_ fusefs.HandleReadDirAller = &dir{}
)

func newDir(
	logger *log.Logger,
	clock timeutil.Clock,
	bucket gcs.Bucket,
	objectPrefix string) *dir {
	return &dir{
		logger:       logger,
		clock:        clock,
		bucket:       bucket,
		objectPrefix: objectPrefix,
	}
}

// Ensure that d.contents is fresh and usable. Must be called before using
// d.contents.
//
// TODO(jacobsa): If contents hasn't expired, return immediately. Otherwise
// list parasitically while holding the lock (why not, we can make this more
// subtle later if we must) and modify the result appropriately.
//
// EXCLUSIVE_LOCKS_REQUIRED(d.mu)
func (d *dir) ensureContents(ctx context.Context) error

func (d *dir) Attr() fuse.Attr {
	return fuse.Attr{
		// TODO(jacobsa): Reflect that we allow writes now. Make sure to test.
		// TODO(jacobsa): Expose ACLs from GCS?
		Mode: os.ModeDir | 0500,
	}
}

func (d *dir) ReadDirAll(ctx context.Context) (ents []fuse.Dirent, err error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	d.logger.Printf("ReadDirAll: [%s]/%s", d.bucket.Name(), d.objectPrefix)

	// Ensure that we can use d.contents.
	if err := d.ensureContents(ctx); err != nil {
		err = fmt.Errorf("d.ensureContents: %v", err)
		return
	}

	// Read out entries from the contents map.
	for name, node := range d.contents {
		ent := fuse.Dirent{
			Name: name,
		}

		if _, ok := node.(*dir); ok {
			ent.Type = fuse.DT_Dir
		} else {
			ent.Type = fuse.DT_File
		}

		ents = append(ents, ent)
	}

	return
}

func (d *dir) Lookup(ctx context.Context, name string) (fusefs.Node, error) {
	d.mu.Lock()
	defer d.mu.Unlock()

	d.logger.Printf("Lookup: ([%s]/%s) %s", d.bucket.Name(), d.objectPrefix, name)

	// Ensure that we can use d.contents.
	if err := d.ensureContents(ctx); err != nil {
		err = fmt.Errorf("d.ensureContents: %v", err)
		return
	}

	// Find the object within the map.
	if n, ok := d.contents[name]; ok {
		return n, nil
	}

	return nil, fuse.ENOENT
}

func (d *dir) Create(
	ctx context.Context,
	req *fuse.CreateRequest,
	resp *fuse.CreateResponse) (
	fusefs.Node,
	fusefs.Handle,
	error) {
	// Tell fuse to use Mknod followed by Open, rather than re-implementing much
	// of Open here.
	return nil, nil, fuse.ENOSYS
}

func (d *dir) Mknod(
	ctx context.Context,
	req *fuse.MknodRequest) (fusefs.Node, error) {
	return nil, errors.New("TODO(jacobsa): Support Mknod.")
}
<|endoftext|>


<commit_before>package ctx

import (
	"fmt"
	"net"
	"os"
	"os/user"
	"runtime"
	"strconv"
	"strings"
)

func NamedZoneZkAddrs(zone string) string {
	parts := strings.Split(ZoneZkAddrs(zone), ",")
	addrs := make([]string, 0, len(parts))
	for _, p := range parts {
		ip, port, err := net.SplitHostPort(p)
		if err != nil {
			fmt.Printf("zone[%s]: %s\n", zone, err)
			os.Exit(1)
		}

		host, present := ReverseDnsLookup(ip, 0)
		if present {
			ip = host
		}

		addrs = append(addrs, net.JoinHostPort(ip, port))
	}
	return strings.Join(addrs, ",")
}

func ZoneZkAddrs(zone string) (zkAddrs string) {
	ensureLogLoaded()

	if z, present := conf.zones[zone]; present {
		return z.Zk
	}

	// should never happen
	fmt.Printf("zone[%s] undefined\n", zone)
	os.Exit(1)
	return ""
}

func Zones() map[string]string {
	ensureLogLoaded()

	r := make(map[string]string, len(conf.zones))
	for name, z := range conf.zones {
		r[name] = z.Zk
	}
	return r
}

func Zone(z string) *zone {
	ensureLogLoaded()

	return conf.zones[z]
}

func ZkDefaultZone() string {
	ensureLogLoaded()
	return conf.zkDefaultZone
}

func Hostname() string {
	ensureLogLoaded()
	return conf.hostname
}

func LogLevel() string {
	ensureLogLoaded()
	return conf.logLevel
}

// UpgradeCenter return the uri where to fetch gk/kguard/kateway/.gafka.cf files.
func UpgradeCenter() string {
	ensureLogLoaded()
	fromEnv := os.Getenv("UPGRADE_CENTER")
	if fromEnv != "" {
		return fromEnv
	}

	return conf.upgradeCenter
}

func ReverseDnsLookup(ip string, port int) (string, bool) {
	ensureLogLoaded()
	hosts, present := conf.reverseDns[ip]
	if !present || len(hosts) == 0 {
		return "", false
	}

	if port <= 0 {
		// ignore port as server name
		return hosts[0], present
	}

	// a single host has multiple services each of which has a different port
	// e,g. k[port][a-z].sit.mycorp.kfk.com/kafka  z[port][a-z].sit.mycorp.zk.com/zk
	for _, name := range hosts {
		p := strings.Split(name, ".")
		if strings.Contains(p[0], strconv.Itoa(port)) {
			return name, true
		}
	}

	return "", false
}

func KafkaHome() string {
	ensureLogLoaded()
	return conf.kafkaHome
}

func SortedZones() []string {
	ensureLogLoaded()
	return conf.sortedZones()
}

func Alias(cmd string) (alias string, present bool) {
	alias, present = conf.aliases[cmd]
	return
}

func Aliases() []string {
	r := make([]string, 0, len(conf.aliases))
	for cmd, _ := range conf.aliases {
		r = append(r, cmd)
	}
	return r
}

func AliasesWithValue() map[string]string {
	return conf.aliases
}

// LocalIP tries to determine a non-loopback address for the local machine
func LocalIP() (net.IP, error) {
	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return nil, err
	}
	for _, addr := range addrs {
		if ipnet, ok := addr.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
			if ipnet.IP.To4() != nil || ipnet.IP.To16() != nil {
				return ipnet.IP, nil
			}
		}
	}
	return nil, nil
}

func NumCPU() int {
	return runtime.NumCPU()
}

func NumCPUStr() string {
	return strconv.Itoa(NumCPU())
}

func CurrentUserIsRoot() bool {
	user, err := user.Current()
	if err != nil {
		return false
	}

	return user.Uid == "0"
}
<commit_msg>for better naming<commit_after>package ctx

import (
	"fmt"
	"net"
	"os"
	"os/user"
	"runtime"
	"strconv"
	"strings"
)

func NamedZoneZkAddrs(zone string) string {
	parts := strings.Split(ZoneZkAddrs(zone), ",")
	addrs := make([]string, 0, len(parts))
	for _, p := range parts {
		ip, port, err := net.SplitHostPort(p)
		if err != nil {
			fmt.Printf("zone[%s]: %s\n", zone, err)
			os.Exit(1)
		}

		host, present := ReverseDnsLookup(ip, 0)
		if present {
			ip = host
		}

		addrs = append(addrs, net.JoinHostPort(ip, port))
	}
	return strings.Join(addrs, ",")
}

func ZoneZkAddrs(zone string) (zkAddrs string) {
	ensureLogLoaded()

	if z, present := conf.zones[zone]; present {
		return z.Zk
	}

	// should never happen
	fmt.Printf("zone[%s] undefined\n", zone)
	os.Exit(1)
	return ""
}

func Zones() map[string]string {
	ensureLogLoaded()

	r := make(map[string]string, len(conf.zones))
	for name, z := range conf.zones {
		r[name] = z.Zk
	}
	return r
}

func Zone(z string) *zone {
	ensureLogLoaded()

	return conf.zones[z]
}

func ZkDefaultZone() string {
	ensureLogLoaded()
	return conf.zkDefaultZone
}

func DefaultZone() string {
	ensureLogLoaded()
	return conf.zkDefaultZone
}

func Hostname() string {
	ensureLogLoaded()
	return conf.hostname
}

func LogLevel() string {
	ensureLogLoaded()
	return conf.logLevel
}

// UpgradeCenter return the uri where to fetch gk/kguard/kateway/.gafka.cf files.
func UpgradeCenter() string {
	ensureLogLoaded()
	fromEnv := os.Getenv("UPGRADE_CENTER")
	if fromEnv != "" {
		return fromEnv
	}

	return conf.upgradeCenter
}

func ReverseDnsLookup(ip string, port int) (string, bool) {
	ensureLogLoaded()
	hosts, present := conf.reverseDns[ip]
	if !present || len(hosts) == 0 {
		return "", false
	}

	if port <= 0 {
		// ignore port as server name
		return hosts[0], present
	}

	// a single host has multiple services each of which has a different port
	// e,g. k[port][a-z].sit.mycorp.kfk.com/kafka  z[port][a-z].sit.mycorp.zk.com/zk
	for _, name := range hosts {
		p := strings.Split(name, ".")
		if strings.Contains(p[0], strconv.Itoa(port)) {
			return name, true
		}
	}

	return "", false
}

func KafkaHome() string {
	ensureLogLoaded()
	return conf.kafkaHome
}

func SortedZones() []string {
	ensureLogLoaded()
	return conf.sortedZones()
}

func Alias(cmd string) (alias string, present bool) {
	alias, present = conf.aliases[cmd]
	return
}

func Aliases() []string {
	r := make([]string, 0, len(conf.aliases))
	for cmd, _ := range conf.aliases {
		r = append(r, cmd)
	}
	return r
}

func AliasesWithValue() map[string]string {
	return conf.aliases
}

// LocalIP tries to determine a non-loopback address for the local machine
func LocalIP() (net.IP, error) {
	addrs, err := net.InterfaceAddrs()
	if err != nil {
		return nil, err
	}
	for _, addr := range addrs {
		if ipnet, ok := addr.(*net.IPNet); ok && ipnet.IP.IsGlobalUnicast() {
			if ipnet.IP.To4() != nil || ipnet.IP.To16() != nil {
				return ipnet.IP, nil
			}
		}
	}
	return nil, nil
}

func NumCPU() int {
	return runtime.NumCPU()
}

func NumCPUStr() string {
	return strconv.Itoa(NumCPU())
}

func CurrentUserIsRoot() bool {
	user, err := user.Current()
	if err != nil {
		return false
	}

	return user.Uid == "0"
}
<|endoftext|>