<commit_before>package command

import (
	"flag"
	"fmt"
	"net"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/funkygao/Go-Redis"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/gofmt"
	log "github.com/funkygao/log4go"
	"github.com/funkygao/termui"
	"github.com/pmylund/sortutil"
	"github.com/ryanuber/columnize"
)

type Redis struct {
	Ui  cli.Ui
	Cmd string

	mu       sync.Mutex
	topInfos []redisTopInfo
}

func (this *Redis) Run(args []string) (exitCode int) {
	var (
		zone   string
		add    string
		list   bool
		byHost int
		del    string
		top    bool
		ping   bool
	)
	cmdFlags := flag.NewFlagSet("redis", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&add, "add", "", "")
	cmdFlags.BoolVar(&list, "list", true, "")
	cmdFlags.IntVar(&byHost, "host", 0, "")
	cmdFlags.BoolVar(&top, "top", false, "")
	cmdFlags.BoolVar(&ping, "ping", false, "")
	cmdFlags.StringVar(&del, "del", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	if top || ping {
		list = false
	}

	if add != "" {
		host, port, err := net.SplitHostPort(add)
		swallow(err)

		nport, err := strconv.Atoi(port)
		swallow(err)
		zkzone.AddRedis(host, nport)
	} else if del != "" {
		host, port, err := net.SplitHostPort(del)
		swallow(err)

		nport, err := strconv.Atoi(port)
		swallow(err)
		zkzone.DelRedis(host, nport)
	} else {
		if top {
			this.runTop(zkzone)
		} else if ping {
			this.runPing(zkzone)
		} else if list {
			machineMap := make(map[string]struct{})
			machinePortMap := make(map[string][]string)
			var machines []string
			hostPorts := zkzone.AllRedis()
			sort.Strings(hostPorts)
			for _, hp := range hostPorts {
				host, port, _ := net.SplitHostPort(hp)
				ips, _ := net.LookupIP(host)
				ip := ips[0].String()
				if _, present := machineMap[ip]; !present {
					machineMap[ip] = struct{}{}
					machinePortMap[ip] = make([]string, 0)

					machines = append(machines, ip)
				}

				if byHost == 0 {
					this.Ui.Output(fmt.Sprintf("%35s %s", host, port))
				} else {
					machinePortMap[ip] = append(machinePortMap[ip], port)
				}

			}

			if byHost > 0 {
				sort.Strings(machines)
				for _, ip := range machines {
					sort.Strings(machinePortMap[ip])
					this.Ui.Info(fmt.Sprintf("%20s %2d ports", ip, len(machinePortMap[ip])))
					if byHost > 1 {
						this.Ui.Output(fmt.Sprintf("%+v", machinePortMap[ip]))
					}

				}
			}

			this.Ui.Output(fmt.Sprintf("Total instances:%d machines:%d", len(hostPorts), len(machines)))
		}
	}

	return
}

type redisTopInfo struct {
	host                       string
	port                       int
	dbsize, ops, rx, tx, conns int64
	t0                         time.Time
	latency                    time.Duration
}

func (this *Redis) runTop(zkzone *zk.ZkZone) {
	termui.Init()
	limit := termui.TermHeight() - 3
	termui.Close()
	this.topInfos = make([]redisTopInfo, 0, 100)
	for {
		var wg sync.WaitGroup
		this.topInfos = this.topInfos[:0]
		for _, hostPort := range zkzone.AllRedis() {
			host, port, err := net.SplitHostPort(hostPort)
			if err != nil {
				log.Error("invalid redis instance: %s", hostPort)
				continue
			}

			nport, err := strconv.Atoi(port)
			if err != nil || nport < 0 {
				log.Error("invalid redis instance: %s", hostPort)
				continue
			}

			wg.Add(1)
			go this.updateRedisInfo(&wg, host, nport)
		}
		wg.Wait()
		refreshScreen()

		sortutil.DescByField(this.topInfos, "ops")
		lines := []string{"Host|Port|dbsize|conns|ops|rx/bps|tx/bps"}

		for i := 0; i < min(limit, len(this.topInfos)); i++ {
			info := this.topInfos[i]
			lines = append(lines, fmt.Sprintf("%s|%d|%s|%s|%s|%s|%s",
				info.host, info.port,
				gofmt.Comma(info.dbsize), gofmt.Comma(info.conns), gofmt.Comma(info.ops),
				gofmt.ByteSize(info.rx*1024/8), gofmt.ByteSize(info.tx*1024/8)))
		}

		this.Ui.Output(columnize.SimpleFormat(lines))

		time.Sleep(time.Second * 5)
	}
}

func (this *Redis) updateRedisInfo(wg *sync.WaitGroup, host string, port int) {
	defer wg.Done()

	spec := redis.DefaultSpec().Host(host).Port(port)
	client, err := redis.NewSynchClientWithSpec(spec)
	if err != nil {
		return
	}
	defer client.Quit()

	infoMap, err := client.Info()
	if err != nil {
		return
	}

	dbSize, _ := client.Dbsize()
	conns, _ := strconv.ParseInt(infoMap["connected_clients"], 10, 64)
	ops, _ := strconv.ParseInt(infoMap["instantaneous_ops_per_sec"], 10, 64)
	rxKbps, _ := strconv.ParseFloat(infoMap["instantaneous_input_kbps"], 64)
	txKbps, _ := strconv.ParseFloat(infoMap["instantaneous_output_kbps"], 64)

	this.mu.Lock()
	this.topInfos = append(this.topInfos, redisTopInfo{
		host:   host,
		port:   port,
		dbsize: dbSize,
		ops:    ops,
		rx:     int64(rxKbps),
		tx:     int64(txKbps),
		conns:  conns,
	})
	this.mu.Unlock()
}

func (this *Redis) runPing(zkzone *zk.ZkZone) {
	var wg sync.WaitGroup
	allRedis := zkzone.AllRedis()
	this.topInfos = make([]redisTopInfo, 0, len(allRedis))

	for _, hostPort := range allRedis {
		host, port, err := net.SplitHostPort(hostPort)
		if err != nil {
			this.Ui.Error(hostPort)
			continue
		}

		nport, err := strconv.Atoi(port)
		if err != nil || nport < 0 {
			this.Ui.Error(hostPort)
			continue
		}

		wg.Add(1)
		go func(wg *sync.WaitGroup, host string, port int) {
			defer wg.Done()

			t0 := time.Now()

			spec := redis.DefaultSpec().Host(host).Port(port)
			client, err := redis.NewSynchClientWithSpec(spec)
			if err != nil {
				this.Ui.Error(fmt.Sprintf("[%s:%d] %v", host, port, err))
				return
			}
			defer client.Quit()

			if err := client.Ping(); err != nil {
				this.Ui.Error(fmt.Sprintf("[%s:%d] %v", host, port, err))
				return
			}

			latency := time.Since(t0)

			this.mu.Lock()
			this.topInfos = append(this.topInfos, redisTopInfo{
				host:    host,
				port:    port,
				t0:      t0,
				latency: latency,
			})
			this.mu.Unlock()
		}(&wg, host, nport)
	}
	wg.Wait()

	latency := metrics.NewRegisteredHistogram("redis.latency", metrics.DefaultRegistry, metrics.NewExpDecaySample(1028, 0.015))

	sortutil.AscByField(this.topInfos, "latency")
	lines := []string{"Host|Port|At|latency"}
	for _, info := range this.topInfos {
		latency.Update(info.latency.Nanoseconds() / 1e6)

		lines = append(lines, fmt.Sprintf("%s|%d|%s|%s",
			info.host, info.port, info.t0, info.latency))
	}
	this.Ui.Output(columnize.SimpleFormat(lines))

	// summary
	ps := latency.Percentiles([]float64{0.90, 0.95, 0.99, 0.999})
	this.Ui.Info(fmt.Sprintf("N:%d Min:%dms Max:%dms Mean:%.1fms 90%%:%.1fms 95%%:%.1fms 99%%:%.1fms",
		latency.Count(), latency.Min(), latency.Max(), latency.Mean(), ps[0], ps[1], ps[2]))
}

func (*Redis) Synopsis() string {
	return "Monitor redis instances"
}

func (this *Redis) Help() string {
	help := fmt.Sprintf(`
Usage: %s redis [options]

    %s

    -z zone

    -list

    -host 1|2
      Work with -list, print host instead of redis instance
      1: only display host
      2: 0 + port info

    -top
      Monitor all redis instances ops

    -ping
      Ping all redis instances

    -add host:port

    -del host:port

`, this.Cmd, this.Synopsis())
	return strings.TrimSpace(help)
}
<commit_msg>display summary of redis top<commit_after>package command

import (
	"flag"
	"fmt"
	"net"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/funkygao/Go-Redis"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/gofmt"
	log "github.com/funkygao/log4go"
	"github.com/funkygao/termui"
	"github.com/pmylund/sortutil"
	"github.com/ryanuber/columnize"
)

type Redis struct {
	Ui  cli.Ui
	Cmd string

	mu       sync.Mutex
	topInfos []redisTopInfo
}

func (this *Redis) Run(args []string) (exitCode int) {
	var (
		zone   string
		add    string
		list   bool
		byHost int
		del    string
		top    bool
		ping   bool
	)
	cmdFlags := flag.NewFlagSet("redis", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&zone, "z", ctx.ZkDefaultZone(), "")
	cmdFlags.StringVar(&add, "add", "", "")
	cmdFlags.BoolVar(&list, "list", true, "")
	cmdFlags.IntVar(&byHost, "host", 0, "")
	cmdFlags.BoolVar(&top, "top", false, "")
	cmdFlags.BoolVar(&ping, "ping", false, "")
	cmdFlags.StringVar(&del, "del", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	zkzone := zk.NewZkZone(zk.DefaultConfig(zone, ctx.ZoneZkAddrs(zone)))
	if top || ping {
		list = false
	}

	if add != "" {
		host, port, err := net.SplitHostPort(add)
		swallow(err)

		nport, err := strconv.Atoi(port)
		swallow(err)
		zkzone.AddRedis(host, nport)
	} else if del != "" {
		host, port, err := net.SplitHostPort(del)
		swallow(err)

		nport, err := strconv.Atoi(port)
		swallow(err)
		zkzone.DelRedis(host, nport)
	} else {
		if top {
			this.runTop(zkzone)
		} else if ping {
			this.runPing(zkzone)
		} else if list {
			machineMap := make(map[string]struct{})
			machinePortMap := make(map[string][]string)
			var machines []string
			hostPorts := zkzone.AllRedis()
			sort.Strings(hostPorts)
			for _, hp := range hostPorts {
				host, port, _ := net.SplitHostPort(hp)
				ips, _ := net.LookupIP(host)
				ip := ips[0].String()
				if _, present := machineMap[ip]; !present {
					machineMap[ip] = struct{}{}
					machinePortMap[ip] = make([]string, 0)

					machines = append(machines, ip)
				}

				if byHost == 0 {
					this.Ui.Output(fmt.Sprintf("%35s %s", host, port))
				} else {
					machinePortMap[ip] = append(machinePortMap[ip], port)
				}

			}

			if byHost > 0 {
				sort.Strings(machines)
				for _, ip := range machines {
					sort.Strings(machinePortMap[ip])
					this.Ui.Info(fmt.Sprintf("%20s %2d ports", ip, len(machinePortMap[ip])))
					if byHost > 1 {
						this.Ui.Output(fmt.Sprintf("%+v", machinePortMap[ip]))
					}

				}
			}

			this.Ui.Output(fmt.Sprintf("Total instances:%d machines:%d", len(hostPorts), len(machines)))
		}
	}

	return
}

type redisTopInfo struct {
	host                       string
	port                       int
	dbsize, ops, rx, tx, conns int64
	t0                         time.Time
	latency                    time.Duration
}

func (this *Redis) runTop(zkzone *zk.ZkZone) {
	termui.Init()
	limit := termui.TermHeight() - 4
	termui.Close()
	this.topInfos = make([]redisTopInfo, 0, 100)
	for {
		var wg sync.WaitGroup
		this.topInfos = this.topInfos[:0]
		for _, hostPort := range zkzone.AllRedis() {
			host, port, err := net.SplitHostPort(hostPort)
			if err != nil {
				log.Error("invalid redis instance: %s", hostPort)
				continue
			}

			nport, err := strconv.Atoi(port)
			if err != nil || nport < 0 {
				log.Error("invalid redis instance: %s", hostPort)
				continue
			}

			wg.Add(1)
			go this.updateRedisInfo(&wg, host, nport)
		}
		wg.Wait()
		refreshScreen()

		sortutil.DescByField(this.topInfos, "ops")
		lines := []string{"Host|Port|dbsize|conns|ops|rx/bps|tx/bps"}

		var (
			sumDbsize, sumConns, sumOps, sumRx, sumTx int64
		)
		for i := 0; i < min(limit, len(this.topInfos)); i++ {
			info := this.topInfos[i]
			lines = append(lines, fmt.Sprintf("%s|%d|%s|%s|%s|%s|%s",
				info.host, info.port,
				gofmt.Comma(info.dbsize), gofmt.Comma(info.conns), gofmt.Comma(info.ops),
				gofmt.ByteSize(info.rx*1024/8), gofmt.ByteSize(info.tx*1024/8)))

			sumDbsize += info.dbsize
			sumConns += info.conns
			sumOps += info.ops
			sumRx += info.rx * 1024 / 8
			sumTx += info.tx * 1024 / 8
		}
		lines = append(lines, fmt.Sprintf("-TOTAL-|-%d-|%s|%s|%s|%s|%s",
			len(this.topInfos),
			gofmt.Comma(sumDbsize), gofmt.Comma(sumConns), gofmt.Comma(sumOps),
			gofmt.ByteSize(sumRx), gofmt.ByteSize(sumTx)))

		this.Ui.Output(columnize.SimpleFormat(lines))

		time.Sleep(time.Second * 5)
	}
}

func (this *Redis) updateRedisInfo(wg *sync.WaitGroup, host string, port int) {
	defer wg.Done()

	spec := redis.DefaultSpec().Host(host).Port(port)
	client, err := redis.NewSynchClientWithSpec(spec)
	if err != nil {
		return
	}
	defer client.Quit()

	infoMap, err := client.Info()
	if err != nil {
		return
	}

	dbSize, _ := client.Dbsize()
	conns, _ := strconv.ParseInt(infoMap["connected_clients"], 10, 64)
	ops, _ := strconv.ParseInt(infoMap["instantaneous_ops_per_sec"], 10, 64)
	rxKbps, _ := strconv.ParseFloat(infoMap["instantaneous_input_kbps"], 64)
	txKbps, _ := strconv.ParseFloat(infoMap["instantaneous_output_kbps"], 64)

	this.mu.Lock()
	this.topInfos = append(this.topInfos, redisTopInfo{
		host:   host,
		port:   port,
		dbsize: dbSize,
		ops:    ops,
		rx:     int64(rxKbps),
		tx:     int64(txKbps),
		conns:  conns,
	})
	this.mu.Unlock()
}

func (this *Redis) runPing(zkzone *zk.ZkZone) {
	var wg sync.WaitGroup
	allRedis := zkzone.AllRedis()
	this.topInfos = make([]redisTopInfo, 0, len(allRedis))

	for _, hostPort := range allRedis {
		host, port, err := net.SplitHostPort(hostPort)
		if err != nil {
			this.Ui.Error(hostPort)
			continue
		}

		nport, err := strconv.Atoi(port)
		if err != nil || nport < 0 {
			this.Ui.Error(hostPort)
			continue
		}

		wg.Add(1)
		go func(wg *sync.WaitGroup, host string, port int) {
			defer wg.Done()

			t0 := time.Now()

			spec := redis.DefaultSpec().Host(host).Port(port)
			client, err := redis.NewSynchClientWithSpec(spec)
			if err != nil {
				this.Ui.Error(fmt.Sprintf("[%s:%d] %v", host, port, err))
				return
			}
			defer client.Quit()

			if err := client.Ping(); err != nil {
				this.Ui.Error(fmt.Sprintf("[%s:%d] %v", host, port, err))
				return
			}

			latency := time.Since(t0)

			this.mu.Lock()
			this.topInfos = append(this.topInfos, redisTopInfo{
				host:    host,
				port:    port,
				t0:      t0,
				latency: latency,
			})
			this.mu.Unlock()
		}(&wg, host, nport)
	}
	wg.Wait()

	latency := metrics.NewRegisteredHistogram("redis.latency", metrics.DefaultRegistry, metrics.NewExpDecaySample(1028, 0.015))

	sortutil.AscByField(this.topInfos, "latency")
	lines := []string{"Host|Port|At|latency"}
	for _, info := range this.topInfos {
		latency.Update(info.latency.Nanoseconds() / 1e6)

		lines = append(lines, fmt.Sprintf("%s|%d|%s|%s",
			info.host, info.port, info.t0, info.latency))
	}
	this.Ui.Output(columnize.SimpleFormat(lines))

	// summary
	ps := latency.Percentiles([]float64{0.90, 0.95, 0.99, 0.999})
	this.Ui.Info(fmt.Sprintf("N:%d Min:%dms Max:%dms Mean:%.1fms 90%%:%.1fms 95%%:%.1fms 99%%:%.1fms",
		latency.Count(), latency.Min(), latency.Max(), latency.Mean(), ps[0], ps[1], ps[2]))
}

func (*Redis) Synopsis() string {
	return "Monitor redis instances"
}

func (this *Redis) Help() string {
	help := fmt.Sprintf(`
Usage: %s redis [options]

    %s

    -z zone

    -list

    -host 1|2
      Work with -list, print host instead of redis instance
      1: only display host
      2: 0 + port info

    -top
      Monitor all redis instances ops

    -ping
      Ping all redis instances

    -add host:port

    -del host:port

`, this.Cmd, this.Synopsis())
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main_test

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"testing"
	"time"
)

var godocTests = []struct {
	args      []string
	matches   []string // regular expressions
	dontmatch []string // regular expressions
}{
	{
		args: []string{"fmt"},
		matches: []string{
			`import "fmt"`,
			`Package fmt implements formatted I/O`,
		},
	},
	{
		args: []string{"io", "WriteString"},
		matches: []string{
			`func WriteString\(`,
			`WriteString writes the contents of the string s to w`,
		},
	},
	{
		args: []string{"nonexistingpkg"},
		matches: []string{
			`no such file or directory|does not exist|cannot find the file`,
		},
	},
	{
		args: []string{"fmt", "NonexistentSymbol"},
		matches: []string{
			`No match found\.`,
		},
	},
	{
		args: []string{"-src", "syscall", "Open"},
		matches: []string{
			`func Open\(`,
		},
		dontmatch: []string{
			`No match found\.`,
		},
	},
}

// buildGodoc builds the godoc executable.
// It returns its path, and a cleanup function.
//
// TODO(adonovan): opt: do this at most once, and do the cleanup
// exactly once.  How though?  There's no atexit.
func buildGodoc(t *testing.T) (bin string, cleanup func()) {
	tmp, err := ioutil.TempDir("", "godoc-regtest-")
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		if cleanup == nil { // probably, go build failed.
			os.RemoveAll(tmp)
		}
	}()

	bin = filepath.Join(tmp, "godoc")
	if runtime.GOOS == "windows" {
		bin += ".exe"
	}
	cmd := exec.Command("go", "build", "-o", bin)
	if err := cmd.Run(); err != nil {
		t.Fatalf("Building godoc: %v", err)
	}

	return bin, func() { os.RemoveAll(tmp) }
}

// Basic regression test for godoc command-line tool.
func TestCLI(t *testing.T) {
	bin, cleanup := buildGodoc(t)
	defer cleanup()
	for _, test := range godocTests {
		cmd := exec.Command(bin, test.args...)
		cmd.Args[0] = "godoc"
		out, err := cmd.CombinedOutput()
		if err != nil {
			t.Errorf("Running with args %#v: %v", test.args, err)
			continue
		}
		for _, pat := range test.matches {
			re := regexp.MustCompile(pat)
			if !re.Match(out) {
				t.Errorf("godoc %v =\n%s\nwanted /%v/", strings.Join(test.args, " "), out, pat)
			}
		}
		for _, pat := range test.dontmatch {
			re := regexp.MustCompile(pat)
			if re.Match(out) {
				t.Errorf("godoc %v =\n%s\ndid not want /%v/", strings.Join(test.args, " "), out, pat)
			}
		}
	}
}

func serverAddress(t *testing.T) string {
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		ln, err = net.Listen("tcp6", "[::1]:0")
	}
	if err != nil {
		t.Fatal(err)
	}
	defer ln.Close()
	return ln.Addr().String()
}

const (
	startTimeout = 5 * time.Minute
	pollInterval = 200 * time.Millisecond
)

var indexingMsg = []byte("Indexing in progress: result may be inaccurate")

func waitForServer(t *testing.T, address string) {
	// "health check" duplicated from x/tools/cmd/tipgodoc/tip.go
	deadline := time.Now().Add(startTimeout)
	for time.Now().Before(deadline) {
		time.Sleep(pollInterval)
		res, err := http.Get(fmt.Sprintf("http://%v/search?q=FALLTHROUGH", address))
		if err != nil {
			continue
		}
		rbody, err := ioutil.ReadAll(res.Body)
		res.Body.Close()
		if err == nil && res.StatusCode == http.StatusOK &&
			!bytes.Contains(rbody, indexingMsg) {
			return
		}
	}
	t.Fatalf("Server %q failed to respond in %v", address, startTimeout)
}

func killAndWait(cmd *exec.Cmd) {
	cmd.Process.Kill()
	cmd.Wait()
}

// Basic integration test for godoc HTTP interface.
func TestWeb(t *testing.T) {
	bin, cleanup := buildGodoc(t)
	defer cleanup()
	addr := serverAddress(t)
	cmd := exec.Command(bin, fmt.Sprintf("-http=%s", addr), "-index", "-index_interval=-1s")
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	cmd.Args[0] = "godoc"
	if err := cmd.Start(); err != nil {
		t.Fatalf("failed to start godoc: %s", err)
	}
	defer killAndWait(cmd)
	waitForServer(t, addr)
	tests := []struct {
		path      string
		match     []string
		dontmatch []string
	}{
		{
			path:  "/",
			match: []string{"Go is an open source programming language"},
		},
		{
			path:  "/pkg/fmt/",
			match: []string{"Package fmt implements formatted I/O"},
		},
		{
			path:  "/src/fmt/",
			match: []string{"scan_test.go"},
		},
		{
			path:  "/src/fmt/print.go",
			match: []string{"// Println formats using"},
		},
		{
			path: "/pkg",
			match: []string{
				"Standard library",
				"Package fmt implements formatted I/O",
			},
			dontmatch: []string{
				"internal/syscall",
				"cmd/gc",
			},
		},
		{
			path: "/pkg/?m=all",
			match: []string{
				"Standard library",
				"Package fmt implements formatted I/O",
				"internal/syscall",
			},
			dontmatch: []string{
				"cmd/gc",
			},
		},
		{
			path: "/search?q=notwithstanding",
			match: []string{
				"/src",
			},
			dontmatch: []string{
				"/pkg/bootstrap",
			},
		},
	}
	for _, test := range tests {
		url := fmt.Sprintf("http://%s%s", addr, test.path)
		resp, err := http.Get(url)
		if err != nil {
			t.Errorf("GET %s failed: %s", url, err)
			continue
		}
		body, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			t.Errorf("GET %s: failed to read body: %s (response: %v)", url, err, resp)
		}
		isErr := false
		for _, substr := range test.match {
			if !bytes.Contains(body, []byte(substr)) {
				t.Errorf("GET %s: wanted substring %q in body", url, substr)
				isErr = true
			}
		}
		for _, substr := range test.dontmatch {
			if bytes.Contains(body, []byte(substr)) {
				t.Errorf("GET %s: didn't want substring %q in body", url, substr)
				isErr = true
			}
		}
		if isErr {
			t.Errorf("GET %s: got:\n%s", url, body)
		}
	}
}

// Basic integration test for godoc -analysis=type (via HTTP interface).
func TestTypeAnalysis(t *testing.T) {
	// Write a fake GOROOT/GOPATH.
	tmpdir, err := ioutil.TempDir("", "godoc-analysis")
	if err != nil {
		t.Fatalf("ioutil.TempDir failed: %s", err)
	}
	defer os.RemoveAll(tmpdir)
	for _, f := range []struct{ file, content string }{
		{"goroot/src/lib/lib.go", `
package lib
type T struct{}
const C = 3
var V T
func (T) F() int { return C }
`},
		{"gopath/src/app/main.go", `
package main
import "lib"
func main() { print(lib.V) }
`},
	} {
		file := filepath.Join(tmpdir, f.file)
		if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil {
			t.Fatalf("MkdirAll(%s) failed: %s", filepath.Dir(file), err)
		}
		if err := ioutil.WriteFile(file, []byte(f.content), 0644); err != nil {
			t.Fatal(err)
		}
	}

	// Start the server.
	bin, cleanup := buildGodoc(t)
	defer cleanup()
	addr := serverAddress(t)
	cmd := exec.Command(bin, fmt.Sprintf("-http=%s", addr), "-analysis=type")
	cmd.Env = append(cmd.Env, fmt.Sprintf("GOROOT=%s", filepath.Join(tmpdir, "goroot")))
	cmd.Env = append(cmd.Env, fmt.Sprintf("GOPATH=%s", filepath.Join(tmpdir, "gopath")))
	for _, e := range os.Environ() {
		if strings.HasPrefix(e, "GOROOT=") || strings.HasPrefix(e, "GOPATH=") {
			continue
		}
		cmd.Env = append(cmd.Env, e)
	}
	cmd.Stdout = os.Stderr
	stderr, err := cmd.StderrPipe()
	if err != nil {
		t.Fatal(err)
	}
	cmd.Args[0] = "godoc"
	if err := cmd.Start(); err != nil {
		t.Fatalf("failed to start godoc: %s", err)
	}
	defer killAndWait(cmd)
	waitForServer(t, addr)

	// Wait for type analysis to complete.
	reader := bufio.NewReader(stderr)
	for {
		s, err := reader.ReadString('\n')
		if err != nil {
			t.Fatal(err)
		}
		fmt.Fprint(os.Stderr, s)
		if strings.Contains(s, "Type analysis complete.") {
			break
		}
	}
	go io.Copy(os.Stderr, reader)

	t0 := time.Now()

	// Make an HTTP request and check for a regular expression match.
	// The patterns are very crude checks that basic type information
	// has been annotated onto the source view.
tryagain:
	for _, test := range []struct{ url, pattern string }{
		{"/src/lib/lib.go", "L2.*package .*Package docs for lib.*/lib"},
		{"/src/lib/lib.go", "L3.*type .*type info for T.*struct"},
		{"/src/lib/lib.go", "L5.*var V .*type T struct"},
		{"/src/lib/lib.go", "L6.*func .*type T struct.*T.*return .*const C untyped int.*C"},

		{"/src/app/main.go", "L2.*package .*Package docs for app"},
		{"/src/app/main.go", "L3.*import .*Package docs for lib.*lib"},
		{"/src/app/main.go", "L4.*func main.*package lib.*lib.*var lib.V lib.T.*V"},
	} {
		url := fmt.Sprintf("http://%s%s", addr, test.url)
		resp, err := http.Get(url)
		if err != nil {
			t.Errorf("GET %s failed: %s", url, err)
			continue
		}
		body, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			t.Errorf("GET %s: failed to read body: %s (response: %v)", url, err, resp)
			continue
		}

		if !bytes.Contains(body, []byte("Static analysis features")) {
			// Type analysis results usually become available within
			// ~4ms after godoc startup (for this input on my machine).
			if elapsed := time.Since(t0); elapsed > 500*time.Millisecond {
				t.Fatalf("type analysis results still unavailable after %s", elapsed)
			}
			time.Sleep(10 * time.Millisecond)
			goto tryagain
		}

		match, err := regexp.Match(test.pattern, body)
		if err != nil {
			t.Errorf("regexp.Match(%q) failed: %s", test.pattern, err)
			continue
		}
		if !match {
			// This is a really ugly failure message.
			t.Errorf("GET %s: body doesn't match %q, got:\n%s",
				url, test.pattern, string(body))
		}
	}
}
<commit_msg>cmd/godoc: skip tests on arm platforms<commit_after>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main_test

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"testing"
	"time"
)

var godocTests = []struct {
	args      []string
	matches   []string // regular expressions
	dontmatch []string // regular expressions
}{
	{
		args: []string{"fmt"},
		matches: []string{
			`import "fmt"`,
			`Package fmt implements formatted I/O`,
		},
	},
	{
		args: []string{"io", "WriteString"},
		matches: []string{
			`func WriteString\(`,
			`WriteString writes the contents of the string s to w`,
		},
	},
	{
		args: []string{"nonexistingpkg"},
		matches: []string{
			`no such file or directory|does not exist|cannot find the file`,
		},
	},
	{
		args: []string{"fmt", "NonexistentSymbol"},
		matches: []string{
			`No match found\.`,
		},
	},
	{
		args: []string{"-src", "syscall", "Open"},
		matches: []string{
			`func Open\(`,
		},
		dontmatch: []string{
			`No match found\.`,
		},
	},
}

// buildGodoc builds the godoc executable.
// It returns its path, and a cleanup function.
//
// TODO(adonovan): opt: do this at most once, and do the cleanup
// exactly once.  How though?  There's no atexit.
func buildGodoc(t *testing.T) (bin string, cleanup func()) {
	if runtime.GOARCH == "arm" {
		t.Skip("skipping test on arm platforms; too slow")
	}
	tmp, err := ioutil.TempDir("", "godoc-regtest-")
	if err != nil {
		t.Fatal(err)
	}
	defer func() {
		if cleanup == nil { // probably, go build failed.
			os.RemoveAll(tmp)
		}
	}()

	bin = filepath.Join(tmp, "godoc")
	if runtime.GOOS == "windows" {
		bin += ".exe"
	}
	cmd := exec.Command("go", "build", "-o", bin)
	if err := cmd.Run(); err != nil {
		t.Fatalf("Building godoc: %v", err)
	}

	return bin, func() { os.RemoveAll(tmp) }
}

// Basic regression test for godoc command-line tool.
func TestCLI(t *testing.T) {
	bin, cleanup := buildGodoc(t)
	defer cleanup()
	for _, test := range godocTests {
		cmd := exec.Command(bin, test.args...)
		cmd.Args[0] = "godoc"
		out, err := cmd.CombinedOutput()
		if err != nil {
			t.Errorf("Running with args %#v: %v", test.args, err)
			continue
		}
		for _, pat := range test.matches {
			re := regexp.MustCompile(pat)
			if !re.Match(out) {
				t.Errorf("godoc %v =\n%s\nwanted /%v/", strings.Join(test.args, " "), out, pat)
			}
		}
		for _, pat := range test.dontmatch {
			re := regexp.MustCompile(pat)
			if re.Match(out) {
				t.Errorf("godoc %v =\n%s\ndid not want /%v/", strings.Join(test.args, " "), out, pat)
			}
		}
	}
}

func serverAddress(t *testing.T) string {
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		ln, err = net.Listen("tcp6", "[::1]:0")
	}
	if err != nil {
		t.Fatal(err)
	}
	defer ln.Close()
	return ln.Addr().String()
}

const (
	startTimeout = 5 * time.Minute
	pollInterval = 200 * time.Millisecond
)

var indexingMsg = []byte("Indexing in progress: result may be inaccurate")

func waitForServer(t *testing.T, address string) {
	// "health check" duplicated from x/tools/cmd/tipgodoc/tip.go
	deadline := time.Now().Add(startTimeout)
	for time.Now().Before(deadline) {
		time.Sleep(pollInterval)
		res, err := http.Get(fmt.Sprintf("http://%v/search?q=FALLTHROUGH", address))
		if err != nil {
			continue
		}
		rbody, err := ioutil.ReadAll(res.Body)
		res.Body.Close()
		if err == nil && res.StatusCode == http.StatusOK &&
			!bytes.Contains(rbody, indexingMsg) {
			return
		}
	}
	t.Fatalf("Server %q failed to respond in %v", address, startTimeout)
}

func killAndWait(cmd *exec.Cmd) {
	cmd.Process.Kill()
	cmd.Wait()
}

// Basic integration test for godoc HTTP interface.
func TestWeb(t *testing.T) {
	bin, cleanup := buildGodoc(t)
	defer cleanup()
	addr := serverAddress(t)
	cmd := exec.Command(bin, fmt.Sprintf("-http=%s", addr), "-index", "-index_interval=-1s")
	cmd.Stdout = os.Stderr
	cmd.Stderr = os.Stderr
	cmd.Args[0] = "godoc"
	if err := cmd.Start(); err != nil {
		t.Fatalf("failed to start godoc: %s", err)
	}
	defer killAndWait(cmd)
	waitForServer(t, addr)
	tests := []struct {
		path      string
		match     []string
		dontmatch []string
	}{
		{
			path:  "/",
			match: []string{"Go is an open source programming language"},
		},
		{
			path:  "/pkg/fmt/",
			match: []string{"Package fmt implements formatted I/O"},
		},
		{
			path:  "/src/fmt/",
			match: []string{"scan_test.go"},
		},
		{
			path:  "/src/fmt/print.go",
			match: []string{"// Println formats using"},
		},
		{
			path: "/pkg",
			match: []string{
				"Standard library",
				"Package fmt implements formatted I/O",
			},
			dontmatch: []string{
				"internal/syscall",
				"cmd/gc",
			},
		},
		{
			path: "/pkg/?m=all",
			match: []string{
				"Standard library",
				"Package fmt implements formatted I/O",
				"internal/syscall",
			},
			dontmatch: []string{
				"cmd/gc",
			},
		},
		{
			path: "/search?q=notwithstanding",
			match: []string{
				"/src",
			},
			dontmatch: []string{
				"/pkg/bootstrap",
			},
		},
	}
	for _, test := range tests {
		url := fmt.Sprintf("http://%s%s", addr, test.path)
		resp, err := http.Get(url)
		if err != nil {
			t.Errorf("GET %s failed: %s", url, err)
			continue
		}
		body, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			t.Errorf("GET %s: failed to read body: %s (response: %v)", url, err, resp)
		}
		isErr := false
		for _, substr := range test.match {
			if !bytes.Contains(body, []byte(substr)) {
				t.Errorf("GET %s: wanted substring %q in body", url, substr)
				isErr = true
			}
		}
		for _, substr := range test.dontmatch {
			if bytes.Contains(body, []byte(substr)) {
				t.Errorf("GET %s: didn't want substring %q in body", url, substr)
				isErr = true
			}
		}
		if isErr {
			t.Errorf("GET %s: got:\n%s", url, body)
		}
	}
}

// Basic integration test for godoc -analysis=type (via HTTP interface).
func TestTypeAnalysis(t *testing.T) {
	// Write a fake GOROOT/GOPATH.
	tmpdir, err := ioutil.TempDir("", "godoc-analysis")
	if err != nil {
		t.Fatalf("ioutil.TempDir failed: %s", err)
	}
	defer os.RemoveAll(tmpdir)
	for _, f := range []struct{ file, content string }{
		{"goroot/src/lib/lib.go", `
package lib
type T struct{}
const C = 3
var V T
func (T) F() int { return C }
`},
		{"gopath/src/app/main.go", `
package main
import "lib"
func main() { print(lib.V) }
`},
	} {
		file := filepath.Join(tmpdir, f.file)
		if err := os.MkdirAll(filepath.Dir(file), 0755); err != nil {
			t.Fatalf("MkdirAll(%s) failed: %s", filepath.Dir(file), err)
		}
		if err := ioutil.WriteFile(file, []byte(f.content), 0644); err != nil {
			t.Fatal(err)
		}
	}

	// Start the server.
	bin, cleanup := buildGodoc(t)
	defer cleanup()
	addr := serverAddress(t)
	cmd := exec.Command(bin, fmt.Sprintf("-http=%s", addr), "-analysis=type")
	cmd.Env = append(cmd.Env, fmt.Sprintf("GOROOT=%s", filepath.Join(tmpdir, "goroot")))
	cmd.Env = append(cmd.Env, fmt.Sprintf("GOPATH=%s", filepath.Join(tmpdir, "gopath")))
	for _, e := range os.Environ() {
		if strings.HasPrefix(e, "GOROOT=") || strings.HasPrefix(e, "GOPATH=") {
			continue
		}
		cmd.Env = append(cmd.Env, e)
	}
	cmd.Stdout = os.Stderr
	stderr, err := cmd.StderrPipe()
	if err != nil {
		t.Fatal(err)
	}
	cmd.Args[0] = "godoc"
	if err := cmd.Start(); err != nil {
		t.Fatalf("failed to start godoc: %s", err)
	}
	defer killAndWait(cmd)
	waitForServer(t, addr)

	// Wait for type analysis to complete.
	reader := bufio.NewReader(stderr)
	for {
		s, err := reader.ReadString('\n')
		if err != nil {
			t.Fatal(err)
		}
		fmt.Fprint(os.Stderr, s)
		if strings.Contains(s, "Type analysis complete.") {
			break
		}
	}
	go io.Copy(os.Stderr, reader)

	t0 := time.Now()

	// Make an HTTP request and check for a regular expression match.
	// The patterns are very crude checks that basic type information
	// has been annotated onto the source view.
tryagain:
	for _, test := range []struct{ url, pattern string }{
		{"/src/lib/lib.go", "L2.*package .*Package docs for lib.*/lib"},
		{"/src/lib/lib.go", "L3.*type .*type info for T.*struct"},
		{"/src/lib/lib.go", "L5.*var V .*type T struct"},
		{"/src/lib/lib.go", "L6.*func .*type T struct.*T.*return .*const C untyped int.*C"},

		{"/src/app/main.go", "L2.*package .*Package docs for app"},
		{"/src/app/main.go", "L3.*import .*Package docs for lib.*lib"},
		{"/src/app/main.go", "L4.*func main.*package lib.*lib.*var lib.V lib.T.*V"},
	} {
		url := fmt.Sprintf("http://%s%s", addr, test.url)
		resp, err := http.Get(url)
		if err != nil {
			t.Errorf("GET %s failed: %s", url, err)
			continue
		}
		body, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()
		if err != nil {
			t.Errorf("GET %s: failed to read body: %s (response: %v)", url, err, resp)
			continue
		}

		if !bytes.Contains(body, []byte("Static analysis features")) {
			// Type analysis results usually become available within
			// ~4ms after godoc startup (for this input on my machine).
			if elapsed := time.Since(t0); elapsed > 500*time.Millisecond {
				t.Fatalf("type analysis results still unavailable after %s", elapsed)
			}
			time.Sleep(10 * time.Millisecond)
			goto tryagain
		}

		match, err := regexp.Match(test.pattern, body)
		if err != nil {
			t.Errorf("regexp.Match(%q) failed: %s", test.pattern, err)
			continue
		}
		if !match {
			// This is a really ugly failure message.
			t.Errorf("GET %s: body doesn't match %q, got:\n%s",
				url, test.pattern, string(body))
		}
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2021 The GoPlus Authors (goplus.org). All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package run implements the ``gop run'' command.
package run

import (
	"fmt"
	"os"
	"os/exec"

	"github.com/goplus/gop/cl"
	"github.com/goplus/gop/cmd/internal/base"
	"github.com/goplus/gop/x/gopproj"
	"github.com/goplus/gop/x/gopprojs"
	"github.com/goplus/gox"
	"github.com/qiniu/x/log"
)

// gop run
var Cmd = &base.Command{
	UsageLine: "gop run [-asm -quiet -debug -nr -gop -prof] package [arguments...]",
	Short:     "Run a Go+ program",
}

var (
	flag        = &Cmd.Flag
	flagAsm     = flag.Bool("asm", false, "generates `asm` code of Go+ bytecode backend")
	flagVerbose = flag.Bool("v", false, "print verbose information")
	flagQuiet   = flag.Bool("quiet", false, "don't generate any compiling stage log")
	flagDebug   = flag.Bool("debug", false, "set log level to debug")
	flagNorun   = flag.Bool("nr", false, "don't run if no change")
	flagRTOE    = flag.Bool("rtoe", false, "remove tempfile on error")
	flagGop     = flag.Bool("gop", false, "parse a .go file as a .gop file")
	flagProf    = flag.Bool("prof", false, "do profile and generate profile report")
)

func init() {
	Cmd.Run = runCmd
}

func runCmd(cmd *base.Command, args []string) {
	err := flag.Parse(args)
	if err != nil {
		log.Fatalln("parse input arguments failed:", err)
	}
	if flag.NArg() < 1 {
		cmd.Usage(os.Stderr)
	}
	gopRun(flag.Args())
}

func gopRun(args []string) {
	proj, args, err := gopprojs.ParseOne(args...)
	if err != nil {
		log.Fatalln(err)
	}

	if *flagQuiet {
		log.SetOutputLevel(0x7000)
	} else if *flagDebug {
		log.SetOutputLevel(log.Ldebug)
		gox.SetDebug(gox.DbgFlagAll)
		cl.SetDebug(cl.DbgFlagAll)
	}
	if *flagVerbose {
		gox.SetDebug(gox.DbgFlagAll &^ gox.DbgFlagComments)
		cl.SetDebug(cl.DbgFlagAll)
		cl.SetDisableRecover(true)
	} else if *flagAsm {
		gox.SetDebug(gox.DbgFlagInstruction)
	}
	if *flagProf {
		panic("TODO: profile not impl")
	}

	flags := 0
	if *flagGop {
		flags = gopproj.FlagGoAsGoPlus
	}
	wd, err := os.Getwd()
	if err != nil {
		fmt.Fprintln(os.Stderr, "Get current directory failed:", err)
		os.Exit(1)
	}
	ctx, goProj, err := gopproj.OpenProject(flags, proj)
	if err != nil {
		fmt.Fprintln(os.Stderr, "OpenProject failed:", err)
		os.Exit(1)
	}
	goProj.ExecArgs = args
	goProj.FlagNRINC = *flagNorun
	goProj.FlagRTOE = *flagRTOE
	if goProj.FlagRTOE {
		goProj.UseDefaultCtx = true
	}
	cmd := ctx.GoCommand("run", goProj)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Env = os.Environ()
	cmd.Dir = wd
	err = cmd.Run()
	if err != nil {
		switch e := err.(type) {
		case *exec.ExitError:
			os.Exit(e.ExitCode())
		default:
			log.Fatalln(err)
		}
	}
}

// -----------------------------------------------------------------------------
<commit_msg>reset gop run dir<commit_after>/*
 * Copyright (c) 2021 The GoPlus Authors (goplus.org). All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Package run implements the ``gop run'' command.
package run

import (
	"fmt"
	"os"
	"os/exec"

	"github.com/goplus/gop/cl"
	"github.com/goplus/gop/cmd/internal/base"
	"github.com/goplus/gop/x/gopproj"
	"github.com/goplus/gop/x/gopprojs"
	"github.com/goplus/gox"
	"github.com/qiniu/x/log"
)

// gop run
var Cmd = &base.Command{
	UsageLine: "gop run [-asm -quiet -debug -nr -gop -prof] package [arguments...]",
	Short:     "Run a Go+ program",
}

var (
	flag        = &Cmd.Flag
	flagAsm     = flag.Bool("asm", false, "generates `asm` code of Go+ bytecode backend")
	flagVerbose = flag.Bool("v", false, "print verbose information")
	flagQuiet   = flag.Bool("quiet", false, "don't generate any compiling stage log")
	flagDebug   = flag.Bool("debug", false, "set log level to debug")
	flagNorun   = flag.Bool("nr", false, "don't run if no change")
	flagRTOE    = flag.Bool("rtoe", false, "remove tempfile on error")
	flagGop     = flag.Bool("gop", false, "parse a .go file as a .gop file")
	flagProf    = flag.Bool("prof", false, "do profile and generate profile report")
)

func init() {
	Cmd.Run = runCmd
}

func runCmd(cmd *base.Command, args []string) {
	err := flag.Parse(args)
	if err != nil {
		log.Fatalln("parse input arguments failed:", err)
	}
	if flag.NArg() < 1 {
		cmd.Usage(os.Stderr)
	}
	gopRun(flag.Args())
}

func gopRun(args []string) {
	proj, args, err := gopprojs.ParseOne(args...)
	if err != nil {
		log.Fatalln(err)
	}

	if *flagQuiet {
		log.SetOutputLevel(0x7000)
	} else if *flagDebug {
		log.SetOutputLevel(log.Ldebug)
		gox.SetDebug(gox.DbgFlagAll)
		cl.SetDebug(cl.DbgFlagAll)
	}
	if *flagVerbose {
		gox.SetDebug(gox.DbgFlagAll &^ gox.DbgFlagComments)
		cl.SetDebug(cl.DbgFlagAll)
		cl.SetDisableRecover(true)
	} else if *flagAsm {
		gox.SetDebug(gox.DbgFlagInstruction)
	}
	if *flagProf {
		panic("TODO: profile not impl")
	}

	flags := 0
	if *flagGop {
		flags = gopproj.FlagGoAsGoPlus
	}
	ctx, goProj, err := gopproj.OpenProject(flags, proj)
	if err != nil {
		fmt.Fprintln(os.Stderr, "OpenProject failed:", err)
		os.Exit(1)
	}
	goProj.ExecArgs = args
	goProj.FlagNRINC = *flagNorun
	goProj.FlagRTOE = *flagRTOE
	if goProj.FlagRTOE {
		goProj.UseDefaultCtx = true
	}
	cmd := ctx.GoCommand("run", goProj)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	cmd.Env = os.Environ()
	err = cmd.Run()
	if err != nil {
		switch e := err.(type) {
		case *exec.ExitError:
			os.Exit(e.ExitCode())
		default:
			log.Fatalln(err)
		}
	}
}

// -----------------------------------------------------------------------------
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"math/rand"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/spf13/cobra"

	"github.com/restic/restic/internal/cache"
	"github.com/restic/restic/internal/checker"
	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/fs"
	"github.com/restic/restic/internal/restic"
)

var cmdCheck = &cobra.Command{
	Use:   "check [flags]",
	Short: "Check the repository for errors",
	Long: `
The "check" command tests the repository for errors and reports any errors it
finds. It can also be used to read all data and therefore simulate a restore.

By default, the "check" command will always load all data directly from the
repository and not use a local cache.

EXIT STATUS
===========

Exit status is 0 if the command was successful, and non-zero if there was any error.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runCheck(checkOptions, globalOptions, args)
	},
	PreRunE: func(cmd *cobra.Command, args []string) error {
		return checkFlags(checkOptions)
	},
}

// CheckOptions bundles all options for the 'check' command.
type CheckOptions struct {
	ReadData       bool
	ReadDataSubset string
	CheckUnused    bool
	WithCache      bool
}

var checkOptions CheckOptions

func init() {
	cmdRoot.AddCommand(cmdCheck)

	f := cmdCheck.Flags()
	f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset")
	f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
	f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache")
}

func checkFlags(opts CheckOptions) error {
	if opts.ReadData && opts.ReadDataSubset != "" {
		return errors.Fatal("check flags --read-data and --read-data-subset cannot be used together")
	}
	if opts.ReadDataSubset != "" {
		dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
		argumentError := errors.Fatal("check flag --read-data-subset has invalid value, please see documentation")
		if err == nil {
			if len(dataSubset) != 2 {
				return argumentError
			}
			if dataSubset[0] == 0 || dataSubset[1] == 0 || dataSubset[0] > dataSubset[1] {
				return errors.Fatal("check flag --read-data-subset=n/t values must be positive integers, and n <= t, e.g. --read-data-subset=1/2")
			}
			if dataSubset[1] > totalBucketsMax {
				return errors.Fatalf("check flag --read-data-subset=n/t t must be at most %d", totalBucketsMax)
			}
		} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
			percentage, err := parsePercentage(opts.ReadDataSubset)
			if err != nil {
				return argumentError
			}

			if percentage <= 0.0 || percentage > 100.0 {
				return errors.Fatal(
					"check flag --read-data-subset=x% x must be above 0.0% and at most 100.0%")
			}

		} else {
			fileSize, err := parseSizeStr(opts.ReadDataSubset)
			if err != nil {
				return argumentError
			}
			if fileSize <= 0.0 {
				return errors.Fatal(
					"check flag --read-data-subset=n n must be above 0")
			}

		}
	}

	return nil
}

// See doReadData in runCheck below for why this is 256.
const totalBucketsMax = 256

// stringToIntSlice converts string to []uint, using '/' as element separator
func stringToIntSlice(param string) (split []uint, err error) {
	if param == "" {
		return nil, nil
	}
	parts := strings.Split(param, "/")
	result := make([]uint, len(parts))
	for idx, part := range parts {
		uintval, err := strconv.ParseUint(part, 10, 0)
		if err != nil {
			return nil, err
		}
		result[idx] = uint(uintval)
	}
	return result, nil
}

// ParsePercentage parses a percentage string of the form "X%" where X is a float constant,
// and returns the value of that constant. It does not check the range of the value.
func parsePercentage(s string) (float64, error) {
	if !strings.HasSuffix(s, "%") {
		return 0, errors.Errorf(`parsePercentage: %q does not end in "%%"`, s)
	}
	s = s[:len(s)-1]

	p, err := strconv.ParseFloat(s, 64)
	if err != nil {
		return 0, errors.Errorf("parsePercentage: %v", err)
	}
	return p, nil
}

// prepareCheckCache configures a special cache directory for check.
//
//  * if --with-cache is specified, the default cache is used
//  * if the user explicitly requested --no-cache, we don't use any cache
//  * if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check
//  * by default, we use a cache in a temporary directory that is deleted after the check
func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func()) {
	cleanup = func() {}
	if opts.WithCache {
		// use the default cache, no setup needed
		return cleanup
	}

	if gopts.NoCache {
		// don't use any cache, no setup needed
		return cleanup
	}

	cachedir := gopts.CacheDir
	if cachedir == "" {
		cachedir = cache.EnvDir()
	}

	// use a cache in a temporary directory
	tempdir, err := ioutil.TempDir(cachedir, "restic-check-cache-")
	if err != nil {
		// if an error occurs, don't use any cache
		Warnf("unable to create temporary directory for cache during check, disabling cache: %v\n", err)
		gopts.NoCache = true
		return cleanup
	}

	gopts.CacheDir = tempdir
	Verbosef("using temporary cache in %v\n", tempdir)

	cleanup = func() {
		err := fs.RemoveAll(tempdir)
		if err != nil {
			Warnf("error removing temporary cache directory: %v\n", err)
		}
	}

	return cleanup
}

func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
	if len(args) != 0 {
		return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
	}

	cleanup := prepareCheckCache(opts, &gopts)
	AddCleanupHandler(func() error {
		cleanup()
		return nil
	})

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if !gopts.NoLock {
		Verbosef("create exclusive lock for repository\n")
		lock, err := lockRepoExclusive(gopts.ctx, repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	chkr := checker.New(repo, opts.CheckUnused)
	err = chkr.LoadSnapshots(gopts.ctx)
	if err != nil {
		return err
	}

	Verbosef("load indexes\n")
	hints, errs := chkr.LoadIndex(gopts.ctx)

	dupFound := false
	for _, hint := range hints {
		Printf("%v\n", hint)
		if _, ok := hint.(*checker.ErrDuplicatePacks); ok {
			dupFound = true
		}
	}

	if dupFound {
		Printf("This is non-critical, you can run `restic rebuild-index' to correct this\n")
	}

	if len(errs) > 0 {
		for _, err := range errs {
			Warnf("error: %v\n", err)
		}
		return errors.Fatal("LoadIndex returned errors")
	}

	errorsFound := false
	orphanedPacks := 0
	errChan := make(chan error)

	Verbosef("check all packs\n")
	go chkr.Packs(gopts.ctx, errChan)

	for err := range errChan {
		if checker.IsOrphanedPack(err) {
			orphanedPacks++
			Verbosef("%v\n", err)
			continue
		}
		errorsFound = true
		Warnf("%v\n", err)
	}

	if orphanedPacks > 0 {
		Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nYou can run `restic prune` to correct this.\n", orphanedPacks)
	}

	Verbosef("check snapshots, trees and blobs\n")
	errChan = make(chan error)
	var wg sync.WaitGroup

	wg.Add(1)
	go func() {
		defer wg.Done()
		bar := newProgressMax(!gopts.Quiet, 0, "snapshots")
		defer bar.Done()
		chkr.Structure(gopts.ctx, bar, errChan)
	}()

	for err := range errChan {
		errorsFound = true
		if e, ok := err.(*checker.TreeError); ok {
			Warnf("error for tree %v:\n", e.ID.Str())
			for _, treeErr := range e.Errors {
				Warnf("  %v\n", treeErr)
			}
		} else {
			Warnf("error: %v\n", err)
		}
	}

	// Wait for the progress bar to be complete before printing more below.
	// Must happen after `errChan` is read from in the above loop to avoid
	// deadlocking in the case of errors.
	wg.Wait()

	if opts.CheckUnused {
		for _, id := range chkr.UnusedBlobs(gopts.ctx) {
			Verbosef("unused blob %v\n", id)
			errorsFound = true
		}
	}

	doReadData := func(packs map[restic.ID]int64) {
		packCount := uint64(len(packs))

		p := newProgressMax(!gopts.Quiet, packCount, "packs")
		errChan := make(chan error)

		go chkr.ReadPacks(gopts.ctx, packs, p, errChan)

		for err := range errChan {
			errorsFound = true
			Warnf("%v\n", err)
		}
		p.Done()
	}

	switch {
	case opts.ReadData:
		Verbosef("read all data\n")
		doReadData(selectPacksByBucket(chkr.GetPacks(), 1, 1))
	case opts.ReadDataSubset != "":
		var packs map[restic.ID]int64
		dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
		if err == nil {
			bucket := dataSubset[0]
			totalBuckets := dataSubset[1]
			packs = selectPacksByBucket(chkr.GetPacks(), bucket, totalBuckets)
			packCount := uint64(len(packs))
			Verbosef("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets)
		} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
			percentage, err := parsePercentage(opts.ReadDataSubset)
			if err == nil {
				packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
				Verbosef("read %.1f%% of data packs\n", percentage)
			}
		} else {
			repoSize := int64(0)
			allPacks := chkr.GetPacks()
			for _, size := range allPacks {
				repoSize += size
			}
			if repoSize == 0 {
				return errors.Fatal("Cannot read from a repository having size 0")
			}
			subsetSize, _ := parseSizeStr(opts.ReadDataSubset)
			if subsetSize > repoSize {
				subsetSize = repoSize
			}
			packs = selectRandomPacksByFileSize(chkr.GetPacks(), subsetSize, repoSize)
			Verbosef("read %d bytes of data packs\n", subsetSize)
		}
		if packs == nil {
			return errors.Fatal("internal error: failed to select packs to check")
		}
		doReadData(packs)
	}

	if errorsFound {
		return errors.Fatal("repository contains errors")
	}

	Verbosef("no errors were found\n")

	return nil
}

// selectPacksByBucket selects subsets of packs by ranges of buckets.
func selectPacksByBucket(allPacks map[restic.ID]int64, bucket, totalBuckets uint) map[restic.ID]int64 {
	packs := make(map[restic.ID]int64)
	for pack, size := range allPacks {
		// If we ever check more than the first byte
		// of pack, update totalBucketsMax.
		if (uint(pack[0]) % totalBuckets) == (bucket - 1) {
			packs[pack] = size
		}
	}
	return packs
}

// selectRandomPacksByPercentage selects the given percentage of packs which are randomly choosen.
func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage float64) map[restic.ID]int64 {
	packCount := len(allPacks)
	packsToCheck := int(float64(packCount) * (percentage / 100.0))
	if packCount > 0 && packsToCheck < 1 {
		packsToCheck = 1
	}
	timeNs := time.Now().UnixNano()
	r := rand.New(rand.NewSource(timeNs))
	idx := r.Perm(packCount)

	var keys []restic.ID
	for k := range allPacks {
		keys = append(keys, k)
	}

	packs := make(map[restic.ID]int64)

	for i := 0; i < packsToCheck; i++ {
		id := keys[idx[i]]
		packs[id] = allPacks[id]
	}
	return packs
}

func selectRandomPacksByFileSize(allPacks map[restic.ID]int64, subsetSize int64, repoSize int64) map[restic.ID]int64 {
	subsetPercentage := (float64(subsetSize) / float64(repoSize)) * 100.0
	packs := selectRandomPacksByPercentage(allPacks, subsetPercentage)
	return packs
}
<commit_msg>check: Better differentiate between warnings and errors<commit_after>package main

import (
	"io/ioutil"
	"math/rand"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/spf13/cobra"

	"github.com/restic/restic/internal/cache"
	"github.com/restic/restic/internal/checker"
	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/fs"
	"github.com/restic/restic/internal/restic"
)

var cmdCheck = &cobra.Command{
	Use:   "check [flags]",
	Short: "Check the repository for errors",
	Long: `
The "check" command tests the repository for errors and reports any errors it
finds. It can also be used to read all data and therefore simulate a restore.

By default, the "check" command will always load all data directly from the
repository and not use a local cache.

EXIT STATUS
===========

Exit status is 0 if the command was successful, and non-zero if there was any error.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runCheck(checkOptions, globalOptions, args)
	},
	PreRunE: func(cmd *cobra.Command, args []string) error {
		return checkFlags(checkOptions)
	},
}

// CheckOptions bundles all options for the 'check' command.
type CheckOptions struct {
	ReadData       bool
	ReadDataSubset string
	CheckUnused    bool
	WithCache      bool
}

var checkOptions CheckOptions

func init() {
	cmdRoot.AddCommand(cmdCheck)

	f := cmdCheck.Flags()
	f.BoolVar(&checkOptions.ReadData, "read-data", false, "read all data blobs")
	f.StringVar(&checkOptions.ReadDataSubset, "read-data-subset", "", "read a `subset` of data packs, specified as 'n/t' for specific part, or either 'x%' or 'x.y%' or a size in bytes with suffixes k/K, m/M, g/G, t/T for a random subset")
	f.BoolVar(&checkOptions.CheckUnused, "check-unused", false, "find unused blobs")
	f.BoolVar(&checkOptions.WithCache, "with-cache", false, "use the cache")
}

func checkFlags(opts CheckOptions) error {
	if opts.ReadData && opts.ReadDataSubset != "" {
		return errors.Fatal("check flags --read-data and --read-data-subset cannot be used together")
	}
	if opts.ReadDataSubset != "" {
		dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
		argumentError := errors.Fatal("check flag --read-data-subset has invalid value, please see documentation")
		if err == nil {
			if len(dataSubset) != 2 {
				return argumentError
			}
			if dataSubset[0] == 0 || dataSubset[1] == 0 || dataSubset[0] > dataSubset[1] {
				return errors.Fatal("check flag --read-data-subset=n/t values must be positive integers, and n <= t, e.g. --read-data-subset=1/2")
			}
			if dataSubset[1] > totalBucketsMax {
				return errors.Fatalf("check flag --read-data-subset=n/t t must be at most %d", totalBucketsMax)
			}
		} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
			percentage, err := parsePercentage(opts.ReadDataSubset)
			if err != nil {
				return argumentError
			}

			if percentage <= 0.0 || percentage > 100.0 {
				return errors.Fatal(
					"check flag --read-data-subset=x% x must be above 0.0% and at most 100.0%")
			}

		} else {
			fileSize, err := parseSizeStr(opts.ReadDataSubset)
			if err != nil {
				return argumentError
			}
			if fileSize <= 0.0 {
				return errors.Fatal(
					"check flag --read-data-subset=n n must be above 0")
			}

		}
	}

	return nil
}

// See doReadData in runCheck below for why this is 256.
const totalBucketsMax = 256

// stringToIntSlice converts string to []uint, using '/' as element separator
func stringToIntSlice(param string) (split []uint, err error) {
	if param == "" {
		return nil, nil
	}
	parts := strings.Split(param, "/")
	result := make([]uint, len(parts))
	for idx, part := range parts {
		uintval, err := strconv.ParseUint(part, 10, 0)
		if err != nil {
			return nil, err
		}
		result[idx] = uint(uintval)
	}
	return result, nil
}

// ParsePercentage parses a percentage string of the form "X%" where X is a float constant,
// and returns the value of that constant. It does not check the range of the value.
func parsePercentage(s string) (float64, error) {
	if !strings.HasSuffix(s, "%") {
		return 0, errors.Errorf(`parsePercentage: %q does not end in "%%"`, s)
	}
	s = s[:len(s)-1]

	p, err := strconv.ParseFloat(s, 64)
	if err != nil {
		return 0, errors.Errorf("parsePercentage: %v", err)
	}
	return p, nil
}

// prepareCheckCache configures a special cache directory for check.
//
//  * if --with-cache is specified, the default cache is used
//  * if the user explicitly requested --no-cache, we don't use any cache
//  * if the user provides --cache-dir, we use a cache in a temporary sub-directory of the specified directory and the sub-directory is deleted after the check
//  * by default, we use a cache in a temporary directory that is deleted after the check
func prepareCheckCache(opts CheckOptions, gopts *GlobalOptions) (cleanup func()) {
	cleanup = func() {}
	if opts.WithCache {
		// use the default cache, no setup needed
		return cleanup
	}

	if gopts.NoCache {
		// don't use any cache, no setup needed
		return cleanup
	}

	cachedir := gopts.CacheDir
	if cachedir == "" {
		cachedir = cache.EnvDir()
	}

	// use a cache in a temporary directory
	tempdir, err := ioutil.TempDir(cachedir, "restic-check-cache-")
	if err != nil {
		// if an error occurs, don't use any cache
		Warnf("unable to create temporary directory for cache during check, disabling cache: %v\n", err)
		gopts.NoCache = true
		return cleanup
	}

	gopts.CacheDir = tempdir
	Verbosef("using temporary cache in %v\n", tempdir)

	cleanup = func() {
		err := fs.RemoveAll(tempdir)
		if err != nil {
			Warnf("error removing temporary cache directory: %v\n", err)
		}
	}

	return cleanup
}

func runCheck(opts CheckOptions, gopts GlobalOptions, args []string) error {
	if len(args) != 0 {
		return errors.Fatal("the check command expects no arguments, only options - please see `restic help check` for usage and flags")
	}

	cleanup := prepareCheckCache(opts, &gopts)
	AddCleanupHandler(func() error {
		cleanup()
		return nil
	})

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if !gopts.NoLock {
		Verbosef("create exclusive lock for repository\n")
		lock, err := lockRepoExclusive(gopts.ctx, repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	chkr := checker.New(repo, opts.CheckUnused)
	err = chkr.LoadSnapshots(gopts.ctx)
	if err != nil {
		return err
	}

	Verbosef("load indexes\n")
	hints, errs := chkr.LoadIndex(gopts.ctx)

	errorsFound := false
	suggestIndexRebuild := false
	for _, hint := range hints {
		switch hint.(type) {
		case *checker.ErrDuplicatePacks, *checker.ErrOldIndexFormat:
			Printf("%v\n", hint)
			suggestIndexRebuild = true
		default:
			Warnf("error: %v\n", hint)
			errorsFound = true
		}
	}

	if suggestIndexRebuild {
		Printf("This is non-critical, you can run `restic rebuild-index' to correct this\n")
	}

	if len(errs) > 0 {
		for _, err := range errs {
			Warnf("error: %v\n", err)
		}
		return errors.Fatal("LoadIndex returned errors")
	}

	orphanedPacks := 0
	errChan := make(chan error)

	Verbosef("check all packs\n")
	go chkr.Packs(gopts.ctx, errChan)

	for err := range errChan {
		if checker.IsOrphanedPack(err) {
			orphanedPacks++
			Verbosef("%v\n", err)
			continue
		}
		errorsFound = true
		Warnf("error: %v\n", err)
	}

	if orphanedPacks > 0 {
		Verbosef("%d additional files were found in the repo, which likely contain duplicate data.\nThis is non-critical, you can run `restic prune` to correct this.\n", orphanedPacks)
	}

	Verbosef("check snapshots, trees and blobs\n")
	errChan = make(chan error)
	var wg sync.WaitGroup

	wg.Add(1)
	go func() {
		defer wg.Done()
		bar := newProgressMax(!gopts.Quiet, 0, "snapshots")
		defer bar.Done()
		chkr.Structure(gopts.ctx, bar, errChan)
	}()

	for err := range errChan {
		errorsFound = true
		if e, ok := err.(*checker.TreeError); ok {
			Warnf("error for tree %v:\n", e.ID.Str())
			for _, treeErr := range e.Errors {
				Warnf("  %v\n", treeErr)
			}
		} else {
			Warnf("error: %v\n", err)
		}
	}

	// Wait for the progress bar to be complete before printing more below.
	// Must happen after `errChan` is read from in the above loop to avoid
	// deadlocking in the case of errors.
	wg.Wait()

	if opts.CheckUnused {
		for _, id := range chkr.UnusedBlobs(gopts.ctx) {
			Verbosef("unused blob %v\n", id)
			errorsFound = true
		}
	}

	doReadData := func(packs map[restic.ID]int64) {
		packCount := uint64(len(packs))

		p := newProgressMax(!gopts.Quiet, packCount, "packs")
		errChan := make(chan error)

		go chkr.ReadPacks(gopts.ctx, packs, p, errChan)

		for err := range errChan {
			errorsFound = true
			Warnf("%v\n", err)
		}
		p.Done()
	}

	switch {
	case opts.ReadData:
		Verbosef("read all data\n")
		doReadData(selectPacksByBucket(chkr.GetPacks(), 1, 1))
	case opts.ReadDataSubset != "":
		var packs map[restic.ID]int64
		dataSubset, err := stringToIntSlice(opts.ReadDataSubset)
		if err == nil {
			bucket := dataSubset[0]
			totalBuckets := dataSubset[1]
			packs = selectPacksByBucket(chkr.GetPacks(), bucket, totalBuckets)
			packCount := uint64(len(packs))
			Verbosef("read group #%d of %d data packs (out of total %d packs in %d groups)\n", bucket, packCount, chkr.CountPacks(), totalBuckets)
		} else if strings.HasSuffix(opts.ReadDataSubset, "%") {
			percentage, err := parsePercentage(opts.ReadDataSubset)
			if err == nil {
				packs = selectRandomPacksByPercentage(chkr.GetPacks(), percentage)
				Verbosef("read %.1f%% of data packs\n", percentage)
			}
		} else {
			repoSize := int64(0)
			allPacks := chkr.GetPacks()
			for _, size := range allPacks {
				repoSize += size
			}
			if repoSize == 0 {
				return errors.Fatal("Cannot read from a repository having size 0")
			}
			subsetSize, _ := parseSizeStr(opts.ReadDataSubset)
			if subsetSize > repoSize {
				subsetSize = repoSize
			}
			packs = selectRandomPacksByFileSize(chkr.GetPacks(), subsetSize, repoSize)
			Verbosef("read %d bytes of data packs\n", subsetSize)
		}
		if packs == nil {
			return errors.Fatal("internal error: failed to select packs to check")
		}
		doReadData(packs)
	}

	if errorsFound {
		return errors.Fatal("repository contains errors")
	}

	Verbosef("no errors were found\n")

	return nil
}

// selectPacksByBucket selects subsets of packs by ranges of buckets.
func selectPacksByBucket(allPacks map[restic.ID]int64, bucket, totalBuckets uint) map[restic.ID]int64 {
	packs := make(map[restic.ID]int64)
	for pack, size := range allPacks {
		// If we ever check more than the first byte
		// of pack, update totalBucketsMax.
		if (uint(pack[0]) % totalBuckets) == (bucket - 1) {
			packs[pack] = size
		}
	}
	return packs
}

// selectRandomPacksByPercentage selects the given percentage of packs which are randomly choosen.
func selectRandomPacksByPercentage(allPacks map[restic.ID]int64, percentage float64) map[restic.ID]int64 {
	packCount := len(allPacks)
	packsToCheck := int(float64(packCount) * (percentage / 100.0))
	if packCount > 0 && packsToCheck < 1 {
		packsToCheck = 1
	}
	timeNs := time.Now().UnixNano()
	r := rand.New(rand.NewSource(timeNs))
	idx := r.Perm(packCount)

	var keys []restic.ID
	for k := range allPacks {
		keys = append(keys, k)
	}

	packs := make(map[restic.ID]int64)

	for i := 0; i < packsToCheck; i++ {
		id := keys[idx[i]]
		packs[id] = allPacks[id]
	}
	return packs
}

func selectRandomPacksByFileSize(allPacks map[restic.ID]int64, subsetSize int64, repoSize int64) map[restic.ID]int64 {
	subsetPercentage := (float64(subsetSize) / float64(repoSize)) * 100.0
	packs := selectRandomPacksByPercentage(allPacks, subsetPercentage)
	return packs
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"github.com/kobtea/go-todoist/cmd/util"
	"github.com/kobtea/go-todoist/todoist"
	"github.com/spf13/cobra"
	"sort"
)

// nextCmd represents the next command
var nextCmd = &cobra.Command{
	Use:   "next",
	Short: "show next 7 days tasks",
	RunE: func(cmd *cobra.Command, args []string) error {
		client, err := util.NewClient()
		if err != nil {
			return err
		}
		items := client.Item.FindByDueDate(todoist.Next7Days())
		sort.Slice(items, func(i, j int) bool {
			return items[i].DueDateUtc.Before(items[j].DueDateUtc)
		})
		relations := client.Relation.Items(items)
		fmt.Println(util.ItemTableString(items, relations, func(i todoist.Item) todoist.Time { return i.DueDateUtc }))
		return nil
	},
}

func init() {
	RootCmd.AddCommand(nextCmd)
}
<commit_msg>filter completed items at next sub-command<commit_after>package cmd

import (
	"fmt"
	"github.com/kobtea/go-todoist/cmd/util"
	"github.com/kobtea/go-todoist/todoist"
	"github.com/spf13/cobra"
	"sort"
)

// nextCmd represents the next command
var nextCmd = &cobra.Command{
	Use:   "next",
	Short: "show next 7 days tasks",
	RunE: func(cmd *cobra.Command, args []string) error {
		client, err := util.NewClient()
		if err != nil {
			return err
		}
		var items []todoist.Item
		for _, i := range client.Item.FindByDueDate(todoist.Next7Days()) {
			if !i.IsChecked() {
				items = append(items, i)
			}
		}
		sort.Slice(items, func(i, j int) bool {
			return items[i].DueDateUtc.Before(items[j].DueDateUtc)
		})
		relations := client.Relation.Items(items)
		fmt.Println(util.ItemTableString(items, relations, func(i todoist.Item) todoist.Time { return i.DueDateUtc }))
		return nil
	},
}

func init() {
	RootCmd.AddCommand(nextCmd)
}
<|endoftext|>


<commit_before>/*
word2vec-client is a tool which uses word2vec.Client to look up similarities (using an external server).
*/
package main

import (
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/sajari/word2vec"
)

var addr string
var addListA, subListA string
var addListB, subListB string
var sim bool
var n int

func init() {
	flag.StringVar(&addr, "addr", "localhost:1234", "server address")
	flag.StringVar(&addListA, "addA", "", "comma separated list of model words to add to the target vector A")
	flag.StringVar(&subListA, "subA", "", "comma separated list of model words to subtract from the target vector A")
	flag.StringVar(&addListB, "addB", "", "comma separated list of model words to add to the target vector B")
	flag.StringVar(&subListB, "subB", "", "comma separated list of model words to subtract from the target vector B")
	flag.BoolVar(&sim, "sim", false, "similarity query")
	flag.IntVar(&n, "n", 10, "return `N` similar items in similarity query")
}

func makeExpr(addList, subList string) (word2vec.Expr, error) {
	if addList == "" && subList == "" {
		return word2vec.Expr{}, fmt.Errorf("must specify 'add' and/or 'sub' component for each target vector; see -h for more details")
	}

	result := word2vec.Expr{}
	if addList != "" {
		for _, w := range strings.Split(addList, ",") {
			result.Add(1, w)
		}
	}
	if subList != "" {
		for _, w := range strings.Split(subList, ",") {
			result.Add(-1, w)
		}
	}
	return result, nil
}

func main() {
	flag.Parse()

	if addr == "" {
		fmt.Println("must specify -addr; see -h for more details")
		os.Exit(1)
	}

	exprA, err := makeExpr(addListA, subListA)
	if err != nil {
		fmt.Printf("error creating target vector for 'A': %v\n", err)
		os.Exit(1)
	}

	if sim {
		c := word2vec.Client{Addr: addr}
		r, err := c.CosN(exprA, n)
		if err != nil {
			fmt.Printf("error looking up similar items: %v\n", err)
			os.Exit(1)
		}
		for _, x := range r {
			fmt.Printf("%9f %#v\n", x.Score, x.Word)
		}
		return
	}

	exprB, err := makeExpr(addListB, subListB)
	if err != nil {
		fmt.Printf("error creating target vector for 'B': %v\n", err)
		os.Exit(1)
	}

	c := word2vec.Client{Addr: addr}

	start := time.Now()
	v, err := c.Cos(exprA, exprB)
	totalTime := time.Since(start)
	if err != nil {
		fmt.Printf("error looking up similarity: %v\n", err)
		os.Exit(1)
	}

	fmt.Printf("cosine similarity: %v (took: %v)\n", v, totalTime)
}
<commit_msg>Polished word-client doc and config variables.<commit_after>/*
word2vec-client is a tool which queries a `word-server` HTTP server to do computations with a word2vec
model.
*/
package main

import (
	"flag"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/sajari/word2vec"
)

var addr string
var addListA, subListA string
var addListB, subListB string
var sim bool
var n int

func init() {
	flag.StringVar(&addr, "addr", "localhost:1234", "server `address`")
	flag.StringVar(&addListA, "addA", "", "comma separated list of model `words` to add to the target vector A")
	flag.StringVar(&subListA, "subA", "", "comma separated list of model `words` to subtract from the target vector A")
	flag.StringVar(&addListB, "addB", "", "comma separated list of model `words` to add to the target vector B")
	flag.StringVar(&subListB, "subB", "", "comma separated list of model `words` to subtract from the target vector B")
	flag.BoolVar(&sim, "sim", false, "similarity query")
	flag.IntVar(&n, "n", 10, "return `N` similar items in similarity query")
}

func makeExpr(addList, subList string) (word2vec.Expr, error) {
	if addList == "" && subList == "" {
		return word2vec.Expr{}, fmt.Errorf("must specify 'add' and/or 'sub' component for each target vector; see -h for more details")
	}

	result := word2vec.Expr{}
	if addList != "" {
		for _, w := range strings.Split(addList, ",") {
			result.Add(1, w)
		}
	}
	if subList != "" {
		for _, w := range strings.Split(subList, ",") {
			result.Add(-1, w)
		}
	}
	return result, nil
}

func main() {
	flag.Parse()

	if addr == "" {
		fmt.Println("must specify -addr; see -h for more details")
		os.Exit(1)
	}

	exprA, err := makeExpr(addListA, subListA)
	if err != nil {
		fmt.Printf("error creating target vector for 'A': %v\n", err)
		os.Exit(1)
	}

	if sim {
		c := word2vec.Client{Addr: addr}
		r, err := c.CosN(exprA, n)
		if err != nil {
			fmt.Printf("error looking up similar items: %v\n", err)
			os.Exit(1)
		}
		for _, x := range r {
			fmt.Printf("%9f %#v\n", x.Score, x.Word)
		}
		return
	}

	exprB, err := makeExpr(addListB, subListB)
	if err != nil {
		fmt.Printf("error creating target vector for 'B': %v\n", err)
		os.Exit(1)
	}

	c := word2vec.Client{Addr: addr}

	start := time.Now()
	v, err := c.Cos(exprA, exprB)
	totalTime := time.Since(start)
	if err != nil {
		fmt.Printf("error looking up similarity: %v\n", err)
		os.Exit(1)
	}

	fmt.Printf("cosine similarity: %v (took: %v)\n", v, totalTime)
}
<|endoftext|>


<commit_before>package sqlexp

import (
	"context"
)

// RawMessage is returned from RowsMessage.
type RawMessage interface{}

// ReturnMessage may be passed into a Query argument.
//
// Drivers must implement driver.NamedValueChecker,
// call ReturnMessageInit on it, save it internally,
// and return driver.ErrOmitArgument to prevent
// this from appearing in the query arguments.
//
// Queries that recieve this message should also not return
// SQL errors from the Query method, but wait to return
// it in a Message.
type ReturnMessage struct {
	queue chan RawMessage
}

// Message is called by clients after Query to dequeue messages.
func (m *ReturnMessage) Message(ctx context.Context) RawMessage {
	select {
	case <-ctx.Done():
		return MsgNextResultSet{}
	case raw := <-m.queue:
		return raw
	}
}

// ReturnMessageEnqueue is called by the driver to enqueue the driver.
// Drivers should not call this until after it returns from Query.
func ReturnMessageEnqueue(ctx context.Context, m *ReturnMessage, raw RawMessage) error {
	select {
	case <-ctx.Done():
		return ctx.Err()
	case m.queue <- raw:
		return nil
	}
}

// ReturnMessageInit is called by database/sql setup the ReturnMessage internals.
func ReturnMessageInit(m *ReturnMessage) {
	m.queue = make(chan RawMessage, 15)
}

type (
	// MsgNextResultSet must be checked for. When received, NextResultSet
	// should be called and if false the message loop should be exited.
	MsgNextResultSet struct{}

	// MsgNext indicates the result set ready to be scanned.
	// This message will often be followed with:
	//
	//	for rows.Next() {
	//		rows.Scan(&v)
	//	}
	MsgNext struct{}

	// MsgRowsAffected returns the number of rows affected.
	// Not all operations that affect rows return results, thus this message
	// may be received multiple times.
	MsgRowsAffected struct{ Count int64 }

	// MsgLastInsertID returns the value of last inserted row. For many
	// database systems and tables this will return int64. Some databases
	// may return a string or GUID equivalent.
	MsgLastInsertID struct{ Value interface{} }

	// MsgNotice is raised from the SQL text and is only informational.
	MsgNotice struct{ Message string }

	// MsgError returns SQL errors from the database system (not transport
	// or other system level errors).
	MsgError struct{ Error error }
)
<commit_msg>sqlexp: replace MsgNotice.Message with an interface<commit_after>package sqlexp

import (
	"context"
	"fmt"
)

// RawMessage is returned from RowsMessage.
type RawMessage interface{}

// ReturnMessage may be passed into a Query argument.
//
// Drivers must implement driver.NamedValueChecker,
// call ReturnMessageInit on it, save it internally,
// and return driver.ErrOmitArgument to prevent
// this from appearing in the query arguments.
//
// Queries that recieve this message should also not return
// SQL errors from the Query method, but wait to return
// it in a Message.
type ReturnMessage struct {
	queue chan RawMessage
}

// Message is called by clients after Query to dequeue messages.
func (m *ReturnMessage) Message(ctx context.Context) RawMessage {
	select {
	case <-ctx.Done():
		return MsgNextResultSet{}
	case raw := <-m.queue:
		return raw
	}
}

// ReturnMessageEnqueue is called by the driver to enqueue the driver.
// Drivers should not call this until after it returns from Query.
func ReturnMessageEnqueue(ctx context.Context, m *ReturnMessage, raw RawMessage) error {
	select {
	case <-ctx.Done():
		return ctx.Err()
	case m.queue <- raw:
		return nil
	}
}

// ReturnMessageInit is called by database/sql setup the ReturnMessage internals.
func ReturnMessageInit(m *ReturnMessage) {
	m.queue = make(chan RawMessage, 15)
}

type (
	// MsgNextResultSet must be checked for. When received, NextResultSet
	// should be called and if false the message loop should be exited.
	MsgNextResultSet struct{}

	// MsgNext indicates the result set ready to be scanned.
	// This message will often be followed with:
	//
	//	for rows.Next() {
	//		rows.Scan(&v)
	//	}
	MsgNext struct{}

	// MsgRowsAffected returns the number of rows affected.
	// Not all operations that affect rows return results, thus this message
	// may be received multiple times.
	MsgRowsAffected struct{ Count int64 }

	// MsgLastInsertID returns the value of last inserted row. For many
	// database systems and tables this will return int64. Some databases
	// may return a string or GUID equivalent.
	MsgLastInsertID struct{ Value interface{} }

	// MsgNotice is raised from the SQL text and is only informational.
	MsgNotice struct{ Message fmt.Stringer }

	// MsgError returns SQL errors from the database system (not transport
	// or other system level errors).
	MsgError struct{ Error error }
)
<|endoftext|>


<commit_before>package slack

// OutgoingMessage is used for the realtime API, and seems incomplete.
type OutgoingMessage struct {
	ID int `json:"id"`
	// channel ID
	Channel         string `json:"channel,omitempty"`
	Text            string `json:"text,omitempty"`
	Type            string `json:"type,omitempty"`
	ThreadTimestamp string `json:"thread_ts,omitempty"`
}

// Message is an auxiliary type to allow us to have a message containing sub messages
type Message struct {
	Msg
	SubMessage *Msg `json:"message,omitempty"`
}

// Msg contains information about a slack message
type Msg struct {
	// Basic Message
	Type            string       `json:"type,omitempty"`
	Channel         string       `json:"channel,omitempty"`
	User            string       `json:"user,omitempty"`
	Text            string       `json:"text,omitempty"`
	Timestamp       string       `json:"ts,omitempty"`
	ThreadTimestamp string       `json:"thread_ts,omitempty"`
	IsStarred       bool         `json:"is_starred,omitempty"`
	PinnedTo        []string     `json:"pinned_to,omitempty"`
	Attachments     []Attachment `json:"attachments,omitempty"`
	Edited          *Edited      `json:"edited,omitempty"`
	LastRead        string       `json:"last_read,omitempty"`
	Subscribed      bool         `json:"subscribed,omitempty"`
	UnreadCount     int          `json:"unread_count,omitempty"`

	// Message Subtypes
	SubType string `json:"subtype,omitempty"`

	// Hidden Subtypes
	Hidden           bool   `json:"hidden,omitempty"`     // message_changed, message_deleted, unpinned_item
	DeletedTimestamp string `json:"deleted_ts,omitempty"` // message_deleted
	EventTimestamp   string `json:"event_ts,omitempty"`

	// bot_message (https://api.slack.com/events/message/bot_message)
	BotID    string `json:"bot_id,omitempty"`
	Username string `json:"username,omitempty"`
	Icons    *Icon  `json:"icons,omitempty"`

	// channel_join, group_join
	Inviter string `json:"inviter,omitempty"`

	// channel_topic, group_topic
	Topic string `json:"topic,omitempty"`

	// channel_purpose, group_purpose
	Purpose string `json:"purpose,omitempty"`

	// channel_name, group_name
	Name    string `json:"name,omitempty"`
	OldName string `json:"old_name,omitempty"`

	// channel_archive, group_archive
	Members []string `json:"members,omitempty"`

	// channels.replies, groups.replies, im.replies, mpim.replies
	ReplyCount   int     `json:"reply_count,omitempty"`
	Replies      []Reply `json:"replies,omitempty"`
	ParentUserId string  `json:"parent_user_id,omitempty"`

	// file_share, file_comment, file_mention
	File *File `json:"file,omitempty"`

	// file_share
	Upload bool `json:"upload,omitempty"`

	// file_comment
	Comment *Comment `json:"comment,omitempty"`

	// pinned_item
	ItemType string `json:"item_type,omitempty"`

	// https://api.slack.com/rtm
	ReplyTo int    `json:"reply_to,omitempty"`
	Team    string `json:"team,omitempty"`

	// reactions
	Reactions []ItemReaction `json:"reactions,omitempty"`

	// slash commands and interactive messages
	ResponseType    string `json:"response_type,omitempty"`
	ReplaceOriginal bool   `json:"replace_original,omitempty"`
	DeleteOriginal  bool   `json:"delete_original,omitempty"`
}

// Icon is used for bot messages
type Icon struct {
	IconURL   string `json:"icon_url,omitempty"`
	IconEmoji string `json:"icon_emoji,omitempty"`
}

// Edited indicates that a message has been edited.
type Edited struct {
	User      string `json:"user,omitempty"`
	Timestamp string `json:"ts,omitempty"`
}

// Reply contains information about a reply for a thread
type Reply struct {
	User      string `json:"user,omitempty"`
	Timestamp string `json:"ts,omitempty"`
}

// Event contains the event type
type Event struct {
	Type string `json:"type,omitempty"`
}

// Ping contains information about a Ping Event
type Ping struct {
	ID   int    `json:"id"`
	Type string `json:"type"`
}

// Pong contains information about a Pong Event
type Pong struct {
	Type    string `json:"type"`
	ReplyTo int    `json:"reply_to"`
}

// NewOutgoingMessage prepares an OutgoingMessage that the user can
// use to send a message. Use this function to properly set the
// messageID.
func (rtm *RTM) NewOutgoingMessage(text string, channelID string) *OutgoingMessage {
	id := rtm.idGen.Next()
	return &OutgoingMessage{
		ID:      id,
		Type:    "message",
		Channel: channelID,
		Text:    text,
	}
}

// NewTypingMessage prepares an OutgoingMessage that the user can
// use to send as a typing indicator. Use this function to properly set the
// messageID.
func (rtm *RTM) NewTypingMessage(channelID string) *OutgoingMessage {
	id := rtm.idGen.Next()
	return &OutgoingMessage{
		ID:      id,
		Type:    "typing",
		Channel: channelID,
	}
}
<commit_msg>Added a timestamp string argument to the rtm.NewOutgoingMessage function so that the function can be used to reply to or start a new thread<commit_after>package slack

// OutgoingMessage is used for the realtime API, and seems incomplete.
type OutgoingMessage struct {
	ID int `json:"id"`
	// channel ID
	Channel         string `json:"channel,omitempty"`
	Text            string `json:"text,omitempty"`
	Type            string `json:"type,omitempty"`
	ThreadTimestamp string `json:"thread_ts,omitempty"`
}

// Message is an auxiliary type to allow us to have a message containing sub messages
type Message struct {
	Msg
	SubMessage *Msg `json:"message,omitempty"`
}

// Msg contains information about a slack message
type Msg struct {
	// Basic Message
	Type            string       `json:"type,omitempty"`
	Channel         string       `json:"channel,omitempty"`
	User            string       `json:"user,omitempty"`
	Text            string       `json:"text,omitempty"`
	Timestamp       string       `json:"ts,omitempty"`
	ThreadTimestamp string       `json:"thread_ts,omitempty"`
	IsStarred       bool         `json:"is_starred,omitempty"`
	PinnedTo        []string     `json:"pinned_to,omitempty"`
	Attachments     []Attachment `json:"attachments,omitempty"`
	Edited          *Edited      `json:"edited,omitempty"`
	LastRead        string       `json:"last_read,omitempty"`
	Subscribed      bool         `json:"subscribed,omitempty"`
	UnreadCount     int          `json:"unread_count,omitempty"`

	// Message Subtypes
	SubType string `json:"subtype,omitempty"`

	// Hidden Subtypes
	Hidden           bool   `json:"hidden,omitempty"`     // message_changed, message_deleted, unpinned_item
	DeletedTimestamp string `json:"deleted_ts,omitempty"` // message_deleted
	EventTimestamp   string `json:"event_ts,omitempty"`

	// bot_message (https://api.slack.com/events/message/bot_message)
	BotID    string `json:"bot_id,omitempty"`
	Username string `json:"username,omitempty"`
	Icons    *Icon  `json:"icons,omitempty"`

	// channel_join, group_join
	Inviter string `json:"inviter,omitempty"`

	// channel_topic, group_topic
	Topic string `json:"topic,omitempty"`

	// channel_purpose, group_purpose
	Purpose string `json:"purpose,omitempty"`

	// channel_name, group_name
	Name    string `json:"name,omitempty"`
	OldName string `json:"old_name,omitempty"`

	// channel_archive, group_archive
	Members []string `json:"members,omitempty"`

	// channels.replies, groups.replies, im.replies, mpim.replies
	ReplyCount   int     `json:"reply_count,omitempty"`
	Replies      []Reply `json:"replies,omitempty"`
	ParentUserId string  `json:"parent_user_id,omitempty"`

	// file_share, file_comment, file_mention
	File *File `json:"file,omitempty"`

	// file_share
	Upload bool `json:"upload,omitempty"`

	// file_comment
	Comment *Comment `json:"comment,omitempty"`

	// pinned_item
	ItemType string `json:"item_type,omitempty"`

	// https://api.slack.com/rtm
	ReplyTo int    `json:"reply_to,omitempty"`
	Team    string `json:"team,omitempty"`

	// reactions
	Reactions []ItemReaction `json:"reactions,omitempty"`

	// slash commands and interactive messages
	ResponseType    string `json:"response_type,omitempty"`
	ReplaceOriginal bool   `json:"replace_original,omitempty"`
	DeleteOriginal  bool   `json:"delete_original,omitempty"`
}

// Icon is used for bot messages
type Icon struct {
	IconURL   string `json:"icon_url,omitempty"`
	IconEmoji string `json:"icon_emoji,omitempty"`
}

// Edited indicates that a message has been edited.
type Edited struct {
	User      string `json:"user,omitempty"`
	Timestamp string `json:"ts,omitempty"`
}

// Reply contains information about a reply for a thread
type Reply struct {
	User      string `json:"user,omitempty"`
	Timestamp string `json:"ts,omitempty"`
}

// Event contains the event type
type Event struct {
	Type string `json:"type,omitempty"`
}

// Ping contains information about a Ping Event
type Ping struct {
	ID   int    `json:"id"`
	Type string `json:"type"`
}

// Pong contains information about a Pong Event
type Pong struct {
	Type    string `json:"type"`
	ReplyTo int    `json:"reply_to"`
}

// NewOutgoingMessage prepares an OutgoingMessage that the user can
// use to send a message. Use this function to properly set the
// messageID.
func (rtm *RTM) NewOutgoingMessage(text string, channelID string, threadTimestamp string) *OutgoingMessage {
	id := rtm.idGen.Next()
	return &OutgoingMessage{
		ID:              id,
		Type:            "message",
		Channel:         channelID,
		Text:            text,
		ThreadTimestamp: threadTimestamp,
	}
}

// NewTypingMessage prepares an OutgoingMessage that the user can
// use to send as a typing indicator. Use this function to properly set the
// messageID.
func (rtm *RTM) NewTypingMessage(channelID string) *OutgoingMessage {
	id := rtm.idGen.Next()
	return &OutgoingMessage{
		ID:      id,
		Type:    "typing",
		Channel: channelID,
	}
}
<|endoftext|>


<commit_before>package vkapi

import (
	"fmt"
	"net/url"
	"strconv"
)

const (
	chatOffset = 2000000000
)

// Dialog describes the structure of the message.
type Dialog struct {
	Unread     int64    `json:"unread"`
	Message    *Message `json:"message"`
	InRead     int64    `json:"in_read"`
	OutRead    int64    `json:"out_read"`
	RealOffset int64    `json:"real_offset"`
}

// Message describes the structure of the message.
type Message struct {
	Id          int64      `json:"id"`
	UserId      int64      `json:"user_id"`
	FromId      int64      `json:"from_id"`
	Date        int64      `json:"date"`
	ReadState   int        `json:"read_state"`
	Out         int        `json:"out"`
	Title       string     `json:"title"`
	Body        string     `json:"body"`
	FwdMessages *[]Message `json:"fwd_messages"`
	Emoji       int        `json:"emoji"`
	Important   int        `json:"important"`
	Deleted     int        `json:"deleted"`
	RandomId    int64      `json:"random_id"`
	ChatId      int64      `json:"chat_id"`
	ChatActive  []int64    `json:"chat_active"`
	UsersCount  int        `json:"users_count"`
	AdminId     int64      `json:"admin_id"`
	Action      string     `json:"action"`
	ActionMid   int64      `json:"action_mid"`   /*идентификатор пользователя (если > 0) или email (если < 0), которого пригласили или исключили (для служебных сообщений с action = chat_invite_user или chat_kick_user). */
	ActionEmail string     `json:"action_email"` /*email, который пригласили или исключили (для служебных сообщений с action = chat_invite_user или chat_kick_user и отрицательным action_mid). */
	ActionText  string     `json:"action_text"`  /*название беседы (для служебных сообщений с action = chat_create или chat_title_update). */
	Photo50     string     `json:"photo_50"`
	Photo100    string     `json:"photo_100"`
	Photo200    string     `json:"photo_200"`
	/*Geo       *Geo {
		type (string) — тип места;
		coordinates (string) — координаты места;
		place (object) — описание места (если оно добавлено), объект с полями:
		id (integer) — идентификатор места (если назначено);
		title (string) — название места (если назначено);
		latitude (number) — географическая широта;
		longitude (number) — географическая долгота;
		created (integer) — дата создания (если назначено);
		icon (string) — URL изображения-иконки;
		country (string) — название страны;
		city (string) — название города;
	} `json:"geo"`*/

	/*Attachments *[]Attachments `json:"attachments"`*/
	/*PushSettings *PushSettings { настройки уведомлений для беседы, если они есть.	} `json:"push_settings"`*/
	/*string	тип действия (если это служебное сообщение). Возможные значения:

	  chat_photo_update — обновлена фотография беседы;
	  chat_photo_remove — удалена фотография беседы;
	  chat_create — создана беседа;
	  chat_title_update — обновлено название беседы;
	  chat_invite_user — приглашен пользователь;
	  chat_kick_user — исключен пользователь.*/

}

// MessageConfig contains the data
// necessary to send a message.
type MessageConfig struct {
	UserID          int64   `json:"user_id"`
	RandomID        int64   `json:"random_id"`
	PeerID          int64   `json:"peer_id"`
	Domain          string  `json:"domain"`
	ChatID          int64   `json:"chat_id"`
	GroupID         int64   `json:"group_id"`
	UserIDs         []int64 `json:"user_ids"`
	Message         string  `json:"message"`
	geo             bool    `json:"-"`
	lat             float64 `json:"lat"`
	long            float64 `json:"long"`
	ForwardMessages []int64 `json:"forward_messages"`
	StickerID       int64   `json:"sticker_id"`
	AccessToken     string  `json:"access_token"`
	//attachment *[]Attachment `json:"attachment"`
}

// SetGeo sets the location.
func (m *MessageConfig) SetGeo(lat float64, long float64) {
	m.geo = true
	m.lat = lat
	m.long = long
}

// NewMessage creates a new message for the user from the text.
func NewMessage(id int64, message string) (config MessageConfig) {
	config.PeerID = id
	config.Message = message
	return
}

// NewMessageToChat creates a new message for the chat from the text.
func NewMessageToChat(id int64, message string) (config MessageConfig) {
	return NewMessage(id+chatOffset, message)
}

// NewMessageToUsers creates a new message for several users from the text.
func NewMessageToUsers(message string, ids ...int64) (config MessageConfig) {
	config.UserIDs = ids
	config.Message = message
	return
}

// SendMessage tries to send a message with the configuration
// from the MessageConfig and returns message ID if it succeeds.
func (client *Client) SendMessage(config MessageConfig) (int64, *Error) {
	var req Request
	req.Token = config.AccessToken
	req.Method = "messages.send"
	v := url.Values{}

	if config.PeerID != 0 {
		v.Add("peer_id", fmt.Sprintf("%d", config.PeerID))
	}

	if config.UserID != 0 {
		v.Add("user_id", fmt.Sprintf("%d", config.UserID))
	}

	if config.Domain != "" {
		v.Add("domain", config.Domain)
	}

	if config.ChatID != 0 {
		v.Add("chat_id", fmt.Sprintf("%d", config.RandomID))
	}

	if len(config.UserIDs) != 0 {
		v.Add("user_ids", ConcatInt64ToString(config.UserIDs...))
	}

	if len(config.ForwardMessages) != 0 {
		v.Add("forward_messages", ConcatInt64ToString(config.ForwardMessages...))
	}

	if config.StickerID != 0 {
		v.Add("sticker_id", fmt.Sprintf("%d", config.StickerID))
	}

	if config.Message != "" {
		v.Add("message", config.Message)
	}

	if config.RandomID != 0 {
		v.Add("random_id", fmt.Sprintf("%d", config.RandomID))
	}

	if config.geo {
		v.Add("lat", strconv.FormatFloat(config.lat, 'f', -1, 64))
		v.Add("long", strconv.FormatFloat(config.long, 'f', -1, 64))
	}

	req.Values = v
	res, err := client.Do(req)
	if err != nil && !err.Code.Is(ErrZero) {
		return 0, err
	}

	answer, error := strconv.ParseInt(res.Response.String(), 10, 64)
	if error != nil {
		return 0, NewError(ErrBadResponseCode, error.Error())
	}

	return answer, nil
}

// SetActivity changes the status of typing by user in the dialog.
// Accepts userID as string or int64, chat as 2000000000 + chatID,
// group as -groupID.
func (client *Client) SetActivity(dst interface{}) *Error {
	values := url.Values{}
	switch dst.(type) {
	case string:
		values.Add("user_id", dst.(string))
	case int64:
		values.Add("peer_id", strconv.FormatInt(dst.(int64), 10))
	case int:
		values.Add("peer_id", strconv.FormatInt(int64(dst.(int)), 10))
	default:
		return NewError(ErrBadCode, "Wrong data")
	}

	values.Add("type", "typing")
	_, err := client.Do(NewRequest("messages.setActivity", "", values))
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Update SetActivity and add funcs NewMC<commit_after>package vkapi

import (
	"fmt"
	"net/url"
	"strconv"
)

const (
	chatOffset = 2000000000
)

// Dialog describes the structure of the message.
type Dialog struct {
	Unread     int64    `json:"unread"`
	Message    *Message `json:"message"`
	InRead     int64    `json:"in_read"`
	OutRead    int64    `json:"out_read"`
	RealOffset int64    `json:"real_offset"`
}

// Message describes the structure of the message.
type Message struct {
	Id          int64      `json:"id"`
	UserId      int64      `json:"user_id"`
	FromId      int64      `json:"from_id"`
	Date        int64      `json:"date"`
	ReadState   int        `json:"read_state"`
	Out         int        `json:"out"`
	Title       string     `json:"title"`
	Body        string     `json:"body"`
	FwdMessages *[]Message `json:"fwd_messages"`
	Emoji       int        `json:"emoji"`
	Important   int        `json:"important"`
	Deleted     int        `json:"deleted"`
	RandomId    int64      `json:"random_id"`
	ChatId      int64      `json:"chat_id"`
	ChatActive  []int64    `json:"chat_active"`
	UsersCount  int        `json:"users_count"`
	AdminId     int64      `json:"admin_id"`
	Action      string     `json:"action"`
	ActionMid   int64      `json:"action_mid"`   /*идентификатор пользователя (если > 0) или email (если < 0), которого пригласили или исключили (для служебных сообщений с action = chat_invite_user или chat_kick_user). */
	ActionEmail string     `json:"action_email"` /*email, который пригласили или исключили (для служебных сообщений с action = chat_invite_user или chat_kick_user и отрицательным action_mid). */
	ActionText  string     `json:"action_text"`  /*название беседы (для служебных сообщений с action = chat_create или chat_title_update). */
	Photo50     string     `json:"photo_50"`
	Photo100    string     `json:"photo_100"`
	Photo200    string     `json:"photo_200"`
	/*Geo       *Geo {
		type (string) — тип места;
		coordinates (string) — координаты места;
		place (object) — описание места (если оно добавлено), объект с полями:
		id (integer) — идентификатор места (если назначено);
		title (string) — название места (если назначено);
		latitude (number) — географическая широта;
		longitude (number) — географическая долгота;
		created (integer) — дата создания (если назначено);
		icon (string) — URL изображения-иконки;
		country (string) — название страны;
		city (string) — название города;
	} `json:"geo"`*/

	/*Attachments *[]Attachments `json:"attachments"`*/
	/*PushSettings *PushSettings { настройки уведомлений для беседы, если они есть.	} `json:"push_settings"`*/
	/*string	тип действия (если это служебное сообщение). Возможные значения:

	  chat_photo_update — обновлена фотография беседы;
	  chat_photo_remove — удалена фотография беседы;
	  chat_create — создана беседа;
	  chat_title_update — обновлено название беседы;
	  chat_invite_user — приглашен пользователь;
	  chat_kick_user — исключен пользователь.*/

}

// MessageConfig contains the data
// necessary to send a message.
type MessageConfig struct {
	UserID          int64   `json:"user_id"`
	RandomID        int64   `json:"random_id"`
	PeerID          int64   `json:"peer_id"`
	Domain          string  `json:"domain"`
	ChatID          int64   `json:"chat_id"`
	GroupID         int64   `json:"group_id"`
	UserIDs         []int64 `json:"user_ids"`
	Message         string  `json:"message"`
	geo             bool    `json:"-"`
	lat             float64 `json:"lat"`
	long            float64 `json:"long"`
	ForwardMessages []int64 `json:"forward_messages"`
	StickerID       int64   `json:"sticker_id"`
	AccessToken     string  `json:"access_token"`
	//attachment *[]Attachment `json:"attachment"`
}

// NewMCFromUserID creates a new MessageConfig instance from userID.
func NewMCFromUserID(userID int64) (config MessageConfig) {
	config.UserID = userID
	return
}

// NewMCFromPeerID creates a new MessageConfig instance from peerID.
func NewMCFromPeerID(peerID int64) (config MessageConfig) {
	config.PeerID = peerID
	return
}

// NewMCFromChatID creates a new MessageConfig instance from chatID.
func NewMCFromChatID(chatID int64) (config MessageConfig) {
	config.ChatID = chatID
	return
}

// NewMCFromGroupID creates a new MessageConfig instance from groupID.
func NewMCFromGroupID(groupID int64) (config MessageConfig) {
	config.GroupID = groupID
	return
}

// NewMCFromDomain creates a new MessageConfig instance from domain.
func NewMCFromDomain(domain string) (config MessageConfig) {
	config.Domain = domain
	return
}

// SetGeo sets the location.
func (m *MessageConfig) SetGeo(lat float64, long float64) {
	m.geo = true
	m.lat = lat
	m.long = long
}

// NewMessage creates a new message for the user from the text.
func NewMessage(id int64, message string) (config MessageConfig) {
	config.PeerID = id
	config.Message = message
	return
}

// NewMessageToChat creates a new message for the chat from the text.
func NewMessageToChat(id int64, message string) (config MessageConfig) {
	return NewMessage(id+chatOffset, message)
}

// NewMessageToUsers creates a new message for several users from the text.
func NewMessageToUsers(message string, ids ...int64) (config MessageConfig) {
	config.UserIDs = ids
	config.Message = message
	return
}

// SendMessage tries to send a message with the configuration
// from the MessageConfig and returns message ID if it succeeds.
func (client *Client) SendMessage(config MessageConfig) (int64, *Error) {
	var req Request
	req.Token = config.AccessToken
	req.Method = "messages.send"
	v := url.Values{}

	if config.PeerID != 0 {
		v.Add("peer_id", fmt.Sprintf("%d", config.PeerID))
	}

	if config.UserID != 0 {
		v.Add("user_id", fmt.Sprintf("%d", config.UserID))
	}

	if config.Domain != "" {
		v.Add("domain", config.Domain)
	}

	if config.ChatID != 0 {
		v.Add("chat_id", fmt.Sprintf("%d", config.RandomID))
	}

	if len(config.UserIDs) != 0 {
		v.Add("user_ids", ConcatInt64ToString(config.UserIDs...))
	}

	if len(config.ForwardMessages) != 0 {
		v.Add("forward_messages", ConcatInt64ToString(config.ForwardMessages...))
	}

	if config.StickerID != 0 {
		v.Add("sticker_id", fmt.Sprintf("%d", config.StickerID))
	}

	if config.Message != "" {
		v.Add("message", config.Message)
	}

	if config.RandomID != 0 {
		v.Add("random_id", fmt.Sprintf("%d", config.RandomID))
	}

	if config.geo {
		v.Add("lat", strconv.FormatFloat(config.lat, 'f', -1, 64))
		v.Add("long", strconv.FormatFloat(config.long, 'f', -1, 64))
	}

	req.Values = v
	res, err := client.Do(req)
	if err != nil && !err.Code.Is(ErrZero) {
		return 0, err
	}

	answer, error := strconv.ParseInt(res.Response.String(), 10, 64)
	if error != nil {
		return 0, NewError(ErrBadResponseCode, error.Error())
	}

	return answer, nil
}

// SetActivity changes the status of typing by user in the dialog.
func (client *Client) SetActivity(config MessageConfig) *Error {
	values := url.Values{}

	switch {
	case config.Domain != "":
		values.Add("user_id", config.Domain)
	case config.UserID != 0:
		values.Add("peer_id", strconv.FormatInt(config.UserID, 10))
	case config.PeerID != 0:
		values.Add("peer_id", strconv.FormatInt(config.PeerID, 10))
	case config.ChatID != 0:
		values.Add("peer_id", strconv.FormatInt(config.ChatID+chatOffset, 10))
	case config.GroupID != 0:
		values.Add("peer_id", strconv.FormatInt(-config.GroupID, 10))
	default:
		return NewError(ErrBadCode, "Wrong data")
	}

	values.Add("type", "typing")
	_, err := client.Do(NewRequest("messages.setActivity", "", values))
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>