<commit_before>package aws

import (
	"fmt"
	"regexp"
	"testing"

	"github.com/hashicorp/terraform-plugin-sdk/helper/acctest"
	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
)

func TestAccAWSEcrDataSource_ecrRepository(t *testing.T) {
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "data.aws_ecr_repository.default"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccCheckAwsEcrRepositoryDataSourceConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					resource.TestMatchResourceAttr(resourceName, "arn", regexp.MustCompile(`^arn:aws:ecr:[a-zA-Z]+-[a-zA-Z]+-\d+:\d+:repository/tf-acc-test-\d+$`)),
					resource.TestCheckResourceAttrSet(resourceName, "registry_id"),
					resource.TestMatchResourceAttr(resourceName, "repository_url", regexp.MustCompile(`^\d+\.dkr\.ecr\.[a-zA-Z]+-[a-zA-Z]+-\d+\.amazonaws\.com/tf-acc-test-\d+$`)),
					resource.TestCheckResourceAttr(resourceName, "tags.%", "2"),
					resource.TestCheckResourceAttr(resourceName, "tags.Usage", "original"),
				),
			},
		},
	})
}

func testAccCheckAwsEcrRepositoryDataSourceConfig(rName string) string {
	return fmt.Sprintf(`
resource "aws_ecr_repository" "default" {
  name = %q

  tags = {
    Environment = "production"
    Usage       = "original"
  }
}

data "aws_ecr_repository" "default" {
  name = "${aws_ecr_repository.default.name}"
}
`, rName)
}
<commit_msg>Updates ECR acceptance tests to use ARN testing check functions<commit_after>package aws

import (
	"fmt"
	"regexp"
	"testing"

	"github.com/hashicorp/terraform-plugin-sdk/helper/acctest"
	"github.com/hashicorp/terraform-plugin-sdk/helper/resource"
)

func TestAccAWSEcrDataSource_ecrRepository(t *testing.T) {
	rName := acctest.RandomWithPrefix("tf-acc-test")
	resourceName := "data.aws_ecr_repository.default"

	resource.ParallelTest(t, resource.TestCase{
		PreCheck:  func() { testAccPreCheck(t) },
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: testAccCheckAwsEcrRepositoryDataSourceConfig(rName),
				Check: resource.ComposeTestCheckFunc(
					testAccMatchResourceAttrRegionalARN(resourceName, "arn", "ecr", regexp.MustCompile(fmt.Sprintf("repository/%s$", rName))),
					resource.TestCheckResourceAttrSet(resourceName, "registry_id"),
					resource.TestMatchResourceAttr(resourceName, "repository_url", regexp.MustCompile(fmt.Sprintf(`^\d+\.dkr\.ecr\.%s\.amazonaws\.com/%s$`, testAccGetRegion(), rName))),
					resource.TestCheckResourceAttr(resourceName, "tags.%", "2"),
					resource.TestCheckResourceAttr(resourceName, "tags.Usage", "original"),
				),
			},
		},
	})
}

func testAccCheckAwsEcrRepositoryDataSourceConfig(rName string) string {
	return fmt.Sprintf(`
resource "aws_ecr_repository" "default" {
  name = %q

  tags = {
    Environment = "production"
    Usage       = "original"
  }
}

data "aws_ecr_repository" "default" {
  name = "${aws_ecr_repository.default.name}"
}
`, rName)
}
<|endoftext|>


<commit_before>// gRPC client
package gcli

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"os"
	"time"

	"github.com/glycerine/blake2b" // vendor https://github.com/dchest/blake2b"
	pb "github.com/glycerine/hnatsd/peer/streambigfile"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"hash"
)

var utclog *log.Logger

func init() {
	utclog = log.New(os.Stderr, "", log.LUTC|log.LstdFlags|log.Lmicroseconds)
}

type client struct {
	hasher     hash.Hash
	nextChunk  int64
	peerClient pb.PeerClient
}

func newClient(conn *grpc.ClientConn) *client {
	h, err := blake2b.New(nil)
	panicOn(err)
	return &client{
		hasher:     h,
		peerClient: pb.NewPeerClient(conn),
	}
}

func (c *client) startNewFile() {
	c.hasher.Reset()
	c.nextChunk = 0
}

func (c *client) runSendFile(path string, data []byte, maxChunkSize int, isBcastSet bool, myID string) error {
	//p("client runSendFile(path='%s') starting", path)

	startOfRunSendFile := time.Now().UTC()
	startOfRunSendFileNanoUint64 := uint64(startOfRunSendFile.UnixNano())

	c.startNewFile()
	stream, err := c.peerClient.SendFile(context.Background())
	if err != nil {
		utclog.Fatalf("%v.SendFile(_) = _, %v", c.peerClient, err)
	}
	n := len(data)
	numChunk := n / maxChunkSize
	if n%maxChunkSize > 0 {
		numChunk++
	}
	nextByte := 0
	lastChunk := numChunk - 1
	//p("'%s' client sees %v chunks of size ~ %v bytes", path, numChunk, intMin(n, maxChunkSize))
	for i := 0; i < numChunk; i++ {
		sendLen := intMin(maxChunkSize, n-(i*maxChunkSize))
		chunk := data[nextByte:(nextByte + sendLen)]
		nextByte += sendLen

		var nk pb.BigFileChunk
		nk.IsBcastSet = isBcastSet
		nk.Filepath = path
		nk.SizeInBytes = int64(sendLen)
		nk.SendTime = uint64(time.Now().UnixNano())
		nk.OriginalStartSendTime = startOfRunSendFileNanoUint64

		// checksums
		c.hasher.Write(chunk)
		nk.Blake2B = blake2bOfBytes(chunk)
		nk.Blake2BCumulative = []byte(c.hasher.Sum(nil))

		nk.Data = chunk
		nk.ChunkNumber = c.nextChunk
		c.nextChunk++
		nk.IsLastChunk = (i == lastChunk)

		//		if nk.ChunkNumber%100 == 0 {
		//p("client, on chunk %v of '%s', checksum='%x', and cumul='%x'", nk.ChunkNumber, nk.Filepath, nk.Blake2B, nk.Blake2BCumulative)
		//		}

		if err := stream.Send(&nk); err != nil {
			// EOF?
			if err == io.EOF {
				if !nk.IsLastChunk {
					panic(fmt.Sprintf("'%s' we got io.EOF before "+
						"the last chunk! At: %v of %v", path, nk.ChunkNumber, numChunk))
				} else {
					break
				}
			}
			panic(err)
			//utclog.Fatalf("%v.Send() = %v", stream, err)
		}
	}
	reply, err := stream.CloseAndRecv()
	if err != nil {
		// EOF ??
		utclog.Printf("%v.CloseAndRecv() got error %v, want %v. reply=%v", stream, err, nil, reply)
		return err
	}

	compared := bytes.Compare(reply.WholeFileBlake2B, []byte(c.hasher.Sum(nil)))
	utclog.Printf("%s client.runSendFile got from stream.CloseAndRecv() a Reply with checksum: '%x'; checksum matches the sent data: %v; size sent = %v, size received = %v. startOfRunSendFile='%v'.", myID, reply.WholeFileBlake2B, compared == 0, len(data), reply.SizeInBytes, startOfRunSendFile)

	if int64(len(data)) != reply.SizeInBytes {
		panic("size mismatch")
	}

	return nil
}

func blake2bOfBytes(by []byte) []byte {
	h, err := blake2b.New(nil)
	panicOn(err)
	h.Write(by)
	return []byte(h.Sum(nil))
}

func intMin(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func SequentialPayload(n int64) []byte {
	if n%8 != 0 {
		panic(fmt.Sprintf("n == %v must be a multiple of 8; has remainder %v", n, n%8))
	}

	k := uint64(n / 8)
	by := make([]byte, n)
	j := uint64(0)
	for i := uint64(0); i < k; i++ {
		j = i * 8
		binary.LittleEndian.PutUint64(by[j:j+8], j)
	}
	return by
}

func (cfg *ClientConfig) ClientSendFile(path string, data []byte, isBcastSet bool, myID string) error {

	var opts []grpc.DialOption
	if cfg.SkipEncryption {
		opts = append(opts, grpc.WithInsecure())
	} else {
		if cfg.UseTLS {
			cfg.setupTLS(&opts)
		} else {
			cfg.setupSSH(&opts)
		}
	}

	serverAddr := fmt.Sprintf("%v:%v", cfg.ServerHost, cfg.ServerPort)

	conn, err := grpc.Dial(serverAddr, opts...)
	if err != nil {
		return err
	}
	defer conn.Close()

	// SendFile
	c := newClient(conn)

	chunkSz := 1 << 20

	t0 := time.Now()
	err = c.runSendFile(path, data, chunkSz, isBcastSet, myID)
	t1 := time.Now()
	elap := t1.Sub(t0)
	if err != nil {
		utclog.Printf("%s ClientSendFile: c.runSendFile sees error '%v' after elap %v", myID, err, elap)
		return err
	}
	mb := float64(len(data)) / float64(1<<20)
	_ = mb
	_ = elap
	utclog.Printf("%s ClientSendFile: elap time to runSendFile(path='%s', len(data)=%v) on %v MB was %v => %.03f MB/sec", myID, path, len(data), mb, elap, mb/(float64(elap)/1e9))
	return nil
}

func (cfg *ClientConfig) setupTLS(opts *[]grpc.DialOption) {
	var sn string
	if cfg.ServerHostOverride != "" {
		sn = cfg.ServerHostOverride
	}
	var creds credentials.TransportCredentials
	if cfg.CertPath != "" {
		var err error
		creds, err = credentials.NewClientTLSFromFile(cfg.CertPath, sn)
		if err != nil {
			utclog.Fatalf("Failed to create TLS credentials %v", err)
		}
	} else {
		creds = credentials.NewClientTLSFromCert(nil, sn)
	}
	*opts = append(*opts, grpc.WithTransportCredentials(creds))
}

func (cfg *ClientConfig) setupSSH(opts *[]grpc.DialOption) {

	destAddr := fmt.Sprintf("%v:%v", cfg.ServerInternalHost, cfg.ServerInternalPort)

	dialer, err := clientSshMain(cfg.AllowNewServer, cfg.TestAllowOneshotConnect, cfg.PrivateKeyPath, cfg.ClientKnownHostsPath, cfg.Username, cfg.ServerHost, destAddr, int64(cfg.ServerPort))
	panicOn(err)

	*opts = append(*opts, grpc.WithDialer(dialer))

	// have to do this too, since we are using an SSH tunnel
	// that grpc doesn't know about:
	*opts = append(*opts, grpc.WithInsecure())
}
<commit_msg>atg. retry for 10 seconds if runSendFile fails<commit_after>// gRPC client
package gcli

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
	"log"
	"os"
	"time"

	"github.com/glycerine/blake2b" // vendor https://github.com/dchest/blake2b"
	pb "github.com/glycerine/hnatsd/peer/streambigfile"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"hash"
)

var utclog *log.Logger

func init() {
	utclog = log.New(os.Stderr, "", log.LUTC|log.LstdFlags|log.Lmicroseconds)
}

type client struct {
	hasher     hash.Hash
	nextChunk  int64
	peerClient pb.PeerClient
}

func newClient(conn *grpc.ClientConn) *client {
	h, err := blake2b.New(nil)
	panicOn(err)
	return &client{
		hasher:     h,
		peerClient: pb.NewPeerClient(conn),
	}
}

func (c *client) startNewFile() {
	c.hasher.Reset()
	c.nextChunk = 0
}

func (c *client) runSendFile(path string, data []byte, maxChunkSize int, isBcastSet bool, myID string) error {
	//p("client runSendFile(path='%s') starting", path)

	startOfRunSendFile := time.Now().UTC()
	startOfRunSendFileNanoUint64 := uint64(startOfRunSendFile.UnixNano())

	c.startNewFile()
	try := 0
	tryLimit := 5
	var err error
	var stream pb.Peer_SendFileClient
	for {
		stream, err = c.peerClient.SendFile(context.Background())
		if err == nil {
			break
		}
		try++
		if try > tryLimit {
			panic(fmt.Sprintf("%v.SendFile(_) = _, %v", c.peerClient, err))
		}
		utclog.Printf("on try %v of %v, %v.SendFile() got err='%v'", try, tryLimit, c.peerClient, err)
		time.Sleep(2 * time.Second)
	}
	n := len(data)
	numChunk := n / maxChunkSize
	if n%maxChunkSize > 0 {
		numChunk++
	}
	nextByte := 0
	lastChunk := numChunk - 1
	//p("'%s' client sees %v chunks of size ~ %v bytes", path, numChunk, intMin(n, maxChunkSize))
	for i := 0; i < numChunk; i++ {
		sendLen := intMin(maxChunkSize, n-(i*maxChunkSize))
		chunk := data[nextByte:(nextByte + sendLen)]
		nextByte += sendLen

		var nk pb.BigFileChunk
		nk.IsBcastSet = isBcastSet
		nk.Filepath = path
		nk.SizeInBytes = int64(sendLen)
		nk.SendTime = uint64(time.Now().UnixNano())
		nk.OriginalStartSendTime = startOfRunSendFileNanoUint64

		// checksums
		c.hasher.Write(chunk)
		nk.Blake2B = blake2bOfBytes(chunk)
		nk.Blake2BCumulative = []byte(c.hasher.Sum(nil))

		nk.Data = chunk
		nk.ChunkNumber = c.nextChunk
		c.nextChunk++
		nk.IsLastChunk = (i == lastChunk)

		//		if nk.ChunkNumber%100 == 0 {
		//p("client, on chunk %v of '%s', checksum='%x', and cumul='%x'", nk.ChunkNumber, nk.Filepath, nk.Blake2B, nk.Blake2BCumulative)
		//		}

		if err := stream.Send(&nk); err != nil {
			// EOF?
			if err == io.EOF {
				if !nk.IsLastChunk {
					panic(fmt.Sprintf("'%s' we got io.EOF before "+
						"the last chunk! At: %v of %v", path, nk.ChunkNumber, numChunk))
				} else {
					break
				}
			}
			panic(err)
			//utclog.Fatalf("%v.Send() = %v", stream, err)
		}
	}
	reply, err := stream.CloseAndRecv()
	if err != nil {
		// EOF ??
		utclog.Printf("%v.CloseAndRecv() got error %v, want %v. reply=%v", stream, err, nil, reply)
		return err
	}

	compared := bytes.Compare(reply.WholeFileBlake2B, []byte(c.hasher.Sum(nil)))
	utclog.Printf("%s client.runSendFile got from stream.CloseAndRecv() a Reply with checksum: '%x'; checksum matches the sent data: %v; size sent = %v, size received = %v. startOfRunSendFile='%v'.", myID, reply.WholeFileBlake2B, compared == 0, len(data), reply.SizeInBytes, startOfRunSendFile)

	if int64(len(data)) != reply.SizeInBytes {
		panic("size mismatch")
	}

	return nil
}

func blake2bOfBytes(by []byte) []byte {
	h, err := blake2b.New(nil)
	panicOn(err)
	h.Write(by)
	return []byte(h.Sum(nil))
}

func intMin(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func SequentialPayload(n int64) []byte {
	if n%8 != 0 {
		panic(fmt.Sprintf("n == %v must be a multiple of 8; has remainder %v", n, n%8))
	}

	k := uint64(n / 8)
	by := make([]byte, n)
	j := uint64(0)
	for i := uint64(0); i < k; i++ {
		j = i * 8
		binary.LittleEndian.PutUint64(by[j:j+8], j)
	}
	return by
}

func (cfg *ClientConfig) ClientSendFile(path string, data []byte, isBcastSet bool, myID string) error {

	var opts []grpc.DialOption
	if cfg.SkipEncryption {
		opts = append(opts, grpc.WithInsecure())
	} else {
		if cfg.UseTLS {
			cfg.setupTLS(&opts)
		} else {
			cfg.setupSSH(&opts)
		}
	}

	serverAddr := fmt.Sprintf("%v:%v", cfg.ServerHost, cfg.ServerPort)

	conn, err := grpc.Dial(serverAddr, opts...)
	if err != nil {
		return err
	}
	defer conn.Close()

	// SendFile
	c := newClient(conn)

	chunkSz := 1 << 20

	t0 := time.Now()
	err = c.runSendFile(path, data, chunkSz, isBcastSet, myID)
	t1 := time.Now()
	elap := t1.Sub(t0)
	if err != nil {
		utclog.Printf("%s ClientSendFile: c.runSendFile sees error '%v' after elap %v", myID, err, elap)
		return err
	}
	mb := float64(len(data)) / float64(1<<20)
	_ = mb
	_ = elap
	utclog.Printf("%s ClientSendFile: elap time to runSendFile(path='%s', len(data)=%v) on %v MB was %v => %.03f MB/sec", myID, path, len(data), mb, elap, mb/(float64(elap)/1e9))
	return nil
}

func (cfg *ClientConfig) setupTLS(opts *[]grpc.DialOption) {
	var sn string
	if cfg.ServerHostOverride != "" {
		sn = cfg.ServerHostOverride
	}
	var creds credentials.TransportCredentials
	if cfg.CertPath != "" {
		var err error
		creds, err = credentials.NewClientTLSFromFile(cfg.CertPath, sn)
		if err != nil {
			utclog.Fatalf("Failed to create TLS credentials %v", err)
		}
	} else {
		creds = credentials.NewClientTLSFromCert(nil, sn)
	}
	*opts = append(*opts, grpc.WithTransportCredentials(creds))
}

func (cfg *ClientConfig) setupSSH(opts *[]grpc.DialOption) {

	destAddr := fmt.Sprintf("%v:%v", cfg.ServerInternalHost, cfg.ServerInternalPort)

	dialer, err := clientSshMain(cfg.AllowNewServer, cfg.TestAllowOneshotConnect, cfg.PrivateKeyPath, cfg.ClientKnownHostsPath, cfg.Username, cfg.ServerHost, destAddr, int64(cfg.ServerPort))
	panicOn(err)

	*opts = append(*opts, grpc.WithDialer(dialer))

	// have to do this too, since we are using an SSH tunnel
	// that grpc doesn't know about:
	*opts = append(*opts, grpc.WithInsecure())
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/jasondelponte/golang-utils/seqgen/perlinnoise"
)

var height = flag.Int("h", 40, "Height of the display")
var width = flag.Int("w", 100, "Width of the display")
var seed = flag.Int64("s", -1, "Seed")

func main() {
	flag.Parse()

	h := *height
	w := *width

	var pn *perlinnoise.PerlinNoise = nil
	if *seed > 0 {
		pn = perlinnoise.New(*seed)
	} else {
		pn = perlinnoise.NewDefault()
	}

	for i := 0; i < h; i++ { // y
		line := ""
		y := float64(i) / float64(h)
		for j := 0; j < w; j++ { // x
			x := float64(j) / float64(w)

			n := pn.Noise(10*x, 10*y, 0.8)

			if n < 0.35 {
				line += "~"

			} else if n >= 0.35 && n < 0.5 {
				line += "."

			} else if n >= 0.5 && n < 0.6 {
				line += ","

			} else if n >= 0.5 && n < 0.8 {
				line += "#"

			} else {
				line += "S"
			}
		}
		fmt.Println(line)
	}
}
<commit_msg>fixed package naming for example executable<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/jasondelponte/perlinnoise"
)

var height = flag.Int("h", 40, "Height of the display")
var width = flag.Int("w", 100, "Width of the display")
var seed = flag.Int64("s", -1, "Seed")

func main() {
	flag.Parse()

	h := *height
	w := *width

	var pn *perlinnoise.PerlinNoise = nil
	if *seed > 0 {
		pn = perlinnoise.New(*seed)
	} else {
		pn = perlinnoise.NewDefault()
	}

	for i := 0; i < h; i++ { // y
		line := ""
		y := float64(i) / float64(h)
		for j := 0; j < w; j++ { // x
			x := float64(j) / float64(w)

			n := pn.Noise(10*x, 10*y, 0.8)

			if n < 0.35 {
				line += "~"

			} else if n >= 0.35 && n < 0.5 {
				line += "."

			} else if n >= 0.5 && n < 0.6 {
				line += ","

			} else if n >= 0.5 && n < 0.8 {
				line += "#"

			} else {
				line += "S"
			}
		}
		fmt.Println(line)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/keltia/ripe-atlas"
	"github.com/urfave/cli"
	"log"
	"os"
)

// init injects our "traceroute" related commands/options.
func init() {
	// Fill-in the various commands
	cliCommands = append(cliCommands, cli.Command{
		Name:        "traceroute",
		Aliases:     []string{"trace"},
		Usage:       "traceroute to given host/IP",
		Description: "Send Traceroute queries to an host/IP",
		Flags: []cli.Flag{
			cli.IntFlag{
				Name:        "max_hops, m",
				Usage:       "Specify a maximum number of hops",
				Destination: &fMaxHops,
			},
			cli.IntFlag{
				Name:        "packet_size, s",
				Usage:       "Sends packets this size",
				Destination: &fPacketSize,
			},
			cli.BoolFlag{
				Name:        "6, ipv6",
				Usage:       "displays only IPv6",
				Destination: &fWant6,
			},
			cli.BoolFlag{
				Name:        "4, ipv4",
				Usage:       "displays only IPv4",
				Destination: &fWant4,
			},
		},
		Action: cmdTraceroute,
	})
}

func cmdTraceroute(c *cli.Context) error {
	// By default do both
	if !fWant4 && !fWant6 {
		fWant6, fWant4 = true, true
	}

	args := c.Args()
	if len(args) == 0 {
		log.Fatal("Error: you must specify a hostname/site!")
	}

	target := args[0]
	var defs []atlas.Definition

	if fWant4 {
		def := atlas.Definition{
			AF:          4,
			Description: fmt.Sprintf("Traceroute - %s", target),
			Type:        "traceroute",
			Target:      target,
		}
		if fPacketSize != 0 {
			def.Size = fPacketSize
		}
		if fMaxHops != 0 {
			def.MaxHops = fMaxHops
		}
		defs = append(defs, def)
	}

	if fWant6 {
		def := atlas.Definition{
			AF:          6,
			Description: fmt.Sprintf("Traceroute6 - %s", target),
			Type:        "traceroute",
			Target:      target,
		}
		if fPacketSize != 0 {
			def.Size = fPacketSize
		}
		if fMaxHops != 0 {
			def.MaxHops = fMaxHops
		}
		defs = append(defs, def)
	}

	req := atlas.MeasurementRequest{
		Definitions: defs,
		IsOneoff:    true,
	}

	// Default set of probes
	probes := atlas.ProbeSet{
		{
			Requested: mycnf.PoolSize,
			Type:      "area",
			Value:     "WW",
			Tags:      nil,
		},
	}

	req.Probes = probes
	log.Printf("req=%#v", req)
	//str := res.Result.Display()

	trc, err := atlas.Traceroute(req)
	if err != nil {
		fmt.Printf("err: %v", err)
		os.Exit(1)
	}
	fmt.Printf("NTP: %#v", trc)

	return nil
}
<commit_msg>Adjust parameters.<commit_after>package main

import (
	"fmt"
	"github.com/keltia/ripe-atlas"
	"github.com/urfave/cli"
	"log"
	"os"
)

const (
	defMaxHops = 30
)

// init injects our "traceroute" related commands/options.
func init() {
	// Fill-in the various commands
	cliCommands = append(cliCommands, cli.Command{
		Name:        "traceroute",
		Aliases:     []string{"trace"},
		Usage:       "traceroute to given host/IP",
		Description: "Send Traceroute queries to an host/IP",
		Flags: []cli.Flag{
			cli.IntFlag{
				Name:        "max_hops, m",
				Usage:       "Specify a maximum number of hops",
				Destination: &fMaxHops,
			},
			cli.IntFlag{
				Name:        "packet_size, s",
				Usage:       "Sends packets this size",
				Destination: &fPacketSize,
			},
			cli.StringFlag{
				Name:        "p, protocol",
				Usage:       "Select UDP or TCP",
				Destination: &fProtocol,
			},
			cli.BoolFlag{
				Name:        "6, ipv6",
				Usage:       "displays only IPv6",
				Destination: &fWant6,
			},
			cli.BoolFlag{
				Name:        "4, ipv4",
				Usage:       "displays only IPv4",
				Destination: &fWant4,
			},
		},
		Action: cmdTraceroute,
	})
}

func cmdTraceroute(c *cli.Context) error {
	var (
		maxHops    int
		packetSize int
		proto      string
	)

	// By default do both
	if !fWant4 && !fWant6 {
		fWant6, fWant4 = true, true
	}

	args := c.Args()
	if len(args) == 0 {
		log.Fatal("Error: you must specify a hostname/site!")
	}

	target := args[0]
	var defs []atlas.Definition

	proto = defProtocol
	maxHops = defMaxHops

	if fPacketSize != 0 {
		packetSize = fPacketSize
	}
	if fMaxHops != 0 {
		maxHops = fMaxHops
	}

	if fWant4 {
		def := atlas.Definition{
			AF:          4,
			Description: fmt.Sprintf("Traceroute - %s", target),
			Type:        "traceroute",
			Target:      target,
			Protocol:    proto,
			MaxHops:     maxHops,
			Size:        packetSize,
		}
		defs = append(defs, def)
	}

	if fWant6 {
		def := atlas.Definition{
			AF:          6,
			Description: fmt.Sprintf("Traceroute6 - %s", target),
			Type:        "traceroute",
			Target:      target,
			Protocol:    proto,
			MaxHops:     maxHops,
			Size:        packetSize,
		}
		defs = append(defs, def)
	}

	req := atlas.MeasurementRequest{
		Definitions: defs,
		IsOneoff:    true,
	}

	// Default set of probes
	probes := atlas.ProbeSet{
		{
			Requested: mycnf.PoolSize,
			Type:      "area",
			Value:     "WW",
			Tags:      nil,
		},
	}

	req.Probes = probes
	log.Printf("req=%#v", req)
	//str := res.Result.Display()

	trc, err := atlas.Traceroute(req)
	if err != nil {
		fmt.Printf("err: %v", err)
		os.Exit(1)
	}
	fmt.Printf("NTP: %#v", trc)

	return nil
}
<|endoftext|>


<commit_before>package main_test

import (
	"github.com/cloudfoundry-incubator/auctioneer"
	"github.com/cloudfoundry-incubator/bbs"
	"github.com/cloudfoundry-incubator/bbs/models"
	"github.com/cloudfoundry-incubator/bbs/models/test/model_helpers"
	"github.com/cloudfoundry-incubator/locket"
	"github.com/cloudfoundry-incubator/rep"
	"github.com/cloudfoundry-incubator/runtime-schema/diego_errors"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

var dummyAction = &models.RunAction{
	User: "me",
	Path: "cat",
	Args: []string{"/tmp/file"},
}

var exampleDesiredLRP = models.DesiredLRP{
	ProcessGuid: "process-guid",
	DiskMb:      1,
	MemoryMb:    1,
	RootFs:      linuxRootFSURL,
	Action:      models.WrapAction(dummyAction),
	Domain:      "test",
	Instances:   2,
}

func exampleTaskDefinition() *models.TaskDefinition {
	taskDef := model_helpers.NewValidTaskDefinition()
	taskDef.RootFs = linuxRootFSURL
	taskDef.Action = models.WrapAction(dummyAction)
	return taskDef
}

var _ = Describe("Auctioneer", func() {
	Context("when etcd is down", func() {
		BeforeEach(func() {
			etcdRunner.Stop()
			auctioneerProcess = ginkgomon.Invoke(runner)
		})

		AfterEach(func() {
			etcdRunner.Start()
		})

		It("starts", func() {
			Consistently(runner).ShouldNot(Exit())
		})
	})

	Context("when a start auction message arrives", func() {
		BeforeEach(func() {
			auctioneerProcess = ginkgomon.Invoke(runner)

			err := auctioneerClient.RequestLRPAuctions([]*auctioneer.LRPStartRequest{{
				ProcessGuid: exampleDesiredLRP.ProcessGuid,
				Domain:      exampleDesiredLRP.Domain,
				Indices:     []int{0},
				Resource: rep.Resource{
					MemoryMB: 5,
					DiskMB:   5,
					RootFs:   exampleDesiredLRP.RootFs,
				},
			}})
			Expect(err).NotTo(HaveOccurred())

			err = auctioneerClient.RequestLRPAuctions([]*auctioneer.LRPStartRequest{{
				ProcessGuid: exampleDesiredLRP.ProcessGuid,
				Domain:      exampleDesiredLRP.Domain,
				Indices:     []int{1},
				Resource: rep.Resource{
					MemoryMB: 5,
					DiskMB:   5,
					RootFs:   exampleDesiredLRP.RootFs,
				},
			}})
			Expect(err).NotTo(HaveOccurred())
		})

		It("should start the process running on reps of the appropriate stack", func() {
			Eventually(linuxCell.LRPs).Should(HaveLen(2))
			Expect(dotNetCell.LRPs()).To(BeEmpty())
		})
	})

	Context("when a task message arrives", func() {
		BeforeEach(func() {
			auctioneerProcess = ginkgomon.Invoke(runner)
		})

		Context("when there are sufficient resources to start the task", func() {
			BeforeEach(func() {
				taskDef := exampleTaskDefinition()
				taskDef.DiskMb = 1
				taskDef.MemoryMb = 1
				err := bbsClient.DesireTask("guid", "domain", taskDef)
				Expect(err).NotTo(HaveOccurred())
			})

			It("should start the task running on reps of the appropriate stack", func() {
				Eventually(linuxCell.Tasks).Should(HaveLen(1))
				Expect(dotNetCell.Tasks()).To(BeEmpty())
			})
		})

		Context("when there are insufficient resources to start the task", func() {
			BeforeEach(func() {
				taskDef := exampleTaskDefinition()
				taskDef.DiskMb = 1000
				taskDef.MemoryMb = 1000

				err := bbsClient.DesireTask("task-guid", "domain", taskDef)
				Expect(err).NotTo(HaveOccurred())
			})

			It("should not start the task on any rep", func() {
				Consistently(linuxCell.Tasks).Should(BeEmpty())
				Consistently(dotNetCell.Tasks).Should(BeEmpty())
			})

			It("should mark the task as failed in the BBS", func() {
				Eventually(func() []*models.Task {
					return getTasksByState(bbsClient, models.Task_Completed)
				}).Should(HaveLen(1))

				completedTasks := getTasksByState(bbsClient, models.Task_Completed)
				completedTask := completedTasks[0]
				Expect(completedTask.TaskGuid).To(Equal("task-guid"))
				Expect(completedTask.Failed).To(BeTrue())
				Expect(completedTask.FailureReason).To(Equal(diego_errors.INSUFFICIENT_RESOURCES_MESSAGE))
			})
		})
	})

	Context("when the auctioneer loses the lock", func() {
		BeforeEach(func() {
			auctioneerProcess = ginkgomon.Invoke(runner)
			consulRunner.Reset()
		})

		It("exits with an error", func() {
			Eventually(runner.ExitCode, 3).Should(Equal(1))
		})
	})

	Context("when the auctioneer cannot acquire the lock on startup", func() {
		var task *rep.Task

		BeforeEach(func() {
			task = &rep.Task{
				TaskGuid: "task-guid",
				Domain:   "test",
				Resource: rep.Resource{
					MemoryMB: 124,
					DiskMB:   456,
					RootFs:   "some-rootfs",
				},
			}
			err := consulSession.AcquireLock(locket.LockSchemaPath("auctioneer_lock"), []byte{})
			Expect(err).NotTo(HaveOccurred())

			runner.StartCheck = "auctioneer.lock-bbs.lock.acquiring-lock"

			auctioneerProcess = ifrit.Background(runner)
		})

		It("should not advertise its presence, and should not be reachable", func() {
			Eventually(func() error {
				return auctioneerClient.RequestTaskAuctions([]*auctioneer.TaskStartRequest{
					&auctioneer.TaskStartRequest{*task},
				})
			}).Should(HaveOccurred())
		})

		It("should eventually come up in the event that the lock is released", func() {
			consulSession.Destroy()
			_, err := consulSession.Recreate()
			Expect(err).NotTo(HaveOccurred())

			Eventually(func() error {
				return auctioneerClient.RequestTaskAuctions([]*auctioneer.TaskStartRequest{
					&auctioneer.TaskStartRequest{*task},
				})
			}).ShouldNot(HaveOccurred())
		})
	})
})

func getTasksByState(client bbs.Client, state models.Task_State) []*models.Task {
	tasks, err := client.Tasks()
	Expect(err).NotTo(HaveOccurred())

	filteredTasks := make([]*models.Task, 0)
	for _, task := range tasks {
		if task.State == state {
			filteredTasks = append(filteredTasks, task)
		}
	}
	return filteredTasks
}
<commit_msg>Use diego_errors from stager on tests<commit_after>package main_test

import (
	"github.com/cloudfoundry-incubator/auctioneer"
	"github.com/cloudfoundry-incubator/bbs"
	"github.com/cloudfoundry-incubator/bbs/models"
	"github.com/cloudfoundry-incubator/bbs/models/test/model_helpers"
	"github.com/cloudfoundry-incubator/locket"
	"github.com/cloudfoundry-incubator/rep"
	"github.com/cloudfoundry-incubator/stager/diego_errors"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gexec"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"
)

var dummyAction = &models.RunAction{
	User: "me",
	Path: "cat",
	Args: []string{"/tmp/file"},
}

var exampleDesiredLRP = models.DesiredLRP{
	ProcessGuid: "process-guid",
	DiskMb:      1,
	MemoryMb:    1,
	RootFs:      linuxRootFSURL,
	Action:      models.WrapAction(dummyAction),
	Domain:      "test",
	Instances:   2,
}

func exampleTaskDefinition() *models.TaskDefinition {
	taskDef := model_helpers.NewValidTaskDefinition()
	taskDef.RootFs = linuxRootFSURL
	taskDef.Action = models.WrapAction(dummyAction)
	return taskDef
}

var _ = Describe("Auctioneer", func() {
	Context("when etcd is down", func() {
		BeforeEach(func() {
			etcdRunner.Stop()
			auctioneerProcess = ginkgomon.Invoke(runner)
		})

		AfterEach(func() {
			etcdRunner.Start()
		})

		It("starts", func() {
			Consistently(runner).ShouldNot(Exit())
		})
	})

	Context("when a start auction message arrives", func() {
		BeforeEach(func() {
			auctioneerProcess = ginkgomon.Invoke(runner)

			err := auctioneerClient.RequestLRPAuctions([]*auctioneer.LRPStartRequest{{
				ProcessGuid: exampleDesiredLRP.ProcessGuid,
				Domain:      exampleDesiredLRP.Domain,
				Indices:     []int{0},
				Resource: rep.Resource{
					MemoryMB: 5,
					DiskMB:   5,
					RootFs:   exampleDesiredLRP.RootFs,
				},
			}})
			Expect(err).NotTo(HaveOccurred())

			err = auctioneerClient.RequestLRPAuctions([]*auctioneer.LRPStartRequest{{
				ProcessGuid: exampleDesiredLRP.ProcessGuid,
				Domain:      exampleDesiredLRP.Domain,
				Indices:     []int{1},
				Resource: rep.Resource{
					MemoryMB: 5,
					DiskMB:   5,
					RootFs:   exampleDesiredLRP.RootFs,
				},
			}})
			Expect(err).NotTo(HaveOccurred())
		})

		It("should start the process running on reps of the appropriate stack", func() {
			Eventually(linuxCell.LRPs).Should(HaveLen(2))
			Expect(dotNetCell.LRPs()).To(BeEmpty())
		})
	})

	Context("when a task message arrives", func() {
		BeforeEach(func() {
			auctioneerProcess = ginkgomon.Invoke(runner)
		})

		Context("when there are sufficient resources to start the task", func() {
			BeforeEach(func() {
				taskDef := exampleTaskDefinition()
				taskDef.DiskMb = 1
				taskDef.MemoryMb = 1
				err := bbsClient.DesireTask("guid", "domain", taskDef)
				Expect(err).NotTo(HaveOccurred())
			})

			It("should start the task running on reps of the appropriate stack", func() {
				Eventually(linuxCell.Tasks).Should(HaveLen(1))
				Expect(dotNetCell.Tasks()).To(BeEmpty())
			})
		})

		Context("when there are insufficient resources to start the task", func() {
			BeforeEach(func() {
				taskDef := exampleTaskDefinition()
				taskDef.DiskMb = 1000
				taskDef.MemoryMb = 1000

				err := bbsClient.DesireTask("task-guid", "domain", taskDef)
				Expect(err).NotTo(HaveOccurred())
			})

			It("should not start the task on any rep", func() {
				Consistently(linuxCell.Tasks).Should(BeEmpty())
				Consistently(dotNetCell.Tasks).Should(BeEmpty())
			})

			It("should mark the task as failed in the BBS", func() {
				Eventually(func() []*models.Task {
					return getTasksByState(bbsClient, models.Task_Completed)
				}).Should(HaveLen(1))

				completedTasks := getTasksByState(bbsClient, models.Task_Completed)
				completedTask := completedTasks[0]
				Expect(completedTask.TaskGuid).To(Equal("task-guid"))
				Expect(completedTask.Failed).To(BeTrue())
				Expect(completedTask.FailureReason).To(Equal(diego_errors.INSUFFICIENT_RESOURCES_MESSAGE))
			})
		})
	})

	Context("when the auctioneer loses the lock", func() {
		BeforeEach(func() {
			auctioneerProcess = ginkgomon.Invoke(runner)
			consulRunner.Reset()
		})

		It("exits with an error", func() {
			Eventually(runner.ExitCode, 3).Should(Equal(1))
		})
	})

	Context("when the auctioneer cannot acquire the lock on startup", func() {
		var task *rep.Task

		BeforeEach(func() {
			task = &rep.Task{
				TaskGuid: "task-guid",
				Domain:   "test",
				Resource: rep.Resource{
					MemoryMB: 124,
					DiskMB:   456,
					RootFs:   "some-rootfs",
				},
			}
			err := consulSession.AcquireLock(locket.LockSchemaPath("auctioneer_lock"), []byte{})
			Expect(err).NotTo(HaveOccurred())

			runner.StartCheck = "auctioneer.lock-bbs.lock.acquiring-lock"

			auctioneerProcess = ifrit.Background(runner)
		})

		It("should not advertise its presence, and should not be reachable", func() {
			Eventually(func() error {
				return auctioneerClient.RequestTaskAuctions([]*auctioneer.TaskStartRequest{
					&auctioneer.TaskStartRequest{*task},
				})
			}).Should(HaveOccurred())
		})

		It("should eventually come up in the event that the lock is released", func() {
			consulSession.Destroy()
			_, err := consulSession.Recreate()
			Expect(err).NotTo(HaveOccurred())

			Eventually(func() error {
				return auctioneerClient.RequestTaskAuctions([]*auctioneer.TaskStartRequest{
					&auctioneer.TaskStartRequest{*task},
				})
			}).ShouldNot(HaveOccurred())
		})
	})
})

func getTasksByState(client bbs.Client, state models.Task_State) []*models.Task {
	tasks, err := client.Tasks()
	Expect(err).NotTo(HaveOccurred())

	filteredTasks := make([]*models.Task, 0)
	for _, task := range tasks {
		if task.State == state {
			filteredTasks = append(filteredTasks, task)
		}
	}
	return filteredTasks
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"fmt"

	"github.com/spf13/cobra"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	_ "k8s.io/client-go/plugin/pkg/client/auth"

	"github.com/jetstack/cert-manager/cmd/controller/app/options"
	_ "github.com/jetstack/cert-manager/pkg/controller/acmechallenges"
	_ "github.com/jetstack/cert-manager/pkg/controller/acmeorders"
	_ "github.com/jetstack/cert-manager/pkg/controller/certificates"
	_ "github.com/jetstack/cert-manager/pkg/controller/clusterissuers"
	_ "github.com/jetstack/cert-manager/pkg/controller/expcertificates/trigger"
	_ "github.com/jetstack/cert-manager/pkg/controller/ingress-shim"
	_ "github.com/jetstack/cert-manager/pkg/controller/issuers"
	_ "github.com/jetstack/cert-manager/pkg/issuer/acme"
	_ "github.com/jetstack/cert-manager/pkg/issuer/ca"
	_ "github.com/jetstack/cert-manager/pkg/issuer/selfsigned"
	_ "github.com/jetstack/cert-manager/pkg/issuer/vault"
	_ "github.com/jetstack/cert-manager/pkg/issuer/venafi"
	logf "github.com/jetstack/cert-manager/pkg/logs"
	"github.com/jetstack/cert-manager/pkg/util"
	utilfeature "github.com/jetstack/cert-manager/pkg/util/feature"
)

type CertManagerControllerOptions struct {
	ControllerOptions *options.ControllerOptions
}

func NewCertManagerControllerOptions() *CertManagerControllerOptions {
	o := &CertManagerControllerOptions{
		ControllerOptions: options.NewControllerOptions(),
	}

	return o
}

// NewCommandStartCertManagerController is a CLI handler for starting cert-manager
func NewCommandStartCertManagerController(stopCh <-chan struct{}) *cobra.Command {
	o := NewCertManagerControllerOptions()

	cmd := &cobra.Command{
		Use:   "cert-manager-controller",
		Short: fmt.Sprintf("Automated TLS controller for Kubernetes (%s) (%s)", util.AppVersion, util.AppGitCommit),
		Long: `
cert-manager is a Kubernetes addon to automate the management and issuance of
TLS certificates from various issuing sources.

It will ensure certificates are valid and up to date periodically, and attempt
to renew certificates at an appropriate time before expiry.`,

		// TODO: Refactor this function from this package
		Run: func(cmd *cobra.Command, args []string) {
			if err := o.Validate(args); err != nil {
				logf.Log.Error(err, "error validating options")
			}

			logf.Log.Info("starting controller", "version", util.AppVersion, "git-commit", util.AppGitCommit)
			o.RunCertManagerController(stopCh)
		},
	}

	flags := cmd.Flags()
	o.ControllerOptions.AddFlags(flags)
	utilfeature.DefaultMutableFeatureGate.AddFlag(flags)

	return cmd
}

func (o CertManagerControllerOptions) Validate(args []string) error {
	errors := []error{}
	errors = append(errors, o.ControllerOptions.Validate())
	return utilerrors.NewAggregate(errors)
}

func (o CertManagerControllerOptions) RunCertManagerController(stopCh <-chan struct{}) {
	Run(o.ControllerOptions, stopCh)
}
<commit_msg>Fix controller bug for not exiting on option error<commit_after>/*
Copyright 2019 The Jetstack cert-manager contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package app

import (
	"fmt"

	"github.com/spf13/cobra"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	_ "k8s.io/client-go/plugin/pkg/client/auth"

	"github.com/jetstack/cert-manager/cmd/controller/app/options"
	_ "github.com/jetstack/cert-manager/pkg/controller/acmechallenges"
	_ "github.com/jetstack/cert-manager/pkg/controller/acmeorders"
	_ "github.com/jetstack/cert-manager/pkg/controller/certificates"
	_ "github.com/jetstack/cert-manager/pkg/controller/clusterissuers"
	_ "github.com/jetstack/cert-manager/pkg/controller/expcertificates/trigger"
	_ "github.com/jetstack/cert-manager/pkg/controller/ingress-shim"
	_ "github.com/jetstack/cert-manager/pkg/controller/issuers"
	_ "github.com/jetstack/cert-manager/pkg/issuer/acme"
	_ "github.com/jetstack/cert-manager/pkg/issuer/ca"
	_ "github.com/jetstack/cert-manager/pkg/issuer/selfsigned"
	_ "github.com/jetstack/cert-manager/pkg/issuer/vault"
	_ "github.com/jetstack/cert-manager/pkg/issuer/venafi"
	logf "github.com/jetstack/cert-manager/pkg/logs"
	"github.com/jetstack/cert-manager/pkg/util"
	utilfeature "github.com/jetstack/cert-manager/pkg/util/feature"
)

type CertManagerControllerOptions struct {
	ControllerOptions *options.ControllerOptions
}

func NewCertManagerControllerOptions() *CertManagerControllerOptions {
	o := &CertManagerControllerOptions{
		ControllerOptions: options.NewControllerOptions(),
	}

	return o
}

// NewCommandStartCertManagerController is a CLI handler for starting cert-manager
func NewCommandStartCertManagerController(stopCh <-chan struct{}) *cobra.Command {
	o := NewCertManagerControllerOptions()

	cmd := &cobra.Command{
		Use:   "cert-manager-controller",
		Short: fmt.Sprintf("Automated TLS controller for Kubernetes (%s) (%s)", util.AppVersion, util.AppGitCommit),
		Long: `
cert-manager is a Kubernetes addon to automate the management and issuance of
TLS certificates from various issuing sources.

It will ensure certificates are valid and up to date periodically, and attempt
to renew certificates at an appropriate time before expiry.`,

		RunE: func(cmd *cobra.Command, args []string) error {
			if err := o.Validate(args); err != nil {
				return fmt.Errorf("error validating options: %s", err)
			}

			logf.Log.Info("starting controller", "version", util.AppVersion, "git-commit", util.AppGitCommit)
			o.RunCertManagerController(stopCh)
			return nil
		},
	}

	flags := cmd.Flags()
	o.ControllerOptions.AddFlags(flags)
	utilfeature.DefaultMutableFeatureGate.AddFlag(flags)

	return cmd
}

func (o CertManagerControllerOptions) Validate(args []string) error {
	errors := []error{}
	errors = append(errors, o.ControllerOptions.Validate())
	return utilerrors.NewAggregate(errors)
}

func (o CertManagerControllerOptions) RunCertManagerController(stopCh <-chan struct{}) {
	Run(o.ControllerOptions, stopCh)
}
<|endoftext|>


<commit_before><commit_msg>Add Timer type that keeps track of start & end<commit_after>package runner

import (
	"time"
)

type Timer struct {
	start time.Time
	end   time.Time
}

func (t *Timer) Start() {
	t.start = time.Now()
}

func (t *Timer) Stop() {
	t.end = time.Now()
}

func (t *Timer) elapsed() time.Duration {
	return t.end.Sub(t.start)
}

func newTimer() *Timer {
	t := &Timer{}
	t.Start()
	return t
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"io/ioutil"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/dummy"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/version"
	"os"
)

type syncToolsSuite struct {
	testing.LoggingSuite
	home testing.FakeHome
}

func (s *syncToolsSuite) SetUpTest(c *C) {
	s.LoggingSuite.SetUpTest(c)
	s.home = testing.MakeEmptyFakeHome(c)
}

func (s *syncToolsSuite) TearDownTest(c *C) {
	dummy.Reset()
	s.home.Restore()
	s.LoggingSuite.TearDownTest(c)
}

var _ = Suite(&syncToolsSuite{})

func runSyncToolsCommand(c *C, args ...string) (*cmd.Context, error) {
	return testing.RunCommand(c, &SyncToolsCommand{}, args)
}

func (s *syncToolsSuite) TestHelp(c *C) {
	ctx, err := runSyncToolsCommand(c, "-h")
	c.Assert(err, ErrorMatches, "flag: help requested")
	c.Assert(ctx, IsNil)
}

func (s *syncToolsSuite) TestReadFromDummy(c *C) {
	dummyAttrs := map[string]interface{}{
		"name":         "test-dummy",
		"type":         "dummy",
		"state-server": false,
		// Note: Without this, you get "no public ssh keys found", which seems
		// a bit odd for the "dummy" environment
		"authorized-keys": "I-am-not-a-real-key",
	}
	env, err := environs.NewFromAttrs(dummyAttrs)
	c.Assert(err, IsNil)
	c.Assert(env, NotNil)
	store := env.PublicStorage().(environs.Storage)
	path := environs.ToolsStoragePath(t1000precise.Binary)
	content := bytes.NewBufferString("content\n")
	err = store.Put(path, content, int64(content.Len()))
	c.Assert(err, IsNil)
	orig := officialBucketAttrs
	defer func() { officialBucketAttrs = orig }()
	officialBucketAttrs = dummyAttrs
	c.Assert(os.Mkdir(testing.HomePath(".juju"), 0775), IsNil)
	jujupath := testing.HomePath(".juju", "environments.yaml")
	err = ioutil.WriteFile(
		jujupath,
		[]byte(`
environments:
    target:
        type: dummy
        state-server: false
        authorized-keys: "not-really-one"
`),
		0660)
	c.Assert(err, IsNil)
	tools, err := environs.ListTools(env, t1000precise.Binary.Major)
	c.Assert(err, IsNil)
	// The one we just uploaded
	c.Assert(tools.Public, HasLen, 2)
	if tools.Public[0].Binary == t1000precise.Binary {
		c.Assert(tools.Public[0].Binary, Equals, t1000precise.Binary)
		c.Assert(tools.Public[1].Binary, Equals, version.Current)
	} else {
		c.Assert(tools.Public[0].Binary, Equals, version.Current)
		c.Assert(tools.Public[1].Binary, Equals, t1000precise.Binary)
	}
	c.Assert(tools.Private, HasLen, 0)
	ctx, err := runSyncToolsCommand(c, "-e", "target")
	c.Assert(err, IsNil)
	c.Assert(ctx, NotNil)
	tools, err = environs.ListTools(env, t1000precise.Binary.Major)
	c.Assert(err, IsNil)
	c.Assert(tools.Private, HasLen, 2)
	c.Assert(tools.Private[0].Binary, Equals, t1000precise.Binary)
	if tools.Private[0].Binary == t1000precise.Binary {
		c.Assert(tools.Private[0].Binary, Equals, t1000precise.Binary)
		c.Assert(tools.Private[1].Binary, Equals, version.Current)
	} else {
		c.Assert(tools.Private[0].Binary, Equals, version.Current)
		c.Assert(tools.Private[1].Binary, Equals, t1000precise.Binary)
	}
}

type toolSuite struct{}

var _ = Suite(&toolSuite{})

var t1000precise = &state.Tools{
	Binary: version.Binary{
		Number: version.Number{1, 0, 0, 0},
		Series: "precise",
		Arch:   "amd64"}}

var t1000quantal = &state.Tools{
	Binary: version.Binary{
		Number: version.Number{1, 0, 0, 0},
		Series: "quantal"}}

var t1900quantal = &state.Tools{
	Binary: version.Binary{
		Number: version.Number{1, 9, 0, 0},
		Series: "quantal"}}

var t2000precise = &state.Tools{
	Binary: version.Binary{
		Number: version.Number{2, 0, 0, 0},
		Series: "precise"}}

func (s *toolSuite) TestFindNewestOneTool(c *C) {
	var onlyOneTests = []struct {
		tools *state.Tools
	}{
		{tools: t1000precise},
		{tools: t1000quantal},
		{tools: t1900quantal},
		{tools: t2000precise},
	}
	for _, t := range onlyOneTests {
		toolList := []*state.Tools{t.tools}
		res := findNewest(toolList)
		c.Assert(res, HasLen, 1)
		c.Assert(res[0], Equals, t.tools)
	}
}

func (s *toolSuite) TestFindNewestOnlyOneBest(c *C) {
	var oneBestTests = []struct {
		all  []*state.Tools
		best *state.Tools
	}{
		{
			all:  []*state.Tools{t1000precise, t1900quantal},
			best: t1900quantal,
		},
	}
	for _, t := range oneBestTests {
		res := findNewest(t.all)
		c.Assert(res, HasLen, 1)
		c.Assert(res[0], Equals, t.best)
	}
}

func (s *toolSuite) TestFindNewestMultipleBest(c *C) {
	var oneBestTests = []struct {
		all  []*state.Tools
		best []*state.Tools
	}{{
		all:  []*state.Tools{t1000precise, t1000quantal},
		best: []*state.Tools{t1000precise, t1000quantal},
	},
	}
	for _, t := range oneBestTests {
		res := findNewest(t.all)
		c.Assert(res, DeepEquals, t.best)
	}
}

func (s *toolSuite) TestFindMissingNoTarget(c *C) {
	var allMissingTests = []struct {
		source []*state.Tools
	}{
		{source: []*state.Tools{t1000precise}},
		{source: []*state.Tools{t1000precise, t1000quantal}},
	}
	for _, t := range allMissingTests {
		res := findMissing(t.source, []*state.Tools(nil))
		c.Assert(res, DeepEquals, t.source)
	}
}

func (s *toolSuite) TestFindMissingSameEntries(c *C) {
	var allMissingTests = []struct {
		source []*state.Tools
	}{
		{source: []*state.Tools{t1000precise}},
		{source: []*state.Tools{t1000precise, t1000quantal}},
	}
	for _, t := range allMissingTests {
		res := findMissing(t.source, t.source)
		c.Assert(res, HasLen, 0)
	}
}

func (s *toolSuite) TestFindHasVersionNotSeries(c *C) {
	res := findMissing(
		[]*state.Tools{t1000precise, t1000quantal},
		[]*state.Tools{t1000quantal})
	c.Assert(res, HasLen, 1)
	c.Assert(res[0], Equals, t1000precise)
	res = findMissing(
		[]*state.Tools{t1000precise, t1000quantal},
		[]*state.Tools{t1000precise})
	c.Assert(res, HasLen, 1)
	c.Assert(res[0], Equals, t1000quantal)
}
<commit_msg>success.<commit_after>package main

import (
	"bytes"
	"io/ioutil"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/environs/dummy"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/version"
	"os"
)

type syncToolsSuite struct {
	testing.LoggingSuite
	home testing.FakeHome
}

func (s *syncToolsSuite) SetUpTest(c *C) {
	s.LoggingSuite.SetUpTest(c)
	s.home = testing.MakeEmptyFakeHome(c)
}

func (s *syncToolsSuite) TearDownTest(c *C) {
	dummy.Reset()
	s.home.Restore()
	s.LoggingSuite.TearDownTest(c)
}

var _ = Suite(&syncToolsSuite{})

func runSyncToolsCommand(c *C, args ...string) (*cmd.Context, error) {
	return testing.RunCommand(c, &SyncToolsCommand{}, args)
}

func (s *syncToolsSuite) TestHelp(c *C) {
	ctx, err := runSyncToolsCommand(c, "-h")
	c.Assert(err, ErrorMatches, "flag: help requested")
	c.Assert(ctx, IsNil)
}

func setupDummyEnvironments(c *C) (env environs.Environ, cleanup func()) {
	dummyAttrs := map[string]interface{}{
		"name":         "test-source",
		"type":         "dummy",
		"state-server": false,
		// Note: Without this, you get "no public ssh keys found", which seems
		// a bit odd for the "dummy" environment
		"authorized-keys": "I-am-not-a-real-key",
	}
	env, err := environs.NewFromAttrs(dummyAttrs)
	c.Assert(err, IsNil)
	c.Assert(env, NotNil)
	store := env.PublicStorage().(environs.Storage)
	// Dummy environments always put fake tools, but we don't want it
	// confusing our state, so we delete them
	fakepath := environs.ToolsStoragePath(version.Current)
	err = store.Remove(fakepath)
	c.Assert(err, IsNil)
	// Upload t1000precise tools
	path := environs.ToolsStoragePath(t1000precise.Binary)
	content := bytes.NewBufferString("content\n")
	err = store.Put(path, content, int64(content.Len()))
	c.Assert(err, IsNil)
	// Save for cleanup
	orig := officialBucketAttrs
	officialBucketAttrs = dummyAttrs
	c.Assert(os.Mkdir(testing.HomePath(".juju"), 0775), IsNil)
	jujupath := testing.HomePath(".juju", "environments.yaml")
	err = ioutil.WriteFile(
		jujupath,
		[]byte(`
environments:
    test-target:
        type: dummy
        state-server: false
        authorized-keys: "not-really-one"
`),
		0660)
	c.Assert(err, IsNil)
	return env, func() { officialBucketAttrs = orig }
}

func (s *syncToolsSuite) TestCopyNewestFromDummy(c *C) {
	sourceEnv, cleanup := setupDummyEnvironments(c)
	defer cleanup()
	sourceTools, err := environs.ListTools(sourceEnv, t1000precise.Binary.Major)
	c.Assert(err, IsNil)
	c.Assert(sourceTools.Public, HasLen, 1)
	c.Assert(sourceTools.Public[0].Binary, Equals, t1000precise.Binary)
	targetEnv, err := environs.NewFromName("test-target")
	c.Assert(err, IsNil)
	targetTools, err := environs.ListTools(targetEnv, t1000precise.Binary.Major)
	// Target env just has the fake tools in the public bucket
	c.Assert(targetTools.Public, HasLen, 1)
	c.Assert(targetTools.Public[0].Binary, Equals, version.Current)
	c.Assert(targetTools.Private, HasLen, 0)
	ctx, err := runSyncToolsCommand(c, "-e", "test-target")
	c.Assert(err, IsNil)
	c.Assert(ctx, NotNil)
	targetTools, err = environs.ListTools(targetEnv, t1000precise.Binary.Major)
	c.Assert(err, IsNil)
	// No change to the Public bucket
	c.Assert(targetTools.Public, HasLen, 1)
	// only the newest added to the private bucket
	c.Assert(targetTools.Private, HasLen, 1)
	c.Assert(targetTools.Private[0].Binary, Equals, t1000precise.Binary)
}

type toolSuite struct{}

var _ = Suite(&toolSuite{})

var t1000precise = &state.Tools{
	Binary: version.Binary{
		Number: version.Number{1, 0, 0, 0},
		Series: "precise",
		Arch:   "amd64"}}

var t1000quantal = &state.Tools{
	Binary: version.Binary{
		Number: version.Number{1, 0, 0, 0},
		Series: "quantal"}}

var t1900quantal = &state.Tools{
	Binary: version.Binary{
		Number: version.Number{1, 9, 0, 0},
		Series: "quantal"}}

var t2000precise = &state.Tools{
	Binary: version.Binary{
		Number: version.Number{2, 0, 0, 0},
		Series: "precise"}}

func (s *toolSuite) TestFindNewestOneTool(c *C) {
	var onlyOneTests = []struct {
		tools *state.Tools
	}{
		{tools: t1000precise},
		{tools: t1000quantal},
		{tools: t1900quantal},
		{tools: t2000precise},
	}
	for _, t := range onlyOneTests {
		toolList := []*state.Tools{t.tools}
		res := findNewest(toolList)
		c.Assert(res, HasLen, 1)
		c.Assert(res[0], Equals, t.tools)
	}
}

func (s *toolSuite) TestFindNewestOnlyOneBest(c *C) {
	var oneBestTests = []struct {
		all  []*state.Tools
		best *state.Tools
	}{
		{
			all:  []*state.Tools{t1000precise, t1900quantal},
			best: t1900quantal,
		},
	}
	for _, t := range oneBestTests {
		res := findNewest(t.all)
		c.Assert(res, HasLen, 1)
		c.Assert(res[0], Equals, t.best)
	}
}

func (s *toolSuite) TestFindNewestMultipleBest(c *C) {
	var oneBestTests = []struct {
		all  []*state.Tools
		best []*state.Tools
	}{{
		all:  []*state.Tools{t1000precise, t1000quantal},
		best: []*state.Tools{t1000precise, t1000quantal},
	},
	}
	for _, t := range oneBestTests {
		res := findNewest(t.all)
		c.Assert(res, DeepEquals, t.best)
	}
}

func (s *toolSuite) TestFindMissingNoTarget(c *C) {
	var allMissingTests = []struct {
		source []*state.Tools
	}{
		{source: []*state.Tools{t1000precise}},
		{source: []*state.Tools{t1000precise, t1000quantal}},
	}
	for _, t := range allMissingTests {
		res := findMissing(t.source, []*state.Tools(nil))
		c.Assert(res, DeepEquals, t.source)
	}
}

func (s *toolSuite) TestFindMissingSameEntries(c *C) {
	var allMissingTests = []struct {
		source []*state.Tools
	}{
		{source: []*state.Tools{t1000precise}},
		{source: []*state.Tools{t1000precise, t1000quantal}},
	}
	for _, t := range allMissingTests {
		res := findMissing(t.source, t.source)
		c.Assert(res, HasLen, 0)
	}
}

func (s *toolSuite) TestFindHasVersionNotSeries(c *C) {
	res := findMissing(
		[]*state.Tools{t1000precise, t1000quantal},
		[]*state.Tools{t1000quantal})
	c.Assert(res, HasLen, 1)
	c.Assert(res[0], Equals, t1000precise)
	res = findMissing(
		[]*state.Tools{t1000precise, t1000quantal},
		[]*state.Tools{t1000precise})
	c.Assert(res, HasLen, 1)
	c.Assert(res[0], Equals, t1000quantal)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"sort"
	"strings"

	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/restic"
	"github.com/restic/restic/internal/ui/table"
	"github.com/spf13/cobra"
)

var cmdSnapshots = &cobra.Command{
	Use:   "snapshots [snapshotID ...]",
	Short: "List all snapshots",
	Long: `
The "snapshots" command lists all snapshots stored in the repository.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runSnapshots(snapshotOptions, globalOptions, args)
	},
}

// SnapshotOptions bundles all options for the snapshots command.
type SnapshotOptions struct {
	Host    string
	Tags    restic.TagLists
	Paths   []string
	Compact bool
	Last    bool
	GroupBy string
}

var snapshotOptions SnapshotOptions

func init() {
	cmdRoot.AddCommand(cmdSnapshots)

	f := cmdSnapshots.Flags()
	f.StringVarP(&snapshotOptions.Host, "host", "H", "", "only consider snapshots for this `host`")
	f.Var(&snapshotOptions.Tags, "tag", "only consider snapshots which include this `taglist` (can be specified multiple times)")
	f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
	f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact format")
	f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path")
	f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "string for grouping snapshots by host,paths,tags")
}

type groupKey struct {
	Hostname string   `json:"hostname"`
	Paths    []string `json:"paths"`
	Tags     []string `json:"tags"`
}

func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if !gopts.NoLock {
		lock, err := lockRepo(repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	// group by hostname and dirs
	snapshotGroups := make(map[string]restic.Snapshots)

	var GroupByTag bool
	var GroupByHost bool
	var GroupByPath bool
	var GroupOptionList []string

	GroupOptionList = strings.Split(opts.GroupBy, ",")

	for _, option := range GroupOptionList {
		switch option {
		case "host":
			GroupByHost = true
		case "paths":
			GroupByPath = true
		case "tags":
			GroupByTag = true
		case "":
		default:
			return errors.Fatal("unknown grouping option: '" + option + "'")
		}
	}

	ctx, cancel := context.WithCancel(gopts.ctx)
	defer cancel()

	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
		// Determining grouping-keys
		var tags []string
		var hostname string
		var paths []string

		if GroupByTag {
			tags = sn.Tags
			sort.StringSlice(tags).Sort()
		}
		if GroupByHost {
			hostname = sn.Hostname
		}
		if GroupByPath {
			paths = sn.Paths
		}

		sort.StringSlice(sn.Paths).Sort()
		var k []byte
		var err error

		k, err = json.Marshal(groupKey{Tags: tags, Hostname: hostname, Paths: paths})

		if err != nil {
			return err
		}
		snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
	}

	for k, list := range snapshotGroups {
		if opts.Last {
			list = FilterLastSnapshots(list)
		}
		sort.Sort(sort.Reverse(list))
		snapshotGroups[k] = list
	}

	if gopts.JSON {
		err := printSnapshotGroupJSON(gopts.stdout, snapshotGroups, GroupByTag || GroupByHost || GroupByPath)
		if err != nil {
			Warnf("error printing snapshots: %v\n", err)
		}
		return nil
	}

	for k, list := range snapshotGroups {
		err := PrintSnapshotGroupHeader(gopts.stdout, k, GroupByTag, GroupByHost, GroupByPath)
		if err != nil {
			Warnf("error printing snapshots: %v\n", err)
			return nil
		}

		PrintSnapshots(gopts.stdout, list, nil, opts.Compact)
	}

	return nil
}

// filterLastSnapshotsKey is used by FilterLastSnapshots.
type filterLastSnapshotsKey struct {
	Hostname    string
	JoinedPaths string
}

// newFilterLastSnapshotsKey initializes a filterLastSnapshotsKey from a Snapshot
func newFilterLastSnapshotsKey(sn *restic.Snapshot) filterLastSnapshotsKey {
	// Shallow slice copy
	var paths = make([]string, len(sn.Paths))
	copy(paths, sn.Paths)
	sort.Strings(paths)
	return filterLastSnapshotsKey{sn.Hostname, strings.Join(paths, "|")}
}

// FilterLastSnapshots filters a list of snapshots to only return the last
// entry for each hostname and path. If the snapshot contains multiple paths,
// they will be joined and treated as one item.
func FilterLastSnapshots(list restic.Snapshots) restic.Snapshots {
	// Sort the snapshots so that the newer ones are listed first
	sort.SliceStable(list, func(i, j int) bool {
		return list[i].Time.After(list[j].Time)
	})

	var results restic.Snapshots
	seen := make(map[filterLastSnapshotsKey]bool)
	for _, sn := range list {
		key := newFilterLastSnapshotsKey(sn)
		if !seen[key] {
			seen[key] = true
			results = append(results, sn)
		}
	}
	return results
}

// PrintSnapshots prints a text table of the snapshots in list to stdout.
func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.KeepReason, compact bool) {
	// keep the reasons a snasphot is being kept in a map, so that it doesn't
	// get lost when the list of snapshots is sorted
	keepReasons := make(map[restic.ID]restic.KeepReason, len(reasons))
	if len(reasons) > 0 {
		for i, sn := range list {
			id := sn.ID()
			keepReasons[*id] = reasons[i]
		}
	}

	// always sort the snapshots so that the newer ones are listed last
	sort.SliceStable(list, func(i, j int) bool {
		return list[i].Time.Before(list[j].Time)
	})

	// Determine the max widths for host and tag.
	maxHost, maxTag := 10, 6
	for _, sn := range list {
		if len(sn.Hostname) > maxHost {
			maxHost = len(sn.Hostname)
		}
		for _, tag := range sn.Tags {
			if len(tag) > maxTag {
				maxTag = len(tag)
			}
		}
	}

	tab := table.New()

	if compact {
		tab.AddColumn("ID", "{{ .ID }}")
		tab.AddColumn("Time", "{{ .Timestamp }}")
		tab.AddColumn("Host", "{{ .Hostname }}")
		tab.AddColumn("Tags  ", `{{ join .Tags "\n" }}`)
	} else {
		tab.AddColumn("ID", "{{ .ID }}")
		tab.AddColumn("Time", "{{ .Timestamp }}")
		tab.AddColumn("Host      ", "{{ .Hostname }}")
		tab.AddColumn("Tags      ", `{{ join .Tags "," }}`)
		if len(reasons) > 0 {
			tab.AddColumn("Reasons", `{{ join .Reasons "\n" }}`)
		}
		tab.AddColumn("Paths", `{{ join .Paths "\n" }}`)
	}

	type snapshot struct {
		ID        string
		Timestamp string
		Hostname  string
		Tags      []string
		Reasons   []string
		Paths     []string
	}

	var multiline bool
	for _, sn := range list {
		data := snapshot{
			ID:        sn.ID().Str(),
			Timestamp: sn.Time.Local().Format(TimeFormat),
			Hostname:  sn.Hostname,
			Tags:      sn.Tags,
			Paths:     sn.Paths,
		}

		if len(reasons) > 0 {
			id := sn.ID()
			data.Reasons = keepReasons[*id].Matches
		}

		if len(sn.Paths) > 1 && !compact {
			multiline = true
		}

		tab.AddRow(data)
	}

	tab.AddFooter(fmt.Sprintf("%d snapshots", len(list)))

	if multiline {
		// print an additional blank line between snapshots

		var last int
		tab.PrintData = func(w io.Writer, idx int, s string) error {
			var err error
			if idx == last {
				_, err = fmt.Fprintf(w, "%s\n", s)
			} else {
				_, err = fmt.Fprintf(w, "\n%s\n", s)
			}
			last = idx
			return err
		}
	}

	tab.Write(stdout)
}

// PrintSnapshotGroupHeader prints which group of the group-by option the
// following snapshots belong to.
// Prints nothing, if we did not group at all.
func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string, GroupByTag bool, GroupByHost bool, GroupByPath bool) error {
	if !GroupByTag && !GroupByHost && !GroupByPath {
		return nil
	}
	var key groupKey
	var err error

	err = json.Unmarshal([]byte(groupKeyJSON), &key)
	if err != nil {
		return err
	}

	// Info
	fmt.Fprintf(stdout, "snapshots")
	var infoStrings []string
	if GroupByTag {
		infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]")
	}
	if GroupByHost {
		infoStrings = append(infoStrings, "host ["+key.Hostname+"]")
	}
	if GroupByPath {
		infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]")
	}
	if infoStrings != nil {
		fmt.Fprintf(stdout, " for (%s)", strings.Join(infoStrings, ", "))
	}
	fmt.Fprintf(stdout, ":\n")

	return nil
}

// Snapshot helps to print Snaphots as JSON with their ID included.
type Snapshot struct {
	*restic.Snapshot

	ID      *restic.ID `json:"id"`
	ShortID string     `json:"short_id"`
}

// SnapshotGroup helps to print SnaphotGroups as JSON with their GroupReasons included.
type SnapshotGroup struct {
	GroupKey  groupKey   `json:"group_key"`
	Snapshots []Snapshot `json:"snapshots"`
}

// printSnapshotsJSON writes the JSON representation of list to stdout.
func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]restic.Snapshots, grouped bool) error {

	if grouped {
		var snapshotGroups []SnapshotGroup

		for k, list := range snGroups {
			var key groupKey
			var err error
			var snapshots []Snapshot

			err = json.Unmarshal([]byte(k), &key)
			if err != nil {
				return err
			}

			for _, sn := range list {
				k := Snapshot{
					Snapshot: sn,
					ID:       sn.ID(),
					ShortID:  sn.ID().Str(),
				}
				snapshots = append(snapshots, k)
			}

			group := SnapshotGroup{
				GroupKey:  key,
				Snapshots: snapshots,
			}
			snapshotGroups = append(snapshotGroups, group)
		}

		return json.NewEncoder(stdout).Encode(snapshotGroups)
	}

	// Old behavior
	var snapshots []Snapshot

	for _, list := range snGroups {
		for _, sn := range list {
			k := Snapshot{
				Snapshot: sn,
				ID:       sn.ID(),
				ShortID:  sn.ID().Str(),
			}
			snapshots = append(snapshots, k)
		}
	}

	return json.NewEncoder(stdout).Encode(snapshots)
}
<commit_msg>Move snapshot grouping code into own function to deduplicate code<commit_after>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"sort"
	"strings"

	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/restic"
	"github.com/restic/restic/internal/ui/table"
	"github.com/spf13/cobra"
)

var cmdSnapshots = &cobra.Command{
	Use:   "snapshots [snapshotID ...]",
	Short: "List all snapshots",
	Long: `
The "snapshots" command lists all snapshots stored in the repository.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runSnapshots(snapshotOptions, globalOptions, args)
	},
}

// SnapshotOptions bundles all options for the snapshots command.
type SnapshotOptions struct {
	Host    string
	Tags    restic.TagLists
	Paths   []string
	Compact bool
	Last    bool
	GroupBy string
}

var snapshotOptions SnapshotOptions

func init() {
	cmdRoot.AddCommand(cmdSnapshots)

	f := cmdSnapshots.Flags()
	f.StringVarP(&snapshotOptions.Host, "host", "H", "", "only consider snapshots for this `host`")
	f.Var(&snapshotOptions.Tags, "tag", "only consider snapshots which include this `taglist` (can be specified multiple times)")
	f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
	f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact format")
	f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path")
	f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "string for grouping snapshots by host,paths,tags")
}

type groupKey struct {
	Hostname string   `json:"hostname"`
	Paths    []string `json:"paths"`
	Tags     []string `json:"tags"`
}

func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if !gopts.NoLock {
		lock, err := lockRepo(repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	ctx, cancel := context.WithCancel(gopts.ctx)
	defer cancel()

	var snapshots restic.Snapshots
	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
		snapshots = append(snapshots, sn)
	}
	snapshotGroups, grouped, err := GroupSnapshots(snapshots, opts.GroupBy)
	if err != nil {
		return err
	}

	for k, list := range snapshotGroups {
		if opts.Last {
			list = FilterLastSnapshots(list)
		}
		sort.Sort(sort.Reverse(list))
		snapshotGroups[k] = list
	}

	if gopts.JSON {
		err := printSnapshotGroupJSON(gopts.stdout, snapshotGroups, grouped)
		if err != nil {
			Warnf("error printing snapshots: %v\n", err)
		}
		return nil
	}

	for k, list := range snapshotGroups {
		if grouped {
			err := PrintSnapshotGroupHeader(gopts.stdout, k)
			if err != nil {
				Warnf("error printing snapshots: %v\n", err)
				return nil
			}
		}
		PrintSnapshots(gopts.stdout, list, nil, opts.Compact)
	}

	return nil
}

// filterLastSnapshotsKey is used by FilterLastSnapshots.
type filterLastSnapshotsKey struct {
	Hostname    string
	JoinedPaths string
}

// newFilterLastSnapshotsKey initializes a filterLastSnapshotsKey from a Snapshot
func newFilterLastSnapshotsKey(sn *restic.Snapshot) filterLastSnapshotsKey {
	// Shallow slice copy
	var paths = make([]string, len(sn.Paths))
	copy(paths, sn.Paths)
	sort.Strings(paths)
	return filterLastSnapshotsKey{sn.Hostname, strings.Join(paths, "|")}
}

// FilterLastSnapshots filters a list of snapshots to only return the last
// entry for each hostname and path. If the snapshot contains multiple paths,
// they will be joined and treated as one item.
func FilterLastSnapshots(list restic.Snapshots) restic.Snapshots {
	// Sort the snapshots so that the newer ones are listed first
	sort.SliceStable(list, func(i, j int) bool {
		return list[i].Time.After(list[j].Time)
	})

	var results restic.Snapshots
	seen := make(map[filterLastSnapshotsKey]bool)
	for _, sn := range list {
		key := newFilterLastSnapshotsKey(sn)
		if !seen[key] {
			seen[key] = true
			results = append(results, sn)
		}
	}
	return results
}

// GroupSnapshots takes a list of snapshots and a grouping criteria and creates
// a group list of snapshots.
func GroupSnapshots(snapshots restic.Snapshots, options string) (map[string]restic.Snapshots, bool, error) {
	// group by hostname and dirs
	snapshotGroups := make(map[string]restic.Snapshots)

	var GroupByTag bool
	var GroupByHost bool
	var GroupByPath bool
	var GroupOptionList []string

	GroupOptionList = strings.Split(options, ",")

	for _, option := range GroupOptionList {
		switch option {
		case "host":
			GroupByHost = true
		case "paths":
			GroupByPath = true
		case "tags":
			GroupByTag = true
		case "":
		default:
			return nil, false, errors.Fatal("unknown grouping option: '" + option + "'")
		}
	}

	for _, sn := range snapshots {
		// Determining grouping-keys
		var tags []string
		var hostname string
		var paths []string

		if GroupByTag {
			tags = sn.Tags
			sort.StringSlice(tags).Sort()
		}
		if GroupByHost {
			hostname = sn.Hostname
		}
		if GroupByPath {
			paths = sn.Paths
		}

		sort.StringSlice(sn.Paths).Sort()
		var k []byte
		var err error

		k, err = json.Marshal(groupKey{Tags: tags, Hostname: hostname, Paths: paths})

		if err != nil {
			return nil, false, err
		}
		snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
	}

	return snapshotGroups, GroupByTag || GroupByHost || GroupByPath, nil
}

// PrintSnapshots prints a text table of the snapshots in list to stdout.
func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.KeepReason, compact bool) {
	// keep the reasons a snasphot is being kept in a map, so that it doesn't
	// get lost when the list of snapshots is sorted
	keepReasons := make(map[restic.ID]restic.KeepReason, len(reasons))
	if len(reasons) > 0 {
		for i, sn := range list {
			id := sn.ID()
			keepReasons[*id] = reasons[i]
		}
	}

	// always sort the snapshots so that the newer ones are listed last
	sort.SliceStable(list, func(i, j int) bool {
		return list[i].Time.Before(list[j].Time)
	})

	// Determine the max widths for host and tag.
	maxHost, maxTag := 10, 6
	for _, sn := range list {
		if len(sn.Hostname) > maxHost {
			maxHost = len(sn.Hostname)
		}
		for _, tag := range sn.Tags {
			if len(tag) > maxTag {
				maxTag = len(tag)
			}
		}
	}

	tab := table.New()

	if compact {
		tab.AddColumn("ID", "{{ .ID }}")
		tab.AddColumn("Time", "{{ .Timestamp }}")
		tab.AddColumn("Host", "{{ .Hostname }}")
		tab.AddColumn("Tags  ", `{{ join .Tags "\n" }}`)
	} else {
		tab.AddColumn("ID", "{{ .ID }}")
		tab.AddColumn("Time", "{{ .Timestamp }}")
		tab.AddColumn("Host      ", "{{ .Hostname }}")
		tab.AddColumn("Tags      ", `{{ join .Tags "," }}`)
		if len(reasons) > 0 {
			tab.AddColumn("Reasons", `{{ join .Reasons "\n" }}`)
		}
		tab.AddColumn("Paths", `{{ join .Paths "\n" }}`)
	}

	type snapshot struct {
		ID        string
		Timestamp string
		Hostname  string
		Tags      []string
		Reasons   []string
		Paths     []string
	}

	var multiline bool
	for _, sn := range list {
		data := snapshot{
			ID:        sn.ID().Str(),
			Timestamp: sn.Time.Local().Format(TimeFormat),
			Hostname:  sn.Hostname,
			Tags:      sn.Tags,
			Paths:     sn.Paths,
		}

		if len(reasons) > 0 {
			id := sn.ID()
			data.Reasons = keepReasons[*id].Matches
		}

		if len(sn.Paths) > 1 && !compact {
			multiline = true
		}

		tab.AddRow(data)
	}

	tab.AddFooter(fmt.Sprintf("%d snapshots", len(list)))

	if multiline {
		// print an additional blank line between snapshots

		var last int
		tab.PrintData = func(w io.Writer, idx int, s string) error {
			var err error
			if idx == last {
				_, err = fmt.Fprintf(w, "%s\n", s)
			} else {
				_, err = fmt.Fprintf(w, "\n%s\n", s)
			}
			last = idx
			return err
		}
	}

	tab.Write(stdout)
}

// PrintSnapshotGroupHeader prints which group of the group-by option the
// following snapshots belong to.
// Prints nothing, if we did not group at all.
func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string) error {
	var key groupKey
	var err error

	err = json.Unmarshal([]byte(groupKeyJSON), &key)
	if err != nil {
		return err
	}

	if key.Hostname == "" && key.Tags == nil && key.Paths == nil {
		return nil
	}

	// Info
	fmt.Fprintf(stdout, "snapshots")
	var infoStrings []string
	if key.Hostname != "" {
		infoStrings = append(infoStrings, "host ["+key.Hostname+"]")
	}
	if key.Tags != nil {
		infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]")
	}
	if key.Paths != nil {
		infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]")
	}
	if infoStrings != nil {
		fmt.Fprintf(stdout, " for (%s)", strings.Join(infoStrings, ", "))
	}
	fmt.Fprintf(stdout, ":\n")

	return nil
}

// Snapshot helps to print Snaphots as JSON with their ID included.
type Snapshot struct {
	*restic.Snapshot

	ID      *restic.ID `json:"id"`
	ShortID string     `json:"short_id"`
}

// SnapshotGroup helps to print SnaphotGroups as JSON with their GroupReasons included.
type SnapshotGroup struct {
	GroupKey  groupKey   `json:"group_key"`
	Snapshots []Snapshot `json:"snapshots"`
}

// printSnapshotsJSON writes the JSON representation of list to stdout.
func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]restic.Snapshots, grouped bool) error {
	if grouped {
		var snapshotGroups []SnapshotGroup

		for k, list := range snGroups {
			var key groupKey
			var err error
			var snapshots []Snapshot

			err = json.Unmarshal([]byte(k), &key)
			if err != nil {
				return err
			}

			for _, sn := range list {
				k := Snapshot{
					Snapshot: sn,
					ID:       sn.ID(),
					ShortID:  sn.ID().Str(),
				}
				snapshots = append(snapshots, k)
			}

			group := SnapshotGroup{
				GroupKey:  key,
				Snapshots: snapshots,
			}
			snapshotGroups = append(snapshotGroups, group)
		}

		return json.NewEncoder(stdout).Encode(snapshotGroups)
	}

	// Old behavior
	var snapshots []Snapshot

	for _, list := range snGroups {
		for _, sn := range list {
			k := Snapshot{
				Snapshot: sn,
				ID:       sn.ID(),
				ShortID:  sn.ID().Str(),
			}
			snapshots = append(snapshots, k)
		}
	}

	return json.NewEncoder(stdout).Encode(snapshots)
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"context"
	"path/filepath"
	"time"

	"github.com/minio/minio/cmd/logger"
)

// commonTime returns a maximally occurring time from a list of time.
func commonTime(modTimes []time.Time) (modTime time.Time, count int) {
	var maxima int // Counter for remembering max occurrence of elements.
	timeOccurenceMap := make(map[time.Time]int)
	// Ignore the uuid sentinel and count the rest.
	for _, time := range modTimes {
		if time == timeSentinel {
			continue
		}
		timeOccurenceMap[time]++
	}
	// Find the common cardinality from previously collected
	// occurrences of elements.
	for time, count := range timeOccurenceMap {
		if count > maxima || (count == maxima && time.After(modTime)) {
			maxima = count
			modTime = time
		}
	}
	// Return the collected common uuid.
	return modTime, maxima
}

// Beginning of unix time is treated as sentinel value here.
var timeSentinel = time.Unix(0, 0).UTC()

// Boot modTimes up to disk count, setting the value to time sentinel.
func bootModtimes(diskCount int) []time.Time {
	modTimes := make([]time.Time, diskCount)
	// Boots up all the modtimes.
	for i := range modTimes {
		modTimes[i] = timeSentinel
	}
	return modTimes
}

// Extracts list of times from xlMetaV1 slice and returns, skips
// slice elements which have errors.
func listObjectModtimes(partsMetadata []xlMetaV1, errs []error) (modTimes []time.Time) {
	modTimes = bootModtimes(len(partsMetadata))
	for index, metadata := range partsMetadata {
		if errs[index] != nil {
			continue
		}
		// Once the file is found, save the uuid saved on disk.
		modTimes[index] = metadata.Stat.ModTime
	}
	return modTimes
}

// Notes:
// There are 5 possible states a disk could be in,
// 1. __online__             - has the latest copy of xl.json - returned by listOnlineDisks
//
// 2. __offline__            - err == errDiskNotFound
//
// 3. __availableWithParts__ - has the latest copy of xl.json and has all
//                             parts with checksums matching; returned by disksWithAllParts
//
// 4. __outdated__           - returned by outDatedDisk, provided []StorageAPI
//                             returned by diskWithAllParts is passed for latestDisks.
//    - has an old copy of xl.json
//    - doesn't have xl.json (errFileNotFound)
//    - has the latest xl.json but one or more parts are corrupt
//
// 5. __missingParts__       - has the latest copy of xl.json but has some parts
// missing.  This is identified separately since this may need manual
// inspection to understand the root cause. E.g, this could be due to
// backend filesystem corruption.

// listOnlineDisks - returns
// - a slice of disks where disk having 'older' xl.json (or nothing)
// are set to nil.
// - latest (in time) of the maximally occurring modTime(s).
func listOnlineDisks(disks []StorageAPI, partsMetadata []xlMetaV1, errs []error) (onlineDisks []StorageAPI, modTime time.Time) {
	onlineDisks = make([]StorageAPI, len(disks))

	// List all the file commit ids from parts metadata.
	modTimes := listObjectModtimes(partsMetadata, errs)

	// Reduce list of UUIDs to a single common value.
	modTime, _ = commonTime(modTimes)

	// Create a new online disks slice, which have common uuid.
	for index, t := range modTimes {
		if t == modTime {
			onlineDisks[index] = disks[index]
		} else {
			onlineDisks[index] = nil
		}
	}
	return onlineDisks, modTime
}

// Returns one of the latest updated xlMeta files and count of total valid xlMeta(s) updated latest
func getLatestXLMeta(partsMetadata []xlMetaV1, errs []error) (xlMetaV1, int) {
	// List all the file commit ids from parts metadata.
	modTimes := listObjectModtimes(partsMetadata, errs)

	// Count all latest updated xlMeta values
	var count int
	var latestXLMeta xlMetaV1

	// Reduce list of UUIDs to a single common value - i.e. the last updated Time
	modTime, _ := commonTime(modTimes)

	// Interate through all the modTimes and count the xlMeta(s) with latest time.
	for index, t := range modTimes {
		if t == modTime && partsMetadata[index].IsValid() {
			latestXLMeta = partsMetadata[index]
			count++
		}
	}
	// Return one of the latest xlMetaData, and the count of lastest updated xlMeta files
	return latestXLMeta, count
}

// disksWithAllParts - This function needs to be called with
// []StorageAPI returned by listOnlineDisks. Returns,
//
// - disks which have all parts specified in the latest xl.json.
//
// - slice of errors about the state of data files on disk - can have
//   a not-found error or a hash-mismatch error.
//
// - non-nil error if any of the disks failed unexpectedly (i.e. error
//   other than file not found and not a checksum error).
func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []xlMetaV1, errs []error, bucket,
	object string) ([]StorageAPI, []error, error) {

	availableDisks := make([]StorageAPI, len(onlineDisks))
	buffer := []byte{}
	dataErrs := make([]error, len(onlineDisks))

	for i, onlineDisk := range onlineDisks {
		if onlineDisk == nil {
			continue
		}

		// disk has a valid xl.json but may not have all the
		// parts. This is considered an outdated disk, since
		// it needs healing too.
		for _, part := range partsMetadata[i].Parts {
			partPath := filepath.Join(object, part.Name)
			checksumInfo := partsMetadata[i].Erasure.GetChecksumInfo(part.Name)
			verifier := NewBitrotVerifier(checksumInfo.Algorithm, checksumInfo.Hash)

			// verification happens even if a 0-length
			// buffer is passed
			_, hErr := onlineDisk.ReadFile(bucket, partPath, 0, buffer, verifier)

			_, isCorrupt := hErr.(hashMismatchError)
			switch {
			case isCorrupt:
				fallthrough
			case hErr == errFileNotFound, hErr == errVolumeNotFound:
				dataErrs[i] = hErr
				break
			case hErr != nil:
				logger.LogIf(ctx, hErr)
				// abort on unhandled errors
				return nil, nil, hErr
			}
		}

		if dataErrs[i] == nil {
			// All parts verified, mark it as all data available.
			availableDisks[i] = onlineDisk
		}
	}

	return availableDisks, dataErrs, nil
}
<commit_msg>Fix type of bitrot mismatch error (#6205)<commit_after>/*
 * Minio Cloud Storage, (C) 2016, 2017 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"context"
	"path/filepath"
	"strings"
	"time"

	"github.com/minio/minio/cmd/logger"
)

// commonTime returns a maximally occurring time from a list of time.
func commonTime(modTimes []time.Time) (modTime time.Time, count int) {
	var maxima int // Counter for remembering max occurrence of elements.
	timeOccurenceMap := make(map[time.Time]int)
	// Ignore the uuid sentinel and count the rest.
	for _, time := range modTimes {
		if time == timeSentinel {
			continue
		}
		timeOccurenceMap[time]++
	}
	// Find the common cardinality from previously collected
	// occurrences of elements.
	for time, count := range timeOccurenceMap {
		if count > maxima || (count == maxima && time.After(modTime)) {
			maxima = count
			modTime = time
		}
	}
	// Return the collected common uuid.
	return modTime, maxima
}

// Beginning of unix time is treated as sentinel value here.
var timeSentinel = time.Unix(0, 0).UTC()

// Boot modTimes up to disk count, setting the value to time sentinel.
func bootModtimes(diskCount int) []time.Time {
	modTimes := make([]time.Time, diskCount)
	// Boots up all the modtimes.
	for i := range modTimes {
		modTimes[i] = timeSentinel
	}
	return modTimes
}

// Extracts list of times from xlMetaV1 slice and returns, skips
// slice elements which have errors.
func listObjectModtimes(partsMetadata []xlMetaV1, errs []error) (modTimes []time.Time) {
	modTimes = bootModtimes(len(partsMetadata))
	for index, metadata := range partsMetadata {
		if errs[index] != nil {
			continue
		}
		// Once the file is found, save the uuid saved on disk.
		modTimes[index] = metadata.Stat.ModTime
	}
	return modTimes
}

// Notes:
// There are 5 possible states a disk could be in,
// 1. __online__             - has the latest copy of xl.json - returned by listOnlineDisks
//
// 2. __offline__            - err == errDiskNotFound
//
// 3. __availableWithParts__ - has the latest copy of xl.json and has all
//                             parts with checksums matching; returned by disksWithAllParts
//
// 4. __outdated__           - returned by outDatedDisk, provided []StorageAPI
//                             returned by diskWithAllParts is passed for latestDisks.
//    - has an old copy of xl.json
//    - doesn't have xl.json (errFileNotFound)
//    - has the latest xl.json but one or more parts are corrupt
//
// 5. __missingParts__       - has the latest copy of xl.json but has some parts
// missing.  This is identified separately since this may need manual
// inspection to understand the root cause. E.g, this could be due to
// backend filesystem corruption.

// listOnlineDisks - returns
// - a slice of disks where disk having 'older' xl.json (or nothing)
// are set to nil.
// - latest (in time) of the maximally occurring modTime(s).
func listOnlineDisks(disks []StorageAPI, partsMetadata []xlMetaV1, errs []error) (onlineDisks []StorageAPI, modTime time.Time) {
	onlineDisks = make([]StorageAPI, len(disks))

	// List all the file commit ids from parts metadata.
	modTimes := listObjectModtimes(partsMetadata, errs)

	// Reduce list of UUIDs to a single common value.
	modTime, _ = commonTime(modTimes)

	// Create a new online disks slice, which have common uuid.
	for index, t := range modTimes {
		if t == modTime {
			onlineDisks[index] = disks[index]
		} else {
			onlineDisks[index] = nil
		}
	}
	return onlineDisks, modTime
}

// Returns one of the latest updated xlMeta files and count of total valid xlMeta(s) updated latest
func getLatestXLMeta(partsMetadata []xlMetaV1, errs []error) (xlMetaV1, int) {
	// List all the file commit ids from parts metadata.
	modTimes := listObjectModtimes(partsMetadata, errs)

	// Count all latest updated xlMeta values
	var count int
	var latestXLMeta xlMetaV1

	// Reduce list of UUIDs to a single common value - i.e. the last updated Time
	modTime, _ := commonTime(modTimes)

	// Interate through all the modTimes and count the xlMeta(s) with latest time.
	for index, t := range modTimes {
		if t == modTime && partsMetadata[index].IsValid() {
			latestXLMeta = partsMetadata[index]
			count++
		}
	}
	// Return one of the latest xlMetaData, and the count of lastest updated xlMeta files
	return latestXLMeta, count
}

// disksWithAllParts - This function needs to be called with
// []StorageAPI returned by listOnlineDisks. Returns,
//
// - disks which have all parts specified in the latest xl.json.
//
// - slice of errors about the state of data files on disk - can have
//   a not-found error or a hash-mismatch error.
//
// - non-nil error if any of the disks failed unexpectedly (i.e. error
//   other than file not found and not a checksum error).
func disksWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []xlMetaV1, errs []error, bucket,
	object string) ([]StorageAPI, []error, error) {

	availableDisks := make([]StorageAPI, len(onlineDisks))
	buffer := []byte{}
	dataErrs := make([]error, len(onlineDisks))

	for i, onlineDisk := range onlineDisks {
		if onlineDisk == nil {
			continue
		}

		// disk has a valid xl.json but may not have all the
		// parts. This is considered an outdated disk, since
		// it needs healing too.
		for _, part := range partsMetadata[i].Parts {
			partPath := filepath.Join(object, part.Name)
			checksumInfo := partsMetadata[i].Erasure.GetChecksumInfo(part.Name)
			verifier := NewBitrotVerifier(checksumInfo.Algorithm, checksumInfo.Hash)

			// verification happens even if a 0-length
			// buffer is passed
			_, hErr := onlineDisk.ReadFile(bucket, partPath, 0, buffer, verifier)

			isCorrupt := false
			if hErr != nil {
				isCorrupt = strings.HasPrefix(hErr.Error(), "Bitrot verification mismatch - expected ")
			}
			switch {
			case isCorrupt:
				fallthrough
			case hErr == errFileNotFound, hErr == errVolumeNotFound:
				dataErrs[i] = hErr
				break
			case hErr != nil:
				logger.LogIf(ctx, hErr)
				// abort on unhandled errors
				return nil, nil, hErr
			}
		}

		if dataErrs[i] == nil {
			// All parts verified, mark it as all data available.
			availableDisks[i] = onlineDisk
		}
	}

	return availableDisks, dataErrs, nil
}
<|endoftext|>


<commit_before>/*
This command is the client portion of the pipeline plotter.

Lineprobe supports two modes. By default, it reads values from Stdin. Each line
is a single value. If the argument `-command` is used, rather than reading from
Stdin, lineprobe runs the given command and reads its output. The command
should return a single value and exit with status 0. The command will be
executed every 'interval' seconds.

Options

These are command-line options.

    -hostname=localhost:3131       Host and port of view server.
    -interval=1.0                  How often to run command.
    -command=                      Command to run every second.
    -axis=default                  Name of axis to associate this line.
    -xlabel=                       X-Label for axis.
    -ylabel=                       Y-Label for axis.
    -label=                        Line label name on axis.
    -color=                        Color of line. Chosen automatically by default.
    -q                             Silence the echo of values sent to the server.

Examples

Reading data from stdin:
    while /usr/bin/true; do ps ax | wc -l ; sleep 1 ; done | lineprobe

Reading data from command output:
    lineprobe --command "ps ax | wc -l" --interval 2.0

*/
package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"time"

	// third-party
	"github.com/stephen-soltesz/go/lineserver"
)

var (
	hostname    = flag.String("hostname", "localhost:3131", "Host and port of view server.")
	command     = flag.String("command", "", "Command to run every second.")
	interval    = flag.Float64("interval", 1.0, "How often to run command.")
	axisName    = flag.String("axis", "default", "Name of axis to associate this line.")
	xlabelName  = flag.String("xlabel", "", "X-Label for axis.")
	ylabelName  = flag.String("ylabel", "", "Y-Label for axis.")
	lineName    = flag.String("label", "", "Line label name on axis.")
	lineColor   = flag.String("color", "", "Color of line. Chosen automatically by default.")
	quietOutput = flag.Bool("q", false, "Whether to echo values sent to server.")
)

type ValueReader struct {
	reader *bufio.Reader
}

func NewValueReader() *ValueReader {
	reader := ValueReader{}
	reader.Setup()
	return &reader
}

func (r *ValueReader) Setup() error {
	var fromStdin bool = ("" == *command)

	if fromStdin {
		r.reader = bufio.NewReader(os.Stdin)
	} else {
		cmd := exec.Command("bash", "-c", *command)
		output, err := cmd.StdoutPipe()
		if err != nil {
			return err
		}
		if err := cmd.Start(); err != nil {
			return err
		}
		go cmd.Wait()
		r.reader = bufio.NewReader(output)
	}
	return nil
}

func (r *ValueReader) GetValue() []byte {
	var err error
	var lineTooLong bool
	var out []byte

	out, lineTooLong, err = r.reader.ReadLine()
	out = append(out, '\n')
	if lineTooLong {
		return nil
	} else if err == io.EOF {
		time.Sleep(time.Duration(*interval) * time.Second)
		err := r.Setup()
		if err != nil {
			log.Fatal(err)
		}
		return r.GetValue()
	} else if err != nil {
		log.Fatal(err)
	}
	return out
}

func SendValue(writer io.Writer, out []byte) {
	_, err := writer.Write(out)
	if err != nil {
		log.Fatal(err)
	}
}

func sendClientSettings(writer *bufio.ReadWriter) {
	axisSetting := "axis:" + *axisName + ":" + *xlabelName + ":" + *ylabelName + "\n"
	SendValue(writer, []byte(axisSetting))
	if *lineName != "" {
		SendValue(writer, []byte(fmt.Sprintf("label:%s\n", *lineName)))
	}
	if *lineColor != "" {
		SendValue(writer, []byte(fmt.Sprintf("color:%s\n", *lineColor)))
	}
}

func main() {
	var writer *bufio.ReadWriter
	var out []byte
	var err error

	flag.Parse()
	if !*quietOutput {
		flag.VisitAll(func(f *flag.Flag) {
			fmt.Printf("%-30s %s\n", fmt.Sprintf("-%s=%s", f.Name, f.Value), f.Usage)
		})
	}

	server := lineserver.NewServer(*hostname)
	if writer, err = server.Connect(); err != nil {
		log.Fatal(err)
	}

	reader := NewValueReader()
	sendClientSettings(writer)

	for {
		if out = reader.GetValue(); out == nil {
			continue
		}
		if !*quietOutput {
			fmt.Print("Sending: ", string(out))
		}
		SendValue(writer, out)
		writer.Flush()
	}

	return
}
<commit_msg>add support for operations (avg, percentile) on raw data.<commit_after>/*
This command is the client portion of the pipeline plotter.

Lineprobe supports two modes. By default, it reads values from Stdin. Each line
is a single value. If the argument `-command` is used, rather than reading from
Stdin, lineprobe runs the given command and reads its output. The command
should return a single value and exit with status 0. The command will be
executed every 'interval' seconds.

Options

These are command-line options.

    -hostname=localhost:3131  Host and port of view server.
    -interval=1.0             How often to run command.
    -command=                 Command to run every second.
    -axis=default             Name of axis to associate this line.
    -xlabel=                  X-Label for axis.
    -ylabel=                  Y-Label for axis.
    -label=                   Line label name on axis.
    -color=                   Color of line. Chosen automatically by default.
    -q                        Silence the echo of values sent to the server.
    -samples=                 Number of samples to use when calculating functions.
    -avg=false                Whether to report the average of the number of samples.
    -percentile=<pct>         Percentile to report over number of samples.

Examples

Reading data from stdin:
    while /usr/bin/true; do ps ax | wc -l ; sleep 1 ; done | lineprobe

Reading data from command output:
    lineprobe --command "ps ax | wc -l" --interval 2.0

*/
package main

import (
	"bufio"
	"errors"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math"
	"os"
	"os/exec"
	"sort"
	"strconv"
	"time"

	// third-party
	"github.com/stephen-soltesz/go/lineserver"
)

var (
	hostname    = flag.String("hostname", "localhost:3131", "Host and port of view server.")
	command     = flag.String("command", "", "Command to run every second.")
	interval    = flag.Float64("interval", 1.0, "How often to run command.")
	axisName    = flag.String("axis", "default", "Name of axis to associate this line.")
	xlabelName  = flag.String("xlabel", "", "X-Label for axis.")
	ylabelName  = flag.String("ylabel", "", "Y-Label for axis.")
	lineName    = flag.String("label", "", "Line label name on axis.")
	lineColor   = flag.String("color", "", "Color of line. Chosen automatically by default.")
	quietOutput = flag.Bool("q", false, "Whether to echo values sent to server.")
	exitServer  = flag.Bool("exit", false, "Tell the lineviewer to exit.")

	calcOperations operationSlice

	debug         = flag.Bool("debug", false, "Enable debug messages on stderr.")
	debugLogger   *log.Logger
)

type sampleSet struct {
	samples []float64
	pos int
}

type ValueWriter struct {
	writer *bufio.ReadWriter
	server *lineserver.Server
	op *Operation
	samples *sampleSet
}

type ValueReader struct {
	reader *bufio.Reader
	cmd *exec.Cmd
}

func initFlags() {
	flag.Var(&calcOperations, "operation", "An operation to run on samples.")
}

func checkFlags() {
	if *debug {
		debugLogger = log.New(os.Stderr, "", 0)
	} else {
		debugLogger = log.New(ioutil.Discard, "", 0)
	}
	if len(calcOperations) == 0 {
		calcOperations = append(calcOperations, &Operation{OpPercentile, 1, 50})
	}
}

func printAllFlagsAndValues() {
	flag.VisitAll(func(f *flag.Flag) {
		debugLogger.Printf("%-30s %s\n", fmt.Sprintf("-%s=%s", f.Name, f.Value), f.Usage)
	})
}

func newSampleSet(sampleCount int64) *sampleSet {
	return &sampleSet{make([]float64, sampleCount), 0}
}

func (ss *sampleSet) Count() int {
	if ss.Full() {
		return len(ss.samples)
	} else {
		return ss.pos
	}
}

func (ss *sampleSet) Full() bool {
	return ss.pos >= len(ss.samples)-1
}

func (ss *sampleSet) Append(val float64) {
	ss.samples[ss.pos%len(ss.samples)] = val
	ss.pos += 1
}

func (ss *sampleSet) Mean() float64 {
	count := ss.Count()
	var s float64 = 0
	for i := 0; i < count; i++ {
		s += ss.samples[i]
	}
	avg := s/float64(count)
	debugLogger.Println("avg:", avg)
	return avg
}

func (ss *sampleSet) Stdev() float64 {
	count := ss.Count()
	avg := ss.Mean()
	var variance float64 = 0

	for i := 0; i < count; i++ {
		variance += math.Pow((ss.samples[i] - avg), 2)
	}
	stdev := math.Sqrt(variance/float64(count))
	debugLogger.Println("stdev:", stdev)
	return stdev
}

func (ss *sampleSet) Percentile(pct int64) float64 {
	count := ss.Count()
	var calc = make([]float64, count)
	copy(calc, ss.samples)

	i := int(math.Ceil(float64(count-1) * float64(pct)/101.0))
	sort.Float64s(calc)
	debugLogger.Println("size", count, "i", i)
	debugLogger.Println("pct:", calc[i])
	return calc[i]
}

func NewValueWriter(op *Operation) *ValueWriter {
	var err error
	writer := ValueWriter{}
	writer.op = op
	fmt.Printf("%#v\n", op)
	writer.samples = newSampleSet(op.samples)
	writer.server = lineserver.NewServer(*hostname)
	if writer.writer, err = writer.server.Connect(); err != nil {
		log.Fatal(err)
	}
	return &writer
}

func SetupValueWriters() []*ValueWriter {
	var writers []*ValueWriter
	for _, op := range calcOperations {
		writer := NewValueWriter(op)
		if err := writer.sendClientSettings(); err != nil {
			log.Fatal(err)
		}
		writers = append(writers, writer)
	}
	return writers
}

func (w *ValueWriter) SendValue(val float64) error {
	var f float64
	w.samples.Append(val)
	if w.op.operation == OpMean {
		f = w.samples.Mean()
	} else if w.op.operation == OpStdev {
		f = w.samples.Stdev()
	} else if w.op.operation == OpPercentile {
		f = w.samples.Percentile(w.op.percentile)
	}

	out := formatFloat(f)
	fmt.Printf("Sending: %s", string(out))
	_, err := w.writer.Write(out)
	if err != nil {
		return err
	}
	w.writer.Flush()
	return nil
}

func (w *ValueWriter) sendClientSettings() error {
	if *exitServer {
		fmt.Println("Sending exit")
		_, err := w.writer.WriteString("EXIT\n")
		if err != nil {
			return err
		}
		w.writer.Flush()
		// create new connection to trigger exit.
		time.Sleep(time.Second)
		NewValueWriter(&Operation{OpNone, 0, 0})
		os.Exit(0)
	}
	axisSetting := "axis:" + *axisName + ":" + *xlabelName + ":" + *ylabelName + "\n"
	_, err := w.writer.WriteString(axisSetting)
	if err != nil {
		return err
	}

	name := ""
	if w.op.operation > OpNone {
		name = fmt.Sprintf("%s-", w.op)
	}

	if *lineName != "" {
		name = "label:" + name + *lineName + "\n"
	} else if *command != "" {
		name = "label:" + name + *command + "\n"
	} else {
		name = "label:" + name + fmt.Sprintf("Thread-%d\n", os.Getpid())
	}

	_, err = w.writer.WriteString(name)
	if err != nil {
		return err
	}

	if *lineColor != "" {
		colorSetting := fmt.Sprintf("color:%s\n", *lineColor)
		_, err := w.writer.WriteString(colorSetting)
		if err != nil {
			return err
		}
	}
	return nil
}

func NewValueReader() *ValueReader {
	reader := ValueReader{}
	reader.Setup()
	return &reader
}

func (r *ValueReader) Setup() error {
	var fromStdin bool = ("" == *command)

	if fromStdin {
		r.reader = bufio.NewReader(os.Stdin)
	} else {
		r.cmd = exec.Command("bash", "-c", *command)
		output, err := r.cmd.StdoutPipe()
		if err != nil {
			return err
		}
		if err := r.cmd.Start(); err != nil {
			return err
		}
		r.reader = bufio.NewReader(output)
	}
	return nil
}

func (r *ValueReader) ReadValue() (float64, error) {
	var err error
	var lineTooLong bool
	var out []byte

	out, lineTooLong, err = r.reader.ReadLine()
	if lineTooLong {
		return 0.0, errors.New("line too long")
	} else if err == io.EOF {
		r.cmd.Wait()	// must run after EOF to avoid races with Read()
		time.Sleep(time.Duration(*interval) * time.Second)
		err := r.Setup()
		if err != nil {
			fmt.Println("reader setup failed:", err)
			return 0.0, err
		}
		return r.ReadValue()
	} else if err != nil {
		fmt.Println("unknown failure:", err)
		return 0.0, err
	}

	f, err := strconv.ParseFloat(string(out), 64)
	if err != nil {
		return 0.0, err
	}
	if !*quietOutput {
		fmt.Printf("Received: %f\n", f)
	}
	return f, nil
}

func parseFloat(val []byte) float64 {
	fmt.Println("%v", val)
	f, err := strconv.ParseFloat(string(val), 64)
	if err != nil {
		debugLogger.Println(err)
		return 0.0
	}
	return f
}

func formatFloat(f float64) []byte {
	return []byte(fmt.Sprintf("%f\n", f))
}

func main() {
	var err error
	var val float64

	initFlags()
	flag.Parse()
	checkFlags()

	if !*quietOutput {
		printAllFlagsAndValues()
	}

	// allocate value reader and writers.
	reader := NewValueReader()
	writers := SetupValueWriters()

	for {
		if val, err = reader.ReadValue(); err != nil {
			log.Println(err)
			break
		}
		for _, writer := range writers {
			if err = writer.SendValue(val); err != nil {
				log.Fatal(err)
			}
		}
	}
	return
}
<|endoftext|>