<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"os"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/relab/raft/pkg/raft"
	pb "github.com/relab/raft/pkg/raft/raftpb"

	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

func main() {
	var id = flag.Uint64("id", 0, "server ID")
	var cluster = flag.String("cluster", ":9201", "comma separated cluster servers")
	var bench = flag.Bool("quiet", false, "Silence log output")
	var recover = flag.Bool("recover", false, "Recover from stable storage")
	var cpuprofile = flag.String("cpuprofile", "", "Write cpu profile to file")
	var slowQuorum = flag.Bool("slowquorum", false, "set quorum size to the number of servers")
	var batch = flag.Bool("batch", true, "enable batching")
	var qrpc = flag.Bool("qrpc", false, "enable QRPC")
	var electionTimeout = flag.Duration("election", 2*time.Second, "How long servers wait before starting an election")
	var heartbeatTimeout = flag.Duration("heartbeat", 250*time.Millisecond, "How often a heartbeat should be sent")
	var maxAppendEntries = flag.Int("maxappend", 5000, "Max entries per AppendEntries message")

	flag.Parse()
	rand.Seed(time.Now().UnixNano())

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
	}

	if *id == 0 {
		fmt.Print("-id argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	nodes := strings.Split(*cluster, ",")

	if len(nodes) == 0 {
		fmt.Print("-cluster argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if !*qrpc && len(nodes) != 3 {
		fmt.Print("only 3 nodes is supported with QRPC enabled\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *maxAppendEntries < 1 {
		fmt.Print("-maxappend must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *bench {
		log.SetOutput(ioutil.Discard)
		silentLogger := log.New(ioutil.Discard, "", log.LstdFlags)
		grpclog.SetLogger(silentLogger)
		grpc.EnableTracing = false
	}

	r, err := raft.NewReplica(&raft.Config{
		ID:               *id,
		Nodes:            nodes,
		Recover:          *recover,
		Batch:            *batch,
		QRPC:             *qrpc,
		SlowQuorum:       *slowQuorum,
		ElectionTimeout:  *electionTimeout,
		HeartbeatTimeout: *heartbeatTimeout,
		MaxAppendEntries: *maxAppendEntries,
	})

	if err != nil {
		log.Fatal(err)
	}

	s := grpc.NewServer()
	pb.RegisterRaftServer(s, r)

	l, err := net.Listen("tcp", nodes[*id-1])

	if err != nil {
		log.Fatal(err)
	}

	go func() {
		err := s.Serve(l)

		if err != nil {
			log.Fatal(err)
		}
	}()

	if *cpuprofile != "" {
		go func() {
			if err := r.Run(); err != nil {
				log.Fatal(err)
			}
		}()

		reader := bufio.NewReader(os.Stdin)
		reader.ReadLine()

		pprof.StopCPUProfile()
	} else {
		if err := r.Run(); err != nil {
			log.Fatal(err)
		}
	}
}
<commit_msg>replica: Group flag variables<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net"
	"os"
	"runtime/pprof"
	"strings"
	"time"

	"github.com/relab/raft/pkg/raft"
	pb "github.com/relab/raft/pkg/raft/raftpb"

	"google.golang.org/grpc"
	"google.golang.org/grpc/grpclog"
)

func main() {
	var (
		id               = flag.Uint64("id", 0, "server ID")
		cluster          = flag.String("cluster", ":9201", "comma separated cluster servers")
		bench            = flag.Bool("quiet", false, "Silence log output")
		recover          = flag.Bool("recover", false, "Recover from stable storage")
		cpuprofile       = flag.String("cpuprofile", "", "Write cpu profile to file")
		slowQuorum       = flag.Bool("slowquorum", false, "set quorum size to the number of servers")
		batch            = flag.Bool("batch", true, "enable batching")
		qrpc             = flag.Bool("qrpc", false, "enable QRPC")
		electionTimeout  = flag.Duration("election", 2*time.Second, "How long servers wait before starting an election")
		heartbeatTimeout = flag.Duration("heartbeat", 250*time.Millisecond, "How often a heartbeat should be sent")
		maxAppendEntries = flag.Int("maxappend", 5000, "Max entries per AppendEntries message")
	)

	flag.Parse()
	rand.Seed(time.Now().UnixNano())

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
	}

	if *id == 0 {
		fmt.Print("-id argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	nodes := strings.Split(*cluster, ",")

	if len(nodes) == 0 {
		fmt.Print("-cluster argument is required\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if !*qrpc && len(nodes) != 3 {
		fmt.Print("only 3 nodes is supported with QRPC enabled\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *maxAppendEntries < 1 {
		fmt.Print("-maxappend must be atleast 1\n\n")
		flag.Usage()
		os.Exit(1)
	}

	if *bench {
		log.SetOutput(ioutil.Discard)
		silentLogger := log.New(ioutil.Discard, "", log.LstdFlags)
		grpclog.SetLogger(silentLogger)
		grpc.EnableTracing = false
	}

	r, err := raft.NewReplica(&raft.Config{
		ID:               *id,
		Nodes:            nodes,
		Recover:          *recover,
		Batch:            *batch,
		QRPC:             *qrpc,
		SlowQuorum:       *slowQuorum,
		ElectionTimeout:  *electionTimeout,
		HeartbeatTimeout: *heartbeatTimeout,
		MaxAppendEntries: *maxAppendEntries,
	})

	if err != nil {
		log.Fatal(err)
	}

	s := grpc.NewServer()
	pb.RegisterRaftServer(s, r)

	l, err := net.Listen("tcp", nodes[*id-1])

	if err != nil {
		log.Fatal(err)
	}

	go func() {
		err := s.Serve(l)

		if err != nil {
			log.Fatal(err)
		}
	}()

	if *cpuprofile != "" {
		go func() {
			if err := r.Run(); err != nil {
				log.Fatal(err)
			}
		}()

		reader := bufio.NewReader(os.Stdin)
		reader.ReadLine()

		pprof.StopCPUProfile()
	} else {
		if err := r.Run(); err != nil {
			log.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"path/filepath"
	"time"

	"gopkg.in/mgo.v2/bson"
	"gopkg.in/vmihailenco/msgpack.v2"
	"gopkg.in/yaml.v2"
	"h12.me/errors"
	"h12.me/sej"
	"h12.me/uuid/hexid"
)

type FileCommand struct {
	JournalFileConfig `positional-args:"yes"  required:"yes"`
}

func (c *FileCommand) Execute(args []string) error {
	jf, err := sej.ParseJournalFileName(path.Split(c.JournalFile))
	if err != nil {
		return err
	}
	firstMsg, err := jf.FirstMessage()
	if err != nil {
		return err
	}
	fmt.Println("first:")
	fmt.Println("    offset:", firstMsg.Offset)
	fmt.Println("    timestamp:", firstMsg.Timestamp)
	lastMsg, err := jf.LastMessage()
	if err != nil {
		return err
	}
	fmt.Println("last:")
	fmt.Println("    offset:", lastMsg.Offset)
	fmt.Println("    timestamp:", lastMsg.Timestamp)
	return nil
}

type TimestampCommand struct {
	JournalDirConfig `positional-args:"yes"  required:"yes"`
	Offset           string `
		long:"offset"
		description:"the offset"`
}

func (d *TimestampCommand) Execute(args []string) error {
	ofsFilename := path.Join(sej.OffsetDirPath(d.Dir), d.Offset) + ".ofs"
	ofsFile, err := os.Open(ofsFilename)
	if err != nil {
		return err
	}
	defer ofsFile.Close()
	offset, err := sej.ReadOffset(ofsFile)
	if err != nil {
		return err
	}

	s, err := sej.NewScanner(d.Dir, offset)
	if err != nil {
		return err
	}
	if s.Offset() != offset {
		return fmt.Errorf("fail to scan to offset %d in %s", offset, d.Dir)
	}
	fmt.Println("offset:", s.Offset())
	fmt.Println("timestamp:", s.Message().Timestamp)
	return nil
}

type ScanCommand struct {
	JournalDirConfig `positional-args:"yes"  required:"yes"`
	Start            Timestamp `
	long:"start"
	description:"start time"`
	End Timestamp `
	long:"end"
	description:"start time"`
	Type byte `
	long:"type"
	description:"message type"`
	Format string `
		long:"format"
		default:"bson"
		description:"encoding format of the message"`
	Count bool `
		long:"count"
		description:"count or print"
	`
}

type CountCommand ScanCommand

func (c *CountCommand) Execute(args []string) error {
	scanCmd := ScanCommand(*c)
	scanCmd.Count = true
	return scanCmd.Execute(args)
}

func (c *ScanCommand) Execute(args []string) error {
	//fmt.Println("couting from", c.Start, c.End, "for type", c.Type)
	dir, err := sej.OpenJournalDir(sej.JournalDirPath(c.Dir))
	if err != nil {
		return err
	}
	startOffset := dir.First().FirstOffset
	for _, file := range dir.Files {
		f, err := os.Open(file.FileName)
		if err != nil {
			return err
		}
		var msg sej.Message
		if _, err := msg.ReadFrom(f); err != nil && err != io.EOF {
			f.Close()
			return err
		}
		f.Close()
		if msg.Timestamp.After(c.Start.Time) {
			break
		}
		startOffset = file.FirstOffset
	}
	if int(startOffset)-5000 > 0 {
		startOffset -= 5000
	}
	s, err := sej.NewScanner(c.Dir, startOffset)
	if err != nil {
		return err
	}
	s.Timeout = time.Second
	cnt := 0
	overCount := 0
	for s.Scan() {
		msg := s.Message()
		if !msg.Timestamp.Before(c.Start.Time) {
			if msg.Timestamp.Before(c.End.Time) {
				if msg.Type == c.Type {
					if !c.Count {
						switch c.Format {
						case "json", "msgpack", "bson":
							line, _ := Format(c.Format).Sprint(msg)
							fmt.Println(line)
						}
					}
					cnt++
				}
			} else {
				overCount++
			}
		}
		if overCount > 5000 {
			break
		}
	}
	if c.Count {
		fmt.Println(cnt)
	}
	return nil
}

type DumpCommand struct {
	JournalFileConfig `positional-args:"yes"  required:"yes"`
}

type JournalFileConfig struct {
	JournalFile string
}

func (d *DumpCommand) Execute(args []string) error {
	var msg sej.Message
	for {
		file, err := os.Open(d.JournalFile)
		if err != nil {
			return errors.Wrap(err)
		}
		defer file.Close()
		if _, err := msg.ReadFrom(file); err != nil {
			if err == io.EOF {
				return nil
			}
			return err
		}
		fmt.Println("offset:", msg.Offset)
		fmt.Printf("message: %x (%s)\n", msg.Value, string(msg.Value))
	}
	return nil
}

type OffsetCommand struct {
	JournalDirConfig `positional-args:"yes"  required:"yes"`
}

func (c *OffsetCommand) Execute(args []string) error {
	dir, err := sej.OpenJournalDir(sej.JournalDirPath(c.Dir))
	if err != nil {
		return err
	}
	firstOffset := dir.First().FirstOffset
	lastOffset, err := dir.Last().LastReadableOffset()
	if err != nil {
		return err
	}
	fmt.Println("first:", firstOffset)
	fmt.Println("last:", lastOffset)
	offsets, err := readOffsets(c.Dir)
	if err != nil {
		return err
	}
	buf, _ := yaml.Marshal(offsets)
	fmt.Println(string(buf))
	return nil
}

type JournalDirConfig struct {
	Dir string
}

type TailCommand struct {
	Count int `
		long:"count"
		description:"the number of tailing messages to print"
		default:"10"`
	Format string `
		long:"format"
		default:"bson"
		description:"encoding format of the message"`
	JournalDirConfig `positional-args:"yes"  required:"yes"`
}

func (c *TailCommand) Execute(args []string) error {
	dir, err := sej.OpenJournalDir(sej.JournalDirPath(c.Dir))
	if err != nil {
		return err
	}
	earlist := dir.First().FirstOffset
	latest, err := dir.Last().LastOffset()
	if err != nil {
		return err
	}
	offset := int(latest) - c.Count
	if offset < int(earlist) {
		offset = int(earlist)
	}
	scanner, err := sej.NewScanner(c.Dir, uint64(offset))
	if err != nil {
		return err
	}
	cnt := 0
	for scanner.Scan() {
		switch c.Format {
		case "json", "msgpack", "bson":
			line, _ := Format(c.Format).Sprint(scanner.Message())
			fmt.Println(line)
		}
		cnt++
		if cnt >= int(c.Count) {
			break
		}
	}
	return nil
}

type Format string

func (format Format) Sprint(msg *sej.Message) (string, error) {
	value := msg.Value
	m := make(map[string]interface{})
	switch format {
	case "msgpack":
		if err := msgpack.Unmarshal(value, &m); err != nil {
			return "", err
		}
	case "bson":
		if err := bson.Unmarshal(value, &m); err != nil {
			return "", err
		}
	default:
		return string(value), nil
	}
	hexid.Restore(m)
	m = map[string]interface{}{
		"key":       string(msg.Key),
		"timestamp": msg.Timestamp,
		"type":      msg.Type,
		"value":     m,
	}
	buf, err := json.Marshal(m)
	if err != nil {
		return "", fmt.Errorf("fail to marshal %#v: %s", m, err.Error())
	}
	return string(buf), nil
}

type OldCommand struct {
	Days int `
		long:"days"
		default:"7"
		description:"max number of days of journal files kept after cleanning"`
	JournalDirConfig `positional-args:"yes"  required:"yes"`
}

func (c *OldCommand) Execute(args []string) error {
	if c.Days < 1 {
		return errors.New("days must be at least 1")
	}
	dir, err := sej.OpenJournalDir(sej.JournalDirPath(c.Dir))
	if err != nil {
		return errors.Wrap(err)
	}

	lastOffset, err := dir.Last().LastReadableOffset()
	if err != nil {
		return errors.Wrap(err)
	}
	slowestReader := ""
	slowestOffset := lastOffset
	offsets, err := readOffsets(c.Dir)
	if err != nil {
		return errors.Wrap(err)
	}
	for ofsFile, offset := range offsets {
		if offset < slowestOffset {
			slowestReader = ofsFile
			slowestOffset = offset
		}
	}

	daysAgo := time.Now().Add(-time.Duration(c.Days) * time.Hour * 24)
	for _, journalFile := range dir.Files[:len(dir.Files)-1] {
		lastMessage, err := journalFile.LastMessage()
		if err != nil {
			return errors.Wrap(err)
		}
		if slowestOffset <= lastMessage.Offset {
			log.Printf("cannot clean %s (%d-%d) because of slow reader %s\n", journalFile.FileName, journalFile.FirstOffset, lastMessage.Offset, slowestReader)
			break
		}
		if !lastMessage.Timestamp.Before(daysAgo) {
			break
		}
		fmt.Println(journalFile.FileName)
	}
	return nil
}

type Timestamp struct {
	time.Time
}

const timeFormat = "2006-01-02T15:04:05"

func (t *Timestamp) UnmarshalFlag(value string) error {
	tm, err := time.Parse(timeFormat, value)
	if err != nil {
		return fmt.Errorf("error parsing %s: %s", value, err.Error())
	}
	t.Time = tm
	return nil
}

func readOffsets(dir string) (map[string]uint64, error) {
	offsets := make(map[string]uint64)
	ofsFiles, err := filepath.Glob(path.Join(sej.OffsetDirPath(dir), "*.ofs"))
	if err != nil {
		return nil, errors.Wrap(err)
	}
	for _, ofsFile := range ofsFiles {
		f, err := os.Open(ofsFile)
		if err != nil {
			return nil, errors.Wrap(err)
		}
		offset, err := sej.ReadOffset(f)
		if err != nil {
			f.Close()
			return nil, errors.Wrap(err)
		}
		f.Close()
		offsets[ofsFile] = offset
	}
	return offsets, nil
}
<commit_msg>fix default value for type<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"path/filepath"
	"time"

	"gopkg.in/mgo.v2/bson"
	"gopkg.in/vmihailenco/msgpack.v2"
	"gopkg.in/yaml.v2"
	"h12.me/errors"
	"h12.me/sej"
	"h12.me/uuid/hexid"
)

type FileCommand struct {
	JournalFileConfig `positional-args:"yes"  required:"yes"`
}

func (c *FileCommand) Execute(args []string) error {
	jf, err := sej.ParseJournalFileName(path.Split(c.JournalFile))
	if err != nil {
		return err
	}
	firstMsg, err := jf.FirstMessage()
	if err != nil {
		return err
	}
	fmt.Println("first:")
	fmt.Println("    offset:", firstMsg.Offset)
	fmt.Println("    timestamp:", firstMsg.Timestamp)
	lastMsg, err := jf.LastMessage()
	if err != nil {
		return err
	}
	fmt.Println("last:")
	fmt.Println("    offset:", lastMsg.Offset)
	fmt.Println("    timestamp:", lastMsg.Timestamp)
	return nil
}

type TimestampCommand struct {
	JournalDirConfig `positional-args:"yes"  required:"yes"`
	Offset           string `
		long:"offset"
		description:"the offset"`
}

func (d *TimestampCommand) Execute(args []string) error {
	ofsFilename := path.Join(sej.OffsetDirPath(d.Dir), d.Offset) + ".ofs"
	ofsFile, err := os.Open(ofsFilename)
	if err != nil {
		return err
	}
	defer ofsFile.Close()
	offset, err := sej.ReadOffset(ofsFile)
	if err != nil {
		return err
	}

	s, err := sej.NewScanner(d.Dir, offset)
	if err != nil {
		return err
	}
	if s.Offset() != offset {
		return fmt.Errorf("fail to scan to offset %d in %s", offset, d.Dir)
	}
	fmt.Println("offset:", s.Offset())
	fmt.Println("timestamp:", s.Message().Timestamp)
	return nil
}

type ScanCommand struct {
	JournalDirConfig `positional-args:"yes"  required:"yes"`
	Start            Timestamp `
	long:"start"
	description:"start time"`
	End Timestamp `
	long:"end"
	description:"start time"`
	Type byte `
	long:"type"
	description:"message type"`
	Format string `
		long:"format"
		default:"bson"
		description:"encoding format of the message"`
	Count bool `
		long:"count"
		description:"count or print"
	`
}

type CountCommand ScanCommand

func (c *CountCommand) Execute(args []string) error {
	scanCmd := ScanCommand(*c)
	scanCmd.Count = true
	return scanCmd.Execute(args)
}

func (c *ScanCommand) Execute(args []string) error {
	// fmt.Fprintln(os.Stderr, "couting from", c.Start, c.End, "for type", c.Type)
	dir, err := sej.OpenJournalDir(sej.JournalDirPath(c.Dir))
	if err != nil {
		return err
	}
	startOffset := dir.First().FirstOffset
	for _, file := range dir.Files {
		f, err := os.Open(file.FileName)
		if err != nil {
			return err
		}
		var msg sej.Message
		if _, err := msg.ReadFrom(f); err != nil && err != io.EOF {
			f.Close()
			return err
		}
		f.Close()
		if msg.Timestamp.After(c.Start.Time) {
			// fmt.Fprintf(os.Stderr, "start from %v, %d\n", msg.Timestamp, startOffset)
			break
		}
		startOffset = file.FirstOffset
	}
	if int(startOffset)-5000 > 0 {
		startOffset -= 5000
	}
	s, err := sej.NewScanner(c.Dir, startOffset)
	if err != nil {
		return err
	}
	s.Timeout = time.Second
	cnt := 0
	overCount := 0
	for s.Scan() {
		msg := s.Message()
		if !msg.Timestamp.Before(c.Start.Time) {
			if msg.Timestamp.Before(c.End.Time) {
				if c.Type == 0 || msg.Type == c.Type {
					if !c.Count {
						switch c.Format {
						case "json", "msgpack", "bson":
							line, _ := Format(c.Format).Sprint(msg)
							fmt.Println(line)
						}
					}
					cnt++
				}
			} else {
				overCount++
			}
		}
		if overCount > 5000 {
			break
		}
	}
	if c.Count {
		fmt.Println(cnt)
	}
	return nil
}

type DumpCommand struct {
	JournalFileConfig `positional-args:"yes"  required:"yes"`
}

type JournalFileConfig struct {
	JournalFile string
}

func (d *DumpCommand) Execute(args []string) error {
	var msg sej.Message
	for {
		file, err := os.Open(d.JournalFile)
		if err != nil {
			return errors.Wrap(err)
		}
		defer file.Close()
		if _, err := msg.ReadFrom(file); err != nil {
			if err == io.EOF {
				return nil
			}
			return err
		}
		fmt.Println("offset:", msg.Offset)
		fmt.Printf("message: %x (%s)\n", msg.Value, string(msg.Value))
	}
	return nil
}

type OffsetCommand struct {
	JournalDirConfig `positional-args:"yes"  required:"yes"`
}

func (c *OffsetCommand) Execute(args []string) error {
	dir, err := sej.OpenJournalDir(sej.JournalDirPath(c.Dir))
	if err != nil {
		return err
	}
	firstOffset := dir.First().FirstOffset
	lastOffset, err := dir.Last().LastReadableOffset()
	if err != nil {
		return err
	}
	fmt.Println("first:", firstOffset)
	fmt.Println("last:", lastOffset)
	offsets, err := readOffsets(c.Dir)
	if err != nil {
		return err
	}
	buf, _ := yaml.Marshal(offsets)
	fmt.Println(string(buf))
	return nil
}

type JournalDirConfig struct {
	Dir string
}

type TailCommand struct {
	Count int `
		long:"count"
		description:"the number of tailing messages to print"
		default:"10"`
	Format string `
		long:"format"
		default:"bson"
		description:"encoding format of the message"`
	JournalDirConfig `positional-args:"yes"  required:"yes"`
}

func (c *TailCommand) Execute(args []string) error {
	dir, err := sej.OpenJournalDir(sej.JournalDirPath(c.Dir))
	if err != nil {
		return err
	}
	earlist := dir.First().FirstOffset
	latest, err := dir.Last().LastOffset()
	if err != nil {
		return err
	}
	offset := int(latest) - c.Count
	if offset < int(earlist) {
		offset = int(earlist)
	}
	scanner, err := sej.NewScanner(c.Dir, uint64(offset))
	if err != nil {
		return err
	}
	cnt := 0
	for scanner.Scan() {
		switch c.Format {
		case "json", "msgpack", "bson":
			line, _ := Format(c.Format).Sprint(scanner.Message())
			fmt.Println(line)
		}
		cnt++
		if cnt >= int(c.Count) {
			break
		}
	}
	return nil
}

type Format string

func (format Format) Sprint(msg *sej.Message) (string, error) {
	value := msg.Value
	m := make(map[string]interface{})
	switch format {
	case "msgpack":
		if err := msgpack.Unmarshal(value, &m); err != nil {
			return "", err
		}
	case "bson":
		if err := bson.Unmarshal(value, &m); err != nil {
			return "", err
		}
	default:
		return string(value), nil
	}
	hexid.Restore(m)
	m = map[string]interface{}{
		"key":       string(msg.Key),
		"timestamp": msg.Timestamp,
		"type":      msg.Type,
		"value":     m,
	}
	buf, err := json.Marshal(m)
	if err != nil {
		return "", fmt.Errorf("fail to marshal %#v: %s", m, err.Error())
	}
	return string(buf), nil
}

type OldCommand struct {
	Days int `
		long:"days"
		default:"7"
		description:"max number of days of journal files kept after cleanning"`
	JournalDirConfig `positional-args:"yes"  required:"yes"`
}

func (c *OldCommand) Execute(args []string) error {
	if c.Days < 1 {
		return errors.New("days must be at least 1")
	}
	dir, err := sej.OpenJournalDir(sej.JournalDirPath(c.Dir))
	if err != nil {
		return errors.Wrap(err)
	}

	lastOffset, err := dir.Last().LastReadableOffset()
	if err != nil {
		return errors.Wrap(err)
	}
	slowestReader := ""
	slowestOffset := lastOffset
	offsets, err := readOffsets(c.Dir)
	if err != nil {
		return errors.Wrap(err)
	}
	for ofsFile, offset := range offsets {
		if offset < slowestOffset {
			slowestReader = ofsFile
			slowestOffset = offset
		}
	}

	daysAgo := time.Now().Add(-time.Duration(c.Days) * time.Hour * 24)
	for _, journalFile := range dir.Files[:len(dir.Files)-1] {
		lastMessage, err := journalFile.LastMessage()
		if err != nil {
			return errors.Wrap(err)
		}
		if slowestOffset <= lastMessage.Offset {
			log.Printf("cannot clean %s (%d-%d) because of slow reader %s\n", journalFile.FileName, journalFile.FirstOffset, lastMessage.Offset, slowestReader)
			break
		}
		if !lastMessage.Timestamp.Before(daysAgo) {
			break
		}
		fmt.Println(journalFile.FileName)
	}
	return nil
}

type Timestamp struct {
	time.Time
}

const timeFormat = "2006-01-02T15:04:05"

func (t *Timestamp) UnmarshalFlag(value string) error {
	tm, err := time.Parse(timeFormat, value)
	if err != nil {
		return fmt.Errorf("error parsing %s: %s", value, err.Error())
	}
	t.Time = tm
	return nil
}

func readOffsets(dir string) (map[string]uint64, error) {
	offsets := make(map[string]uint64)
	ofsFiles, err := filepath.Glob(path.Join(sej.OffsetDirPath(dir), "*.ofs"))
	if err != nil {
		return nil, errors.Wrap(err)
	}
	for _, ofsFile := range ofsFiles {
		f, err := os.Open(ofsFile)
		if err != nil {
			return nil, errors.Wrap(err)
		}
		offset, err := sej.ReadOffset(f)
		if err != nil {
			f.Close()
			return nil, errors.Wrap(err)
		}
		f.Close()
		offsets[ofsFile] = offset
	}
	return offsets, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"

	"fmt"
	"io/ioutil"
	"strings"

	"github.com/j-delaney/typewriter"
)

func main() {
	flag.Usage = func() {
		fmt.Printf("Usage: side_by_side [options] leftFile rightFile\nOptions:\n")
		flag.PrintDefaults()
	}

	padding := flag.Int("padding", 5, "The padding between the two columns")
	separator := flag.String("separator", "", "Character to separate the two columns")
	markDifference := flag.Bool("diff", false, "Mark the first difference found")
	lineNumbers := flag.Bool("linenums", false, "Show line numbers")
	showHeader := flag.Bool("header", false, "Use the filenames as headers")

	flag.Parse()

	if flag.NArg() != 2 {
		flag.Usage()
		os.Exit(2)
	}

	filePath1 := flag.Arg(0)
	filePath2 := flag.Arg(1)

	bytes1, err := ioutil.ReadFile(filePath1)
	if err != nil {
		fmt.Fprint(os.Stderr, "Could not read %v: %v", filePath1, err)
		os.Exit(1)
	}

	bytes2, err := ioutil.ReadFile(filePath2)
	if err != nil {
		fmt.Fprint(os.Stderr, "Could not read %v: %v", filePath2, err)
		os.Exit(1)
	}

	lines1 := strings.Split(string(bytes1), "\n")
	lines2 := strings.Split(string(bytes2), "\n")

	config := typewriter.Config{
		Padding:   *padding,
		Separator: *separator,

		MarkFirstDifference: *markDifference,
		ShowLineNumbers:     *lineNumbers,
	}

	if *showHeader {
		config.LeftHeader = filePath1
		config.RightHeader = filePath2
	}

	s := typewriter.Run(lines1, lines2, config)

	fmt.Print(s)
}
<commit_msg>Fix printing issue<commit_after>package main

import (
	"flag"
	"os"

	"fmt"
	"io/ioutil"
	"strings"

	"github.com/j-delaney/typewriter"
)

func main() {
	flag.Usage = func() {
		fmt.Printf("Usage: side_by_side [options] leftFile rightFile\nOptions:\n")
		flag.PrintDefaults()
	}

	padding := flag.Int("padding", 5, "The padding between the two columns")
	separator := flag.String("separator", "", "Character to separate the two columns")
	markDifference := flag.Bool("diff", false, "Mark the first difference found")
	lineNumbers := flag.Bool("linenums", false, "Show line numbers")
	showHeader := flag.Bool("header", false, "Use the filenames as headers")

	flag.Parse()

	if flag.NArg() != 2 {
		flag.Usage()
		os.Exit(2)
	}

	filePath1 := flag.Arg(0)
	filePath2 := flag.Arg(1)

	bytes1, err := ioutil.ReadFile(filePath1)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Could not read %v: %v", filePath1, err)
		os.Exit(1)
	}

	bytes2, err := ioutil.ReadFile(filePath2)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Could not read %v: %v", filePath2, err)
		os.Exit(1)
	}

	lines1 := strings.Split(string(bytes1), "\n")
	lines2 := strings.Split(string(bytes2), "\n")

	config := typewriter.Config{
		Padding:   *padding,
		Separator: *separator,

		MarkFirstDifference: *markDifference,
		ShowLineNumbers:     *lineNumbers,
	}

	if *showHeader {
		config.LeftHeader = filePath1
		config.RightHeader = filePath2
	}

	s := typewriter.Run(lines1, lines2, config)

	fmt.Print(s)
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/miekg/dns"
	"github.com/spf13/pflag"
	"k8s.io/kubernetes/pkg/util/flag"
	"k8s.io/kubernetes/pkg/util/logs"
	"k8s.io/kubernetes/pkg/version/verflag"

	"k8s.io/dns/pkg/sidecar"
	"k8s.io/dns/pkg/version"
)

const (
	defaultProbeInterval = 5 * time.Second
)

func main() {
	options := sidecar.NewOptions()
	configureFlags(options, pflag.CommandLine)
	flag.InitFlags()

	logs.InitLogs()
	defer logs.FlushLogs()

	glog.Infof("Version v%s", version.VERSION)

	verflag.PrintAndExitIfRequested()

	server := sidecar.NewServer()
	server.Run(options)
}

type probeOptions []sidecar.DNSProbeOption

func (po *probeOptions) String() string {
	return fmt.Sprintf("%+v", *po)
}

func (po *probeOptions) Set(value string) error {
	splits := strings.Split(value, ",")
	if !(3 <= len(splits) && len(splits) <= 5) {
		return fmt.Errorf("invalid format to --probe")
	}

	option := sidecar.DNSProbeOption{
		Label:    splits[0],
		Server:   splits[1],
		Name:     splits[2],
		Interval: defaultProbeInterval,
		Type:     dns.TypeANY,
	}

	const labelRegexp = "^[a-zA-Z0-9_]+"
	if !regexp.MustCompile(labelRegexp).MatchString(option.Label) {
		return fmt.Errorf("label must be of format " + labelRegexp)
	}

	if !strings.Contains(option.Server, ":") {
		option.Server = option.Server + ":53"
	}

	if !strings.HasSuffix(option.Name, ".") {
		// dns package requires a fully qualified (e.g. terminal '.') name
		option.Name = option.Name + "."
	}

	if len(splits) >= 4 {
		if interval, err := strconv.Atoi(splits[3]); err == nil {
			option.Interval = time.Duration(interval) * time.Second
		} else {
			return err
		}
	}

	if len(splits) >= 5 {
		switch splits[4] {
		case "A":
			option.Type = dns.TypeA
			break
		case "AAAA":
			option.Type = dns.TypeAAAA
			break
		case "ANY":
			option.Type = dns.TypeANY
			break
		default:
			return fmt.Errorf("invalid type for DNS: %v", splits[5])
		}
	}

	*po = append(*po, option)

	return nil
}

func (po *probeOptions) Type() string {
	return "string"
}

var _ pflag.Value = (*probeOptions)(nil)

func configureFlags(opt *sidecar.Options, flagSet *pflag.FlagSet) {
	flagSet.StringVar(
		&opt.DnsMasqAddr, "dnsmasq-addr", opt.DnsMasqAddr,
		"address that the dnsmasq server is listening on")
	flagSet.IntVar(
		&opt.DnsMasqPort, "dnsmasq-port", opt.DnsMasqPort,
		"port that the dnsmasq server is listening on")
	flagSet.IntVar(
		&opt.DnsMasqPollIntervalMs, "dnsmasq-poll-interval-ms", opt.DnsMasqPollIntervalMs,
		"interval with which to poll dnsmasq for stats")
	flagSet.Var(
		(*probeOptions)(&opt.Probes), "probe",
		"probe the given DNS server with the DNS name and export probe"+
			" metrics and healthcheck URI. Specified as"+
			" <label>,<server>,<dns name>[,<interval_seconds>][,<type>]."+
			" Healthcheck url will be exported under /healthcheck/<label>."+
			" interval_seconds is optional."+
			" This option may be specified multiple times to check multiple servers."+
			" <type> is one of ANY, A, AAAA."+
			" Example: 'mydns,127.0.0.1:53,example.com,10,A'.")
	flagSet.StringVar(
		&opt.PrometheusAddr, "prometheus-addr", opt.PrometheusAddr,
		"http addr to bind metrics server to")
	flagSet.IntVar(
		&opt.PrometheusPort, "prometheus-port", opt.PrometheusPort,
		"http port to use to export prometheus metrics")
	flagSet.StringVar(
		&opt.PrometheusPath, "prometheus-path", opt.PrometheusPath,
		"http path used to export metrics")
	flagSet.StringVar(
		&opt.PrometheusNamespace, "prometheus-namespace", opt.PrometheusNamespace,
		"prometheus metric namespace")
}
<commit_msg>Use string format instead of append<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/miekg/dns"
	"github.com/spf13/pflag"
	"k8s.io/kubernetes/pkg/util/flag"
	"k8s.io/kubernetes/pkg/util/logs"
	"k8s.io/kubernetes/pkg/version/verflag"

	"k8s.io/dns/pkg/sidecar"
	"k8s.io/dns/pkg/version"
)

const (
	defaultProbeInterval = 5 * time.Second
)

func main() {
	options := sidecar.NewOptions()
	configureFlags(options, pflag.CommandLine)
	flag.InitFlags()

	logs.InitLogs()
	defer logs.FlushLogs()

	glog.Infof("Version v%s", version.VERSION)

	verflag.PrintAndExitIfRequested()

	server := sidecar.NewServer()
	server.Run(options)
}

type probeOptions []sidecar.DNSProbeOption

func (po *probeOptions) String() string {
	return fmt.Sprintf("%+v", *po)
}

func (po *probeOptions) Set(value string) error {
	splits := strings.Split(value, ",")
	if !(3 <= len(splits) && len(splits) <= 5) {
		return fmt.Errorf("invalid format to --probe")
	}

	option := sidecar.DNSProbeOption{
		Label:    splits[0],
		Server:   splits[1],
		Name:     splits[2],
		Interval: defaultProbeInterval,
		Type:     dns.TypeANY,
	}

	const labelRegexp = "^[a-zA-Z0-9_]+"
	if !regexp.MustCompile(labelRegexp).MatchString(option.Label) {
		return fmt.Errorf("label must be of format %v", labelRegexp)
	}

	if !strings.Contains(option.Server, ":") {
		option.Server = option.Server + ":53"
	}

	if !strings.HasSuffix(option.Name, ".") {
		// dns package requires a fully qualified (e.g. terminal '.') name
		option.Name = option.Name + "."
	}

	if len(splits) >= 4 {
		if interval, err := strconv.Atoi(splits[3]); err == nil {
			option.Interval = time.Duration(interval) * time.Second
		} else {
			return err
		}
	}

	if len(splits) >= 5 {
		switch splits[4] {
		case "A":
			option.Type = dns.TypeA
			break
		case "AAAA":
			option.Type = dns.TypeAAAA
			break
		case "ANY":
			option.Type = dns.TypeANY
			break
		default:
			return fmt.Errorf("invalid type for DNS: %v", splits[5])
		}
	}

	*po = append(*po, option)

	return nil
}

func (po *probeOptions) Type() string {
	return "string"
}

var _ pflag.Value = (*probeOptions)(nil)

func configureFlags(opt *sidecar.Options, flagSet *pflag.FlagSet) {
	flagSet.StringVar(
		&opt.DnsMasqAddr, "dnsmasq-addr", opt.DnsMasqAddr,
		"address that the dnsmasq server is listening on")
	flagSet.IntVar(
		&opt.DnsMasqPort, "dnsmasq-port", opt.DnsMasqPort,
		"port that the dnsmasq server is listening on")
	flagSet.IntVar(
		&opt.DnsMasqPollIntervalMs, "dnsmasq-poll-interval-ms", opt.DnsMasqPollIntervalMs,
		"interval with which to poll dnsmasq for stats")
	flagSet.Var(
		(*probeOptions)(&opt.Probes), "probe",
		"probe the given DNS server with the DNS name and export probe"+
			" metrics and healthcheck URI. Specified as"+
			" <label>,<server>,<dns name>[,<interval_seconds>][,<type>]."+
			" Healthcheck url will be exported under /healthcheck/<label>."+
			" interval_seconds is optional."+
			" This option may be specified multiple times to check multiple servers."+
			" <type> is one of ANY, A, AAAA."+
			" Example: 'mydns,127.0.0.1:53,example.com,10,A'.")
	flagSet.StringVar(
		&opt.PrometheusAddr, "prometheus-addr", opt.PrometheusAddr,
		"http addr to bind metrics server to")
	flagSet.IntVar(
		&opt.PrometheusPort, "prometheus-port", opt.PrometheusPort,
		"http port to use to export prometheus metrics")
	flagSet.StringVar(
		&opt.PrometheusPath, "prometheus-path", opt.PrometheusPath,
		"http path used to export metrics")
	flagSet.StringVar(
		&opt.PrometheusNamespace, "prometheus-namespace", opt.PrometheusNamespace,
		"prometheus metric namespace")
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"os"
	"sort"
	"strings"
	"time"

	"github.com/sahib/brig/cmd/tabwriter"

	"github.com/fatih/color"
	"github.com/sahib/brig/client"
	"github.com/urfave/cli"
)

func handleReset(ctx *cli.Context, ctl *client.Client) error {
	force := ctx.Bool("force")
	path := ctx.Args().First()
	rev := "HEAD"

	if len(ctx.Args()) > 1 {
		rev = ctx.Args().Get(1)
	}

	if err := ctl.Reset(path, rev, force); err != nil {
		return ExitCode{UnknownError, fmt.Sprintf("unpin: %v", err)}
	}

	return nil
}

func commitName(cmt *client.Commit) string {
	if len(cmt.Tags) > 0 {
		return strings.ToUpper(cmt.Tags[0])
	}

	return cmt.Hash.ShortB58()
}

func handleHistory(ctx *cli.Context, ctl *client.Client) error {
	path := ctx.Args().First()

	history, err := ctl.History(path)
	if err != nil {
		return ExitCode{UnknownError, fmt.Sprintf("history: %v", err)}
	}

	if _, err := ctl.Stat(path); err != nil {
		fmt.Printf("%s %s",
			color.YellowString("WARNING:"),
			`This file is not part of this commit, but there's still history for it.
         Most likely this file was moved or removed in the past.

`)
	}

	tabW := tabwriter.NewWriter(
		os.Stdout, 0, 0, 2, ' ',
		tabwriter.StripEscape,
	)

	containsMoves := false
	for _, entry := range history {
		for _, detail := range entry.Mask {
			if detail == "moved" {
				containsMoves = true
				break
			}
		}

		if containsMoves {
			break
		}
	}

	if len(history) != 0 {
		if containsMoves {
			fmt.Fprintf(tabW, "CHANGE\tFROM\tTO\tHOW\tWHEN\t\n")
		} else {
			fmt.Fprintf(tabW, "CHANGE\tFROM\tTO\t\tWHEN\t\n")
		}
	}

	for idx, entry := range history {
		what := ""
		printLine := true

		for _, detail := range entry.Mask {
			// If it was moved, let's display what moved.
			if detail == "moved" && idx+1 < len(history) {
				src := history[idx+1].Path
				dst := entry.Path

				if entry.ReferTo != "" {
					dst = entry.ReferTo
				}

				what = fmt.Sprintf(
					"%s → %s", color.RedString(src), color.RedString(dst),
				)
			}

			// Only display empty changes if nothing happened.
			if detail == "none" && !ctx.Bool("empty") {
				printLine = false
			}
		}
		if !printLine {
			continue
		}

		changeDesc := color.YellowString(strings.Join(entry.Mask, ", "))
		when := color.MagentaString(entry.Head.Date.Format(time.Stamp))

		fmt.Fprintf(
			tabW,
			"%s\t%s\t%s\t%s\t%s\t\n",
			changeDesc,
			color.CyanString(commitName(entry.Next)),
			color.GreenString(commitName(entry.Head)),
			what,
			when,
		)
	}

	return tabW.Flush()
}

func printDiffTree(diff *client.Diff) {
	const (
		diffTypeNone = iota
		diffTypeAdded
		diffTypeRemoved
		diffTypeMoved
		diffTypeIgnored
		diffTypeConflict
		diffTypeMerged
	)

	type diffEntry struct {
		typ  int
		pair client.DiffPair
	}

	entries := []client.StatInfo{}
	types := make(map[string]diffEntry)

	// Singular types:
	for _, info := range diff.Added {
		types[info.Path] = diffEntry{typ: diffTypeAdded}
		entries = append(entries, info)
	}
	for _, info := range diff.Removed {
		types[info.Path] = diffEntry{typ: diffTypeRemoved}
		entries = append(entries, info)
	}
	for _, info := range diff.Ignored {
		types[info.Path] = diffEntry{typ: diffTypeIgnored}
		entries = append(entries, info)
	}

	// Pair types:
	for _, pair := range diff.Moved {
		types[pair.Dst.Path] = diffEntry{
			typ:  diffTypeMoved,
			pair: pair,
		}
		entries = append(entries, pair.Dst)
	}
	for _, pair := range diff.Conflict {
		types[pair.Dst.Path] = diffEntry{
			typ:  diffTypeConflict,
			pair: pair,
		}
		entries = append(entries, pair.Dst)
	}
	for _, pair := range diff.Merged {
		types[pair.Dst.Path] = diffEntry{
			typ:  diffTypeMerged,
			pair: pair,
		}
		entries = append(entries, pair.Dst)
	}

	if len(entries) == 0 {
		// Nothing to show:
		return
	}

	sort.Slice(entries, func(i, j int) bool {
		return entries[i].Path < entries[j].Path
	})

	// Called to format each name in the resulting tree:
	formatter := func(n *treeNode) string {
		if n.name == "/" {
			return color.MagentaString("•")
		}

		if diffEntry, ok := types[n.entry.Path]; ok {
			switch diffEntry.typ {
			case diffTypeAdded:
				return color.GreenString(" + " + n.name)
			case diffTypeRemoved:
				return color.RedString(" - " + n.name)
			case diffTypeIgnored:
				return color.YellowString(" * " + n.name)
			case diffTypeMoved:
				// TODO: Print base(src) and relTo(src, dst)
				name := fmt.Sprintf(
					" %s → %s",
					diffEntry.pair.Src.Path,
					diffEntry.pair.Dst.Path,
				)
				return color.BlueString(name)
			case diffTypeMerged:
				name := fmt.Sprintf(
					" %s ⇄ %s",
					diffEntry.pair.Src.Path,
					diffEntry.pair.Dst.Path,
				)
				return color.CyanString(name)
			case diffTypeConflict:
				name := fmt.Sprintf(
					" %s ⚡ %s",
					diffEntry.pair.Src.Path,
					diffEntry.pair.Dst.Path,
				)
				return color.MagentaString(name)
			}
		}

		return n.name
	}

	// Render the tree:
	showTree(entries, &treeCfg{
		format:  formatter,
		showPin: false,
	})
}

func printDiff(diff *client.Diff) {
	simpleSection := func(heading string, infos []client.StatInfo) {
		if len(infos) == 0 {
			return
		}

		fmt.Println(heading)
		for _, info := range infos {
			fmt.Printf("  %s\n", info.Path)
		}

		fmt.Println()
	}

	pairSection := func(heading string, infos []client.DiffPair) {
		if len(infos) == 0 {
			return
		}

		fmt.Println(heading)
		for _, pair := range infos {
			fmt.Printf("  %s <-> %s\n", pair.Src.Path, pair.Dst.Path)
		}

		fmt.Println()
	}

	simpleSection(color.GreenString("Added:"), diff.Added)
	simpleSection(color.YellowString("Ignored:"), diff.Ignored)
	simpleSection(color.RedString("Removed:"), diff.Removed)

	pairSection(color.BlueString("Moved:"), diff.Moved)
	pairSection(color.CyanString("Resolveable Conflicts:"), diff.Merged)
	pairSection(color.MagentaString("Conflicts:"), diff.Conflict)
}

func handleDiff(ctx *cli.Context, ctl *client.Client) error {
	if ctx.NArg() > 4 {
		fmt.Println("More than four arguments can't be handled.")
	}

	self, err := ctl.Whoami()
	if err != nil {
		return err
	}

	localName := self.CurrentUser
	remoteName := self.CurrentUser

	localRev := "CURR"
	remoteRev := "HEAD"

	switch n := ctx.NArg(); n {
	case 1:
		remoteRev = ctx.Args().Get(0)
	case 2:
		localRev = ctx.Args().Get(0)
		remoteRev = ctx.Args().Get(1)
	case 3:
		remoteName = ctx.Args().Get(0)
		localRev = ctx.Args().Get(1)
		remoteRev = ctx.Args().Get(2)
	case 4:
		localName = ctx.Args().Get(0)
		remoteName = ctx.Args().Get(1)
		localRev = ctx.Args().Get(2)
		remoteRev = ctx.Args().Get(3)
	}

	diff, err := ctl.MakeDiff(localName, remoteName, localRev, remoteRev)
	if err != nil {
		return ExitCode{UnknownError, fmt.Sprintf("diff: %v", err)}
	}

	if ctx.Bool("list") {
		printDiff(diff)
	} else {
		printDiffTree(diff)
	}

	return nil
}

func handleFetch(ctx *cli.Context, ctl *client.Client) error {
	who := ctx.Args().First()
	return ctl.Fetch(who)
}

func handleSync(ctx *cli.Context, ctl *client.Client) error {
	who := ctx.Args().First()

	needFetch := true
	if ctx.Bool("no-fetch") {
		needFetch = false
	}

	return ctl.Sync(who, needFetch)
}

func handleStatus(ctx *cli.Context, ctl *client.Client) error {
	self, err := ctl.Whoami()
	if err != nil {
		return err
	}

	curr := self.CurrentUser
	diff, err := ctl.MakeDiff(curr, curr, "HEAD", "CURR")
	if err != nil {
		return err
	}

	if ctx.Bool("tree") {
		printDiffTree(diff)
	} else {
		printDiff(diff)
	}

	return nil
}

func handleBecome(ctx *cli.Context, ctl *client.Client) error {
	becomeSelf := ctx.Bool("self")
	if !becomeSelf && ctx.NArg() < 1 {
		return fmt.Errorf("become needs at least one argument without -s")
	}

	whoami, err := ctl.Whoami()
	if err != nil {
		return err
	}

	who := ctx.Args().First()
	if becomeSelf {
		who = whoami.Owner
	}

	if whoami.CurrentUser == who {
		fmt.Printf("You are already %s.\n", color.GreenString(who))
		return nil
	}

	if err := ctl.Become(who); err != nil {
		return err
	}

	suffix := "Changes will be local only."
	if who == whoami.Owner {
		suffix = "Welcome back!"
	}

	fmt.Printf(
		"You are viewing %s's data now. %s\n",
		color.GreenString(who), suffix,
	)
	return nil
}

func handleCommit(ctx *cli.Context, ctl *client.Client) error {
	msg := ""
	if msg = ctx.String("message"); msg == "" {
		msg = fmt.Sprintf("manual commit")
	}

	if err := ctl.MakeCommit(msg); err != nil {
		return ExitCode{UnknownError, fmt.Sprintf("commit: %v", err)}
	}

	return nil
}

func handleTag(ctx *cli.Context, ctl *client.Client) error {
	if ctx.Bool("delete") {
		name := ctx.Args().Get(0)

		if err := ctl.Untag(name); err != nil {
			return ExitCode{
				UnknownError,
				fmt.Sprintf("untag: %v", err),
			}
		}
	} else {
		if len(ctx.Args()) < 2 {
			return ExitCode{BadArgs, "tag needs at least two arguments"}
		}

		rev := ctx.Args().Get(0)
		name := ctx.Args().Get(1)

		if err := ctl.Tag(rev, name); err != nil {
			return ExitCode{
				UnknownError,
				fmt.Sprintf("tag: %v", err),
			}
		}
	}

	return nil
}

func handleLog(ctx *cli.Context, ctl *client.Client) error {
	entries, err := ctl.Log()
	if err != nil {
		return ExitCode{UnknownError, fmt.Sprintf("commit: %v", err)}
	}

	for _, entry := range entries {
		tags := ""
		if len(entry.Tags) > 0 {
			tags = fmt.Sprintf(" (%s)", strings.Join(entry.Tags, ", "))
		}

		msg := entry.Msg
		if msg == "" {
			msg = color.RedString("•")
		}

		entry.Hash.ShortB58()

		fmt.Printf(
			"%s %s %s%s\n",
			color.GreenString(entry.Hash.ShortB58()),
			color.YellowString(entry.Date.Format(time.Stamp)),
			msg,
			color.CyanString(tags),
		)
	}

	return nil
}
<commit_msg>cmd: diff: display moves, merges and conflicts as relative paths<commit_after>package cmd

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"github.com/sahib/brig/cmd/tabwriter"

	"github.com/fatih/color"
	"github.com/sahib/brig/client"
	"github.com/urfave/cli"
)

func handleReset(ctx *cli.Context, ctl *client.Client) error {
	force := ctx.Bool("force")
	path := ctx.Args().First()
	rev := "HEAD"

	if len(ctx.Args()) > 1 {
		rev = ctx.Args().Get(1)
	}

	if err := ctl.Reset(path, rev, force); err != nil {
		return ExitCode{UnknownError, fmt.Sprintf("unpin: %v", err)}
	}

	return nil
}

func commitName(cmt *client.Commit) string {
	if len(cmt.Tags) > 0 {
		return strings.ToUpper(cmt.Tags[0])
	}

	return cmt.Hash.ShortB58()
}

func handleHistory(ctx *cli.Context, ctl *client.Client) error {
	path := ctx.Args().First()

	history, err := ctl.History(path)
	if err != nil {
		return ExitCode{UnknownError, fmt.Sprintf("history: %v", err)}
	}

	if _, err := ctl.Stat(path); err != nil {
		fmt.Printf("%s %s",
			color.YellowString("WARNING:"),
			`This file is not part of this commit, but there's still history for it.
         Most likely this file was moved or removed in the past.

`)
	}

	tabW := tabwriter.NewWriter(
		os.Stdout, 0, 0, 2, ' ',
		tabwriter.StripEscape,
	)

	containsMoves := false
	for _, entry := range history {
		for _, detail := range entry.Mask {
			if detail == "moved" {
				containsMoves = true
				break
			}
		}

		if containsMoves {
			break
		}
	}

	if len(history) != 0 {
		if containsMoves {
			fmt.Fprintf(tabW, "CHANGE\tFROM\tTO\tHOW\tWHEN\t\n")
		} else {
			fmt.Fprintf(tabW, "CHANGE\tFROM\tTO\t\tWHEN\t\n")
		}
	}

	for idx, entry := range history {
		what := ""
		printLine := true

		for _, detail := range entry.Mask {
			// If it was moved, let's display what moved.
			if detail == "moved" && idx+1 < len(history) {
				src := history[idx+1].Path
				dst := entry.Path

				if entry.ReferTo != "" {
					dst = entry.ReferTo
				}

				what = fmt.Sprintf(
					"%s → %s", color.RedString(src), color.RedString(dst),
				)
			}

			// Only display empty changes if nothing happened.
			if detail == "none" && !ctx.Bool("empty") {
				printLine = false
			}
		}
		if !printLine {
			continue
		}

		changeDesc := color.YellowString(strings.Join(entry.Mask, ", "))
		when := color.MagentaString(entry.Head.Date.Format(time.Stamp))

		fmt.Fprintf(
			tabW,
			"%s\t%s\t%s\t%s\t%s\t\n",
			changeDesc,
			color.CyanString(commitName(entry.Next)),
			color.GreenString(commitName(entry.Head)),
			what,
			when,
		)
	}

	return tabW.Flush()
}

// makePathAbbrev tries to abbreviate the `dst` path if
// both are in the same directory.
func makePathAbbrev(src, dst string) string {
	if path.Dir(src) == path.Dir(dst) {
		return path.Base(dst)
	}

	relPath, err := filepath.Rel(path.Dir(src), dst)
	if err != nil {
		fmt.Println("Failed to get relatie path: ", err)
		return dst
	}

	// We could also possibly check here if relPath is longer than dst
	// and only display the relative version then. But being consistent
	// is more valueable here I think.
	return relPath
}

func printDiffTree(diff *client.Diff) {
	const (
		diffTypeNone = iota
		diffTypeAdded
		diffTypeRemoved
		diffTypeMoved
		diffTypeIgnored
		diffTypeConflict
		diffTypeMerged
	)

	type diffEntry struct {
		typ  int
		pair client.DiffPair
	}

	entries := []client.StatInfo{}
	types := make(map[string]diffEntry)

	// Singular types:
	for _, info := range diff.Added {
		types[info.Path] = diffEntry{typ: diffTypeAdded}
		entries = append(entries, info)
	}
	for _, info := range diff.Removed {
		types[info.Path] = diffEntry{typ: diffTypeRemoved}
		entries = append(entries, info)
	}
	for _, info := range diff.Ignored {
		types[info.Path] = diffEntry{typ: diffTypeIgnored}
		entries = append(entries, info)
	}

	// Pair types:
	for _, pair := range diff.Moved {
		types[pair.Dst.Path] = diffEntry{
			typ:  diffTypeMoved,
			pair: pair,
		}
		entries = append(entries, pair.Dst)
	}
	for _, pair := range diff.Conflict {
		types[pair.Dst.Path] = diffEntry{
			typ:  diffTypeConflict,
			pair: pair,
		}
		entries = append(entries, pair.Dst)
	}
	for _, pair := range diff.Merged {
		types[pair.Dst.Path] = diffEntry{
			typ:  diffTypeMerged,
			pair: pair,
		}
		entries = append(entries, pair.Dst)
	}

	if len(entries) == 0 {
		// Nothing to show:
		return
	}

	sort.Slice(entries, func(i, j int) bool {
		return entries[i].Path < entries[j].Path
	})

	// Called to format each name in the resulting tree:
	formatter := func(n *treeNode) string {
		if n.name == "/" {
			return color.MagentaString("•")
		}

		if diffEntry, ok := types[n.entry.Path]; ok {
			switch diffEntry.typ {
			case diffTypeAdded:
				return color.GreenString(" + " + n.name)
			case diffTypeRemoved:
				return color.RedString(" - " + n.name)
			case diffTypeIgnored:
				return color.YellowString(" * " + n.name)
			case diffTypeMoved:
				// TODO: Print base(src) and relTo(src, dst)
				dstPath := makePathAbbrev(
					diffEntry.pair.Src.Path,
					diffEntry.pair.Dst.Path,
				)
				name := fmt.Sprintf(
					" %s → %s",
					path.Base(diffEntry.pair.Src.Path),
					dstPath,
				)
				return color.BlueString(name)
			case diffTypeMerged:
				dstPath := makePathAbbrev(
					diffEntry.pair.Src.Path,
					diffEntry.pair.Dst.Path,
				)
				name := fmt.Sprintf(
					" %s ⇄ %s",
					path.Base(diffEntry.pair.Src.Path),
					dstPath,
				)
				return color.CyanString(name)
			case diffTypeConflict:
				dstPath := makePathAbbrev(
					diffEntry.pair.Src.Path,
					diffEntry.pair.Dst.Path,
				)
				name := fmt.Sprintf(
					" %s ⚡ %s",
					path.Base(diffEntry.pair.Src.Path),
					dstPath,
				)
				return color.MagentaString(name)
			}
		}

		return n.name
	}

	// Render the tree:
	showTree(entries, &treeCfg{
		format:  formatter,
		showPin: false,
	})
}

func printDiff(diff *client.Diff) {
	simpleSection := func(heading string, infos []client.StatInfo) {
		if len(infos) == 0 {
			return
		}

		fmt.Println(heading)
		for _, info := range infos {
			fmt.Printf("  %s\n", info.Path)
		}

		fmt.Println()
	}

	pairSection := func(heading string, infos []client.DiffPair) {
		if len(infos) == 0 {
			return
		}

		fmt.Println(heading)
		for _, pair := range infos {
			fmt.Printf("  %s <-> %s\n", pair.Src.Path, pair.Dst.Path)
		}

		fmt.Println()
	}

	simpleSection(color.GreenString("Added:"), diff.Added)
	simpleSection(color.YellowString("Ignored:"), diff.Ignored)
	simpleSection(color.RedString("Removed:"), diff.Removed)

	pairSection(color.BlueString("Moved:"), diff.Moved)
	pairSection(color.CyanString("Resolveable Conflicts:"), diff.Merged)
	pairSection(color.MagentaString("Conflicts:"), diff.Conflict)
}

func handleDiff(ctx *cli.Context, ctl *client.Client) error {
	if ctx.NArg() > 4 {
		fmt.Println("More than four arguments can't be handled.")
	}

	self, err := ctl.Whoami()
	if err != nil {
		return err
	}

	localName := self.CurrentUser
	remoteName := self.CurrentUser

	localRev := "CURR"
	remoteRev := "HEAD"

	switch n := ctx.NArg(); n {
	case 1:
		remoteRev = ctx.Args().Get(0)
	case 2:
		localRev = ctx.Args().Get(0)
		remoteRev = ctx.Args().Get(1)
	case 3:
		remoteName = ctx.Args().Get(0)
		localRev = ctx.Args().Get(1)
		remoteRev = ctx.Args().Get(2)
	case 4:
		localName = ctx.Args().Get(0)
		remoteName = ctx.Args().Get(1)
		localRev = ctx.Args().Get(2)
		remoteRev = ctx.Args().Get(3)
	}

	diff, err := ctl.MakeDiff(localName, remoteName, localRev, remoteRev)
	if err != nil {
		return ExitCode{UnknownError, fmt.Sprintf("diff: %v", err)}
	}

	if ctx.Bool("list") {
		printDiff(diff)
	} else {
		printDiffTree(diff)
	}

	return nil
}

func handleFetch(ctx *cli.Context, ctl *client.Client) error {
	who := ctx.Args().First()
	return ctl.Fetch(who)
}

func handleSync(ctx *cli.Context, ctl *client.Client) error {
	who := ctx.Args().First()

	needFetch := true
	if ctx.Bool("no-fetch") {
		needFetch = false
	}

	return ctl.Sync(who, needFetch)
}

func handleStatus(ctx *cli.Context, ctl *client.Client) error {
	self, err := ctl.Whoami()
	if err != nil {
		return err
	}

	curr := self.CurrentUser
	diff, err := ctl.MakeDiff(curr, curr, "HEAD", "CURR")
	if err != nil {
		return err
	}

	if ctx.Bool("tree") {
		printDiffTree(diff)
	} else {
		printDiff(diff)
	}

	return nil
}

func handleBecome(ctx *cli.Context, ctl *client.Client) error {
	becomeSelf := ctx.Bool("self")
	if !becomeSelf && ctx.NArg() < 1 {
		return fmt.Errorf("become needs at least one argument without -s")
	}

	whoami, err := ctl.Whoami()
	if err != nil {
		return err
	}

	who := ctx.Args().First()
	if becomeSelf {
		who = whoami.Owner
	}

	if whoami.CurrentUser == who {
		fmt.Printf("You are already %s.\n", color.GreenString(who))
		return nil
	}

	if err := ctl.Become(who); err != nil {
		return err
	}

	suffix := "Changes will be local only."
	if who == whoami.Owner {
		suffix = "Welcome back!"
	}

	fmt.Printf(
		"You are viewing %s's data now. %s\n",
		color.GreenString(who), suffix,
	)
	return nil
}

func handleCommit(ctx *cli.Context, ctl *client.Client) error {
	msg := ""
	if msg = ctx.String("message"); msg == "" {
		msg = fmt.Sprintf("manual commit")
	}

	if err := ctl.MakeCommit(msg); err != nil {
		return ExitCode{UnknownError, fmt.Sprintf("commit: %v", err)}
	}

	return nil
}

func handleTag(ctx *cli.Context, ctl *client.Client) error {
	if ctx.Bool("delete") {
		name := ctx.Args().Get(0)

		if err := ctl.Untag(name); err != nil {
			return ExitCode{
				UnknownError,
				fmt.Sprintf("untag: %v", err),
			}
		}
	} else {
		if len(ctx.Args()) < 2 {
			return ExitCode{BadArgs, "tag needs at least two arguments"}
		}

		rev := ctx.Args().Get(0)
		name := ctx.Args().Get(1)

		if err := ctl.Tag(rev, name); err != nil {
			return ExitCode{
				UnknownError,
				fmt.Sprintf("tag: %v", err),
			}
		}
	}

	return nil
}

func handleLog(ctx *cli.Context, ctl *client.Client) error {
	entries, err := ctl.Log()
	if err != nil {
		return ExitCode{UnknownError, fmt.Sprintf("commit: %v", err)}
	}

	for _, entry := range entries {
		tags := ""
		if len(entry.Tags) > 0 {
			tags = fmt.Sprintf(" (%s)", strings.Join(entry.Tags, ", "))
		}

		msg := entry.Msg
		if msg == "" {
			msg = color.RedString("•")
		}

		entry.Hash.ShortB58()

		fmt.Printf(
			"%s %s %s%s\n",
			color.GreenString(entry.Hash.ShortB58()),
			color.YellowString(entry.Date.Format(time.Stamp)),
			msg,
			color.CyanString(tags),
		)
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"v.io/x/lib/cmdline"
)

const (
	testDir    = "../../lib/vdl/testdata/base"
	outPkgPath = "v.io/x/ref/lib/vdl/testdata/base"
)

//go:generate v23 test generate

// Compares generated VDL files against the copy in the repo.
func TestVDLGenerator(t *testing.T) {
	// Use vdl to generate Go code from input, into a temporary directory.
	outDir, err := ioutil.TempDir("", "vdltest")
	if err != nil {
		t.Fatalf("TempDir() failed: %v", err)
	}
	defer os.RemoveAll(outDir)
	// TODO(toddw): test the generated java and javascript files too.
	outOpt := fmt.Sprintf("--go-out-dir=%s", outDir)
	env := cmdline.NewEnv()
	if err := cmdline.ParseAndRun(cmdVDL, env, []string{"generate", "--lang=go", outOpt, testDir}); err != nil {
		t.Fatalf("Execute() failed: %v", err)
	}
	// Check that each *.vdl.go file in the testDir matches the generated output.
	entries, err := ioutil.ReadDir(testDir)
	if err != nil {
		t.Fatalf("ReadDir(%v) failed: %v", testDir, err)
	}
	numEqual := 0
	for _, entry := range entries {
		if !strings.HasSuffix(entry.Name(), ".vdl.go") {
			continue
		}
		testFile := filepath.Join(testDir, entry.Name())
		testBytes, err := ioutil.ReadFile(testFile)
		if err != nil {
			t.Fatalf("ReadFile(%v) failed: %v", testFile, err)
		}
		outFile := filepath.Join(outDir, outPkgPath, entry.Name())
		outBytes, err := ioutil.ReadFile(outFile)
		if err != nil {
			t.Fatalf("ReadFile(%v) failed: %v", outFile, err)
		}
		if !bytes.Equal(outBytes, testBytes) {
			t.Fatalf("GOT:\n%v\n\nWANT:\n%v\n", string(outBytes), string(testBytes))
		}
		numEqual++
	}
	if numEqual == 0 {
		t.Fatalf("testDir %s has no golden files *.vdl.go", testDir)
	}
}
<commit_msg>ref: Rename cmdline.NewEnv to cmdline.EnvFromOS<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"v.io/x/lib/cmdline"
)

const (
	testDir    = "../../lib/vdl/testdata/base"
	outPkgPath = "v.io/x/ref/lib/vdl/testdata/base"
)

//go:generate v23 test generate

// Compares generated VDL files against the copy in the repo.
func TestVDLGenerator(t *testing.T) {
	// Use vdl to generate Go code from input, into a temporary directory.
	outDir, err := ioutil.TempDir("", "vdltest")
	if err != nil {
		t.Fatalf("TempDir() failed: %v", err)
	}
	defer os.RemoveAll(outDir)
	// TODO(toddw): test the generated java and javascript files too.
	outOpt := fmt.Sprintf("--go-out-dir=%s", outDir)
	env := cmdline.EnvFromOS()
	if err := cmdline.ParseAndRun(cmdVDL, env, []string{"generate", "--lang=go", outOpt, testDir}); err != nil {
		t.Fatalf("Execute() failed: %v", err)
	}
	// Check that each *.vdl.go file in the testDir matches the generated output.
	entries, err := ioutil.ReadDir(testDir)
	if err != nil {
		t.Fatalf("ReadDir(%v) failed: %v", testDir, err)
	}
	numEqual := 0
	for _, entry := range entries {
		if !strings.HasSuffix(entry.Name(), ".vdl.go") {
			continue
		}
		testFile := filepath.Join(testDir, entry.Name())
		testBytes, err := ioutil.ReadFile(testFile)
		if err != nil {
			t.Fatalf("ReadFile(%v) failed: %v", testFile, err)
		}
		outFile := filepath.Join(outDir, outPkgPath, entry.Name())
		outBytes, err := ioutil.ReadFile(outFile)
		if err != nil {
			t.Fatalf("ReadFile(%v) failed: %v", outFile, err)
		}
		if !bytes.Equal(outBytes, testBytes) {
			t.Fatalf("GOT:\n%v\n\nWANT:\n%v\n", string(outBytes), string(testBytes))
		}
		numEqual++
	}
	if numEqual == 0 {
		t.Fatalf("testDir %s has no golden files *.vdl.go", testDir)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"math/rand"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"text/template"
	"time"

	inKafkaMdm "github.com/grafana/metrictank/input/kafkamdm"
	"github.com/grafana/metrictank/stats"
	"github.com/raintank/worldping-api/pkg/log"
	"github.com/rakyll/globalconf"
	"gopkg.in/raintank/schema.v1"
)

var (
	confFile = flag.String("config", "/etc/metrictank/metrictank.ini", "configuration file path")
	format   = flag.String("format", "{{.First.Seen}} {{.First.Time}} | {{.Seen}} {{.Time}} {{.Part}} {{.OrgId}} {{.Id}} {{.Name}} {{.Metric}} {{.Interval}} {{.Value}} {{.Unit}} {{.Mtype}} {{.Tags}}", "template to render the data with")
	prefix   = flag.String("prefix", "", "only show metrics that have this prefix")
	substr   = flag.String("substr", "", "only show metrics that have this substring")
)

type Data struct {
	Part int32
	Seen time.Time
	schema.MetricData
}

type TplData struct {
	Data       // currently seen
	First Data // seen the first time
}

// find out of order metrics
type inputOOOFinder struct {
	template.Template
	data map[string]Data // by metric id
	lock sync.Mutex
}

func newInputOOOFinder(format string) *inputOOOFinder {
	tpl := template.Must(template.New("format").Parse(format + "\n"))
	return &inputOOOFinder{
		*tpl,
		make(map[string]Data),
		sync.Mutex{},
	}
}

func (ip *inputOOOFinder) Process(metric *schema.MetricData, partition int32) {
	if *prefix != "" && !strings.HasPrefix(metric.Metric, *prefix) {
		return
	}
	if *substr != "" && !strings.Contains(metric.Metric, *substr) {
		return
	}
	now := Data{
		Part:       partition,
		Seen:       time.Now(),
		MetricData: *metric,
	}
	ip.lock.Lock()
	first, ok := ip.data[metric.Id]
	if !ok {
		ip.data[metric.Id] = now
	} else {
		if metric.Time > first.Time {
			ip.data[metric.Id] = now
		} else {
			t := TplData{now, first}
			err := ip.Execute(os.Stdout, t)
			if err != nil {
				log.Error(0, "executing template: %s", err)
			}
		}
	}
	ip.lock.Unlock()
}

func main() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "mt-kafka-mdm-sniff-out-of-order")
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "Inspects what's flowing through kafka (in mdm format) and reports it to you")
		fmt.Fprintf(os.Stderr, "\nFlags:\n\n")
		flag.PrintDefaults()
	}
	flag.Parse()
	log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":false}`, 2))
	instance := "mt-kafka-mdm-sniff-out-of-order" + strconv.Itoa(rand.Int())

	// Only try and parse the conf file if it exists
	path := ""
	if _, err := os.Stat(*confFile); err == nil {
		path = *confFile
	}
	conf, err := globalconf.NewWithOptions(&globalconf.Options{
		Filename:  path,
		EnvPrefix: "MT_",
	})
	if err != nil {
		log.Fatal(4, "error with configuration file: %s", err)
		os.Exit(1)
	}
	inKafkaMdm.ConfigSetup()
	conf.ParseAll()

	// config may have had it disabled
	inKafkaMdm.Enabled = true
	// important: we don't want to share the same offset tracker as the mdm input of MT itself
	inKafkaMdm.DataDir = "/tmp/" + instance

	inKafkaMdm.ConfigProcess(instance)

	stats.NewDevnull() // make sure metrics don't pile up without getting discarded

	mdm := inKafkaMdm.New()
	mdm.Start(newInputOOOFinder(*format))
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
	<-sigChan
	log.Info("stopping")
	mdm.Stop()
}
<commit_msg>fix terminology: compare against last added point<commit_after>package main

import (
	"flag"
	"fmt"
	"math/rand"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"
	"syscall"
	"text/template"
	"time"

	inKafkaMdm "github.com/grafana/metrictank/input/kafkamdm"
	"github.com/grafana/metrictank/stats"
	"github.com/raintank/worldping-api/pkg/log"
	"github.com/rakyll/globalconf"
	"gopkg.in/raintank/schema.v1"
)

var (
	confFile = flag.String("config", "/etc/metrictank/metrictank.ini", "configuration file path")
	format   = flag.String("format", "{{.Last.Seen}} {{.Last.Time}} | {{.Seen}} {{.Time}} {{.Part}} {{.OrgId}} {{.Id}} {{.Name}} {{.Metric}} {{.Interval}} {{.Value}} {{.Unit}} {{.Mtype}} {{.Tags}}", "template to render the data with. data under .Last represents the 'head' of the series, the most recent successfully added point (e.g. it had a higher timestamp than previous values)")
	prefix   = flag.String("prefix", "", "only show metrics that have this prefix")
	substr   = flag.String("substr", "", "only show metrics that have this substring")
)

type Data struct {
	Part int32
	Seen time.Time
	schema.MetricData
}

type TplData struct {
	Data      // currently seen
	Last Data // last added point that could be added
}

// find out of order metrics
type inputOOOFinder struct {
	template.Template
	data map[string]Data // by metric id
	lock sync.Mutex
}

func newInputOOOFinder(format string) *inputOOOFinder {
	tpl := template.Must(template.New("format").Parse(format + "\n"))
	return &inputOOOFinder{
		*tpl,
		make(map[string]Data),
		sync.Mutex{},
	}
}

func (ip *inputOOOFinder) Process(metric *schema.MetricData, partition int32) {
	if *prefix != "" && !strings.HasPrefix(metric.Metric, *prefix) {
		return
	}
	if *substr != "" && !strings.Contains(metric.Metric, *substr) {
		return
	}
	now := Data{
		Part:       partition,
		Seen:       time.Now(),
		MetricData: *metric,
	}
	ip.lock.Lock()
	last, ok := ip.data[metric.Id]
	if !ok {
		ip.data[metric.Id] = now
	} else {
		if metric.Time > last.Time {
			ip.data[metric.Id] = now
		} else {
			// if metric time <= last time, print output
			t := TplData{now, last}
			err := ip.Execute(os.Stdout, t)
			if err != nil {
				log.Error(0, "executing template: %s", err)
			}
		}
	}
	ip.lock.Unlock()
}

func main() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "mt-kafka-mdm-sniff-out-of-order")
		fmt.Fprintln(os.Stderr)
		fmt.Fprintln(os.Stderr, "Inspects what's flowing through kafka (in mdm format) and reports it to you")
		fmt.Fprintf(os.Stderr, "\nFlags:\n\n")
		flag.PrintDefaults()
	}
	flag.Parse()
	log.NewLogger(0, "console", fmt.Sprintf(`{"level": %d, "formatting":false}`, 2))
	instance := "mt-kafka-mdm-sniff-out-of-order" + strconv.Itoa(rand.Int())

	// Only try and parse the conf file if it exists
	path := ""
	if _, err := os.Stat(*confFile); err == nil {
		path = *confFile
	}
	conf, err := globalconf.NewWithOptions(&globalconf.Options{
		Filename:  path,
		EnvPrefix: "MT_",
	})
	if err != nil {
		log.Fatal(4, "error with configuration file: %s", err)
		os.Exit(1)
	}
	inKafkaMdm.ConfigSetup()
	conf.ParseAll()

	// config may have had it disabled
	inKafkaMdm.Enabled = true
	// important: we don't want to share the same offset tracker as the mdm input of MT itself
	inKafkaMdm.DataDir = "/tmp/" + instance

	inKafkaMdm.ConfigProcess(instance)

	stats.NewDevnull() // make sure metrics don't pile up without getting discarded

	mdm := inKafkaMdm.New()
	mdm.Start(newInputOOOFinder(*format))
	sigChan := make(chan os.Signal, 1)
	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
	<-sigChan
	log.Info("stopping")
	mdm.Stop()
}
<|endoftext|>


<commit_before>// +build !windows

// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.
package statemanager_test

import (
	"io/ioutil"
	"os"
	"testing"

	"github.com/aws/amazon-ecs-agent/agent/api"
	"github.com/aws/amazon-ecs-agent/agent/config"
	"github.com/aws/amazon-ecs-agent/agent/engine"
	"github.com/aws/amazon-ecs-agent/agent/engine/dockerstate"
	engine_testutils "github.com/aws/amazon-ecs-agent/agent/engine/testutils"
	"github.com/aws/amazon-ecs-agent/agent/statemanager"
	"github.com/stretchr/testify/assert"
)

func TestStateManager(t *testing.T) {
	tmpDir, err := ioutil.TempDir("/tmp", "ecs_statemanager_test")
	assert.Nil(t, err)
	defer os.RemoveAll(tmpDir)
	cfg := &config.Config{DataDir: tmpDir}
	manager, err := statemanager.NewStateManager(cfg)
	assert.Nil(t, err, "Error loading manager")

	err = manager.Load()
	assert.Nil(t, err, "Expected loading a non-existant file to not be an error")

	// Now let's make some state to save
	containerInstanceArn := ""
	taskEngine := engine.NewTaskEngine(&config.Config{}, nil, nil, nil, nil, dockerstate.NewDockerTaskEngineState())

	manager, err = statemanager.NewStateManager(cfg, statemanager.AddSaveable("TaskEngine", taskEngine), statemanager.AddSaveable("ContainerInstanceArn", &containerInstanceArn))
	if err != nil {
		t.Fatal(err)
	}

	containerInstanceArn = "containerInstanceArn"

	testTask := &api.Task{Arn: "test-arn"}
	taskEngine.(*engine.DockerTaskEngine).State().AddTask(testTask)

	err = manager.Save()
	if err != nil {
		t.Fatal("Error saving state", err)
	}

	// Now make sure we can load that state sanely
	loadedTaskEngine := engine.NewTaskEngine(&config.Config{}, nil, nil, nil, nil, dockerstate.NewDockerTaskEngineState())
	var loadedContainerInstanceArn string

	manager, err = statemanager.NewStateManager(cfg, statemanager.AddSaveable("TaskEngine", &loadedTaskEngine), statemanager.AddSaveable("ContainerInstanceArn", &loadedContainerInstanceArn))
	if err != nil {
		t.Fatal(err)
	}

	err = manager.Load()
	if err != nil {
		t.Fatal("Error loading state", err)
	}

	if loadedContainerInstanceArn != containerInstanceArn {
		t.Error("Did not load containerInstanceArn correctly; got ", loadedContainerInstanceArn, " instead of ", containerInstanceArn)
	}

	if !engine_testutils.DockerTaskEnginesEqual(loadedTaskEngine.(*engine.DockerTaskEngine), (taskEngine.(*engine.DockerTaskEngine))) {
		t.Error("Did not load taskEngine correctly")
	}

	// I'd rather double check .Equal there; let's make sure ListTasks agrees.
	tasks, err := loadedTaskEngine.ListTasks()
	if err != nil {
		t.Error("Error listing tasks", err)
	}
	if len(tasks) != 1 {
		t.Error("Should have a task!")
	} else {
		if tasks[0].Arn != "test-arn" {
			t.Error("Wrong arn, expected test-arn but got ", tasks[0].Arn)
		}
	}
}
<commit_msg>statemanager: Assert state file mode 0600<commit_after>// +build !windows

// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.
package statemanager_test

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/aws/amazon-ecs-agent/agent/api"
	"github.com/aws/amazon-ecs-agent/agent/config"
	"github.com/aws/amazon-ecs-agent/agent/engine"
	"github.com/aws/amazon-ecs-agent/agent/engine/dockerstate"
	engine_testutils "github.com/aws/amazon-ecs-agent/agent/engine/testutils"
	"github.com/aws/amazon-ecs-agent/agent/statemanager"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestStateManager(t *testing.T) {
	tmpDir, err := ioutil.TempDir("/tmp", "ecs_statemanager_test")
	assert.Nil(t, err)
	defer os.RemoveAll(tmpDir)
	cfg := &config.Config{DataDir: tmpDir}
	manager, err := statemanager.NewStateManager(cfg)
	assert.Nil(t, err, "Error loading manager")

	err = manager.Load()
	assert.Nil(t, err, "Expected loading a non-existant file to not be an error")

	// Now let's make some state to save
	containerInstanceArn := ""
	taskEngine := engine.NewTaskEngine(&config.Config{}, nil, nil, nil, nil, dockerstate.NewDockerTaskEngineState())

	manager, err = statemanager.NewStateManager(cfg, statemanager.AddSaveable("TaskEngine", taskEngine), statemanager.AddSaveable("ContainerInstanceArn", &containerInstanceArn))
	require.Nil(t, err)

	containerInstanceArn = "containerInstanceArn"

	testTask := &api.Task{Arn: "test-arn"}
	taskEngine.(*engine.DockerTaskEngine).State().AddTask(testTask)

	err = manager.Save()
	require.Nil(t, err, "Error saving state")

	assertFileMode(t, filepath.Join(tmpDir, "ecs_agent_data.json"))

	// Now make sure we can load that state sanely
	loadedTaskEngine := engine.NewTaskEngine(&config.Config{}, nil, nil, nil, nil, dockerstate.NewDockerTaskEngineState())
	var loadedContainerInstanceArn string

	manager, err = statemanager.NewStateManager(cfg, statemanager.AddSaveable("TaskEngine", &loadedTaskEngine), statemanager.AddSaveable("ContainerInstanceArn", &loadedContainerInstanceArn))
	require.Nil(t, err)

	err = manager.Load()
	require.Nil(t, err, "Error loading state")

	assert.Equal(t, containerInstanceArn, loadedContainerInstanceArn, "Did not load containerInstanceArn correctly")

	if !engine_testutils.DockerTaskEnginesEqual(loadedTaskEngine.(*engine.DockerTaskEngine), (taskEngine.(*engine.DockerTaskEngine))) {
		t.Error("Did not load taskEngine correctly")
	}

	// I'd rather double check .Equal there; let's make sure ListTasks agrees.
	tasks, err := loadedTaskEngine.ListTasks()
	assert.Nil(t, err, "Error listing tasks")
	require.Equal(t, 1, len(tasks), "Should have a task!")
	assert.Equal(t, "test-arn", tasks[0].Arn, "Wrong arn")
}

func assertFileMode(t *testing.T, path string) {
	info, err := os.Stat(path)
	assert.Nil(t, err)

	mode := info.Mode()
	assert.Equal(t, os.FileMode(0600), mode, "Wrong file mode")
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package etcdserver

import (
	"io"
	"log"
	"os"
	"path"

	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
	"github.com/coreos/etcd/migrate"
	"github.com/coreos/etcd/pkg/pbutil"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/snap"
	"github.com/coreos/etcd/version"
	"github.com/coreos/etcd/wal"
	"github.com/coreos/etcd/wal/walpb"
)

type Storage interface {
	// Save function saves ents and state to the underlying stable storage.
	// Save MUST block until st and ents are on stable storage.
	Save(st raftpb.HardState, ents []raftpb.Entry) error
	// SaveSnap function saves snapshot to the underlying stable storage.
	SaveSnap(snap raftpb.Snapshot) error
	// Close closes the Storage and performs finalization.
	Close() error
}

type storage struct {
	*wal.WAL
	*snap.Snapshotter
}

func NewStorage(w *wal.WAL, s *snap.Snapshotter) Storage {
	return &storage{w, s}
}

// SaveSnap saves the snapshot to disk and release the locked
// wal files since they will not be used.
func (st *storage) SaveSnap(snap raftpb.Snapshot) error {
	err := st.Snapshotter.SaveSnap(snap)
	if err != nil {
		return err
	}
	walsnap := walpb.Snapshot{
		Index: snap.Metadata.Index,
		Term:  snap.Metadata.Term,
	}
	err = st.WAL.SaveSnapshot(walsnap)
	if err != nil {
		return err
	}
	err = st.WAL.ReleaseLockTo(snap.Metadata.Index)
	if err != nil {
		return err
	}
	return nil
}

func readWAL(waldir string, snap walpb.Snapshot) (w *wal.WAL, id, cid types.ID, st raftpb.HardState, ents []raftpb.Entry) {
	var (
		err       error
		wmetadata []byte
	)

	for i := 0; i < 2; i++ {
		if w, err = wal.Open(waldir, snap); err != nil {
			log.Fatalf("etcdserver: open wal error: %v", err)
		}
		if wmetadata, st, ents, err = w.ReadAll(); err != nil {
			w.Close()
			if i != 0 || err != io.ErrUnexpectedEOF {
				log.Fatalf("etcdserver: read wal error: %v", err)
			}
			if !wal.Repair(waldir) {
				log.Fatalf("etcdserver: WAL error (%v) cannot be repaired", err)
			} else {
				log.Printf("etcdserver: repaired WAL error (%v)", err)
			}
			continue
		}
		break
	}
	var metadata pb.Metadata
	pbutil.MustUnmarshal(&metadata, wmetadata)
	id = types.ID(metadata.NodeID)
	cid = types.ID(metadata.ClusterID)
	return
}

// upgradeWAL converts an older version of the etcdServer data to the newest version.
// It must ensure that, after upgrading, the most recent version is present.
func upgradeDataDir(baseDataDir string, name string, ver version.DataDirVersion) error {
	switch ver {
	case version.DataDir0_4:
		log.Print("etcdserver: converting v0.4 log to v2.0")
		err := migrate.Migrate4To2(baseDataDir, name)
		if err != nil {
			log.Fatalf("etcdserver: failed migrating data-dir: %v", err)
			return err
		}
		fallthrough
	case version.DataDir2_0:
		err := makeMemberDir(baseDataDir)
		if err != nil {
			return err
		}
		fallthrough
	case version.DataDir2_0_1:
		fallthrough
	default:
		log.Printf("etcdserver: datadir is valid for the 2.0.1 format")
	}
	return nil
}

func makeMemberDir(dir string) error {
	membdir := path.Join(dir, "member")
	_, err := os.Stat(membdir)
	switch {
	case err == nil:
		return nil
	case !os.IsNotExist(err):
		return err
	}
	if err := os.MkdirAll(membdir, 0700); err != nil {
		return err
	}
	names := []string{"snap", "wal"}
	for _, name := range names {
		if err := os.Rename(path.Join(dir, name), path.Join(membdir, name)); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>etcdserver: save snapshot into wal first<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package etcdserver

import (
	"io"
	"log"
	"os"
	"path"

	pb "github.com/coreos/etcd/etcdserver/etcdserverpb"
	"github.com/coreos/etcd/migrate"
	"github.com/coreos/etcd/pkg/pbutil"
	"github.com/coreos/etcd/pkg/types"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/snap"
	"github.com/coreos/etcd/version"
	"github.com/coreos/etcd/wal"
	"github.com/coreos/etcd/wal/walpb"
)

type Storage interface {
	// Save function saves ents and state to the underlying stable storage.
	// Save MUST block until st and ents are on stable storage.
	Save(st raftpb.HardState, ents []raftpb.Entry) error
	// SaveSnap function saves snapshot to the underlying stable storage.
	SaveSnap(snap raftpb.Snapshot) error
	// Close closes the Storage and performs finalization.
	Close() error
}

type storage struct {
	*wal.WAL
	*snap.Snapshotter
}

func NewStorage(w *wal.WAL, s *snap.Snapshotter) Storage {
	return &storage{w, s}
}

// SaveSnap saves the snapshot to disk and release the locked
// wal files since they will not be used.
func (st *storage) SaveSnap(snap raftpb.Snapshot) error {
	walsnap := walpb.Snapshot{
		Index: snap.Metadata.Index,
		Term:  snap.Metadata.Term,
	}
	err := st.WAL.SaveSnapshot(walsnap)
	if err != nil {
		return err
	}
	err = st.Snapshotter.SaveSnap(snap)
	if err != nil {
		return err
	}
	err = st.WAL.ReleaseLockTo(snap.Metadata.Index)
	if err != nil {
		return err
	}
	return nil
}

func readWAL(waldir string, snap walpb.Snapshot) (w *wal.WAL, id, cid types.ID, st raftpb.HardState, ents []raftpb.Entry) {
	var (
		err       error
		wmetadata []byte
	)

	for i := 0; i < 2; i++ {
		if w, err = wal.Open(waldir, snap); err != nil {
			log.Fatalf("etcdserver: open wal error: %v", err)
		}
		if wmetadata, st, ents, err = w.ReadAll(); err != nil {
			w.Close()
			if i != 0 || err != io.ErrUnexpectedEOF {
				log.Fatalf("etcdserver: read wal error: %v", err)
			}
			if !wal.Repair(waldir) {
				log.Fatalf("etcdserver: WAL error (%v) cannot be repaired", err)
			} else {
				log.Printf("etcdserver: repaired WAL error (%v)", err)
			}
			continue
		}
		break
	}
	var metadata pb.Metadata
	pbutil.MustUnmarshal(&metadata, wmetadata)
	id = types.ID(metadata.NodeID)
	cid = types.ID(metadata.ClusterID)
	return
}

// upgradeWAL converts an older version of the etcdServer data to the newest version.
// It must ensure that, after upgrading, the most recent version is present.
func upgradeDataDir(baseDataDir string, name string, ver version.DataDirVersion) error {
	switch ver {
	case version.DataDir0_4:
		log.Print("etcdserver: converting v0.4 log to v2.0")
		err := migrate.Migrate4To2(baseDataDir, name)
		if err != nil {
			log.Fatalf("etcdserver: failed migrating data-dir: %v", err)
			return err
		}
		fallthrough
	case version.DataDir2_0:
		err := makeMemberDir(baseDataDir)
		if err != nil {
			return err
		}
		fallthrough
	case version.DataDir2_0_1:
		fallthrough
	default:
		log.Printf("etcdserver: datadir is valid for the 2.0.1 format")
	}
	return nil
}

func makeMemberDir(dir string) error {
	membdir := path.Join(dir, "member")
	_, err := os.Stat(membdir)
	switch {
	case err == nil:
		return nil
	case !os.IsNotExist(err):
		return err
	}
	if err := os.MkdirAll(membdir, 0700); err != nil {
		return err
	}
	names := []string{"snap", "wal"}
	for _, name := range names {
		if err := os.Rename(path.Join(dir, name), path.Join(membdir, name)); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Package build implements Pipe and can build Go projects for
// several platforms, with pre and post hook support.
package build

import (
	"errors"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/goreleaser/goreleaser/context"
	"golang.org/x/sync/errgroup"
)

// Pipe for build
type Pipe struct{}

// Description of the pipe
func (Pipe) Description() string {
	return "Building binaries"
}

// Run the pipe
func (Pipe) Run(ctx *context.Context) error {
	if err := runHook(ctx.Config.Build.Hooks.Pre); err != nil {
		return err
	}
	var g errgroup.Group
	for _, goos := range ctx.Config.Build.Goos {
		for _, goarch := range ctx.Config.Build.Goarch {
			goos := goos
			goarch := goarch
			if !valid(goos, goarch) {
				continue
			}
			name, err := nameFor(ctx, goos, goarch)
			if err != nil {
				return err
			}
			ctx.Archives[goos+goarch] = name
			g.Go(func() error {
				return build(name, goos, goarch, ctx)
			})
		}
	}
	if err := g.Wait(); err != nil {
		return err
	}
	return runHook(ctx.Config.Build.Hooks.Post)
}

func runHook(hook string) error {
	if hook == "" {
		return nil
	}
	log.Println("Running hook", hook)
	cmd := strings.Fields(hook)
	return run(runtime.GOOS, runtime.GOARCH, cmd)
}

func build(name, goos, goarch string, ctx *context.Context) error {
	output := filepath.Join(
		ctx.Config.Dist,
		name,
		ctx.Config.Build.Binary+extFor(goos),
	)
	log.Println("Building", output)
	cmd := []string{"go", "build"}
	if ctx.Config.Build.Flags != "" {
		cmd = append(cmd, strings.Fields(ctx.Config.Build.Flags)...)
	}
	flags, err := ldflags(ctx)
	if err != nil {
		return err
	}
	cmd = append(cmd, "-ldflags="+flags, "-o", output, ctx.Config.Build.Main)
	return run(goos, goarch, cmd)
}

func run(goos, goarch string, command []string) error {
	cmd := exec.Command(command[0], command[1:]...)
	cmd.Env = append(cmd.Env, os.Environ()...)
	cmd.Env = append(cmd.Env, "GOOS="+goos, "GOARCH="+goarch)
	if out, err := cmd.CombinedOutput(); err != nil {
		return errors.New(string(out))
	}
	return nil
}
<commit_msg>Reduce the number of concurrent builds<commit_after>// Package build implements Pipe and can build Go projects for
// several platforms, with pre and post hook support.
package build

import (
	"errors"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/goreleaser/goreleaser/context"
	"golang.org/x/sync/errgroup"
)

// Pipe for build
type Pipe struct{}

// Description of the pipe
func (Pipe) Description() string {
	return "Building binaries"
}

// Run the pipe
func (Pipe) Run(ctx *context.Context) error {
	if err := runHook(ctx.Config.Build.Hooks.Pre); err != nil {
		return err
	}
	sem := make(chan bool, 4)
	var g errgroup.Group
	for _, goos := range ctx.Config.Build.Goos {
		for _, goarch := range ctx.Config.Build.Goarch {
			sem <- true
			goos := goos
			goarch := goarch
			if !valid(goos, goarch) {
				continue
			}
			name, err := nameFor(ctx, goos, goarch)
			if err != nil {
				return err
			}
			ctx.Archives[goos+goarch] = name
			g.Go(func() error {
				defer func() {
					<-sem
				}()
				return build(name, goos, goarch, ctx)
			})
		}
	}
	if err := g.Wait(); err != nil {
		return err
	}
	return runHook(ctx.Config.Build.Hooks.Post)
}

func runHook(hook string) error {
	if hook == "" {
		return nil
	}
	log.Println("Running hook", hook)
	cmd := strings.Fields(hook)
	return run(runtime.GOOS, runtime.GOARCH, cmd)
}

func build(name, goos, goarch string, ctx *context.Context) error {
	output := filepath.Join(
		ctx.Config.Dist,
		name,
		ctx.Config.Build.Binary+extFor(goos),
	)
	log.Println("Building", output)
	cmd := []string{"go", "build"}
	if ctx.Config.Build.Flags != "" {
		cmd = append(cmd, strings.Fields(ctx.Config.Build.Flags)...)
	}
	flags, err := ldflags(ctx)
	if err != nil {
		return err
	}
	cmd = append(cmd, "-ldflags="+flags, "-o", output, ctx.Config.Build.Main)
	return run(goos, goarch, cmd)
}

func run(goos, goarch string, command []string) error {
	cmd := exec.Command(command[0], command[1:]...)
	cmd.Env = append(cmd.Env, os.Environ()...)
	cmd.Env = append(cmd.Env, "GOOS="+goos, "GOARCH="+goarch)
	if out, err := cmd.CombinedOutput(); err != nil {
		return errors.New(string(out))
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/ContextLogic/eventmaster/eventmaster"
	log "github.com/Sirupsen/logrus"
	"github.com/jessevdk/go-flags"
	"github.com/julienschmidt/httprouter"
	metrics "github.com/rcrowley/go-metrics"
	"github.com/soheilhy/cmux"
	"google.golang.org/grpc"
)

type dbConfig struct {
	CassandraAddr  []string `json:"cassandra_addr"`
	Keyspace       string   `json:"cassandra_keyspace"`
	Consistency    string   `json:"consistency"`
	ESAddr         []string `json:"es_addr"`
	ESUsername     string   `json:"es_username"`
	ESPassword     string   `json:"es_password"`
	FlushInterval  int      `json:"flush_interval"`
	UpdateInterval int      `json:"update_interval"`
	CertFile       string   `json:"cert_file"`
	KeyFile        string   `json:"key_file"`
	CAFile         string   `json:"ca_file"`
}

func getConfig() dbConfig {
	dbConf := dbConfig{}
	confFile, err := ioutil.ReadFile("db_config.json")
	if err != nil {
		fmt.Println("No db_config file specified")
	} else {
		err = json.Unmarshal(confFile, &dbConf)
		if err != nil {
			fmt.Println("Error parsing db_config.json, using defaults:", err)
		}
	}
	if dbConf.Keyspace == "" {
		dbConf.Keyspace = "event_master"
	}
	if dbConf.Consistency == "" {
		dbConf.Consistency = "local_quorum"
	}
	if dbConf.FlushInterval == 0 {
		dbConf.FlushInterval = 5
	}
	if dbConf.UpdateInterval == 0 {
		dbConf.UpdateInterval = 5
	}
	if dbConf.CassandraAddr == nil || len(dbConf.CassandraAddr) == 0 {
		dbConf.CassandraAddr = append(dbConf.CassandraAddr, "127.0.0.1:9042")
	}
	if dbConf.ESAddr == nil || len(dbConf.ESAddr) == 0 {
		dbConf.ESAddr = append(dbConf.ESAddr, "http://127.0.0.1:9200")
	}
	return dbConf
}

func getHTTPServer(store *EventStore, registry metrics.Registry) *http.Server {
	r := httprouter.New()
	h := httpHandler{
		store: store,
	}

	// API endpoints
	r.POST("/v1/event", wrapHandler(h.handleAddEvent, registry))
	r.GET("/v1/event", wrapHandler(h.handleGetEvent, registry))
	r.POST("/v1/topic", wrapHandler(h.handleAddTopic, registry))
	r.PUT("/v1/topic/:name", wrapHandler(h.handleUpdateTopic, registry))
	r.GET("/v1/topic", wrapHandler(h.handleGetTopic, registry))
	r.DELETE("/v1/topic/:name", wrapHandler(h.handleDeleteTopic, registry))
	r.POST("/v1/dc", wrapHandler(h.handleAddDc, registry))
	r.PUT("/v1/dc/:name", wrapHandler(h.handleUpdateDc, registry))
	r.GET("/v1/dc/", wrapHandler(h.handleGetDc, registry))

	// GitHub webhook endpoint
	r.POST("/v1/github_event", wrapHandler(h.handleGitHubEvent, registry))

	// UI endpoints
	r.GET("/", HandleMainPage)
	r.GET("/add_event", HandleCreatePage)
	r.GET("/topic", HandleTopicPage)
	r.GET("/dc", HandleDcPage)

	// JS file endpoints
	r.ServeFiles("/js/*filepath", http.Dir("ui/js"))
	r.ServeFiles("/bootstrap/*filepath", http.Dir("ui/bootstrap"))
	r.ServeFiles("/css/*filepath", http.Dir("ui/css"))

	return &http.Server{
		Handler: r,
	}
}

func main() {
	var config Config
	parser := flags.NewParser(&config, flags.Default)
	if _, err := parser.Parse(); err != nil {
		log.Fatalf("Error parsing flags: %v", err)
	}

	r := metrics.NewRegistry()

	// Set up event store
	dbConf := getConfig()
	store, err := NewEventStore(dbConf, config, r)
	if err != nil {
		log.Fatalf("Unable to create event store: %v", err)
	}
	if err := store.Update(); err != nil {
		fmt.Println("Error loading dcs and topics from Cassandra", err)
	}

	// Create listening socket for grpc server
	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", config.Port))
	if err != nil {
		log.Fatalf("failed to listen: %v", err)
	}

	mux := cmux.New(lis)
	httpL := mux.Match(cmux.HTTP1Fast())
	grpcL := mux.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))

	httpS := getHTTPServer(store, r)

	// Create the EventMaster server
	grpcServer, err := NewGRPCServer(&config, store, r)
	if err != nil {
		log.Fatalf("Unable to start server: %v", err)
	}

	maxMsgSizeOpt := grpc.MaxMsgSize(1024 * 1024 * 100)
	// Create the gRPC server and register our service
	grpcS := grpc.NewServer(maxMsgSizeOpt)
	eventmaster.RegisterEventMasterServer(grpcS, grpcServer)

	go httpS.Serve(httpL)
	go grpcS.Serve(grpcL)

	go func() {
		fmt.Println("Starting server on port", config.Port)
		if err := mux.Serve(); err != nil {
			log.Fatalf("Error starting server: %v", err)
		}
	}()

	flushTicker := time.NewTicker(time.Second * time.Duration(dbConf.FlushInterval))
	go func() {
		for range flushTicker.C {
			if err := store.FlushToES(); err != nil {
				fmt.Println("Error flushing events from temp_event to ES:", err)
			}
		}
	}()

	updateTicker := time.NewTicker(time.Second * time.Duration(dbConf.UpdateInterval))
	go func() {
		for range updateTicker.C {
			if err := store.Update(); err != nil {
				fmt.Println("Error updating dcs and topics from cassandra:", err)
			}
		}
	}()

	stopChan := make(chan os.Signal)
	signal.Notify(stopChan, syscall.SIGTERM, syscall.SIGINT)

	<-stopChan
	fmt.Println("Got shutdown signal, gracefully shutting down")
	flushTicker.Stop()
	updateTicker.Stop()
	store.CloseSession()
	grpcS.GracefulStop()
	lis.Close()
}
<commit_msg>Expose metrics on metrics client using expvar<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/ContextLogic/eventmaster/eventmaster"
	log "github.com/Sirupsen/logrus"
	"github.com/jessevdk/go-flags"
	"github.com/julienschmidt/httprouter"
	metrics "github.com/rcrowley/go-metrics"
	"github.com/rcrowley/go-metrics/exp"
	"github.com/soheilhy/cmux"
	"google.golang.org/grpc"
)

type dbConfig struct {
	CassandraAddr  []string `json:"cassandra_addr"`
	Keyspace       string   `json:"cassandra_keyspace"`
	Consistency    string   `json:"consistency"`
	ESAddr         []string `json:"es_addr"`
	ESUsername     string   `json:"es_username"`
	ESPassword     string   `json:"es_password"`
	FlushInterval  int      `json:"flush_interval"`
	UpdateInterval int      `json:"update_interval"`
	CertFile       string   `json:"cert_file"`
	KeyFile        string   `json:"key_file"`
	CAFile         string   `json:"ca_file"`
}

func getConfig() dbConfig {
	dbConf := dbConfig{}
	confFile, err := ioutil.ReadFile("db_config.json")
	if err != nil {
		fmt.Println("No db_config file specified")
	} else {
		err = json.Unmarshal(confFile, &dbConf)
		if err != nil {
			fmt.Println("Error parsing db_config.json, using defaults:", err)
		}
	}
	if dbConf.Keyspace == "" {
		dbConf.Keyspace = "event_master"
	}
	if dbConf.Consistency == "" {
		dbConf.Consistency = "local_quorum"
	}
	if dbConf.FlushInterval == 0 {
		dbConf.FlushInterval = 5
	}
	if dbConf.UpdateInterval == 0 {
		dbConf.UpdateInterval = 5
	}
	if dbConf.CassandraAddr == nil || len(dbConf.CassandraAddr) == 0 {
		dbConf.CassandraAddr = append(dbConf.CassandraAddr, "127.0.0.1:9042")
	}
	if dbConf.ESAddr == nil || len(dbConf.ESAddr) == 0 {
		dbConf.ESAddr = append(dbConf.ESAddr, "http://127.0.0.1:9200")
	}
	return dbConf
}

func getHTTPServer(store *EventStore, registry metrics.Registry) *http.Server {
	r := httprouter.New()
	h := httpHandler{
		store: store,
	}

	// API endpoints
	r.POST("/v1/event", wrapHandler(h.handleAddEvent, registry))
	r.GET("/v1/event", wrapHandler(h.handleGetEvent, registry))
	r.POST("/v1/topic", wrapHandler(h.handleAddTopic, registry))
	r.PUT("/v1/topic/:name", wrapHandler(h.handleUpdateTopic, registry))
	r.GET("/v1/topic", wrapHandler(h.handleGetTopic, registry))
	r.DELETE("/v1/topic/:name", wrapHandler(h.handleDeleteTopic, registry))
	r.POST("/v1/dc", wrapHandler(h.handleAddDc, registry))
	r.PUT("/v1/dc/:name", wrapHandler(h.handleUpdateDc, registry))
	r.GET("/v1/dc/", wrapHandler(h.handleGetDc, registry))

	// GitHub webhook endpoint
	r.POST("/v1/github_event", wrapHandler(h.handleGitHubEvent, registry))

	// UI endpoints
	r.GET("/", HandleMainPage)
	r.GET("/add_event", HandleCreatePage)
	r.GET("/topic", HandleTopicPage)
	r.GET("/dc", HandleDcPage)

	// JS file endpoints
	r.ServeFiles("/js/*filepath", http.Dir("ui/js"))
	r.ServeFiles("/bootstrap/*filepath", http.Dir("ui/bootstrap"))
	r.ServeFiles("/css/*filepath", http.Dir("ui/css"))

	return &http.Server{
		Handler: r,
	}
}

func main() {
	var config Config
	parser := flags.NewParser(&config, flags.Default)
	if _, err := parser.Parse(); err != nil {
		log.Fatalf("Error parsing flags: %v", err)
	}

	r := metrics.NewRegistry()

	// Set up event store
	dbConf := getConfig()
	store, err := NewEventStore(dbConf, config, r)
	if err != nil {
		log.Fatalf("Unable to create event store: %v", err)
	}
	if err := store.Update(); err != nil {
		fmt.Println("Error loading dcs and topics from Cassandra", err)
	}

	exp.Exp(metrics.DefaultRegistry)
	sock, err := net.Listen("tcp", "0.0.0.0:12345")
	if err != nil {
		log.Fatalf("failed to listen: %v", err)
	}
	go func() {
		fmt.Println("go-metrics server listening at port 12345")
		http.Serve(sock, nil)
	}()

	// Create listening socket for grpc server
	lis, err := net.Listen("tcp", fmt.Sprintf(":%d", config.Port))
	if err != nil {
		log.Fatalf("failed to listen: %v", err)
	}

	mux := cmux.New(lis)
	httpL := mux.Match(cmux.HTTP1Fast())
	grpcL := mux.Match(cmux.HTTP2HeaderField("content-type", "application/grpc"))

	httpS := getHTTPServer(store, r)

	// Create the EventMaster server
	grpcServer, err := NewGRPCServer(&config, store, r)
	if err != nil {
		log.Fatalf("Unable to start server: %v", err)
	}

	maxMsgSizeOpt := grpc.MaxMsgSize(1024 * 1024 * 100)
	// Create the gRPC server and register our service
	grpcS := grpc.NewServer(maxMsgSizeOpt)
	eventmaster.RegisterEventMasterServer(grpcS, grpcServer)

	go httpS.Serve(httpL)
	go grpcS.Serve(grpcL)

	go func() {
		fmt.Println("Starting server on port", config.Port)
		if err := mux.Serve(); err != nil {
			log.Fatalf("Error starting server: %v", err)
		}
	}()

	flushTicker := time.NewTicker(time.Second * time.Duration(dbConf.FlushInterval))
	go func() {
		for range flushTicker.C {
			if err := store.FlushToES(); err != nil {
				fmt.Println("Error flushing events from temp_event to ES:", err)
			}
		}
	}()

	updateTicker := time.NewTicker(time.Second * time.Duration(dbConf.UpdateInterval))
	go func() {
		for range updateTicker.C {
			if err := store.Update(); err != nil {
				fmt.Println("Error updating dcs and topics from cassandra:", err)
			}
		}
	}()

	stopChan := make(chan os.Signal)
	signal.Notify(stopChan, syscall.SIGTERM, syscall.SIGINT)

	<-stopChan
	fmt.Println("Got shutdown signal, gracefully shutting down")
	flushTicker.Stop()
	updateTicker.Stop()
	store.CloseSession()
	grpcS.GracefulStop()
	lis.Close()
}
<|endoftext|>