<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package main

import (
	"flag"
	"fmt"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/google/mtail/internal/metrics"
	"github.com/google/mtail/internal/mtail"
	"github.com/google/mtail/internal/watcher"
)

type seqStringFlag []string

func (f *seqStringFlag) String() string {
	return fmt.Sprint(*f)
}

func (f *seqStringFlag) Set(value string) error {
	for _, v := range strings.Split(value, ",") {
		*f = append(*f, v)
	}
	return nil
}

var logs seqStringFlag

var (
	port    = flag.String("port", "3903", "HTTP port to listen on.")
	address = flag.String("address", "", "Host or IP address on which to bind HTTP listener")
	progs   = flag.String("progs", "", "Name of the directory containing mtail programs")

	version = flag.Bool("version", false, "Print mtail version information.")

	// Compiler behaviour flags
	oneShot      = flag.Bool("one_shot", false, "Compile the programs, then read the contents of the provided logs from start until EOF, print the values of the metrics store and exit. This is a debugging flag only, not for production use.")
	compileOnly  = flag.Bool("compile_only", false, "Compile programs only, do not load the virtual machine.")
	dumpAst      = flag.Bool("dump_ast", false, "Dump AST of programs after parse (to INFO log).")
	dumpAstTypes = flag.Bool("dump_ast_types", false, "Dump AST of programs with type annotation after typecheck (to INFO log).")
	dumpBytecode = flag.Bool("dump_bytecode", false, "Dump bytecode of programs (to INFO log).")

	// VM Runtime behaviour flags
	syslogUseCurrentYear = flag.Bool("syslog_use_current_year", true, "Patch yearless timestamps with the present year.")
	overrideTimezone     = flag.String("override_timezone", "", "If set, use the provided timezone in timestamp conversion, instead of UTC.")
	emitProgLabel        = flag.Bool("emit_prog_label", true, "Emit the 'prog' label in variable exports.")
	emitMetricTimestamp  = flag.Bool("emit_metric_timestamp", false, "Emit the recorded timestamp of a metric.  If disabled (the default) no explicit timestamp is sent to a collector.")

	// Ops flags
	pollInterval                = flag.Duration("poll_interval", 0, "Set the interval to poll all log files for data; must be positive, or zero to disable polling.  With polling mode, only the files found at mtail startup will be polled.")
	disableFsnotify             = flag.Bool("disable_fsnotify", false, "EXPERIMENTAL: When enabled no fsnotify watcher is created, and mtail falls back to polling mode only.  Only the files known at program startup will be polled.")
	expiredMetricGcTickInterval = flag.Duration("expired_metrics_gc_interval", time.Hour, "interval between expired metric garbage collection runs")
	staleLogGcTickInterval      = flag.Duration("stale_log_gc_interval", time.Hour, "interval between stale log garbage collection runs")

	// Debugging flags
	blockProfileRate     = flag.Int("block_profile_rate", 0, "Nanoseconds of block time before goroutine blocking events reported. 0 turns off.  See https://golang.org/pkg/runtime/#SetBlockProfileRate")
	mutexProfileFraction = flag.Int("mutex_profile_fraction", 0, "Fraction of mutex contention events reported.  0 turns off.  See http://golang.org/pkg/runtime/#SetMutexProfileFraction")
)

func init() {
	flag.Var(&logs, "logs", "List of log files to monitor, separated by commas.  This flag may be specified multiple times.")
}

var (
	// Version and Revision are supplied by the linker
	Branch   string = "invalid-use-make"
	Version  string = "invalid-use-make"
	Revision string = "invalid-use-make"
)

func main() {
	buildInfo := mtail.BuildInfo{
		Branch:   Branch,
		Version:  Version,
		Revision: Revision,
	}

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "%s\n", buildInfo.String())
		fmt.Fprintf(os.Stderr, "\nUsage:\n")
		flag.PrintDefaults()
	}
	flag.Parse()
	if *version {
		fmt.Println(buildInfo.String())
		os.Exit(1)
	}
	glog.Info(buildInfo.String())
	glog.Infof("Commandline: %q", os.Args)
	if len(flag.Args()) > 0 {
		glog.Exitf("Too many extra arguments specified: %q\n(the logs flag can be repeated, or the filenames separated by commas.)", flag.Args())
	}
	loc, err := time.LoadLocation(*overrideTimezone)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Couldn't parse timezone %q: %s", *overrideTimezone, err)
		os.Exit(1)
	}
	if *blockProfileRate > 0 {
		glog.Infof("Setting block profile rate to %d", *blockProfileRate)
		runtime.SetBlockProfileRate(*blockProfileRate)
	}
	if *mutexProfileFraction > 0 {
		glog.Infof("Setting mutex profile fraction to %d", *mutexProfileFraction)
		runtime.SetMutexProfileFraction(*mutexProfileFraction)
	}
	if *progs == "" {
		glog.Exitf("mtail requires programs that in instruct it how to extract metrics from logs; please use the flag -progs to specify the directory containing the programs.")
	}
	if !(*dumpBytecode || *dumpAst || *dumpAstTypes || *compileOnly) {
		if len(logs) == 0 {
			glog.Exitf("mtail requires the names of logs to follow in order to extract logs from them; please use the flag -logs one or more times to specify glob patterns describing these logs.")
		}
	}
	w, err := watcher.NewLogWatcher(*pollInterval, !*disableFsnotify)
	if err != nil {
		glog.Exitf("Failure to create log watcher: %s", err)
	}
	opts := []func(*mtail.Server) error{
		mtail.ProgramPath(*progs),
		mtail.LogPathPatterns(logs...),
		mtail.BindAddress(*address, *port),
		mtail.SetBuildInfo(buildInfo),
		mtail.OverrideLocation(loc),
		mtail.ExpiredMetricGcTickInterval(*expiredMetricGcTickInterval),
		mtail.StaleLogGcTickInterval(*staleLogGcTickInterval),
	}
	if *oneShot {
		opts = append(opts, mtail.OneShot)
	}
	if *compileOnly {
		opts = append(opts, mtail.CompileOnly)
	}
	if *dumpAst {
		opts = append(opts, mtail.DumpAst)
	}
	if *dumpAstTypes {
		opts = append(opts, mtail.DumpAstTypes)
	}
	if *dumpBytecode {
		opts = append(opts, mtail.DumpBytecode)
	}
	if *syslogUseCurrentYear {
		opts = append(opts, mtail.SyslogUseCurrentYear)
	}
	if !*emitProgLabel {
		opts = append(opts, mtail.OmitProgLabel)
	}
	if *emitMetricTimestamp {
		opts = append(opts, mtail.EmitMetricTimestamp)
	}
	m, err := mtail.New(metrics.NewStore(), w, opts...)
	if err != nil {
		glog.Error(err)
		os.Exit(1)
	}
	err = m.Run()
	if err != nil {
		glog.Error(err)
		os.Exit(1)
	}
}
<commit_msg>Hack up zipkin trace reporting.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"runtime"
	"strings"
	"time"

	openzipkin "github.com/openzipkin/zipkin-go"
	zipkinHTTP "github.com/openzipkin/zipkin-go/reporter/http"

	"github.com/golang/glog"
	"github.com/google/mtail/internal/metrics"
	"github.com/google/mtail/internal/mtail"
	"github.com/google/mtail/internal/watcher"
	"go.opencensus.io/exporter/zipkin"
	"go.opencensus.io/trace"
)

type seqStringFlag []string

func (f *seqStringFlag) String() string {
	return fmt.Sprint(*f)
}

func (f *seqStringFlag) Set(value string) error {
	for _, v := range strings.Split(value, ",") {
		*f = append(*f, v)
	}
	return nil
}

var logs seqStringFlag

var (
	port    = flag.String("port", "3903", "HTTP port to listen on.")
	address = flag.String("address", "", "Host or IP address on which to bind HTTP listener")
	progs   = flag.String("progs", "", "Name of the directory containing mtail programs")

	version = flag.Bool("version", false, "Print mtail version information.")

	// Compiler behaviour flags
	oneShot      = flag.Bool("one_shot", false, "Compile the programs, then read the contents of the provided logs from start until EOF, print the values of the metrics store and exit. This is a debugging flag only, not for production use.")
	compileOnly  = flag.Bool("compile_only", false, "Compile programs only, do not load the virtual machine.")
	dumpAst      = flag.Bool("dump_ast", false, "Dump AST of programs after parse (to INFO log).")
	dumpAstTypes = flag.Bool("dump_ast_types", false, "Dump AST of programs with type annotation after typecheck (to INFO log).")
	dumpBytecode = flag.Bool("dump_bytecode", false, "Dump bytecode of programs (to INFO log).")

	// VM Runtime behaviour flags
	syslogUseCurrentYear = flag.Bool("syslog_use_current_year", true, "Patch yearless timestamps with the present year.")
	overrideTimezone     = flag.String("override_timezone", "", "If set, use the provided timezone in timestamp conversion, instead of UTC.")
	emitProgLabel        = flag.Bool("emit_prog_label", true, "Emit the 'prog' label in variable exports.")
	emitMetricTimestamp  = flag.Bool("emit_metric_timestamp", false, "Emit the recorded timestamp of a metric.  If disabled (the default) no explicit timestamp is sent to a collector.")

	// Ops flags
	pollInterval                = flag.Duration("poll_interval", 0, "Set the interval to poll all log files for data; must be positive, or zero to disable polling.  With polling mode, only the files found at mtail startup will be polled.")
	disableFsnotify             = flag.Bool("disable_fsnotify", false, "EXPERIMENTAL: When enabled no fsnotify watcher is created, and mtail falls back to polling mode only.  Only the files known at program startup will be polled.")
	expiredMetricGcTickInterval = flag.Duration("expired_metrics_gc_interval", time.Hour, "interval between expired metric garbage collection runs")
	staleLogGcTickInterval      = flag.Duration("stale_log_gc_interval", time.Hour, "interval between stale log garbage collection runs")

	// Debugging flags
	blockProfileRate     = flag.Int("block_profile_rate", 0, "Nanoseconds of block time before goroutine blocking events reported. 0 turns off.  See https://golang.org/pkg/runtime/#SetBlockProfileRate")
	mutexProfileFraction = flag.Int("mutex_profile_fraction", 0, "Fraction of mutex contention events reported.  0 turns off.  See http://golang.org/pkg/runtime/#SetMutexProfileFraction")
)

func init() {
	flag.Var(&logs, "logs", "List of log files to monitor, separated by commas.  This flag may be specified multiple times.")
}

var (
	// Version and Revision are supplied by the linker
	Branch   string = "invalid-use-make"
	Version  string = "invalid-use-make"
	Revision string = "invalid-use-make"
)

func main() {
	buildInfo := mtail.BuildInfo{
		Branch:   Branch,
		Version:  Version,
		Revision: Revision,
	}

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "%s\n", buildInfo.String())
		fmt.Fprintf(os.Stderr, "\nUsage:\n")
		flag.PrintDefaults()
	}
	flag.Parse()
	if *version {
		fmt.Println(buildInfo.String())
		os.Exit(1)
	}
	glog.Info(buildInfo.String())
	glog.Infof("Commandline: %q", os.Args)
	if len(flag.Args()) > 0 {
		glog.Exitf("Too many extra arguments specified: %q\n(the logs flag can be repeated, or the filenames separated by commas.)", flag.Args())
	}
	loc, err := time.LoadLocation(*overrideTimezone)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Couldn't parse timezone %q: %s", *overrideTimezone, err)
		os.Exit(1)
	}
	if *blockProfileRate > 0 {
		glog.Infof("Setting block profile rate to %d", *blockProfileRate)
		runtime.SetBlockProfileRate(*blockProfileRate)
	}
	if *mutexProfileFraction > 0 {
		glog.Infof("Setting mutex profile fraction to %d", *mutexProfileFraction)
		runtime.SetMutexProfileFraction(*mutexProfileFraction)
	}
	if *progs == "" {
		glog.Exitf("mtail requires programs that in instruct it how to extract metrics from logs; please use the flag -progs to specify the directory containing the programs.")
	}
	if !(*dumpBytecode || *dumpAst || *dumpAstTypes || *compileOnly) {
		if len(logs) == 0 {
			glog.Exitf("mtail requires the names of logs to follow in order to extract logs from them; please use the flag -logs one or more times to specify glob patterns describing these logs.")
		}
	}

	localEndpoint, err := openzipkin.NewEndpoint("mtail", "127.0.0.1:3903")
	if err != nil {
		log.Fatalf("Failed to create the local zipkinEndpoint: %v", err)
	}
	reporter := zipkinHTTP.NewReporter("http://localhost:9411/api/v2/spans")
	ze := zipkin.NewExporter(reporter, localEndpoint)
	trace.RegisterExporter(ze)

	trace.ApplyConfig(trace.Config{DefaultSampler: trace.AlwaysSample()})

	w, err := watcher.NewLogWatcher(*pollInterval, !*disableFsnotify)
	if err != nil {
		glog.Exitf("Failure to create log watcher: %s", err)
	}
	opts := []func(*mtail.Server) error{
		mtail.ProgramPath(*progs),
		mtail.LogPathPatterns(logs...),
		mtail.BindAddress(*address, *port),
		mtail.SetBuildInfo(buildInfo),
		mtail.OverrideLocation(loc),
		mtail.ExpiredMetricGcTickInterval(*expiredMetricGcTickInterval),
		mtail.StaleLogGcTickInterval(*staleLogGcTickInterval),
	}
	if *oneShot {
		opts = append(opts, mtail.OneShot)
	}
	if *compileOnly {
		opts = append(opts, mtail.CompileOnly)
	}
	if *dumpAst {
		opts = append(opts, mtail.DumpAst)
	}
	if *dumpAstTypes {
		opts = append(opts, mtail.DumpAstTypes)
	}
	if *dumpBytecode {
		opts = append(opts, mtail.DumpBytecode)
	}
	if *syslogUseCurrentYear {
		opts = append(opts, mtail.SyslogUseCurrentYear)
	}
	if !*emitProgLabel {
		opts = append(opts, mtail.OmitProgLabel)
	}
	if *emitMetricTimestamp {
		opts = append(opts, mtail.EmitMetricTimestamp)
	}
	m, err := mtail.New(metrics.NewStore(), w, opts...)
	if err != nil {
		glog.Error(err)
		os.Exit(1)
	}
	err = m.Run()
	if err != nil {
		glog.Error(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strings"
	"sync"

	"github.com/dhowden/tchaik/index"
)

type LibraryAPI struct {
	index.Library

	collections map[string]index.Collection
	filters     map[string][]index.FilterItem
	recent      []index.Path
	searcher    index.Searcher
	players     *players
}

type libraryFileSystem struct {
	http.FileSystem
	index.Library
}

// Open implements http.FileSystem and rewrites TrackID values to their corresponding Location
// values using the index.Library
func (l *libraryFileSystem) Open(path string) (http.File, error) {
	t, ok := l.Library.Track(strings.Trim(path, "/")) // TrackIDs arrive with leading slash
	if !ok {
		return nil, fmt.Errorf("could not find track: %v", path)
	}

	loc := t.GetString("Location")
	if loc == "" {
		return nil, fmt.Errorf("invalid (empty) location for track: %v", path)
	}
	return l.FileSystem.Open(loc)
}

type group struct {
	Name        string
	Key         index.Key
	TotalTime   interface{} `json:",omitempty"`
	Artist      interface{} `json:",omitempty"`
	AlbumArtist interface{} `json:",omitempty"`
	Composer    interface{} `json:",omitempty"`
	ListStyle   interface{} `json:",omitempty"`
	TrackID     interface{} `json:",omitempty"`
	Year        interface{} `json:",omitempty"`
	Groups      []group     `json:",omitempty"`
	Tracks      []track     `json:",omitempty"`
}

type track struct {
	TrackID     string `json:",omitempty"`
	Name        string `json:",omitempty"`
	Album       string `json:",omitempty"`
	Artist      string `json:",omitempty"`
	AlbumArtist string `json:",omitempty"`
	Composer    string `json:",omitempty"`
	Year        int    `json:",omitempty"`
	DiscNumber  int    `json:",omitempty"`
	TotalTime   int    `json:",omitempty"`
}

func buildCollection(h group, c index.Collection) group {
	for _, k := range c.Keys() {
		g := c.Get(k)
		g = index.FirstTrackAttr(index.StringAttr("AlbumArtist"), g)
		h.Groups = append(h.Groups, group{
			Name:        g.Name(),
			Key:         k,
			AlbumArtist: g.Field("AlbumArtist"),
		})
	}
	return h
}

func build(g index.Group, key index.Key) group {
	h := group{
		Name:        g.Name(),
		Key:         key,
		TotalTime:   g.Field("TotalTime"),
		Artist:      g.Field("Artist"),
		AlbumArtist: g.Field("AlbumArtist"),
		Composer:    g.Field("Composer"),
		Year:        g.Field("Year"),
		ListStyle:   g.Field("ListStyle"),
		TrackID:     g.Field("TrackID"),
	}

	if c, ok := g.(index.Collection); ok {
		return buildCollection(h, c)
	}

	getString := func(t index.Track, field string) string {
		if g.Field(field) != "" {
			return ""
		}
		return t.GetString(field)
	}

	getInt := func(t index.Track, field string) int {
		if g.Field(field) != 0 {
			return 0
		}
		return t.GetInt(field)
	}

	for _, t := range g.Tracks() {
		h.Tracks = append(h.Tracks, track{
			TrackID:    t.GetString("TrackID"),
			Name:       t.GetString("Name"),
			TotalTime:  t.GetInt("TotalTime"),
			DiscNumber: t.GetInt("DiscNumber"),
			// Potentially common fields (don't want to re-transmit everything)
			Album:       getString(t, "Album"),
			Artist:      getString(t, "Artist"),
			AlbumArtist: getString(t, "AlbumArtist"),
			Composer:    getString(t, "Composer"),
			Year:        getInt(t, "Year"),
		})
	}
	return h
}

func (l *LibraryAPI) Fetch(c index.Collection, path []string) (group, error) {
	if len(path) == 0 {
		return build(c, index.Key("Root")), nil
	}

	var g index.Group = c
	k := index.Key(path[0])
	g = c.Get(k)

	if g == nil {
		return group{}, fmt.Errorf("invalid path: near '%v'", path[0])
	}

	index.Sort(g.Tracks(), index.MultiSort(index.SortByInt("DiscNumber"), index.SortByInt("TrackNumber")))
	c = index.Collect(g, index.ByPrefix("Name"))
	g = index.SubTransform(c, index.TrimEnumPrefix)
	g = index.SumGroupIntAttr("TotalTime", g)
	commonFields := []index.Attr{
		index.StringAttr("Album"),
		index.StringAttr("Artist"),
		index.StringAttr("AlbumArtist"),
		index.StringAttr("Composer"),
		index.IntAttr("Year"),
	}
	g = index.CommonGroupAttr(commonFields, g)
	g = index.RemoveEmptyCollections(g)

	for i, p := range path[1:] {
		var ok bool
		c, ok = g.(index.Collection)
		if !ok {
			return group{}, fmt.Errorf("retrieved Group is not a Collection")
		}
		k = index.Key(p)
		g = c.Get(k)

		if g == nil {
			return group{}, fmt.Errorf("invalid path near '%v'", path[1:][i])
		}

		if _, ok = g.(index.Collection); !ok {
			if i == len(path[1:])-1 {
				break
			}
			return group{}, fmt.Errorf("retrieved Group isn't a Collection: %v", p)
		}
	}
	if g == nil {
		return group{}, fmt.Errorf("could not find group")
	}
	g = index.FirstTrackAttr(index.StringAttr("TrackID"), g)

	return build(g, k), nil
}

func (l *LibraryAPI) FileSystem(fs http.FileSystem) http.FileSystem {
	return &libraryFileSystem{fs, l.Library}
}

type players struct {
	sync.RWMutex
	m map[string]Player
}

func newPlayers() *players {
	return &players{m: make(map[string]Player)}
}

func (s *players) add(p Player) {
	s.Lock()
	defer s.Unlock()

	s.m[p.Key()] = p
}

func (s *players) remove(key string) {
	s.Lock()
	defer s.Unlock()

	delete(s.m, key)
}

func (s *players) get(key string) Player {
	s.RLock()
	defer s.RUnlock()

	return s.m[key]
}

func createPlayer(l LibraryAPI, w http.ResponseWriter, r *http.Request) {
	dec := json.NewDecoder(r.Body)
	defer r.Body.Close()

	postData := struct {
		Key        string
		PlayerKeys []string
	}{}
	err := dec.Decode(&postData)
	if err != nil {
		http.Error(w, fmt.Sprintf("error parsing JSON: %v", err), http.StatusBadRequest)
		return
	}

	if p := l.players.get(postData.Key); p != nil {
		http.Error(w, "player key already exists", http.StatusBadRequest)
		return
	}

	if postData.PlayerKeys == nil || len(postData.PlayerKeys) == 0 {
		http.Error(w, "no player keys specified", http.StatusBadRequest)
		return
	}

	var players []Player
	for _, pk := range postData.PlayerKeys {
		p := l.players.get(pk)
		if p == nil {
			http.Error(w, fmt.Sprintf("invalid player key: %v", pk), http.StatusBadRequest)
			return
		}
		players = append(players, p)
	}
	l.players.add(MultiPlayer(postData.Key, players...))
	w.WriteHeader(http.StatusCreated)
}

func playerAction(p Player, w http.ResponseWriter, r *http.Request) {
	dec := json.NewDecoder(r.Body)
	defer r.Body.Close()

	putData := struct {
		Action string
		Value  interface{}
	}{}
	err := dec.Decode(&putData)
	if err != nil {
		http.Error(w, fmt.Sprintf("error parsing JSON: %v", err), http.StatusBadRequest)
		return
	}

	switch putData.Action {
	case "play":
		err = p.Play()

	case "pause":
		err = p.Pause()

	case "next":
		err = p.NextTrack()

	case "prev":
		err = p.PreviousTrack()

	case "togglePlayPause":
		err = p.TogglePlayPause()

	case "toggleMute":
		err = p.ToggleMute()

	case "setVolume":
		f, ok := putData.Value.(float64)
		if !ok {
			err = InvalidValueError("invalid volume value: expected float")
			break
		}
		err = p.SetVolume(f)

	case "setMute":
		b, ok := putData.Value.(bool)
		if !ok {
			err = InvalidValueError("invalid mute value: expected boolean")
			break
		}
		err = p.SetMute(b)

	case "setTime":
		f, ok := putData.Value.(float64)
		if !ok {
			err = InvalidValueError("invalid time value: expected float")
			break
		}
		err = p.SetTime(f)

	default:
		err = InvalidValueError("invalid action")
		return
	}

	if err != nil {
		if err, ok := err.(InvalidValueError); ok {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		http.Error(w, fmt.Sprintf("error sending player command: %v", err), http.StatusInternalServerError)
	}
}

func playerView(p Player, w http.ResponseWriter, t *http.Request) {
	enc := json.NewEncoder(w)
	err := enc.Encode(p)
	if err != nil {
		log.Errorf("error encoding player data: %v", err)
		return
	}
}

func playersHandler(l LibraryAPI) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "" && r.Method == "POST" {
			createPlayer(l, w, r)
			return
		}

		paths := strings.Split(r.URL.Path, "/")

		if len(paths) != 1 {
			http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
			return
		}

		p := l.players.get(paths[0])
		if p == nil {
			http.Error(w, "invalid player key", http.StatusBadRequest)
			return
		}

		switch r.Method {
		case "DELETE":
			l.players.remove(paths[0])
			w.WriteHeader(http.StatusNoContent)
			return

		case "PUT":
			playerAction(p, w, r)

		case "GET":
			playerView(p, w, r)
		}
	})
}
<commit_msg>Fixed incorrect log statement<commit_after>// Copyright 2015, David Howden
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strings"
	"sync"

	"github.com/dhowden/tchaik/index"
)

type LibraryAPI struct {
	index.Library

	collections map[string]index.Collection
	filters     map[string][]index.FilterItem
	recent      []index.Path
	searcher    index.Searcher
	players     *players
}

type libraryFileSystem struct {
	http.FileSystem
	index.Library
}

// Open implements http.FileSystem and rewrites TrackID values to their corresponding Location
// values using the index.Library
func (l *libraryFileSystem) Open(path string) (http.File, error) {
	t, ok := l.Library.Track(strings.Trim(path, "/")) // TrackIDs arrive with leading slash
	if !ok {
		return nil, fmt.Errorf("could not find track: %v", path)
	}

	loc := t.GetString("Location")
	if loc == "" {
		return nil, fmt.Errorf("invalid (empty) location for track: %v", path)
	}
	return l.FileSystem.Open(loc)
}

type group struct {
	Name        string
	Key         index.Key
	TotalTime   interface{} `json:",omitempty"`
	Artist      interface{} `json:",omitempty"`
	AlbumArtist interface{} `json:",omitempty"`
	Composer    interface{} `json:",omitempty"`
	ListStyle   interface{} `json:",omitempty"`
	TrackID     interface{} `json:",omitempty"`
	Year        interface{} `json:",omitempty"`
	Groups      []group     `json:",omitempty"`
	Tracks      []track     `json:",omitempty"`
}

type track struct {
	TrackID     string `json:",omitempty"`
	Name        string `json:",omitempty"`
	Album       string `json:",omitempty"`
	Artist      string `json:",omitempty"`
	AlbumArtist string `json:",omitempty"`
	Composer    string `json:",omitempty"`
	Year        int    `json:",omitempty"`
	DiscNumber  int    `json:",omitempty"`
	TotalTime   int    `json:",omitempty"`
}

func buildCollection(h group, c index.Collection) group {
	for _, k := range c.Keys() {
		g := c.Get(k)
		g = index.FirstTrackAttr(index.StringAttr("AlbumArtist"), g)
		h.Groups = append(h.Groups, group{
			Name:        g.Name(),
			Key:         k,
			AlbumArtist: g.Field("AlbumArtist"),
		})
	}
	return h
}

func build(g index.Group, key index.Key) group {
	h := group{
		Name:        g.Name(),
		Key:         key,
		TotalTime:   g.Field("TotalTime"),
		Artist:      g.Field("Artist"),
		AlbumArtist: g.Field("AlbumArtist"),
		Composer:    g.Field("Composer"),
		Year:        g.Field("Year"),
		ListStyle:   g.Field("ListStyle"),
		TrackID:     g.Field("TrackID"),
	}

	if c, ok := g.(index.Collection); ok {
		return buildCollection(h, c)
	}

	getString := func(t index.Track, field string) string {
		if g.Field(field) != "" {
			return ""
		}
		return t.GetString(field)
	}

	getInt := func(t index.Track, field string) int {
		if g.Field(field) != 0 {
			return 0
		}
		return t.GetInt(field)
	}

	for _, t := range g.Tracks() {
		h.Tracks = append(h.Tracks, track{
			TrackID:    t.GetString("TrackID"),
			Name:       t.GetString("Name"),
			TotalTime:  t.GetInt("TotalTime"),
			DiscNumber: t.GetInt("DiscNumber"),
			// Potentially common fields (don't want to re-transmit everything)
			Album:       getString(t, "Album"),
			Artist:      getString(t, "Artist"),
			AlbumArtist: getString(t, "AlbumArtist"),
			Composer:    getString(t, "Composer"),
			Year:        getInt(t, "Year"),
		})
	}
	return h
}

func (l *LibraryAPI) Fetch(c index.Collection, path []string) (group, error) {
	if len(path) == 0 {
		return build(c, index.Key("Root")), nil
	}

	var g index.Group = c
	k := index.Key(path[0])
	g = c.Get(k)

	if g == nil {
		return group{}, fmt.Errorf("invalid path: near '%v'", path[0])
	}

	index.Sort(g.Tracks(), index.MultiSort(index.SortByInt("DiscNumber"), index.SortByInt("TrackNumber")))
	c = index.Collect(g, index.ByPrefix("Name"))
	g = index.SubTransform(c, index.TrimEnumPrefix)
	g = index.SumGroupIntAttr("TotalTime", g)
	commonFields := []index.Attr{
		index.StringAttr("Album"),
		index.StringAttr("Artist"),
		index.StringAttr("AlbumArtist"),
		index.StringAttr("Composer"),
		index.IntAttr("Year"),
	}
	g = index.CommonGroupAttr(commonFields, g)
	g = index.RemoveEmptyCollections(g)

	for i, p := range path[1:] {
		var ok bool
		c, ok = g.(index.Collection)
		if !ok {
			return group{}, fmt.Errorf("retrieved Group is not a Collection")
		}
		k = index.Key(p)
		g = c.Get(k)

		if g == nil {
			return group{}, fmt.Errorf("invalid path near '%v'", path[1:][i])
		}

		if _, ok = g.(index.Collection); !ok {
			if i == len(path[1:])-1 {
				break
			}
			return group{}, fmt.Errorf("retrieved Group isn't a Collection: %v", p)
		}
	}
	if g == nil {
		return group{}, fmt.Errorf("could not find group")
	}
	g = index.FirstTrackAttr(index.StringAttr("TrackID"), g)

	return build(g, k), nil
}

func (l *LibraryAPI) FileSystem(fs http.FileSystem) http.FileSystem {
	return &libraryFileSystem{fs, l.Library}
}

type players struct {
	sync.RWMutex
	m map[string]Player
}

func newPlayers() *players {
	return &players{m: make(map[string]Player)}
}

func (s *players) add(p Player) {
	s.Lock()
	defer s.Unlock()

	s.m[p.Key()] = p
}

func (s *players) remove(key string) {
	s.Lock()
	defer s.Unlock()

	delete(s.m, key)
}

func (s *players) get(key string) Player {
	s.RLock()
	defer s.RUnlock()

	return s.m[key]
}

func createPlayer(l LibraryAPI, w http.ResponseWriter, r *http.Request) {
	dec := json.NewDecoder(r.Body)
	defer r.Body.Close()

	postData := struct {
		Key        string
		PlayerKeys []string
	}{}
	err := dec.Decode(&postData)
	if err != nil {
		http.Error(w, fmt.Sprintf("error parsing JSON: %v", err), http.StatusBadRequest)
		return
	}

	if p := l.players.get(postData.Key); p != nil {
		http.Error(w, "player key already exists", http.StatusBadRequest)
		return
	}

	if postData.PlayerKeys == nil || len(postData.PlayerKeys) == 0 {
		http.Error(w, "no player keys specified", http.StatusBadRequest)
		return
	}

	var players []Player
	for _, pk := range postData.PlayerKeys {
		p := l.players.get(pk)
		if p == nil {
			http.Error(w, fmt.Sprintf("invalid player key: %v", pk), http.StatusBadRequest)
			return
		}
		players = append(players, p)
	}
	l.players.add(MultiPlayer(postData.Key, players...))
	w.WriteHeader(http.StatusCreated)
}

func playerAction(p Player, w http.ResponseWriter, r *http.Request) {
	dec := json.NewDecoder(r.Body)
	defer r.Body.Close()

	putData := struct {
		Action string
		Value  interface{}
	}{}
	err := dec.Decode(&putData)
	if err != nil {
		http.Error(w, fmt.Sprintf("error parsing JSON: %v", err), http.StatusBadRequest)
		return
	}

	switch putData.Action {
	case "play":
		err = p.Play()

	case "pause":
		err = p.Pause()

	case "next":
		err = p.NextTrack()

	case "prev":
		err = p.PreviousTrack()

	case "togglePlayPause":
		err = p.TogglePlayPause()

	case "toggleMute":
		err = p.ToggleMute()

	case "setVolume":
		f, ok := putData.Value.(float64)
		if !ok {
			err = InvalidValueError("invalid volume value: expected float")
			break
		}
		err = p.SetVolume(f)

	case "setMute":
		b, ok := putData.Value.(bool)
		if !ok {
			err = InvalidValueError("invalid mute value: expected boolean")
			break
		}
		err = p.SetMute(b)

	case "setTime":
		f, ok := putData.Value.(float64)
		if !ok {
			err = InvalidValueError("invalid time value: expected float")
			break
		}
		err = p.SetTime(f)

	default:
		err = InvalidValueError("invalid action")
		return
	}

	if err != nil {
		if err, ok := err.(InvalidValueError); ok {
			http.Error(w, err.Error(), http.StatusBadRequest)
			return
		}
		http.Error(w, fmt.Sprintf("error sending player command: %v", err), http.StatusInternalServerError)
	}
}

func playerView(p Player, w http.ResponseWriter, t *http.Request) {
	enc := json.NewEncoder(w)
	err := enc.Encode(p)
	if err != nil {
		log.Printf("error encoding player data: %v", err)
		return
	}
}

func playersHandler(l LibraryAPI) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "" && r.Method == "POST" {
			createPlayer(l, w, r)
			return
		}

		paths := strings.Split(r.URL.Path, "/")

		if len(paths) != 1 {
			http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
			return
		}

		p := l.players.get(paths[0])
		if p == nil {
			http.Error(w, "invalid player key", http.StatusBadRequest)
			return
		}

		switch r.Method {
		case "DELETE":
			l.players.remove(paths[0])
			w.WriteHeader(http.StatusNoContent)
			return

		case "PUT":
			playerAction(p, w, r)

		case "GET":
			playerView(p, w, r)
		}
	})
}
<|endoftext|>


<commit_before>package cephfs

import (
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/ceph/go-ceph/rados"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var (
	testMdsName = "Z"
)

func init() {
	mdsName := os.Getenv("GO_CEPH_TEST_MDS_NAME")
	if mdsName != "" {
		testMdsName = mdsName
	}
}

func TestCreateMount(t *testing.T) {
	mount, err := CreateMount()
	assert.NoError(t, err)
	assert.NotNil(t, mount)
	assert.NoError(t, mount.Release())
}

func fsConnect(t require.TestingT) *MountInfo {
	mount, err := CreateMount()
	require.NoError(t, err)
	require.NotNil(t, mount)

	err = mount.ReadDefaultConfigFile()
	require.NoError(t, err)

	timeout := time.After(time.Second * 5)
	ch := make(chan error)
	go func(mount *MountInfo) {
		ch <- mount.Mount()
	}(mount)
	select {
	case err = <-ch:
	case <-timeout:
		err = fmt.Errorf("timed out waiting for connect")
	}
	require.NoError(t, err)
	return mount
}

func fsDisconnect(t assert.TestingT, mount *MountInfo) {
	assert.NoError(t, mount.Unmount())
	assert.NoError(t, mount.Release())
}

func TestMountRoot(t *testing.T) {
	mount := fsConnect(t)
	fsDisconnect(t, mount)
}

func TestSyncFs(t *testing.T) {
	mount := fsConnect(t)
	defer fsDisconnect(t, mount)

	err := mount.SyncFs()
	assert.NoError(t, err)
}

func TestUnmountMount(t *testing.T) {
	t.Run("neverMounted", func(t *testing.T) {
		mount, err := CreateMount()
		require.NoError(t, err)
		require.NotNil(t, mount)
		assert.False(t, mount.IsMounted())
		assert.NoError(t, mount.Release())
	})
	t.Run("mountUnmount", func(t *testing.T) {
		mount := fsConnect(t)
		defer func() { assert.NoError(t, mount.Release()) }()
		assert.True(t, mount.IsMounted())

		err := mount.Unmount()
		assert.NoError(t, err)
		assert.False(t, mount.IsMounted())
	})
}

func TestReleaseMount(t *testing.T) {
	mount, err := CreateMount()
	assert.NoError(t, err)
	require.NotNil(t, mount)

	assert.NoError(t, mount.Release())
	// call release again to ensure idempotency of the func
	assert.NoError(t, mount.Release())
}

func radosConnect(t *testing.T) *rados.Conn {
	conn, err := rados.NewConn()
	require.NoError(t, err)
	err = conn.ReadDefaultConfigFile()
	require.NoError(t, err)

	timeout := time.After(time.Second * 5)
	ch := make(chan error)
	go func(conn *rados.Conn) {
		ch <- conn.Connect()
	}(conn)
	select {
	case err = <-ch:
	case <-timeout:
		err = fmt.Errorf("timed out waiting for connect")
	}
	require.NoError(t, err)
	return conn
}

func TestCreateFromRados(t *testing.T) {
	conn := radosConnect(t)
	mount, err := CreateFromRados(conn)
	assert.NoError(t, err)
	assert.NotNil(t, mount)
}

func TestCreateMountWithId(t *testing.T) {
	mount, err := CreateMountWithId("bobolink")
	assert.NoError(t, err)
	assert.NotNil(t, mount)
	defer func() { assert.NoError(t, mount.Release()) }()

	err = mount.ReadDefaultConfigFile()
	assert.NoError(t, err)

	err = mount.Mount()
	assert.NoError(t, err)
	defer func() { assert.NoError(t, mount.Unmount()) }()

	// verify the custom entity_id is visible in the 'session ls' output
	// of mds.
	cmd := []byte(`{"prefix": "session ls"}`)
	buf, info, err := mount.MdsCommand(
		testMdsName,
		[][]byte{cmd})
	assert.NoError(t, err)
	assert.NotEqual(t, "", string(buf))
	assert.Equal(t, "", string(info))
	assert.Contains(t, string(buf), `"bobolink"`)
}

func TestMountWithRoot(t *testing.T) {
	bMount := fsConnect(t)
	defer fsDisconnect(t, bMount)

	dir1 := "/test-mount-with-root"
	err := bMount.MakeDir(dir1, 0755)
	assert.NoError(t, err)
	defer bMount.RemoveDir(dir1)

	sub1 := "/i.was.here"
	dir2 := dir1 + sub1
	err = bMount.MakeDir(dir2, 0755)
	assert.NoError(t, err)
	defer bMount.RemoveDir(dir2)

	t.Run("withRoot", func(t *testing.T) {
		mount, err := CreateMount()
		require.NoError(t, err)
		require.NotNil(t, mount)
		defer func() {
			assert.NoError(t, mount.Release())
		}()

		err = mount.ReadDefaultConfigFile()
		require.NoError(t, err)

		err = mount.MountWithRoot(dir1)
		assert.NoError(t, err)
		defer func() {
			assert.NoError(t, mount.Unmount())
		}()

		err = mount.ChangeDir(sub1)
		assert.NoError(t, err)
	})
	t.Run("badRoot", func(t *testing.T) {
		mount, err := CreateMount()
		require.NoError(t, err)
		require.NotNil(t, mount)
		defer func() {
			assert.NoError(t, mount.Release())
		}()

		err = mount.ReadDefaultConfigFile()
		require.NoError(t, err)

		err = mount.MountWithRoot("/i-yam-what-i-yam")
		assert.Error(t, err)
	})
}

func TestGetSetConfigOption(t *testing.T) {
	// we don't need an active connection for this, just the handle
	mount, err := CreateMount()
	require.NoError(t, err)
	require.NotNil(t, mount)
	defer func() { assert.NoError(t, mount.Release()) }()

	err = mount.SetConfigOption("__dne__", "value")
	assert.Error(t, err)
	_, err = mount.GetConfigOption("__dne__")
	assert.Error(t, err)

	origVal, err := mount.GetConfigOption("log_file")
	assert.NoError(t, err)

	err = mount.SetConfigOption("log_file", "/dev/null")
	assert.NoError(t, err)
	currVal, err := mount.GetConfigOption("log_file")
	assert.NoError(t, err)
	assert.Equal(t, "/dev/null", currVal)

	err = mount.SetConfigOption("log_file", origVal)
	assert.NoError(t, err)
	currVal, err = mount.GetConfigOption("log_file")
	assert.NoError(t, err)
	assert.Equal(t, origVal, currVal)
}

func TestParseConfigArgv(t *testing.T) {
	mount, err := CreateMount()
	require.NoError(t, err)
	require.NotNil(t, mount)
	defer func() { assert.NoError(t, mount.Release()) }()

	origVal, err := mount.GetConfigOption("log_file")
	assert.NoError(t, err)

	err = mount.ParseConfigArgv(
		[]string{"cephfs.test", "--log_file", "/dev/null"})
	assert.NoError(t, err)

	currVal, err := mount.GetConfigOption("log_file")
	assert.NoError(t, err)
	assert.Equal(t, "/dev/null", currVal)
	assert.NotEqual(t, "/dev/null", origVal)

	// ensure that an empty slice triggers an error (not a crash)
	err = mount.ParseConfigArgv([]string{})
	assert.Error(t, err)

	// ensure we get an error for an invalid mount value
	badMount := &MountInfo{}
	err = badMount.ParseConfigArgv(
		[]string{"cephfs.test", "--log_file", "/dev/null"})
	assert.Error(t, err)
}

func TestParseDefaultConfigEnv(t *testing.T) {
	mount, err := CreateMount()
	require.NoError(t, err)
	require.NotNil(t, mount)
	defer func() { assert.NoError(t, mount.Release()) }()

	origVal, err := mount.GetConfigOption("log_file")
	assert.NoError(t, err)

	err = os.Setenv("CEPH_ARGS", "--log_file /dev/null")
	assert.NoError(t, err)
	err = mount.ParseDefaultConfigEnv()
	assert.NoError(t, err)

	currVal, err := mount.GetConfigOption("log_file")
	assert.NoError(t, err)
	assert.Equal(t, "/dev/null", currVal)
	assert.NotEqual(t, "/dev/null", origVal)
}

func TestValidate(t *testing.T) {
	mount, err := CreateMount()
	assert.NoError(t, err)
	assert.NotNil(t, mount)
	defer assert.NoError(t, mount.Release())

	t.Run("mountCurrentDir", func(t *testing.T) {
		path := mount.CurrentDir()
		assert.Equal(t, path, "")
	})

	t.Run("mountChangeDir", func(t *testing.T) {
		err := mount.ChangeDir("someDir")
		assert.Error(t, err)
		assert.Equal(t, err, ErrNotConnected)
	})

	t.Run("mountMakeDir", func(t *testing.T) {
		err := mount.MakeDir("someName", 0444)
		assert.Error(t, err)
		assert.Equal(t, err, ErrNotConnected)
	})

	t.Run("mountRemoveDir", func(t *testing.T) {
		err := mount.RemoveDir("someDir")
		assert.Error(t, err)
		assert.Equal(t, err, ErrNotConnected)
	})

	t.Run("mountLink", func(t *testing.T) {
		err := mount.Link("/", "/")
		assert.Error(t, err)
		assert.Equal(t, err, ErrNotConnected)
	})

	t.Run("mountUnlink", func(t *testing.T) {
		err := mount.Unlink("someFile")
		assert.Error(t, err)
		assert.Equal(t, err, ErrNotConnected)
	})

	t.Run("mountSymlink", func(t *testing.T) {
		err := mount.Symlink("/", "/")
		assert.Error(t, err)
		assert.Equal(t, err, ErrNotConnected)
	})

	t.Run("mountReadlink", func(t *testing.T) {
		_, err := mount.Readlink("somePath")
		assert.Error(t, err)
		assert.Equal(t, err, ErrNotConnected)
	})
}
<commit_msg>cephfs: add test case for ReadConfigFile<commit_after>package cephfs

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"testing"
	"time"

	"github.com/ceph/go-ceph/rados"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

var (
	testMdsName = "Z"
)

func init() {
	mdsName := os.Getenv("GO_CEPH_TEST_MDS_NAME")
	if mdsName != "" {
		testMdsName = mdsName
	}
}

func TestCreateMount(t *testing.T) {
	mount, err := CreateMount()
	assert.NoError(t, err)
	assert.NotNil(t, mount)
	assert.NoError(t, mount.Release())
}

func fsConnect(t require.TestingT) *MountInfo {
	mount, err := CreateMount()
	require.NoError(t, err)
	require.NotNil(t, mount)

	err = mount.ReadDefaultConfigFile()
	require.NoError(t, err)

	timeout := time.After(time.Second * 5)
	ch := make(chan error)
	go func(mount *MountInfo) {
		ch <- mount.Mount()
	}(mount)
	select {
	case err = <-ch:
	case <-timeout:
		err = fmt.Errorf("timed out waiting for connect")
	}
	require.NoError(t, err)
	return mount
}

func fsDisconnect(t assert.TestingT, mount *MountInfo) {
	assert.NoError(t, mount.Unmount())
	assert.NoError(t, mount.Release())
}

func TestMountRoot(t *testing.T) {
	mount := fsConnect(t)
	fsDisconnect(t, mount)
}

func TestSyncFs(t *testing.T) {
	mount := fsConnect(t)
	defer fsDisconnect(t, mount)

	err := mount.SyncFs()
	assert.NoError(t, err)
}

func TestUnmountMount(t *testing.T) {
	t.Run("neverMounted", func(t *testing.T) {
		mount, err := CreateMount()
		require.NoError(t, err)
		require.NotNil(t, mount)
		assert.False(t, mount.IsMounted())
		assert.NoError(t, mount.Release())
	})
	t.Run("mountUnmount", func(t *testing.T) {
		mount := fsConnect(t)
		defer func() { assert.NoError(t, mount.Release()) }()
		assert.True(t, mount.IsMounted())

		err := mount.Unmount()
		assert.NoError(t, err)
		assert.False(t, mount.IsMounted())
	})
}

func TestReleaseMount(t *testing.T) {
	mount, err := CreateMount()
	assert.NoError(t, err)
	require.NotNil(t, mount)

	assert.NoError(t, mount.Release())
	// call release again to ensure idempotency of the func
	assert.NoError(t, mount.Release())
}

func radosConnect(t *testing.T) *rados.Conn {
	conn, err := rados.NewConn()
	require.NoError(t, err)
	err = conn.ReadDefaultConfigFile()
	require.NoError(t, err)

	timeout := time.After(time.Second * 5)
	ch := make(chan error)
	go func(conn *rados.Conn) {
		ch <- conn.Connect()
	}(conn)
	select {
	case err = <-ch:
	case <-timeout:
		err = fmt.Errorf("timed out waiting for connect")
	}
	require.NoError(t, err)
	return conn
}

func TestCreateFromRados(t *testing.T) {
	conn := radosConnect(t)
	mount, err := CreateFromRados(conn)
	assert.NoError(t, err)
	assert.NotNil(t, mount)
}

func TestCreateMountWithId(t *testing.T) {
	mount, err := CreateMountWithId("bobolink")
	assert.NoError(t, err)
	assert.NotNil(t, mount)
	defer func() { assert.NoError(t, mount.Release()) }()

	err = mount.ReadDefaultConfigFile()
	assert.NoError(t, err)

	err = mount.Mount()
	assert.NoError(t, err)
	defer func() { assert.NoError(t, mount.Unmount()) }()

	// verify the custom entity_id is visible in the 'session ls' output
	// of mds.
	cmd := []byte(`{"prefix": "session ls"}`)
	buf, info, err := mount.MdsCommand(
		testMdsName,
		[][]byte{cmd})
	assert.NoError(t, err)
	assert.NotEqual(t, "", string(buf))
	assert.Equal(t, "", string(info))
	assert.Contains(t, string(buf), `"bobolink"`)
}

func TestMountWithRoot(t *testing.T) {
	bMount := fsConnect(t)
	defer fsDisconnect(t, bMount)

	dir1 := "/test-mount-with-root"
	err := bMount.MakeDir(dir1, 0755)
	assert.NoError(t, err)
	defer bMount.RemoveDir(dir1)

	sub1 := "/i.was.here"
	dir2 := dir1 + sub1
	err = bMount.MakeDir(dir2, 0755)
	assert.NoError(t, err)
	defer bMount.RemoveDir(dir2)

	t.Run("withRoot", func(t *testing.T) {
		mount, err := CreateMount()
		require.NoError(t, err)
		require.NotNil(t, mount)
		defer func() {
			assert.NoError(t, mount.Release())
		}()

		err = mount.ReadDefaultConfigFile()
		require.NoError(t, err)

		err = mount.MountWithRoot(dir1)
		assert.NoError(t, err)
		defer func() {
			assert.NoError(t, mount.Unmount())
		}()

		err = mount.ChangeDir(sub1)
		assert.NoError(t, err)
	})
	t.Run("badRoot", func(t *testing.T) {
		mount, err := CreateMount()
		require.NoError(t, err)
		require.NotNil(t, mount)
		defer func() {
			assert.NoError(t, mount.Release())
		}()

		err = mount.ReadDefaultConfigFile()
		require.NoError(t, err)

		err = mount.MountWithRoot("/i-yam-what-i-yam")
		assert.Error(t, err)
	})
}

func TestGetSetConfigOption(t *testing.T) {
	// we don't need an active connection for this, just the handle
	mount, err := CreateMount()
	require.NoError(t, err)
	require.NotNil(t, mount)
	defer func() { assert.NoError(t, mount.Release()) }()

	err = mount.SetConfigOption("__dne__", "value")
	assert.Error(t, err)
	_, err = mount.GetConfigOption("__dne__")
	assert.Error(t, err)

	origVal, err := mount.GetConfigOption("log_file")
	assert.NoError(t, err)

	err = mount.SetConfigOption("log_file", "/dev/null")
	assert.NoError(t, err)
	currVal, err := mount.GetConfigOption("log_file")
	assert.NoError(t, err)
	assert.Equal(t, "/dev/null", currVal)

	err = mount.SetConfigOption("log_file", origVal)
	assert.NoError(t, err)
	currVal, err = mount.GetConfigOption("log_file")
	assert.NoError(t, err)
	assert.Equal(t, origVal, currVal)
}

func TestParseConfigArgv(t *testing.T) {
	mount, err := CreateMount()
	require.NoError(t, err)
	require.NotNil(t, mount)
	defer func() { assert.NoError(t, mount.Release()) }()

	origVal, err := mount.GetConfigOption("log_file")
	assert.NoError(t, err)

	err = mount.ParseConfigArgv(
		[]string{"cephfs.test", "--log_file", "/dev/null"})
	assert.NoError(t, err)

	currVal, err := mount.GetConfigOption("log_file")
	assert.NoError(t, err)
	assert.Equal(t, "/dev/null", currVal)
	assert.NotEqual(t, "/dev/null", origVal)

	// ensure that an empty slice triggers an error (not a crash)
	err = mount.ParseConfigArgv([]string{})
	assert.Error(t, err)

	// ensure we get an error for an invalid mount value
	badMount := &MountInfo{}
	err = badMount.ParseConfigArgv(
		[]string{"cephfs.test", "--log_file", "/dev/null"})
	assert.Error(t, err)
}

func TestParseDefaultConfigEnv(t *testing.T) {
	mount, err := CreateMount()
	require.NoError(t, err)
	require.NotNil(t, mount)
	defer func() { assert.NoError(t, mount.Release()) }()

	origVal, err := mount.GetConfigOption("log_file")
	assert.NoError(t, err)

	err = os.Setenv("CEPH_ARGS", "--log_file /dev/null")
	assert.NoError(t, err)
	err = mount.ParseDefaultConfigEnv()
	assert.NoError(t, err)

	currVal, err := mount.GetConfigOption("log_file")
	assert.NoError(t, err)
	assert.Equal(t, "/dev/null", currVal)
	assert.NotEqual(t, "/dev/null", origVal)
}

func TestValidate(t *testing.T) {
	mount, err := CreateMount()
	assert.NoError(t, err)
	assert.NotNil(t, mount)
	defer assert.NoError(t, mount.Release())

	t.Run("mountCurrentDir", func(t *testing.T) {
		path := mount.CurrentDir()
		assert.Equal(t, path, "")
	})

	t.Run("mountChangeDir", func(t *testing.T) {
		err := mount.ChangeDir("someDir")
		assert.Error(t, err)
		assert.Equal(t, err, ErrNotConnected)
	})

	t.Run("mountMakeDir", func(t *testing.T) {
		err := mount.MakeDir("someName", 0444)
		assert.Error(t, err)
		assert.Equal(t, err, ErrNotConnected)
	})

	t.Run("mountRemoveDir", func(t *testing.T) {
		err := mount.RemoveDir("someDir")
		assert.Error(t, err)
		assert.Equal(t, err, ErrNotConnected)
	})

	t.Run("mountLink", func(t *testing.T) {
		err := mount.Link("/", "/")
		assert.Error(t, err)
		assert.Equal(t, err, ErrNotConnected)
	})

	t.Run("mountUnlink", func(t *testing.T) {
		err := mount.Unlink("someFile")
		assert.Error(t, err)
		assert.Equal(t, err, ErrNotConnected)
	})

	t.Run("mountSymlink", func(t *testing.T) {
		err := mount.Symlink("/", "/")
		assert.Error(t, err)
		assert.Equal(t, err, ErrNotConnected)
	})

	t.Run("mountReadlink", func(t *testing.T) {
		_, err := mount.Readlink("somePath")
		assert.Error(t, err)
		assert.Equal(t, err, ErrNotConnected)
	})
}

func TestReadConfigFile(t *testing.T) {
	file, err := ioutil.TempFile("/tmp", "cephfs.conf")
	require.NoError(t, err)
	defer func() {
		assert.NoError(t, file.Close())
		assert.NoError(t, os.Remove(file.Name()))
	}()
	_, err = io.WriteString(
		file, "[global]\nfsid = 04862775-14d5-46e0-a015-000000000000\n")
	require.NoError(t, err)

	mount, err := CreateMount()
	require.NoError(t, err)
	require.NotNil(t, mount)

	err = mount.ReadConfigFile(file.Name())
	require.NoError(t, err)

	v, err := mount.GetConfigOption("fsid")
	assert.NoError(t, err)
	assert.Equal(t, "04862775-14d5-46e0-a015-000000000000", v)
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Only build with go1.7 or above because b.Run did not exist before.
// +build go1.7

package main

import (
	"flag"
	"path/filepath"
	"testing"

	"github.com/google/mtail/mtail"
	"github.com/google/mtail/watcher"
)

var (
	recordBenchmark = flag.Bool("record_benchmark", false, "Record the benchmark results to 'benchmark_results.csv'.")
)

func BenchmarkProgram(b *testing.B) {
	// exampleProgramTests live in ex_test.go
	for _, bm := range exampleProgramTests {
		prog := filepath.Base(bm.programfile)
		log := filepath.Base(bm.logfile)
		b.Run(prog+"+"+log, func(b *testing.B) {
			b.ReportAllocs()

			w := watcher.NewFakeWatcher()
			o := mtail.Options{Progs: bm.programfile, W: w}
			mtail, err := mtail.New(o)
			if err != nil {
				b.Fatalf("Failed to create mtail: %s", err)
			}

			var lines int64
			b.ResetTimer()
			for i := 0; i < b.N; i++ {
				count, err := mtail.OneShot(bm.logfile, false)
				if err != nil {
					b.Errorf("OneShot log parse failed: %s", err)
				}
				lines += count
			}
			b.StopTimer()
			mtail.Close()
			if err != nil {
				b.Fatalf("strconv.ParseInt failed: %s", err)
				return
			}
			b.SetBytes(lines)
		})
	}
}
<commit_msg>Change the names of the benchmarks to the same style as in ex_test.<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Only build with go1.7 or above because b.Run did not exist before.
// +build go1.7

package main

import (
	"flag"
	"fmt"
	"testing"

	"github.com/google/mtail/mtail"
	"github.com/google/mtail/watcher"
)

var (
	recordBenchmark = flag.Bool("record_benchmark", false, "Record the benchmark results to 'benchmark_results.csv'.")
)

func BenchmarkProgram(b *testing.B) {
	// exampleProgramTests live in ex_test.go
	for _, bm := range exampleProgramTests {
		b.Run(fmt.Sprintf("%s on %s", bm.programfile, bm.logfile), func(b *testing.B) {
			b.ReportAllocs()

			w := watcher.NewFakeWatcher()
			o := mtail.Options{Progs: bm.programfile, W: w}
			mtail, err := mtail.New(o)
			if err != nil {
				b.Fatalf("Failed to create mtail: %s", err)
			}

			var lines int64
			b.ResetTimer()
			for i := 0; i < b.N; i++ {
				count, err := mtail.OneShot(bm.logfile, false)
				if err != nil {
					b.Errorf("OneShot log parse failed: %s", err)
				}
				lines += count
			}
			b.StopTimer()
			mtail.Close()
			if err != nil {
				b.Fatalf("strconv.ParseInt failed: %s", err)
				return
			}
			// The bytes recorded is really the number of lines read.
			b.SetBytes(lines)
		})
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"sync/atomic"
	"testing"

	"github.com/bmatsuo/lmdb-go/lmdb"
	"github.com/dgraph-io/badger"
	"github.com/dgraph-io/badger-bench/store"
	"github.com/dgraph-io/badger/table"
	"github.com/dgraph-io/badger/y"
)

var (
	ctx           = context.Background()
	numKeys       = flag.Float64("keys_mil", 10.0, "How many million keys to write.")
	flagDir       = flag.String("dir", "bench-tmp", "Where data is temporarily stored.")
	flagValueSize = flag.Int("valsz", 128, "Size of each value.")
)

const Mi int = 1000000
const Mf float64 = 1000000

func getBadger() (*badger.KV, error) {
	opt := badger.DefaultOptions
	opt.MapTablesTo = table.LoadToRAM
	opt.Dir = *flagDir + "/badger"
	opt.ValueDir = opt.Dir
	fmt.Println(opt.Dir)
	opt.DoNotCompact = true
	opt.ValueGCThreshold = 0.0
	return badger.NewKV(&opt)
}

func getRocks() *store.Store {
	rdb, err := store.NewReadOnlyStore(*flagDir + "/rocks")
	y.Check(err)
	return rdb
}

func getLmdb() *lmdb.Env {
	lmdbEnv, err := lmdb.NewEnv()
	y.Check(err)
	err = lmdbEnv.SetMaxDBs(1)
	y.Check(err)
	err = lmdbEnv.SetMapSize(1 << 38) // ~273Gb
	y.Check(err)

	err = lmdbEnv.Open(*flagDir+"/lmdb", lmdb.Readonly, 0777)
	y.Check(err)
	return lmdbEnv
}

func newKey() []byte {
	k := rand.Int() % int(*numKeys*Mf)
	key := fmt.Sprintf("vsz=%05d-k=%010d", *flagValueSize, k) // 22 bytes.
	return []byte(key)
}

func print(count int) {
	if count%100000 == 0 {
		fmt.Printf(".")
	} else if count%Mi == 0 {
		fmt.Printf("-")
	}
}

func BenchmarkReadRandomBadger(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	defer bdb.Close()

	var maxCount uint64
	b.Run("read-random-badger", func(b *testing.B) {
		var totalCount uint64
		b.RunParallel(func(pb *testing.PB) {
			var count uint64
			var val badger.KVItem
			for pb.Next() {
				key := newKey()
				if err := bdb.Get(key, &val); err == nil && val.Value() != nil {
					if len(val.Value()) != *flagValueSize {
						b.Logf("Value size for key %s - %d != %d", string(key), len(val.Value()), *flagValueSize)
						b.FailNow()
					}
					count++
				}
			}
			atomic.AddUint64(&totalCount, count)
		})
		if totalCount > maxCount {
			maxCount = totalCount
		}
	})
	b.Logf("badger: Maximum no. of keys found: %d", maxCount)
}

func BenchmarkReadRandomRocks(b *testing.B) {
	rdb := getRocks()
	defer rdb.Close()

	var maxCount uint64
	b.Run("read-random-rocks", func(b *testing.B) {
		var totalCount uint64
		b.RunParallel(func(pb *testing.PB) {
			var count uint64
			for pb.Next() {
				key := newKey()
				if v, err := rdb.Get(key); err == nil {

					if len(v.Data()) != *flagValueSize {
						b.Logf("Value size %d != %d", len(v.Data()), *flagValueSize)
						b.FailNow()
					}
					count++
				}
			}
			atomic.AddUint64(&totalCount, count)
		})
		if totalCount > maxCount {
			maxCount = totalCount
		}
	})
	b.Logf("rocks: Maximum no. of keys found: %d", maxCount)
}

func BenchmarkReadRandomLmdb(b *testing.B) {
	lmdbEnv := getLmdb()
	defer lmdbEnv.Close()

	var lmdbDBI lmdb.DBI
	// Acquire handle
	err := lmdbEnv.View(func(txn *lmdb.Txn) error {
		var err error
		lmdbDBI, err = txn.OpenDBI("bench", 0)
		return err
	})
	y.Check(err)
	defer lmdbEnv.CloseDBI(lmdbDBI)

	var maxCount uint64
	b.Run("read-random-lmdb", func(b *testing.B) {
		var totalCount uint64
		b.RunParallel(func(pb *testing.PB) {
			var count uint64
			for pb.Next() {
				key := newKey()
				_ = lmdbEnv.View(func(txn *lmdb.Txn) error {
					v, err := txn.Get(lmdbDBI, key)
					if err != nil {
						return err
					}
					if len(v) != *flagValueSize {
						b.Logf("Value size %d != %d", v, *flagValueSize)
						b.FailNow()
					}
					count++
					return nil
				})
			}
			atomic.AddUint64(&totalCount, count)
		})
		if totalCount > maxCount {
			maxCount = totalCount
		}
	})
	b.Logf("lmdb: Maximum no. of keys found: %d", maxCount)
}

func safecopy(dst []byte, src []byte) []byte {
	if cap(dst) < len(src) {
		dst = make([]byte, len(src))
	}
	dst = dst[0:len(src)]
	copy(dst, src)
	return dst
}

func BenchmarkIterateRocks(b *testing.B) {
	rdb := getRocks()
	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("rocksdb-iterate", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			itr := rdb.NewIterator()
			var count int
			for itr.SeekToFirst(); itr.Valid(); itr.Next() {
				{
					// do some processing.
					k = safecopy(k, itr.Key().Data())
					v = safecopy(v, itr.Value().Data())
				}
				if len(v) != *flagValueSize {
					b.Logf("Value size %d != %d", v, *flagValueSize)
					b.FailNow()
				}

				count++
				print(count)
				if count > 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateLmdb(b *testing.B) {
	lmdbEnv := getLmdb()

	var lmdbDBI lmdb.DBI
	// Acquire handle
	err := lmdbEnv.View(func(txn *lmdb.Txn) error {
		var err error
		lmdbDBI, err = txn.OpenDBI("bench", 0)
		return err
	})
	y.Check(err)
	defer lmdbEnv.CloseDBI(lmdbDBI)

	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("lmdb-iterate", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			err = lmdbEnv.View(func(txn *lmdb.Txn) error {
				cur, err := txn.OpenCursor(lmdbDBI)
				if err != nil {
					return err
				}
				defer cur.Close()

				for {
					k1, v1, err := cur.Get(nil, nil, lmdb.Next)
					if lmdb.IsNotFound(err) {
						return nil
					}
					if err != nil {
						return err
					}
					if len(v1) != *flagValueSize {
						b.Logf("Value size %d != %d", v, *flagValueSize)
						b.FailNow()
					}

					//fmt.Printf("%s %s\n", k, v)

					// do some processing.
					k = safecopy(k, k1)
					v = safecopy(v, v1)

					count++
					print(count)
					if count > 2*Mi {
						break
					}
				}
				return nil
			})
			y.Check(err)
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateBadgerOnlyKeys(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	k := make([]byte, 1024)
	b.ResetTimer()

	b.Run("badger-iterate-onlykeys", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			// 100 = size, 0 = num workers, false = fwd direction.
			opt := badger.IteratorOptions{}
			opt.PrefetchSize = 10000
			itr := bdb.NewIterator(opt)
			for itr.Rewind(); itr.Valid(); itr.Next() {
				item := itr.Item()
				{
					// do some processing.
					k = safecopy(k, item.Key())
				}
				count++
				print(count)
				if count > 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateBadgerWithValues(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("badger-iterate-withvals", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			opt := badger.IteratorOptions{}
			opt.PrefetchSize = 10000
			opt.FetchValues = true
			itr := bdb.NewIterator(opt)
			for itr.Rewind(); itr.Valid(); itr.Next() {
				item := itr.Item()
				{
					// do some processing.
					k = safecopy(k, item.Key())
					v = safecopy(v, item.Value())
				}
				if len(v) != *flagValueSize {
					b.Logf("Value size %d != %d", v, *flagValueSize)
					b.FailNow()
				}

				count++
				print(count)
				if count >= 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func TestMain(m *testing.M) {
	flag.Parse()
	// call flag.Parse() here if TestMain uses flags
	go http.ListenAndServe(":8080", nil)
	os.Exit(m.Run())
}
<commit_msg>Clean up after debugging session<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"math/rand"
	"net/http"
	"os"
	"sync/atomic"
	"testing"

	"github.com/bmatsuo/lmdb-go/lmdb"
	"github.com/dgraph-io/badger"
	"github.com/dgraph-io/badger-bench/store"
	"github.com/dgraph-io/badger/table"
	"github.com/dgraph-io/badger/y"
)

var (
	ctx           = context.Background()
	numKeys       = flag.Float64("keys_mil", 10.0, "How many million keys to write.")
	flagDir       = flag.String("dir", "bench-tmp", "Where data is temporarily stored.")
	flagValueSize = flag.Int("valsz", 128, "Size of each value.")
)

const Mi int = 1000000
const Mf float64 = 1000000

func getBadger() (*badger.KV, error) {
	opt := badger.DefaultOptions
	opt.MapTablesTo = table.LoadToRAM
	opt.Dir = *flagDir + "/badger"
	opt.ValueDir = opt.Dir
	fmt.Println(opt.Dir)
	opt.DoNotCompact = true
	opt.ValueGCThreshold = 0.0
	return badger.NewKV(&opt)
}

func getRocks() *store.Store {
	rdb, err := store.NewReadOnlyStore(*flagDir + "/rocks")
	y.Check(err)
	return rdb
}

func getLmdb() *lmdb.Env {
	lmdbEnv, err := lmdb.NewEnv()
	y.Check(err)
	err = lmdbEnv.SetMaxDBs(1)
	y.Check(err)
	err = lmdbEnv.SetMapSize(1 << 38) // ~273Gb
	y.Check(err)

	err = lmdbEnv.Open(*flagDir+"/lmdb", lmdb.Readonly, 0777)
	y.Check(err)
	return lmdbEnv
}

func newKey() []byte {
	k := rand.Int() % int(*numKeys*Mf)
	key := fmt.Sprintf("vsz=%05d-k=%010d", *flagValueSize, k) // 22 bytes.
	return []byte(key)
}

func print(count int) {
	if count%100000 == 0 {
		fmt.Printf(".")
	} else if count%Mi == 0 {
		fmt.Printf("-")
	}
}

func BenchmarkReadRandomBadger(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	defer bdb.Close()

	var totalFound uint64
	var totalErr uint64
	var totalNotFound uint64
	b.Run("read-random-badger", func(b *testing.B) {
		b.RunParallel(func(pb *testing.PB) {
			var found, error_, notFound uint64
			for pb.Next() {
				key := newKey()
				var val badger.KVItem
				if err := bdb.Get(key, &val); err == nil && val.Value() != nil {
					found++
				} else if err != nil {
					error_++
				} else {
					notFound++
				}
			}
			atomic.AddUint64(&totalFound, found)
			atomic.AddUint64(&totalErr, error_)
			atomic.AddUint64(&totalNotFound, notFound)
		})
	})
	b.Logf("badger %d keys had valid values.", totalFound)
	b.Logf("badger %d keys had no values", totalNotFound)
	b.Logf("badger %d keys had errors", totalErr)
	b.Logf("badger hit rate : %.2f: ", float64(totalFound)/float64(totalFound+totalNotFound+totalErr))
}

func BenchmarkReadRandomRocks(b *testing.B) {
	rdb := getRocks()
	defer rdb.Close()

	var totalCount uint64
	b.Run("read-random-rocks", func(b *testing.B) {
		b.RunParallel(func(pb *testing.PB) {
			var count uint64
			for pb.Next() {
				key := newKey()
				if _, err := rdb.Get(key); err == nil {
					count++
				}
			}
			atomic.AddUint64(&totalCount, count)
		})
	})
	b.Logf("rocks %d keys had valid values.", totalCount)
}

func BenchmarkReadRandomLmdb(b *testing.B) {
	lmdbEnv := getLmdb()
	defer lmdbEnv.Close()

	var lmdbDBI lmdb.DBI
	// Acquire handle
	err := lmdbEnv.View(func(txn *lmdb.Txn) error {
		var err error
		lmdbDBI, err = txn.OpenDBI("bench", 0)
		return err
	})
	y.Check(err)
	defer lmdbEnv.CloseDBI(lmdbDBI)

	var totalFound uint64
	var totalErr uint64
	var totalNotFound uint64
	b.Run("read-random-lmdb", func(b *testing.B) {
		b.RunParallel(func(pb *testing.PB) {
			var found, error_, notFound uint64

			for pb.Next() {
				key := newKey()
				_ = lmdbEnv.View(func(txn *lmdb.Txn) error {
					_, err := txn.Get(lmdbDBI, key)
					if lmdb.IsNotFound(err) {
						notFound++
						return nil

					} else if err != nil {
						error_++
						return err
					}
					found++
					return nil
				})
			}
			atomic.AddUint64(&totalFound, found)
			atomic.AddUint64(&totalErr, error_)
			atomic.AddUint64(&totalNotFound, notFound)
		})
	})
	b.Logf("lmdb %d keys had valid values.", totalFound)
	b.Logf("lmdb %d keys had no values", totalNotFound)
	b.Logf("lmdb %d keys had errors", totalErr)
	b.Logf("lmdb hit rate : %.2f", float64(totalFound)/float64(totalFound+totalNotFound+totalErr))
}

func safecopy(dst []byte, src []byte) []byte {
	if cap(dst) < len(src) {
		dst = make([]byte, len(src))
	}
	dst = dst[0:len(src)]
	copy(dst, src)
	return dst
}

func BenchmarkIterateRocks(b *testing.B) {
	rdb := getRocks()
	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("rocksdb-iterate", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			itr := rdb.NewIterator()
			var count int
			for itr.SeekToFirst(); itr.Valid(); itr.Next() {
				{
					// do some processing.
					k = safecopy(k, itr.Key().Data())
					v = safecopy(v, itr.Value().Data())
				}
				count++
				print(count)
				if count > 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateLmdb(b *testing.B) {
	lmdbEnv := getLmdb()

	var lmdbDBI lmdb.DBI
	// Acquire handle
	err := lmdbEnv.View(func(txn *lmdb.Txn) error {
		var err error
		lmdbDBI, err = txn.OpenDBI("bench", 0)
		return err
	})
	y.Check(err)
	defer lmdbEnv.CloseDBI(lmdbDBI)

	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("lmdb-iterate", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			err = lmdbEnv.View(func(txn *lmdb.Txn) error {
				cur, err := txn.OpenCursor(lmdbDBI)
				if err != nil {
					return err
				}
				defer cur.Close()

				for {
					k1, v1, err := cur.Get(nil, nil, lmdb.Next)
					if lmdb.IsNotFound(err) {
						return nil
					}
					if err != nil {
						return err
					}

					//fmt.Printf("%s %s\n", k, v)

					// do some processing.
					k = safecopy(k, k1)
					v = safecopy(v, v1)

					count++
					print(count)
					if count > 2*Mi {
						break
					}
				}
				return nil
			})
			y.Check(err)
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateBadgerOnlyKeys(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	k := make([]byte, 1024)
	b.ResetTimer()

	b.Run("badger-iterate-onlykeys", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			// 100 = size, 0 = num workers, false = fwd direction.
			opt := badger.IteratorOptions{}
			opt.PrefetchSize = 10000
			itr := bdb.NewIterator(opt)
			for itr.Rewind(); itr.Valid(); itr.Next() {
				item := itr.Item()
				{
					// do some processing.
					k = safecopy(k, item.Key())
				}
				count++
				print(count)
				if count > 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func BenchmarkIterateBadgerWithValues(b *testing.B) {
	bdb, err := getBadger()
	y.Check(err)
	k := make([]byte, 1024)
	v := make([]byte, Mi)
	b.ResetTimer()

	b.Run("badger-iterate-withvals", func(b *testing.B) {
		for j := 0; j < b.N; j++ {
			var count int
			opt := badger.IteratorOptions{}
			opt.PrefetchSize = 10000
			opt.FetchValues = true
			itr := bdb.NewIterator(opt)
			for itr.Rewind(); itr.Valid(); itr.Next() {
				item := itr.Item()
				{
					// do some processing.
					k = safecopy(k, item.Key())
					v = safecopy(v, item.Value())
				}
				count++
				print(count)
				if count >= 2*Mi {
					break
				}
			}
			b.Logf("[%d] Counted %d keys\n", j, count)
		}
	})
}

func TestMain(m *testing.M) {
	flag.Parse()
	// call flag.Parse() here if TestMain uses flags
	go http.ListenAndServe(":8080", nil)
	os.Exit(m.Run())
}
<|endoftext|>


<commit_before>// Copyright 2014 <chaishushan{AT}gmail.com>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package webp

import (
	"bytes"
	"testing"
)

func BenchmarkEncodeAndDecode(b *testing.B) {
	var buf bytes.Buffer

	img, err := loadImage("1_webp_ll.png")
	if err != nil {
		b.Fatal(err)
	}
	s := img.Bounds().Size()
	b.SetBytes(int64(s.X * s.Y * 4))

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if err = Encode(&buf, img, nil); err != nil {
			b.Fatal(err)
		}
		if _, err = Decode(&buf); err != nil {
			b.Fatal(err)
		}
	}
}
<commit_msg>more benchmark<commit_after>// Copyright 2014 <chaishushan{AT}gmail.com>. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package webp

import (
	"bytes"
	"io/ioutil"
	"testing"

	xwebp "golang.org/x/image/webp"
)

func BenchmarkGetInfo(b *testing.B) {
	data, err := ioutil.ReadFile("./testdata/1_webp_ll.webp")
	if err != nil {
		b.Fatal(err)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		GetInfoEx(data, nil)
	}
	b.StopTimer()
}

func BenchmarkGetInfoEx(b *testing.B) {
	data, err := ioutil.ReadFile("./testdata/1_webp_ll.webp")
	if err != nil {
		b.Fatal(err)
	}
	cbuf := NewCBuffer(len(data))
	copy(cbuf.CData(), data)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		GetInfoEx(cbuf.CData(), cbuf)
	}
	b.StopTimer()
}

func BenchmarkDecodeGray(b *testing.B) {
	data, err := ioutil.ReadFile("./testdata/1_webp_ll.webp")
	if err != nil {
		b.Fatal(err)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		m, err := DecodeGray(data)
		if err != nil {
			b.Fatal(err)
		}
		_ = m
	}
	b.StopTimer()
}

func BenchmarkDecodeGrayEx(b *testing.B) {
	data, err := ioutil.ReadFile("./testdata/1_webp_ll.webp")
	if err != nil {
		b.Fatal(err)
	}
	cbuf := NewCBuffer(len(data))
	copy(cbuf.CData(), data)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		m, pix, err := DecodeGrayEx(cbuf.CData(), cbuf)
		if err != nil {
			b.Fatal(err)
		}
		_ = m
		b.StopTimer()
		pix.Close()
		b.StartTimer()
	}
	b.StopTimer()
}

func BenchmarkDecodeRGB(b *testing.B) {
	data, err := ioutil.ReadFile("./testdata/1_webp_ll.webp")
	if err != nil {
		b.Fatal(err)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		m, err := DecodeRGB(data)
		if err != nil {
			b.Fatal(err)
		}
		_ = m
	}
	b.StopTimer()
}

func BenchmarkDecodeRGBEx(b *testing.B) {
	data, err := ioutil.ReadFile("./testdata/1_webp_ll.webp")
	if err != nil {
		b.Fatal(err)
	}
	cbuf := NewCBuffer(len(data))
	copy(cbuf.CData(), data)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		m, pix, err := DecodeRGBEx(cbuf.CData(), cbuf)
		if err != nil {
			b.Fatal(err)
		}
		_ = m
		b.StopTimer()
		pix.Close()
		b.StartTimer()
	}
	b.StopTimer()
}

func BenchmarkDecodeRGBA(b *testing.B) {
	data, err := ioutil.ReadFile("./testdata/1_webp_ll.webp")
	if err != nil {
		b.Fatal(err)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		m, err := DecodeRGBA(data)
		if err != nil {
			b.Fatal(err)
		}
		_ = m
	}
	b.StopTimer()
}

func BenchmarkDecodeRGBAEx(b *testing.B) {
	data, err := ioutil.ReadFile("./testdata/1_webp_ll.webp")
	if err != nil {
		b.Fatal(err)
	}
	cbuf := NewCBuffer(len(data))
	copy(cbuf.CData(), data)
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		m, pix, err := DecodeRGBAEx(cbuf.CData(), cbuf)
		if err != nil {
			b.Fatal(err)
		}
		_ = m
		b.StopTimer()
		pix.Close()
		b.StartTimer()
	}
	b.StopTimer()
}

func BenchmarkDecodeRGBA_x_image_webp(b *testing.B) {
	data, err := ioutil.ReadFile("./testdata/1_webp_ll.webp")
	if err != nil {
		b.Fatal(err)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		m, err := xwebp.Decode(bytes.NewReader(data))
		if err != nil {
			b.Fatal(err)
		}
		_ = m
	}
	b.StopTimer()
}

func BenchmarkEncodeAndDecode(b *testing.B) {
	var buf bytes.Buffer

	img, err := loadImage("1_webp_ll.png")
	if err != nil {
		b.Fatal(err)
	}
	s := img.Bounds().Size()
	b.SetBytes(int64(s.X * s.Y * 4))

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		if err = Encode(&buf, img, nil); err != nil {
			b.Fatal(err)
		}
		if _, err = Decode(&buf); err != nil {
			b.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>package controllers

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"os/exec"
	"syscall"
	"time"

	"github.com/learnin/go-multilog"
	"github.com/zenazn/goji/web"

	"github.com/learnin/batch-rest-controller/helpers"
)

type JobsController struct {
	DS     *helpers.DataSource
	Logger *multilog.MultiLogger
}

type Request struct {
	Async         bool
	RequireResult bool
	Command       string
	Args          string
}

type Job struct {
	Id            int64
	Async         bool
	RequireResult bool
	Command       string
	Args          string
	CreatedAt     time.Time
}

type JobMessage struct {
	JobId     int64
	Seq       int64
	Type      int
	Message   string
	CreatedAt time.Time
}

func (controller *JobsController) Show(c web.C, w http.ResponseWriter, r *http.Request) {
	// jobId := c.URLParams["jobId"]
	encoder := json.NewEncoder(w)
	encoder.Encode(response{Error: false, Messages: []string{}})
}

func (controller *JobsController) Run(c web.C, w http.ResponseWriter, r *http.Request) {
	req := Request{}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		controller.Logger.Error(err)
		http.Error(w, "", http.StatusBadRequest)
		return
	}
	if req.Async && req.RequireResult {
		job := Job{
			Async:         req.Async,
			RequireResult: req.RequireResult,
			Command:       req.Command,
			Args:          req.Args,
		}
		if err := controller.DS.GetDB().Save(&job).Error; err != nil {
			controller.Logger.Errorf("jobs テーブル登録時にエラーが発生しました。error=%v", err)
			sendEroorResponse(w, err, "")
			return
		}
		out := make(chan string)
		errout := make(chan string)
		jobquit := make(chan bool)

		go func() {
			nowSeq := int64(0)
		loop:
			for {
				select {
				case <-jobquit:
					break loop
				case stdout := <-out:
					nowSeq++
					jobMsg := JobMessage{
						JobId:   job.Id,
						Seq:     nowSeq,
						Type:    1,
						Message: stdout,
					}
					if err := controller.DS.GetDB().Save(&jobMsg).Error; err != nil {
						controller.Logger.Errorf("job_messages テーブル登録時にエラーが発生しました。error=%v", err)
						return
					}
				case stderr := <-errout:
					nowSeq++
					jobMsg := JobMessage{
						JobId:   job.Id,
						Seq:     nowSeq,
						Type:    2,
						Message: stderr,
					}
					if err := controller.DS.GetDB().Save(&jobMsg).Error; err != nil {
						controller.Logger.Errorf("job_messages テーブル登録時にエラーが発生しました。error=%v", err)
						return
					}
				}
			}
		}()
		go func() {
			cmd := exec.Command("ps", "-ef")
			stdout, err := cmd.StdoutPipe()
			if err != nil {
				// TODO テーブルに登録する
				fmt.Println(err)
				jobquit <- true
			}
			stderr, err := cmd.StderrPipe()
			if err != nil {
				// TODO テーブルに登録する
				fmt.Println(err)
				jobquit <- true
			}

			if err := cmd.Start(); err != nil {
				// TODO テーブルに登録する
				fmt.Println(err)
				jobquit <- true
			}

			go func() {
				scanner := bufio.NewScanner(stdout)
				for scanner.Scan() {
					out <- scanner.Text()
				}
			}()
			go func() {
				scanner := bufio.NewScanner(stderr)
				for scanner.Scan() {
					errout <- scanner.Text()
				}
			}()
			if err := cmd.Wait(); err != nil {
				if err2, ok := err.(*exec.ExitError); ok {
					if s, ok := err2.Sys().(syscall.WaitStatus); ok {
						fmt.Println(err)
						// TODO テーブルに登録する
						fmt.Println(s.ExitStatus())
					} else {
						// Unix や Winodws とは異なり、 exec.ExitError.Sys() が syscall.WaitStatus ではないOSの場合
						fmt.Println(err)
					}
				} else {
					// may be returned for I/O problems.
					fmt.Println(err)
				}
			} else {
				// TODO exitStatus = 0. テーブルに登録する
				fmt.Println(0)
			}
			jobquit <- true
		}()
		fmt.Fprintf(w, "{\"id\": %v}", job.Id)
		return
	} else if !req.Async {
		cmd := exec.Command("ps", "-ef")
		var stdout bytes.Buffer
		var stderr bytes.Buffer
		cmd.Stdout = &stdout
		cmd.Stderr = &stderr

		if err := cmd.Run(); err != nil {
			if err2, ok := err.(*exec.ExitError); ok {
				if s, ok := err2.Sys().(syscall.WaitStatus); ok {
					sendEroorResponse(w, err, fmt.Sprintf("バッチが正常終了しませんでした。exitStatus=%v stdout=%v stderr=%v", s.ExitStatus(), stdout.String(), stderr.String()))
					return
				} else {
					// Unix や Winodws とは異なり、 exec.ExitError.Sys() が syscall.WaitStatus ではないOSの場合
					sendEroorResponse(w, err, fmt.Sprintf("バッチが正常終了しませんでした。stdout=%v stderr=%v", stdout.String(), stderr.String()))
					return
				}
			} else {
				// may be returned for I/O problems.
				sendEroorResponse(w, err, "バッチ実行時にエラーが発生しました。")
				return
			}
		}
		fmt.Fprintf(w, "{\"exitStatus\": 0, \"stdout\": \"%v\", \"stderr\": \"%v\"}", stdout.String(), stderr.String())
		return
	}
	encoder := json.NewEncoder(w)
	encoder.Encode(response{Error: false, Messages: []string{}})
}
<commit_msg>Revise a job async execution processing<commit_after>package controllers

import (
	"bufio"
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os/exec"
	"syscall"
	"time"

	"github.com/learnin/go-multilog"
	"github.com/zenazn/goji/web"

	"github.com/learnin/batch-rest-controller/helpers"
)

type JobsController struct {
	DS     *helpers.DataSource
	Logger *multilog.MultiLogger
}

type Request struct {
	Async         bool
	RequireResult bool
	Command       string
	Args          string
}

type Job struct {
	Id            int64
	Async         bool
	RequireResult bool
	Command       string
	Args          string
	CreatedAt     time.Time
}

type JobMessage struct {
	JobId     int64
	Seq       int64
	Type      int
	Message   string
	CreatedAt time.Time
}

func (controller *JobsController) Show(c web.C, w http.ResponseWriter, r *http.Request) {
	// jobId := c.URLParams["jobId"]
	encoder := json.NewEncoder(w)
	encoder.Encode(response{Error: false, Messages: []string{}})
}

func (controller *JobsController) Run(c web.C, w http.ResponseWriter, r *http.Request) {
	req := Request{}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
		controller.Logger.Error(err)
		http.Error(w, "", http.StatusBadRequest)
		return
	}
	if req.Async && req.RequireResult {
		job := Job{
			Async:         req.Async,
			RequireResult: req.RequireResult,
			Command:       req.Command,
			Args:          req.Args,
		}
		if err := controller.DS.GetDB().Save(&job).Error; err != nil {
			controller.Logger.Errorf("jobs テーブル登録時にエラーが発生しました。error=%v", err)
			sendEroorResponse(w, err, "")
			return
		}
		cmd := exec.Command("ps", "-ef")
		stdout, err := cmd.StdoutPipe()
		if err != nil {
			controller.Logger.Errorf("標準出力パイプ取得時にエラーが発生しました。error=%v", err)
			sendEroorResponse(w, err, "")
			return
		}
		stderr, err := cmd.StderrPipe()
		if err != nil {
			controller.Logger.Errorf("標準エラー出力パイプ取得時にエラーが発生しました。error=%v", err)
			sendEroorResponse(w, err, "")
			return
		}

		go func(stdout *io.ReadCloser, stderr *io.ReadCloser) {
			if err := cmd.Start(); err != nil {
				jobMsg := JobMessage{
					JobId:   job.Id,
					Seq:     1,
					Type:    2,
					Message: err.Error(),
				}
				if err := controller.DS.GetDB().Save(&jobMsg).Error; err != nil {
					controller.Logger.Errorf("job_messages テーブル登録時にエラーが発生しました。error=%v", err)
					return
				}
				return
			}

			out := make(chan string)
			errout := make(chan string)
			jobquit := make(chan bool)

			go func() {
				nowSeq := int64(0)
			loop:
				for {
					select {
					case <-jobquit:
						break loop
					case stdout := <-out:
						nowSeq++
						jobMsg := JobMessage{
							JobId:   job.Id,
							Seq:     nowSeq,
							Type:    1,
							Message: stdout,
						}
						if err := controller.DS.GetDB().Save(&jobMsg).Error; err != nil {
							controller.Logger.Errorf("job_messages テーブル登録時にエラーが発生しました。error=%v", err)
							return
						}
					case stderr := <-errout:
						nowSeq++
						jobMsg := JobMessage{
							JobId:   job.Id,
							Seq:     nowSeq,
							Type:    2,
							Message: stderr,
						}
						if err := controller.DS.GetDB().Save(&jobMsg).Error; err != nil {
							controller.Logger.Errorf("job_messages テーブル登録時にエラーが発生しました。error=%v", err)
							return
						}
					}
				}
			}()
			go func() {
				scanner := bufio.NewScanner(*stdout)
				for scanner.Scan() {
					out <- scanner.Text()
				}
			}()
			go func() {
				scanner := bufio.NewScanner(*stderr)
				for scanner.Scan() {
					errout <- scanner.Text()
				}
			}()

			if err := cmd.Wait(); err != nil {
				if err2, ok := err.(*exec.ExitError); ok {
					if s, ok := err2.Sys().(syscall.WaitStatus); ok {
						fmt.Println(err)
						// TODO テーブルに登録する
						fmt.Println(s.ExitStatus())
					} else {
						// Unix や Winodws とは異なり、 exec.ExitError.Sys() が syscall.WaitStatus ではないOSの場合
						fmt.Println(err)
					}
				} else {
					// may be returned for I/O problems.
					fmt.Println(err)
				}
			} else {
				// TODO exitStatus = 0. テーブルに登録する
				fmt.Println(0)
			}
			jobquit <- true
		}(&stdout, &stderr)

		fmt.Fprintf(w, "{\"id\": %v}", job.Id)
		return
	} else if !req.Async {
		cmd := exec.Command("ps", "-ef")
		var stdout bytes.Buffer
		var stderr bytes.Buffer
		cmd.Stdout = &stdout
		cmd.Stderr = &stderr

		if err := cmd.Run(); err != nil {
			if err2, ok := err.(*exec.ExitError); ok {
				if s, ok := err2.Sys().(syscall.WaitStatus); ok {
					sendEroorResponse(w, err, fmt.Sprintf("バッチが正常終了しませんでした。exitStatus=%v stdout=%v stderr=%v", s.ExitStatus(), stdout.String(), stderr.String()))
					return
				} else {
					// Unix や Winodws とは異なり、 exec.ExitError.Sys() が syscall.WaitStatus ではないOSの場合
					sendEroorResponse(w, err, fmt.Sprintf("バッチが正常終了しませんでした。stdout=%v stderr=%v", stdout.String(), stderr.String()))
					return
				}
			} else {
				// may be returned for I/O problems.
				sendEroorResponse(w, err, "バッチ実行時にエラーが発生しました。")
				return
			}
		}
		fmt.Fprintf(w, "{\"exitStatus\": 0, \"stdout\": \"%v\", \"stderr\": \"%v\"}", stdout.String(), stderr.String())
		return
	}
	encoder := json.NewEncoder(w)
	encoder.Encode(response{Error: false, Messages: []string{}})
}
<|endoftext|>


<commit_before>package openstack

import (
	"errors"
	"fmt"
	"github.com/mitchellh/multistep"
	"github.com/rackspace/gophercloud"
	"log"
	"time"
)

// StateRefreshFunc is a function type used for StateChangeConf that is
// responsible for refreshing the item being watched for a state change.
//
// It returns three results. `result` is any object that will be returned
// as the final object after waiting for state change. This allows you to
// return the final updated object, for example an openstack instance after
// refreshing it.
//
// `state` is the latest state of that object. And `err` is any error that
// may have happened while refreshing the state.
type StateRefreshFunc func() (result interface{}, state string, progress int, err error)

// StateChangeConf is the configuration struct used for `WaitForState`.
type StateChangeConf struct {
	Pending   []string
	Refresh   StateRefreshFunc
	StepState multistep.StateBag
	Target    string
}

// ServerStateRefreshFunc returns a StateRefreshFunc that is used to watch
// an openstacn server.
func ServerStateRefreshFunc(csp gophercloud.CloudServersProvider, s *gophercloud.Server) StateRefreshFunc {
	return func() (interface{}, string, int, error) {
		resp, err := csp.ServerById(s.Id)
		if err != nil {
			log.Printf("Error on ServerStateRefresh: %s", err)
			return nil, "", 0, err
		}

		return resp, resp.Status, resp.Progress, nil
	}
}

// WaitForState watches an object and waits for it to achieve a certain
// state.
func WaitForState(conf *StateChangeConf) (i interface{}, err error) {
	log.Printf("Waiting for state to become: %s", conf.Target)

	for {
		var currentProgress int
		var currentState string
		i, currentState, currentProgress, err = conf.Refresh()
		if err != nil {
			return
		}

		if currentState == conf.Target {
			return
		}

		if conf.StepState != nil {
			if _, ok := conf.StepState.GetOk(multistep.StateCancelled); ok {
				return nil, errors.New("interrupted")
			}
		}

		found := false
		for _, allowed := range conf.Pending {
			if currentState == allowed {
				found = true
				break
			}
		}

		if !found {
			fmt.Errorf("unexpected state '%s', wanted target '%s'", currentState, conf.Target)
			return
		}

		log.Printf("Waiting for state to become: %s currently %s (%d%%)", conf.Target, currentState, currentProgress)
		time.Sleep(2 * time.Second)
	}

	return
}
<commit_msg>Properly return error when invalid instance state is found<commit_after>package openstack

import (
	"errors"
	"fmt"
	"github.com/mitchellh/multistep"
	"github.com/rackspace/gophercloud"
	"log"
	"time"
)

// StateRefreshFunc is a function type used for StateChangeConf that is
// responsible for refreshing the item being watched for a state change.
//
// It returns three results. `result` is any object that will be returned
// as the final object after waiting for state change. This allows you to
// return the final updated object, for example an openstack instance after
// refreshing it.
//
// `state` is the latest state of that object. And `err` is any error that
// may have happened while refreshing the state.
type StateRefreshFunc func() (result interface{}, state string, progress int, err error)

// StateChangeConf is the configuration struct used for `WaitForState`.
type StateChangeConf struct {
	Pending   []string
	Refresh   StateRefreshFunc
	StepState multistep.StateBag
	Target    string
}

// ServerStateRefreshFunc returns a StateRefreshFunc that is used to watch
// an openstacn server.
func ServerStateRefreshFunc(csp gophercloud.CloudServersProvider, s *gophercloud.Server) StateRefreshFunc {
	return func() (interface{}, string, int, error) {
		resp, err := csp.ServerById(s.Id)
		if err != nil {
			log.Printf("Error on ServerStateRefresh: %s", err)
			return nil, "", 0, err
		}

		return resp, resp.Status, resp.Progress, nil
	}
}

// WaitForState watches an object and waits for it to achieve a certain
// state.
func WaitForState(conf *StateChangeConf) (i interface{}, err error) {
	log.Printf("Waiting for state to become: %s", conf.Target)

	for {
		var currentProgress int
		var currentState string
		i, currentState, currentProgress, err = conf.Refresh()
		if err != nil {
			return
		}

		if currentState == conf.Target {
			return
		}

		if conf.StepState != nil {
			if _, ok := conf.StepState.GetOk(multistep.StateCancelled); ok {
				return nil, errors.New("interrupted")
			}
		}

		found := false
		for _, allowed := range conf.Pending {
			if currentState == allowed {
				found = true
				break
			}
		}

		if !found {
			return nil, fmt.Errorf("unexpected state '%s', wanted target '%s'", currentState, conf.Target)
		}

		log.Printf("Waiting for state to become: %s currently %s (%d%%)", conf.Target, currentState, currentProgress)
		time.Sleep(2 * time.Second)
	}

	return
}
<|endoftext|>


<commit_before>package livereload

type (
  message struct {
	Command string `json:"command"`
}

updateMessage struct {
		message
		Url string
	  }
	reloadMessage struct {
		message
		Path    string `json:"path"`
		LiveCSS bool `json:"liveCSS"`
	  }

	alertMessage struct {
		message
		Message string `json:"message"`
	}

	helloMessage struct {
		message
		Protocols []string `json:"protocols"`
	}

	serverHello struct {
		message
		Protocols  []string `json:"protocols"`
		ServerName string   `json:"serverName"`
	}
)
<commit_msg>go fmt.<commit_after>package livereload

type (
	message struct {
		Command string `json:"command"`
	}

	updateMessage struct {
		message
		Url string
	}
	reloadMessage struct {
		message
		Path    string `json:"path"`
		LiveCSS bool   `json:"liveCSS"`
	}

	alertMessage struct {
		message
		Message string `json:"message"`
	}

	helloMessage struct {
		message
		Protocols []string `json:"protocols"`
	}

	serverHello struct {
		message
		Protocols  []string `json:"protocols"`
		ServerName string   `json:"serverName"`
	}
)
<|endoftext|>


<commit_before>package slack

// OutgoingMessage is used for the realtime API, and seems incomplete.
type OutgoingMessage struct {
	ID      int    `json:"id"`
	Channel string `json:"channel,omitempty"`
	Text    string `json:"text,omitempty"`
	Type    string `json:"type,omitempty"`
}

// Message is an auxiliary type to allow us to have a message containing sub messages
type Message struct {
	Msg
	SubMessage *Msg `json:"message,omitempty"`
}

// Msg contains information about a slack message
type Msg struct {
	// Basic Message
	Type        string       `json:"type,omitempty"`
	Channel     string       `json:"channel,omitempty"`
	User        string       `json:"user,omitempty"`
	Text        string       `json:"text,omitempty"`
	Timestamp   string       `json:"ts,omitempty"`
	IsStarred   bool         `json:"is_starred,omitempty"`
	PinnedTo    []string     `json:"pinned_to, omitempty"`
	Attachments []Attachment `json:"attachments,omitempty"`
	Edited      *Edited      `json:"edited,omitempty"`

	// Message Subtypes
	SubType string `json:"subtype,omitempty"`

	// Hidden Subtypes
	Hidden           bool   `json:"hidden,omitempty"`     // message_changed, message_deleted, unpinned_item
	DeletedTimestamp string `json:"deleted_ts,omitempty"` // message_deleted
	EventTimestamp   string `json:"event_ts,omitempty"`

	// bot_message (https://api.slack.com/events/message/bot_message)
	BotID    string `json:"bot_id,omitempty"`
	Username string `json:"username,omitempty"`
	Icons    *Icon  `json:"icons,omitempty"`

	// channel_join, group_join
	Inviter string `json:"inviter,omitempty"`

	// channel_topic, group_topic
	Topic string `json:"topic,omitempty"`

	// channel_purpose, group_purpose
	Purpose string `json:"purpose,omitempty"`

	// channel_name, group_name
	Name    string `json:"name,omitempty"`
	OldName string `json:"old_name,omitempty"`

	// channel_archive, group_archive
	Members []string `json:"members,omitempty"`

	// file_share, file_comment, file_mention
	File *File `json:"file,omitempty"`

	// file_share
	Upload bool `json:"upload,omitempty"`

	// file_comment
	Comment *Comment `json:"comment,omitempty"`

	// pinned_item
	ItemType string `json:"item_type,omitempty"`

	// https://api.slack.com/rtm
	ReplyTo int    `json:"reply_to,omitempty"`
	Team    string `json:"team,omitempty"`
}

// Icon is used for bot messages
type Icon struct {
	IconURL   string `json:"icon_url,omitempty"`
	IconEmoji string `json:"icon_emoji,omitempty"`
}

// Edited indicates that a message has been edited.
type Edited struct {
	User      string `json:"user,omitempty"`
	Timestamp string `json:"ts,omitempty"`
}

// Event contains the event type
type Event struct {
	Type string `json:"type,omitempty"`
}

// Ping contains information about a Ping Event
type Ping struct {
	ID   int    `json:"id"`
	Type string `json:"type"`
}

// Pong contains information about a Pong Event
type Pong struct {
	Type    string `json:"type"`
	ReplyTo int    `json:"reply_to"`
}

// NewOutgoingMessage prepares an OutgoingMessage that the user can
// use to send a message. Use this function to properly set the
// messageID.
func (rtm *RTM) NewOutgoingMessage(text string, channel string) *OutgoingMessage {
	id := rtm.idGen.Next()
	return &OutgoingMessage{
		ID:      id,
		Type:    "message",
		Channel: channel,
		Text:    text,
	}
}
<commit_msg>add NewTypingMessage to RTM API<commit_after>package slack

// OutgoingMessage is used for the realtime API, and seems incomplete.
type OutgoingMessage struct {
	ID      int    `json:"id"`
	Channel string `json:"channel,omitempty"`
	Text    string `json:"text,omitempty"`
	Type    string `json:"type,omitempty"`
}

// Message is an auxiliary type to allow us to have a message containing sub messages
type Message struct {
	Msg
	SubMessage *Msg `json:"message,omitempty"`
}

// Msg contains information about a slack message
type Msg struct {
	// Basic Message
	Type        string       `json:"type,omitempty"`
	Channel     string       `json:"channel,omitempty"`
	User        string       `json:"user,omitempty"`
	Text        string       `json:"text,omitempty"`
	Timestamp   string       `json:"ts,omitempty"`
	IsStarred   bool         `json:"is_starred,omitempty"`
	PinnedTo    []string     `json:"pinned_to, omitempty"`
	Attachments []Attachment `json:"attachments,omitempty"`
	Edited      *Edited      `json:"edited,omitempty"`

	// Message Subtypes
	SubType string `json:"subtype,omitempty"`

	// Hidden Subtypes
	Hidden           bool   `json:"hidden,omitempty"`     // message_changed, message_deleted, unpinned_item
	DeletedTimestamp string `json:"deleted_ts,omitempty"` // message_deleted
	EventTimestamp   string `json:"event_ts,omitempty"`

	// bot_message (https://api.slack.com/events/message/bot_message)
	BotID    string `json:"bot_id,omitempty"`
	Username string `json:"username,omitempty"`
	Icons    *Icon  `json:"icons,omitempty"`

	// channel_join, group_join
	Inviter string `json:"inviter,omitempty"`

	// channel_topic, group_topic
	Topic string `json:"topic,omitempty"`

	// channel_purpose, group_purpose
	Purpose string `json:"purpose,omitempty"`

	// channel_name, group_name
	Name    string `json:"name,omitempty"`
	OldName string `json:"old_name,omitempty"`

	// channel_archive, group_archive
	Members []string `json:"members,omitempty"`

	// file_share, file_comment, file_mention
	File *File `json:"file,omitempty"`

	// file_share
	Upload bool `json:"upload,omitempty"`

	// file_comment
	Comment *Comment `json:"comment,omitempty"`

	// pinned_item
	ItemType string `json:"item_type,omitempty"`

	// https://api.slack.com/rtm
	ReplyTo int    `json:"reply_to,omitempty"`
	Team    string `json:"team,omitempty"`
}

// Icon is used for bot messages
type Icon struct {
	IconURL   string `json:"icon_url,omitempty"`
	IconEmoji string `json:"icon_emoji,omitempty"`
}

// Edited indicates that a message has been edited.
type Edited struct {
	User      string `json:"user,omitempty"`
	Timestamp string `json:"ts,omitempty"`
}

// Event contains the event type
type Event struct {
	Type string `json:"type,omitempty"`
}

// Ping contains information about a Ping Event
type Ping struct {
	ID   int    `json:"id"`
	Type string `json:"type"`
}

// Pong contains information about a Pong Event
type Pong struct {
	Type    string `json:"type"`
	ReplyTo int    `json:"reply_to"`
}

// NewOutgoingMessage prepares an OutgoingMessage that the user can
// use to send a message. Use this function to properly set the
// messageID.
func (rtm *RTM) NewOutgoingMessage(text string, channel string) *OutgoingMessage {
	id := rtm.idGen.Next()
	return &OutgoingMessage{
		ID:      id,
		Type:    "message",
		Channel: channel,
		Text:    text,
	}
}

// NewTypingMessage prepares an OutgoingMessage that the user can
// use to send as a typing indicator. Use this function to properly set the
// messageID.
func (rtm *RTM) NewTypingMessage(channel string) *OutgoingMessage {
	id := rtm.idGen.Next()
	return &OutgoingMessage{
		ID:      id,
		Type:    "typing",
		Channel: channel,
	}
}
<|endoftext|>