<commit_before>package curator

import (
	"sync"
)

type ConnectionStateListener interface {
	// Called when there is a state change in the connection
	StateChanged(client CuratorFramework, newState ConnectionState)
}

// Receives notifications about errors and background events
type CuratorListener interface {
	// Called when a background task has completed or a watch has triggered
	EventReceived(client CuratorFramework, event CuratorEvent) error
}

type UnhandledErrorListener interface {
	// Called when an exception is caught in a background thread, handler, etc.
	UnhandledError(err error)
}

type connectionStateListenerCallback func(client CuratorFramework, newState ConnectionState)

type connectionStateListenerStub struct {
	callback connectionStateListenerCallback
}

func NewConnectionStateListener(callback connectionStateListenerCallback) ConnectionStateListener {
	return &connectionStateListenerStub{callback}
}

func (l *connectionStateListenerStub) StateChanged(client CuratorFramework, newState ConnectionState) {
	l.callback(client, newState)
}

type curatorListenerCallback func(client CuratorFramework, event CuratorEvent) error

type curatorListenerStub struct {
	callback curatorListenerCallback
}

func NewCuratorListener(callback curatorListenerCallback) CuratorListener {
	return &curatorListenerStub{callback}
}

func (l *curatorListenerStub) EventReceived(client CuratorFramework, event CuratorEvent) error {
	return l.callback(client, event)
}

type unhandledErrorListenerCallback func(err error)

type unhandledErrorListenerStub struct {
	callback unhandledErrorListenerCallback
}

func NewUnhandledErrorListener(callback unhandledErrorListenerCallback) UnhandledErrorListener {
	return &unhandledErrorListenerStub{callback}
}

func (l *unhandledErrorListenerStub) UnhandledError(err error) {
	l.callback(err)
}

// Abstracts a listenable object
type Listenable /* [T] */ interface {
	Len() int

	Clear()

	ForEach(callback func(interface{}))
}

type ConnectionStateListenable interface {
	Listenable /* [T] */

	AddListener(listener ConnectionStateListener)

	RemoveListener(listener ConnectionStateListener)
}

type CuratorListenable interface {
	Listenable /* [T] */

	AddListener(listener CuratorListener)

	RemoveListener(listener CuratorListener)
}

type UnhandledErrorListenable interface {
	Listenable /* [T] */

	AddListener(listener UnhandledErrorListener)

	RemoveListener(listener UnhandledErrorListener)
}

type ListenerContainer struct {
	lock      sync.RWMutex
	listeners []interface{}
}

func (c *ListenerContainer) Add(listener interface{}) {
	c.lock.Lock()

	c.listeners = append(c.listeners, listener)

	c.lock.Unlock()
}

func (c *ListenerContainer) Remove(listener interface{}) {
	c.lock.Lock()

	for i, l := range c.listeners {
		if l == listener {
			copy(c.listeners[i:], c.listeners[i+1:])
			c.listeners = c.listeners[:len(c.listeners)-1]
			break
		}
	}

	c.lock.Unlock()
}

func (c *ListenerContainer) Len() int {
	return len(c.listeners)
}

func (c *ListenerContainer) Clear() {
	c.lock.Lock()

	c.listeners = nil

	c.lock.Unlock()
}

func (c *ListenerContainer) ForEach(callback func(interface{})) {
	c.lock.RLock()

	for _, listener := range c.listeners {
		callback(listener)
	}

	c.lock.RUnlock()
}

type connectionStateListenerContainer struct {
	ListenerContainer
}

func (c *connectionStateListenerContainer) AddListener(listener ConnectionStateListener) {
	c.Add(listener)
}

func (c *connectionStateListenerContainer) RemoveListener(listener ConnectionStateListener) {
	c.Remove(listener)
}

type curatorListenerContainer struct {
	ListenerContainer
}

func (c *curatorListenerContainer) AddListener(listener CuratorListener) {
	c.Add(listener)
}

func (c *curatorListenerContainer) RemoveListener(listener CuratorListener) {
	c.Remove(listener)
}

type unhandledErrorListenerContainer struct {
	ListenerContainer
}

func (c *unhandledErrorListenerContainer) AddListener(listener UnhandledErrorListener) {
	c.Add(listener)
}

func (c *unhandledErrorListenerContainer) RemoveListener(listener UnhandledErrorListener) {
	c.Remove(listener)
}
<commit_msg>use append to remove element from slice<commit_after>package curator

import (
	"sync"
)

type ConnectionStateListener interface {
	// Called when there is a state change in the connection
	StateChanged(client CuratorFramework, newState ConnectionState)
}

// Receives notifications about errors and background events
type CuratorListener interface {
	// Called when a background task has completed or a watch has triggered
	EventReceived(client CuratorFramework, event CuratorEvent) error
}

type UnhandledErrorListener interface {
	// Called when an exception is caught in a background thread, handler, etc.
	UnhandledError(err error)
}

type connectionStateListenerCallback func(client CuratorFramework, newState ConnectionState)

type connectionStateListenerStub struct {
	callback connectionStateListenerCallback
}

func NewConnectionStateListener(callback connectionStateListenerCallback) ConnectionStateListener {
	return &connectionStateListenerStub{callback}
}

func (l *connectionStateListenerStub) StateChanged(client CuratorFramework, newState ConnectionState) {
	l.callback(client, newState)
}

type curatorListenerCallback func(client CuratorFramework, event CuratorEvent) error

type curatorListenerStub struct {
	callback curatorListenerCallback
}

func NewCuratorListener(callback curatorListenerCallback) CuratorListener {
	return &curatorListenerStub{callback}
}

func (l *curatorListenerStub) EventReceived(client CuratorFramework, event CuratorEvent) error {
	return l.callback(client, event)
}

type unhandledErrorListenerCallback func(err error)

type unhandledErrorListenerStub struct {
	callback unhandledErrorListenerCallback
}

func NewUnhandledErrorListener(callback unhandledErrorListenerCallback) UnhandledErrorListener {
	return &unhandledErrorListenerStub{callback}
}

func (l *unhandledErrorListenerStub) UnhandledError(err error) {
	l.callback(err)
}

// Abstracts a listenable object
type Listenable /* [T] */ interface {
	Len() int

	Clear()

	ForEach(callback func(interface{}))
}

type ConnectionStateListenable interface {
	Listenable /* [T] */

	AddListener(listener ConnectionStateListener)

	RemoveListener(listener ConnectionStateListener)
}

type CuratorListenable interface {
	Listenable /* [T] */

	AddListener(listener CuratorListener)

	RemoveListener(listener CuratorListener)
}

type UnhandledErrorListenable interface {
	Listenable /* [T] */

	AddListener(listener UnhandledErrorListener)

	RemoveListener(listener UnhandledErrorListener)
}

type ListenerContainer struct {
	lock      sync.RWMutex
	listeners []interface{}
}

func (c *ListenerContainer) Add(listener interface{}) {
	c.lock.Lock()

	c.listeners = append(c.listeners, listener)

	c.lock.Unlock()
}

func (c *ListenerContainer) Remove(listener interface{}) {
	c.lock.Lock()

	for i, l := range c.listeners {
		if l == listener {
			c.listeners = append(c.listeners[:i], c.listeners[i+1:]...)
			break
		}
	}

	c.lock.Unlock()
}

func (c *ListenerContainer) Len() int {
	return len(c.listeners)
}

func (c *ListenerContainer) Clear() {
	c.lock.Lock()

	c.listeners = nil

	c.lock.Unlock()
}

func (c *ListenerContainer) ForEach(callback func(interface{})) {
	c.lock.RLock()

	for _, listener := range c.listeners {
		callback(listener)
	}

	c.lock.RUnlock()
}

type connectionStateListenerContainer struct {
	ListenerContainer
}

func (c *connectionStateListenerContainer) AddListener(listener ConnectionStateListener) {
	c.Add(listener)
}

func (c *connectionStateListenerContainer) RemoveListener(listener ConnectionStateListener) {
	c.Remove(listener)
}

type curatorListenerContainer struct {
	ListenerContainer
}

func (c *curatorListenerContainer) AddListener(listener CuratorListener) {
	c.Add(listener)
}

func (c *curatorListenerContainer) RemoveListener(listener CuratorListener) {
	c.Remove(listener)
}

type unhandledErrorListenerContainer struct {
	ListenerContainer
}

func (c *unhandledErrorListenerContainer) AddListener(listener UnhandledErrorListener) {
	c.Add(listener)
}

func (c *unhandledErrorListenerContainer) RemoveListener(listener UnhandledErrorListener) {
	c.Remove(listener)
}
<|endoftext|>


<commit_before>// This is an example of using DiscordGo to obtain the
// authentication token for a given user account.
package main

import (
	"fmt"
	"os"

	"github.com/bwmarrin/discordgo"
)

func main() {

	// Check for Username and Password CLI arguments.
	if len(os.Args) != 3 {
		fmt.Println("You must provide username and password as arguments. See below example.")
		fmt.Println(os.Args[0], " [email] [password]")
		return
	}

	// Create a New Discord session and login with the provided
	// email and password.
	dg, err := discordgo.New(os.Args[1], os.Args[2])
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Printf("Your Authentication Token is:\n\n%s\n", dg.Token)
}
<commit_msg>Updates to example<commit_after>package main

import (
	"flag"
	"fmt"

	"github.com/bwmarrin/discordgo"
)

var (
	Email    string
	Password string
)

func init() {

	flag.StringVar(&Email, "e", "", "Account Email")
	flag.StringVar(&Password, "p", "", "Account Password")
	flag.Parse()
}

func main() {

	// Create a new Discord session using the provided login information.
	dg, err := discordgo.New(Email, Password)
	if err != nil {
		fmt.Println("error creating Discord session,", err)
		return
	}

	fmt.Printf("Your Authentication Token is:\n\n%s\n", dg.Token)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"log/syslog"
	"net"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/go.net/websocket"
	"github.com/fsouza/go-dockerclient"
	"github.com/go-martini/martini"
)

var debugMode bool

func debug(v ...interface{}) {
	if debugMode {
		log.Println(v...)
	}
}

func assert(err error, context string) {
	if err != nil {
		log.Fatal(context+": ", err)
	}
}

func getopt(name, dfault string) string {
	value := os.Getenv(name)
	if value == "" {
		value = dfault
	}
	return value
}

type Colorizer map[string]int

// returns up to 14 color escape codes (then repeats) for each unique key
func (c Colorizer) Get(key string) string {
	i, exists := c[key]
	if !exists {
		c[key] = len(c)
		i = c[key]
	}
	bright := "1;"
	if i%14 > 6 {
		bright = ""
	}
	return "\x1b[" + bright + "3" + strconv.Itoa(7-(i%7)) + "m"
}

func externalIP() (string, error) {
	ifaces, err := net.Interfaces()
	if err != nil {
		return "", err
	}
	for _, iface := range ifaces {
		if iface.Flags&net.FlagUp == 0 {
			continue // interface down
		}
		if iface.Flags&net.FlagLoopback != 0 {
			continue // loopback interface
		}
		addrs, err := iface.Addrs()
		if err != nil {
			return "", err
		}
		for _, addr := range addrs {
			var ip net.IP
			switch v := addr.(type) {
			case *net.IPNet:
				ip = v.IP
			case *net.IPAddr:
				ip = v.IP
			}
			if ip == nil || ip.IsLoopback() {
				continue
			}
			ip = ip.To4()
			if ip == nil {
				continue // not an ipv4 address
			}
			return ip.String(), nil
		}
	}
	return "", errors.New("No network connection?")
}

func httpPostStreamer(target Target, types []string, logstream chan *Log) {
	typestr := "," + strings.Join(types, ",") + ","

	client := &http.Client{}

	hostname, err := os.Hostname()
	if err != nil {
		debug("httpPostStreamer - error getting hostname:", err)
		hostname = "<unknown>"
	}

	ip, err := externalIP()
	if err != nil {
		debug("httpPostStreamer - error getting external IP address:", err)
		ip = "<unknown>"
	}

	typestr := "," + strings.Join(types, ",") + ","

	url := target.Type + "://" + target.Addr + target.Path
	debug("httpPostStreamer - typestr:", typestr)
	debug("httpPostStreamer - URL:", url)
	for logline := range logstream {
		if typestr != ",," && !strings.Contains(typestr, logline.Type) {
			continue
		}

		//tag := logline.Name + target.AppendTag
		//debug("httpPostStreamer - tag: ", tag)

		//debug("httpPostStreamer - logline.ID: ", logline.ID)
		//debug("httpPostStreamer - logline.Name: ", logline.Name)
		//debug("httpPostStreamer - logline.Type: ", logline.Type)
		//debug("httpPostStreamer - logline.Data: ", logline.Data)

		messageTime := time.Now()
		message := fmt.Sprintf("%s hostname=%s ip=%s id=%s name=%s %s",
			messageTime, hostname, ip, logline.ID, logline.Name,
			logline.Data)
		req, err := http.NewRequest("POST", url, strings.NewReader(message))
		if err != nil {
			debug("httpPostStreamer - Error on http.NewRequest: ", err, url)
		}
		resp, err := client.Do(req)
		if err != nil {
			debug("httpPostStreamer - Error on client.Do: ", err, url)
		}
		io.Copy(ioutil.Discard, resp.Body)
		resp.Body.Close()
	}
}

func syslogStreamer(target Target, types []string, logstream chan *Log) {
	typestr := "," + strings.Join(types, ",") + ","
	for logline := range logstream {
		if typestr != ",," && !strings.Contains(typestr, logline.Type) {
			continue
		}
		tag := logline.Name + target.AppendTag
		remote, err := syslog.Dial("udp", target.Addr, syslog.LOG_USER|syslog.LOG_INFO, tag)
		assert(err, "syslog")
		io.WriteString(remote, logline.Data)
	}
}

func udpStreamer(target Target, types []string, logstream chan *Log) {
	typestr := "," + strings.Join(types, ",") + ","
	addr, err := net.ResolveUDPAddr("udp", target.Addr)
	assert(err, "resolve udp failed")
	conn, err := net.DialUDP("udp", nil, addr)
	assert(err, "connect udp failed")
	encoder := json.NewEncoder(conn)
	defer conn.Close()
	for logline := range logstream {
		if typestr != ",," && !strings.Contains(typestr, logline.Type) {
			continue
		}
		encoder.Encode(logline)
	}
}

func websocketStreamer(w http.ResponseWriter, req *http.Request, logstream chan *Log, closer chan bool) {
	websocket.Handler(func(conn *websocket.Conn) {
		for logline := range logstream {
			if req.URL.Query().Get("type") != "" && logline.Type != req.URL.Query().Get("type") {
				continue
			}
			_, err := conn.Write(append(marshal(logline), '\n'))
			if err != nil {
				closer <- true
				return
			}
		}
	}).ServeHTTP(w, req)
}

func httpStreamer(w http.ResponseWriter, req *http.Request, logstream chan *Log, multi bool) {
	var colors Colorizer
	var usecolor, usejson bool
	nameWidth := 16
	if req.URL.Query().Get("colors") != "off" {
		colors = make(Colorizer)
		usecolor = true
	}
	if req.Header.Get("Accept") == "application/json" {
		w.Header().Add("Content-Type", "application/json")
		usejson = true
	} else {
		w.Header().Add("Content-Type", "text/plain")
	}
	for logline := range logstream {
		if req.URL.Query().Get("types") != "" && logline.Type != req.URL.Query().Get("types") {
			continue
		}
		if usejson {
			w.Write(append(marshal(logline), '\n'))
		} else {
			if multi {
				if len(logline.Name) > nameWidth {
					nameWidth = len(logline.Name)
				}
				if usecolor {
					w.Write([]byte(fmt.Sprintf(
						"%s%"+strconv.Itoa(nameWidth)+"s|%s\x1b[0m\n",
						colors.Get(logline.Name), logline.Name, logline.Data,
					)))
				} else {
					w.Write([]byte(fmt.Sprintf(
						"%"+strconv.Itoa(nameWidth)+"s|%s\n", logline.Name, logline.Data,
					)))
				}
			} else {
				w.Write(append([]byte(logline.Data), '\n'))
			}
		}
		w.(http.Flusher).Flush()
	}
}

func main() {
	debugMode = getopt("DEBUG", "") != ""
	port := getopt("PORT", "8000")
	endpoint := getopt("DOCKER_HOST", "unix:///var/run/docker.sock")
	routespath := getopt("ROUTESPATH", "/var/lib/logspout")

	client, err := docker.NewClient(endpoint)
	assert(err, "docker")
	attacher := NewAttachManager(client)
	router := NewRouteManager(attacher)

	if len(os.Args) > 1 {
		u, err := url.Parse(os.Args[1])
		assert(err, "url")
		log.Println("routing all to " + os.Args[1])
		router.Add(&Route{Target: Target{Type: u.Scheme, Addr: u.Host, Path: u.Path}})
	}

	if _, err := os.Stat(routespath); err == nil {
		log.Println("loading and persisting routes in " + routespath)
		assert(router.Load(RouteFileStore(routespath)), "persistor")
	}

	m := martini.Classic()

	m.Get("/logs(?:/(?P<predicate>[a-zA-Z]+):(?P<value>.+))?", func(w http.ResponseWriter, req *http.Request, params martini.Params) {
		source := new(Source)
		switch {
		case params["predicate"] == "id" && params["value"] != "":
			source.ID = params["value"][:12]
		case params["predicate"] == "name" && params["value"] != "":
			source.Name = params["value"]
		case params["predicate"] == "filter" && params["value"] != "":
			source.Filter = params["value"]
		}

		if source.ID != "" && attacher.Get(source.ID) == nil {
			http.NotFound(w, req)
			return
		}

		logstream := make(chan *Log)
		defer close(logstream)

		var closer <-chan bool
		if req.Header.Get("Upgrade") == "websocket" {
			closerBi := make(chan bool)
			go websocketStreamer(w, req, logstream, closerBi)
			closer = closerBi
		} else {
			go httpStreamer(w, req, logstream, source.All() || source.Filter != "")
			closer = w.(http.CloseNotifier).CloseNotify()
		}

		attacher.Listen(source, logstream, closer)
	})

	m.Get("/routes", func(w http.ResponseWriter, req *http.Request) {
		w.Header().Add("Content-Type", "application/json")
		routes, _ := router.GetAll()
		w.Write(append(marshal(routes), '\n'))
	})

	m.Post("/routes", func(w http.ResponseWriter, req *http.Request) (int, string) {
		route := new(Route)
		if err := unmarshal(req.Body, route); err != nil {
			return http.StatusBadRequest, "Bad request: " + err.Error()
		}

		// TODO: validate?
		router.Add(route)

		w.Header().Add("Content-Type", "application/json")
		return http.StatusCreated, string(append(marshal(route), '\n'))
	})

	m.Get("/routes/:id", func(w http.ResponseWriter, req *http.Request, params martini.Params) {
		route, _ := router.Get(params["id"])
		if route == nil {
			http.NotFound(w, req)
			return
		}
		w.Write(append(marshal(route), '\n'))
	})

	m.Delete("/routes/:id", func(w http.ResponseWriter, req *http.Request, params martini.Params) {
		if ok := router.Remove(params["id"]); !ok {
			http.NotFound(w, req)
		}
	})

	log.Println("logspout serving http on :" + port)
	log.Fatal(http.ListenAndServe(":"+port, m))
}
<commit_msg>More cleanup<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"log/syslog"
	"net"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"code.google.com/p/go.net/websocket"
	"github.com/fsouza/go-dockerclient"
	"github.com/go-martini/martini"
)

var debugMode bool

func debug(v ...interface{}) {
	if debugMode {
		log.Println(v...)
	}
}

func assert(err error, context string) {
	if err != nil {
		log.Fatal(context+": ", err)
	}
}

func getopt(name, dfault string) string {
	value := os.Getenv(name)
	if value == "" {
		value = dfault
	}
	return value
}

type Colorizer map[string]int

// returns up to 14 color escape codes (then repeats) for each unique key
func (c Colorizer) Get(key string) string {
	i, exists := c[key]
	if !exists {
		c[key] = len(c)
		i = c[key]
	}
	bright := "1;"
	if i%14 > 6 {
		bright = ""
	}
	return "\x1b[" + bright + "3" + strconv.Itoa(7-(i%7)) + "m"
}

func httpPostStreamer(target Target, types []string, logstream chan *Log) {
	typestr := "," + strings.Join(types, ",") + ","

	client := &http.Client{}

	hostname, err := os.Hostname()
	if err != nil {
		debug("httpPostStreamer - error getting hostname:", err)
		hostname = "<unknown>"
	}

	url := target.Type + "://" + target.Addr + target.Path
	debug("httpPostStreamer - typestr:", typestr)
	debug("httpPostStreamer - URL:", url)
	for logline := range logstream {
		if typestr != ",," && !strings.Contains(typestr, logline.Type) {
			continue
		}

		//tag := logline.Name + target.AppendTag
		//debug("httpPostStreamer - tag: ", tag)

		//debug("httpPostStreamer - logline.ID: ", logline.ID)
		//debug("httpPostStreamer - logline.Name: ", logline.Name)
		//debug("httpPostStreamer - logline.Type: ", logline.Type)
		//debug("httpPostStreamer - logline.Data: ", logline.Data)

		messageTime := time.Now()
		message := fmt.Sprintf("%s hostname=%s ip=%s id=%s name=%s %s",
			messageTime, hostname, ip, logline.ID, logline.Name,
			logline.Data)
		req, err := http.NewRequest("POST", url, strings.NewReader(message))
		if err != nil {
			debug("httpPostStreamer - Error on http.NewRequest: ", err, url)
		}
		resp, err := client.Do(req)
		if err != nil {
			debug("httpPostStreamer - Error on client.Do: ", err, url)
		}
		io.Copy(ioutil.Discard, resp.Body)
		resp.Body.Close()
	}
}

func syslogStreamer(target Target, types []string, logstream chan *Log) {
	typestr := "," + strings.Join(types, ",") + ","
	for logline := range logstream {
		if typestr != ",," && !strings.Contains(typestr, logline.Type) {
			continue
		}
		tag := logline.Name + target.AppendTag
		remote, err := syslog.Dial("udp", target.Addr, syslog.LOG_USER|syslog.LOG_INFO, tag)
		assert(err, "syslog")
		io.WriteString(remote, logline.Data)
	}
}

func udpStreamer(target Target, types []string, logstream chan *Log) {
	typestr := "," + strings.Join(types, ",") + ","
	addr, err := net.ResolveUDPAddr("udp", target.Addr)
	assert(err, "resolve udp failed")
	conn, err := net.DialUDP("udp", nil, addr)
	assert(err, "connect udp failed")
	encoder := json.NewEncoder(conn)
	defer conn.Close()
	for logline := range logstream {
		if typestr != ",," && !strings.Contains(typestr, logline.Type) {
			continue
		}
		encoder.Encode(logline)
	}
}

func websocketStreamer(w http.ResponseWriter, req *http.Request, logstream chan *Log, closer chan bool) {
	websocket.Handler(func(conn *websocket.Conn) {
		for logline := range logstream {
			if req.URL.Query().Get("type") != "" && logline.Type != req.URL.Query().Get("type") {
				continue
			}
			_, err := conn.Write(append(marshal(logline), '\n'))
			if err != nil {
				closer <- true
				return
			}
		}
	}).ServeHTTP(w, req)
}

func httpStreamer(w http.ResponseWriter, req *http.Request, logstream chan *Log, multi bool) {
	var colors Colorizer
	var usecolor, usejson bool
	nameWidth := 16
	if req.URL.Query().Get("colors") != "off" {
		colors = make(Colorizer)
		usecolor = true
	}
	if req.Header.Get("Accept") == "application/json" {
		w.Header().Add("Content-Type", "application/json")
		usejson = true
	} else {
		w.Header().Add("Content-Type", "text/plain")
	}
	for logline := range logstream {
		if req.URL.Query().Get("types") != "" && logline.Type != req.URL.Query().Get("types") {
			continue
		}
		if usejson {
			w.Write(append(marshal(logline), '\n'))
		} else {
			if multi {
				if len(logline.Name) > nameWidth {
					nameWidth = len(logline.Name)
				}
				if usecolor {
					w.Write([]byte(fmt.Sprintf(
						"%s%"+strconv.Itoa(nameWidth)+"s|%s\x1b[0m\n",
						colors.Get(logline.Name), logline.Name, logline.Data,
					)))
				} else {
					w.Write([]byte(fmt.Sprintf(
						"%"+strconv.Itoa(nameWidth)+"s|%s\n", logline.Name, logline.Data,
					)))
				}
			} else {
				w.Write(append([]byte(logline.Data), '\n'))
			}
		}
		w.(http.Flusher).Flush()
	}
}

func main() {
	debugMode = getopt("DEBUG", "") != ""
	port := getopt("PORT", "8000")
	endpoint := getopt("DOCKER_HOST", "unix:///var/run/docker.sock")
	routespath := getopt("ROUTESPATH", "/var/lib/logspout")

	client, err := docker.NewClient(endpoint)
	assert(err, "docker")
	attacher := NewAttachManager(client)
	router := NewRouteManager(attacher)

	if len(os.Args) > 1 {
		u, err := url.Parse(os.Args[1])
		assert(err, "url")
		log.Println("routing all to " + os.Args[1])
		router.Add(&Route{Target: Target{Type: u.Scheme, Addr: u.Host, Path: u.Path}})
	}

	if _, err := os.Stat(routespath); err == nil {
		log.Println("loading and persisting routes in " + routespath)
		assert(router.Load(RouteFileStore(routespath)), "persistor")
	}

	m := martini.Classic()

	m.Get("/logs(?:/(?P<predicate>[a-zA-Z]+):(?P<value>.+))?", func(w http.ResponseWriter, req *http.Request, params martini.Params) {
		source := new(Source)
		switch {
		case params["predicate"] == "id" && params["value"] != "":
			source.ID = params["value"][:12]
		case params["predicate"] == "name" && params["value"] != "":
			source.Name = params["value"]
		case params["predicate"] == "filter" && params["value"] != "":
			source.Filter = params["value"]
		}

		if source.ID != "" && attacher.Get(source.ID) == nil {
			http.NotFound(w, req)
			return
		}

		logstream := make(chan *Log)
		defer close(logstream)

		var closer <-chan bool
		if req.Header.Get("Upgrade") == "websocket" {
			closerBi := make(chan bool)
			go websocketStreamer(w, req, logstream, closerBi)
			closer = closerBi
		} else {
			go httpStreamer(w, req, logstream, source.All() || source.Filter != "")
			closer = w.(http.CloseNotifier).CloseNotify()
		}

		attacher.Listen(source, logstream, closer)
	})

	m.Get("/routes", func(w http.ResponseWriter, req *http.Request) {
		w.Header().Add("Content-Type", "application/json")
		routes, _ := router.GetAll()
		w.Write(append(marshal(routes), '\n'))
	})

	m.Post("/routes", func(w http.ResponseWriter, req *http.Request) (int, string) {
		route := new(Route)
		if err := unmarshal(req.Body, route); err != nil {
			return http.StatusBadRequest, "Bad request: " + err.Error()
		}

		// TODO: validate?
		router.Add(route)

		w.Header().Add("Content-Type", "application/json")
		return http.StatusCreated, string(append(marshal(route), '\n'))
	})

	m.Get("/routes/:id", func(w http.ResponseWriter, req *http.Request, params martini.Params) {
		route, _ := router.Get(params["id"])
		if route == nil {
			http.NotFound(w, req)
			return
		}
		w.Write(append(marshal(route), '\n'))
	})

	m.Delete("/routes/:id", func(w http.ResponseWriter, req *http.Request, params martini.Params) {
		if ok := router.Remove(params["id"]); !ok {
			http.NotFound(w, req)
		}
	})

	log.Println("logspout serving http on :" + port)
	log.Fatal(http.ListenAndServe(":"+port, m))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"os"

	"github.com/spf13/cobra"
)

// Cmd is the only command of mdc
var Cmd = &cobra.Command{
	Use:   "mdc",
	Short: "Generate markdown from godocs recursively",
	Long:  "This tool produces the goa.design reference content",
	Run:   run,
}

func main() {
	if _, err := exec.LookPath("godoc2md"); err != nil {
		fatal("could not find godoc2md in path, please install godoc2md with go get github.com/davecheney/godoc2md")
	}
	if err := Cmd.Execute(); err != nil {
		fatal(err)
	}
}

func run(cmd *cobra.Command, args []string) {
	if len(args) < 2 {
		fatal("usage: %s INPUTDIR OUTPUTDIR", os.Args[0])
	}
	root, err := filepath.Abs(args[0])
	if err != nil {
		fatal(err)
	}
	out, err := filepath.Abs(args[1])
	if err != nil {
		fatal(err)
	}
	if err := os.MkdirAll(out, 0755); err != nil {
		fatal(err)
	}
	var packagePath string
	gopaths := filepath.SplitList(os.Getenv("GOPATH"))
	for _, gopath := range gopaths {
		if g, err := filepath.Abs(gopath); err == nil {
			gopath = g
		}
		if strings.HasPrefix(root, gopath) {
			packagePath = filepath.ToSlash(strings.TrimPrefix(root, filepath.Join(gopath, "src")))[1:]
			break
		}
	}
	if packagePath == "" {
		fatal("path %s is not in a Go workspace", root)
	}
	fmt.Printf("* Packages root: %s\n* Output dir: %s\n", root, out)
	err = filepath.Walk(root, func(p string, i os.FileInfo, _ error) error {
		if i.Name() == ".git" || strings.HasPrefix(i.Name(), "_") {
			return filepath.SkipDir
		}
		if !i.IsDir() {
			return nil
		}
		rel, err := filepath.Rel(root, p)
		if err != nil {
			return err
		}
		mdPath := filepath.Join(out, filepath.Base(root), rel)
		if err := os.MkdirAll(filepath.Dir(mdPath), 0755); err != nil {
			return err
		}
		pkg := path.Join(packagePath, filepath.ToSlash(rel))
		fullMdPath := fmt.Sprintf("%s.%s", mdPath, "md")
		err = godoc2md(pkg, fullMdPath)
		if err == nil {
			fmt.Println("OK")
		} else {
			fmt.Printf("FAIL: %s\n", err)
		}
		return nil
	})
	if err != nil {
		fatal(err)
	}
}

func godoc2md(pkg, filename string) error {
	fmt.Printf("godoc2md %s > %s...", pkg, filename)
	cmd := exec.Command("godoc2md", pkg)
	b, err := cmd.Output()
	if err != nil {
		fatal(err)
	}
	f, err := os.Create(filename)
	if err != nil {
		fatal(err)
	}
	defer f.Close()
	f.Write(b)

	return nil

}

func fatal(format interface{}, val ...interface{}) {
	var f string
	if err, ok := format.(error); ok {
		f = err.Error()
	} else if s, ok := format.(string); ok {
		f = s
	} else {
		f = fmt.Sprintf("%v", format)
	}
	fmt.Fprintf(os.Stderr, f, val...)
	os.Exit(-1)
}
<commit_msg>Update mdc so it does what we need<commit_after>package main

import (
	"fmt"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"text/template"
	"time"

	"os"

	"github.com/spf13/cobra"
)

// Cmd is the only command of mdc
var Cmd = &cobra.Command{
	Use:   "mdc",
	Short: "Generate markdown from godocs recursively",
	Long:  "This tool produces the goa.design reference content",
	Run:   run,
}

// List of paths to exclude from processing
var excludes []string

func init() {
	Cmd.Flags().StringSliceVarP(&excludes, "exclude", "x", nil, "list of paths to exclude from processing")
}

func main() {
	if _, err := exec.LookPath("godoc2md"); err != nil {
		fatal("could not find godoc2md in path, please install godoc2md with go get github.com/davecheney/godoc2md")
	}
	if err := Cmd.Execute(); err != nil {
		fatal(err)
	}
}

func run(cmd *cobra.Command, args []string) {
	if len(args) < 2 {
		fatal("usage: %s PACKAGE OUTPUTDIR [flags]", os.Args[0])
	}
	packagePath := args[0]
	var fullPath string
	gopaths := filepath.SplitList(os.Getenv("GOPATH"))
	for _, gopath := range gopaths {
		candidate := filepath.Join(gopath, "src", packagePath)
		if c, err := filepath.Abs(candidate); err == nil {
			candidate = c
		}
		if _, err := os.Stat(candidate); err == nil {
			fullPath = candidate
			break
		}
	}
	if fullPath == "" {
		fatal("could not find package %s in %s", packagePath, os.Getenv("GOPATH"))
	}
	root, err := filepath.Abs(fullPath)
	if err != nil {
		fatal(err)
	}
	out, err := filepath.Abs(args[1])
	if err != nil {
		fatal(err)
	}
	if err := os.MkdirAll(out, 0755); err != nil {
		fatal(err)
	}
	ex := make(map[string]bool)
	for _, e := range excludes {
		ex[e] = true
	}
	fmt.Printf("* Packages root: %s\n* Output dir: %s\n", root, out)
	if len(excludes) > 0 {
		fmt.Printf("* Excludes: %s\n", strings.Join(excludes, ", "))
	}
	err = filepath.Walk(root, func(p string, i os.FileInfo, _ error) error {
		if i.Name() == ".git" || strings.HasPrefix(i.Name(), "_") || ex[i.Name()] {
			return filepath.SkipDir
		}
		if !i.IsDir() {
			return nil
		}
		rel, err := filepath.Rel(root, p)
		if err != nil {
			return err
		}
		mdPath := filepath.Join(out, filepath.Base(root), rel)
		if err := os.MkdirAll(filepath.Dir(mdPath), 0755); err != nil {
			return err
		}
		pkg := path.Join(packagePath, filepath.ToSlash(rel))
		fullMdPath := fmt.Sprintf("%s.%s", mdPath, "md")
		err = godoc2md(pkg, fullMdPath)
		if err == nil {
			fmt.Println("OK")
		} else {
			fmt.Printf("FAIL: %s\n", err)
		}
		return nil
	})
	if err != nil {
		fatal(err)
	}
}

func godoc2md(pkg, filename string) error {
	fmt.Printf("godoc2md %s > %s...", pkg, filename)
	cmd := exec.Command("godoc2md", pkg)
	b, err := cmd.Output()
	if err != nil {
		fatal(err)
	}
	f, err := os.Create(filename)
	if err != nil {
		fatal(err)
	}
	defer f.Close()
	t, err := template.New("header").Parse(headerT)
	if err != nil {
		fatal(err)
	}
	data := map[string]interface{}{
		"PackagePath": pkg,
		"Date":        time.Now().Format(time.RFC3339),
		"PackageName": path.Base(pkg),
	}
	err = t.Execute(f, data)
	if err != nil {
		fatal(err)
	}
	f.Write(b)

	return nil

}

func fatal(format interface{}, val ...interface{}) {
	var f string
	if err, ok := format.(error); ok {
		f = err.Error()
	} else if s, ok := format.(string); ok {
		f = s
	} else {
		f = fmt.Sprintf("%v", format)
	}
	fmt.Fprintf(os.Stderr, f, val...)
	os.Exit(-1)
}

const headerT = `+++
title="{{.PackagePath}}"
date="{{.Date}}"
description="godoc for {{.PackagePath}}"
categories=["godoc"]
tags=["godoc", "{{.PackageName}}"]
+++
`
<|endoftext|>


<commit_before>package golexer

import "reflect"

// #开头的行注释
type UnixStyleCommentMatcher struct {
	baseMatcher
}

func (self *UnixStyleCommentMatcher) String() string {
	return reflect.TypeOf(self).Elem().Name()
}

func (self *UnixStyleCommentMatcher) Match(tz *Tokenizer) (Token, error) {
	if tz.Current() != '#' {
		return EmptyToken, nil
	}

	tz.ConsumeOne()

	begin := tz.Index()

	for {

		tz.ConsumeOne()

		if tz.Current() == '\n' || tz.Current() == 0 {
			break
		}

	}

	return NewToken(self, tz, tz.StringRange(begin, tz.index-1), ""), nil
}

func NewUnixStyleCommentMatcher(id int) TokenMatcher {
	return &UnixStyleCommentMatcher{
		baseMatcher{id},
	}
}
<commit_msg>修复缺字符问题<commit_after>package golexer

import "reflect"

// #开头的行注释
type UnixStyleCommentMatcher struct {
	baseMatcher
}

func (self *UnixStyleCommentMatcher) String() string {
	return reflect.TypeOf(self).Elem().Name()
}

func (self *UnixStyleCommentMatcher) Match(tz *Tokenizer) (Token, error) {
	if tz.Current() != '#' {
		return EmptyToken, nil
	}

	tz.ConsumeOne()

	begin := tz.Index()

	for {

		tz.ConsumeOne()

		if tz.Current() == '\n' || tz.Current() == 0 {
			break
		}

	}

	return NewToken(self, tz, tz.StringRange(begin, tz.index), ""), nil
}

func NewUnixStyleCommentMatcher(id int) TokenMatcher {
	return &UnixStyleCommentMatcher{
		baseMatcher{id},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/grafana/grafana/pkg/log"
	"sort"
)

// represents a data "archive", i.e. the raw one, or an aggregated series
type archive struct {
	title      string
	interval   uint32
	pointCount uint32
	comment    string
}

func (b archive) String() string {
	return fmt.Sprintf("<archive %s> int:%d, comment: %s", b.title, b.interval, b.comment)
}

type archives []archive

func (a archives) Len() int           { return len(a) }
func (a archives) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a archives) Less(i, j int) bool { return a[i].interval < a[j].interval }

func findMetricsForRequests(reqs []Req, metaCache *MetaCache) error {
	for i := range reqs {
		err := metaCache.UpdateReq(&reqs[i])
		if err != nil {
			return err
		}
	}
	return nil
}

// updates the requests with all details for fetching, making sure all metrics are in the same, optimal interval
// luckily, all metrics still use the same aggSettings, making this a bit simpler
// for all requests, sets archive, numPoints, interval (and rawInterval as a side effect)
// note: it is assumed that all requests have the same from, to and maxdatapoints!
func alignRequests(reqs []Req, aggSettings []aggSetting) ([]Req, error) {

	// model all the archives for each requested metric
	// the 0th archive is always the raw series, with highest res (lowest interval)
	aggs := aggSettingsSpanAsc(aggSettings)
	sort.Sort(aggs)

	options := make([]archive, len(aggs)+1)

	minInterval := uint32(0)
	rawIntervals := make(map[uint32]bool)
	for _, req := range reqs {
		if minInterval == 0 || minInterval > req.rawInterval {
			minInterval = req.rawInterval
		}
		rawIntervals[req.rawInterval] = true
	}
	tsRange := (reqs[0].to - reqs[0].from)

	options[0] = archive{"raw", minInterval, tsRange / minInterval, ""}
	// now model the archives we get from the aggregations
	for j, agg := range aggs {
		options[j+1] = archive{fmt.Sprintf("agg %d", j), agg.span, tsRange / agg.span, ""}
	}

	// find the first option with a pointCount < maxDataPoints
	selected := len(options) - 1
	for i, opt := range options {
		if opt.pointCount < reqs[0].maxPoints {
			selected = i
			break
		}
	}

	/*
	   do a quick calculation of the ratio between pointCount and maxDatapoints of
	   the selected option, and the option before that.  eg. with a time range of 1hour,
	   our pointCounts for each option are:
	   10s   | 360
	   600s  | 6
	   7200s | 0

	   if maxPoints is 100, then selected will be 1, our 600s rollups.
	   We then calculate the ratio between maxPoints and our
	   selected pointCount "6" and the previous option "360".
	   belowMaxDataPointsRatio  =  16  #(100/6)
	   aboveMaxDataPointsRatio  =  3   #(360/100)

	   As the maxDataPoint requested is much closer to 360 then it is to 6,
	   we will use 360 and do runtime consolidation.
	*/
	runTimeConsolidate := false
	if selected > 0 {
		belowMaxDataPointsRatio := float64(reqs[0].maxPoints) / float64(options[selected].pointCount)
		aboveMaxDataPointsRatio := float64(options[selected-1].pointCount) / float64(reqs[0].maxPoints)

		if aboveMaxDataPointsRatio < belowMaxDataPointsRatio {
			selected--
			runTimeConsolidate = true
		}
	}

	chosenInterval := options[selected].interval

	// if we are using raw metrics, we need to find an interval that all request intervals work with.
	if selected == 0 && len(rawIntervals) > 1 {
		runTimeConsolidate = true
		var keys []int
		for k := range rawIntervals {
			keys = append(keys, int(k))
		}
		sort.Ints(keys)
		chosenInterval = uint32(keys[0])
		for i := 1; i < len(keys); i++ {
			var a, b uint32
			if uint32(keys[i]) > chosenInterval {
				a = uint32(keys[i])
				b = chosenInterval
			} else {
				a = chosenInterval
				b = uint32(keys[i])
			}
			r := a % b
			if r != 0 {
				for j := uint32(2); j <= b; j++ {
					if (j*a)%b == 0 {
						chosenInterval = j * a
						break
					}
				}
			} else {

				chosenInterval = uint32(a)
			}
			options[0].pointCount = tsRange / chosenInterval
			options[0].interval = chosenInterval
		}
		//make sure that the calculated interval is not greater then the interval of the fist rollup.
		if len(options) > 1 && chosenInterval >= options[1].interval {
			selected = 1
			chosenInterval = options[1].interval
		}
	}

	options[selected].comment = "<-- chosen"
	for _, archive := range options {
		log.Debug("%-6s %-6d %-6d %s", archive.title, archive.interval, tsRange/archive.interval, archive.comment)
	}

	/* we now just need to update the archiveInterval, outInterval and aggNum of each req.
	   archInterval uint32 // the interval corresponding to the archive we'll fetch
	   outInterval  uint32 // the interval of the output data, after any runtime consolidation
	   aggNum       uint32 // how many points to consolidate together at runtime, after fetching from the archive
	*/
	for i, _ := range reqs {
		req := &reqs[i]
		req.archive = selected
		req.archInterval = options[selected].interval
		req.outInterval = chosenInterval
		aggNum := uint32(1)
		if runTimeConsolidate {
			ptCount := options[selected].pointCount

			aggNum = ptCount / req.maxPoints
			if ptCount%req.maxPoints != 0 {
				aggNum++
			}
			if selected == 0 {
				// Handle RAW interval
				req.archInterval = req.rawInterval

				// each request can have a different rawInterval. So the aggNum is vairable.
				if chosenInterval != req.rawInterval {
					aggNum = aggNum * (chosenInterval / req.rawInterval)
				}
			}

			req.outInterval = req.archInterval * aggNum
		}

		req.aggNum = aggNum
	}
	return reqs, nil
}
<commit_msg>cleanups, clarifications, fixups for alignRequests<commit_after>package main

import (
	"fmt"
	"github.com/grafana/grafana/pkg/log"
	"sort"
)

// represents a data "archive", i.e. the raw one, or an aggregated series
type archive struct {
	title      string
	interval   uint32
	pointCount uint32
	comment    string
}

func (b archive) String() string {
	return fmt.Sprintf("<archive %s> int:%d, comment: %s", b.title, b.interval, b.comment)
}

type archives []archive

func (a archives) Len() int           { return len(a) }
func (a archives) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a archives) Less(i, j int) bool { return a[i].interval < a[j].interval }

func findMetricsForRequests(reqs []Req, metaCache *MetaCache) error {
	for i := range reqs {
		err := metaCache.UpdateReq(&reqs[i])
		if err != nil {
			return err
		}
	}
	return nil
}

// updates the requests with all details for fetching, making sure all metrics are in the same, optimal interval
// luckily, all metrics still use the same aggSettings, making this a bit simpler
// for all requests, sets archive, numPoints, interval (and rawInterval as a side effect)
// note: it is assumed that all requests have the same from, to and maxdatapoints!
func alignRequests(reqs []Req, aggSettings []aggSetting) ([]Req, error) {

	// model all the archives for each requested metric
	// the 0th archive is always the raw series, with highest res (lowest interval)
	aggs := aggSettingsSpanAsc(aggSettings)
	sort.Sort(aggs)

	options := make([]archive, len(aggs)+1)

	minInterval := uint32(0)
	rawIntervals := make(map[uint32]bool)
	for _, req := range reqs {
		if minInterval == 0 || minInterval > req.rawInterval {
			minInterval = req.rawInterval
		}
		rawIntervals[req.rawInterval] = true
	}
	tsRange := (reqs[0].to - reqs[0].from)

	// note: not all series necessarily have the same raw settings, will be fixed further down
	options[0] = archive{"raw", minInterval, tsRange / minInterval, ""}
	// now model the archives we get from the aggregations
	for j, agg := range aggs {
		options[j+1] = archive{fmt.Sprintf("agg %d", j), agg.span, tsRange / agg.span, ""}
	}

	// find the first, i.e. highest-res option with a pointCount <= maxDataPoints
	selected := len(options) - 1
	for i, opt := range options {
		if opt.pointCount <= reqs[0].maxPoints {
			selected = i
			break
		}
	}

	/*
	   do a quick calculation of the ratio between pointCount and maxDatapoints of
	   the selected option, and the option before that.  eg. with a time range of 1hour,
	   our pointCounts for each option are:
	   10s   | 360
	   600s  | 6
	   7200s | 0

	   if maxPoints is 100, then selected will be 1, our 600s rollups.
	   We then calculate the ratio between maxPoints and our
	   selected pointCount "6" and the previous option "360".
	   belowMaxDataPointsRatio  =  16  #(100/6)
	   aboveMaxDataPointsRatio  =  3   #(360/100)

	   As the maxDataPoint requested is much closer to 360 then it is to 6,
	   we will use 360 and do runtime consolidation.
	*/
	runTimeConsolidate := false
	if selected > 0 {
		belowMaxDataPointsRatio := float64(reqs[0].maxPoints) / float64(options[selected].pointCount)
		aboveMaxDataPointsRatio := float64(options[selected-1].pointCount) / float64(reqs[0].maxPoints)

		if aboveMaxDataPointsRatio < belowMaxDataPointsRatio {
			selected--
			runTimeConsolidate = true
		}
	}

	chosenInterval := options[selected].interval

	// if we are using raw metrics, we need to find an interval that all request intervals work with.
	if selected == 0 && len(rawIntervals) > 1 {
		runTimeConsolidate = true
		var keys []int
		for k := range rawIntervals {
			keys = append(keys, int(k))
		}
		sort.Ints(keys)
		chosenInterval = uint32(keys[0])
		for i := 1; i < len(keys); i++ {
			a := max(uint32(keys[i]), chosenInterval)
			b := min(uint32(keys[i]), chosenInterval)
			r := a % b
			if r != 0 {
				for j := uint32(2); j <= b; j++ {
					if (j*a)%b == 0 {
						chosenInterval = j * a
						break
					}
				}
			} else {

				chosenInterval = a
			}
			options[0].pointCount = tsRange / chosenInterval
			options[0].interval = chosenInterval
		}
		//make sure that the calculated interval is not greater then the interval of the fist rollup.
		if len(options) > 1 && chosenInterval >= options[1].interval {
			selected = 1
			chosenInterval = options[1].interval
		}
	}

	options[selected].comment = "<-- chosen"
	for _, archive := range options {
		log.Debug("%-6s %-6d %-6d %s", archive.title, archive.interval, tsRange/archive.interval, archive.comment)
	}

	/* we now just need to update the archiveInterval, outInterval and aggNum of each req.
	   archInterval uint32 // the interval corresponding to the archive we'll fetch
	   outInterval  uint32 // the interval of the output data, after any runtime consolidation
	   aggNum       uint32 // how many points to consolidate together at runtime, after fetching from the archive
	*/
	for i, _ := range reqs {
		req := &reqs[i]
		req.archive = selected
		req.archInterval = options[selected].interval
		req.outInterval = chosenInterval
		aggNum := uint32(1)
		if runTimeConsolidate {
			ptCount := options[selected].pointCount

			aggNum = ptCount / req.maxPoints
			if ptCount%req.maxPoints != 0 {
				aggNum++
			}
			if selected == 0 {
				// Handle RAW interval
				req.archInterval = req.rawInterval

				// each request can have a different rawInterval. So the aggNum is variable.
				if chosenInterval != req.rawInterval {
					aggNum = aggNum * (chosenInterval / req.rawInterval)
				}
			}

			req.outInterval = req.archInterval * aggNum
		}

		req.aggNum = aggNum
	}
	return reqs, nil
}
<|endoftext|>


<commit_before>// +build darwin

package darwin

import (
	"time"

	"github.com/mackerelio/go-osstat/network"
	"github.com/mackerelio/golib/logging"
	"github.com/mackerelio/mackerel-agent/metrics"
)

/*
collect network interface I/O

`interface.{interface}.{metric}.delta`: The increased amount of network I/O per minute retrieved from the result of netstat -bni

interface = "en0", "en1" and so on...
*/

// InterfaceGenerator XXX
type InterfaceGenerator struct {
	Interval time.Duration
}

// metrics for posting to Mackerel

var interfaceLogger = logging.GetLogger("metrics.interface")

// Generate XXX
func (g *InterfaceGenerator) Generate() (metrics.Values, error) {
	prevValues, err := g.collectIntarfacesValues()
	if err != nil {
		return nil, err
	}

	time.Sleep(g.Interval)

	currValues, err := g.collectIntarfacesValues()
	if err != nil {
		return nil, err
	}

	ret := make(map[string]float64)
	for name, value := range prevValues {
		currValue, ok := currValues[name]
		if ok {
			ret[name+".delta"] = float64(currValue-value) / g.Interval.Seconds()
		}
	}

	return metrics.Values(ret), nil
}

func (g *InterfaceGenerator) collectIntarfacesValues() (map[string]uint64, error) {
	networks, err := network.Get()
	if err != nil {
		interfaceLogger.Errorf("failed to get network statistics: %s", err)
		return nil, err
	}
	if len(networks) == 0 {
		return nil, nil
	}
	results := make(map[string]uint64, len(networks)*2)
	for _, network := range networks {
		results["interface."+network.Name+".rxBytes"] = network.RxBytes
		results["interface."+network.Name+".txBytes"] = network.TxBytes
	}
	return results, nil
}
<commit_msg>fix typo for interface generator of darwin<commit_after>// +build darwin

package darwin

import (
	"time"

	"github.com/mackerelio/go-osstat/network"
	"github.com/mackerelio/golib/logging"
	"github.com/mackerelio/mackerel-agent/metrics"
)

/*
collect network interface I/O

`interface.{interface}.{metric}.delta`: The increased amount of network I/O per minute retrieved from the result of netstat -bni

interface = "en0", "en1" and so on...
*/

// InterfaceGenerator XXX
type InterfaceGenerator struct {
	Interval time.Duration
}

// metrics for posting to Mackerel

var interfaceLogger = logging.GetLogger("metrics.interface")

// Generate XXX
func (g *InterfaceGenerator) Generate() (metrics.Values, error) {
	prevValues, err := g.collectInterfacesValues()
	if err != nil {
		return nil, err
	}

	time.Sleep(g.Interval)

	currValues, err := g.collectInterfacesValues()
	if err != nil {
		return nil, err
	}

	ret := make(map[string]float64)
	for name, value := range prevValues {
		currValue, ok := currValues[name]
		if ok {
			ret[name+".delta"] = float64(currValue-value) / g.Interval.Seconds()
		}
	}

	return metrics.Values(ret), nil
}

func (g *InterfaceGenerator) collectInterfacesValues() (map[string]uint64, error) {
	networks, err := network.Get()
	if err != nil {
		interfaceLogger.Errorf("failed to get network statistics: %s", err)
		return nil, err
	}
	if len(networks) == 0 {
		return nil, nil
	}
	results := make(map[string]uint64, len(networks)*2)
	for _, network := range networks {
		results["interface."+network.Name+".rxBytes"] = network.RxBytes
		results["interface."+network.Name+".txBytes"] = network.TxBytes
	}
	return results, nil
}
<|endoftext|>


<commit_before>package render

import (
	"path/filepath"
	"strings"

	"github.com/webx-top/echo"
	"github.com/webx-top/echo/handler/mvc/static/resource"
	"github.com/webx-top/echo/middleware"
	"github.com/webx-top/echo/middleware/render/driver"
	"github.com/webx-top/echo/middleware/tplfunc"
)

type Config struct {
	TmplDir              string
	Theme                string
	Engine               string
	Style                string
	Reload               bool
	ParseStrings         map[string]string
	ErrorPages           map[int]string
	DefaultHTTPErrorCode int
	StaticOptions        *middleware.StaticOptions
	Debug                bool
	renderer             driver.Driver
	errorPageFuncSetter  []echo.HandlerFunc
	FuncMapSkipper       echo.Skipper
}

var DefaultFuncMapSkipper = func(c echo.Context) bool {
	return c.Format() != `html`
}

func (t *Config) SetFuncMapSkipper(skipper echo.Skipper) *Config {
	t.FuncMapSkipper = skipper
	return t
}

func (t *Config) Parser() func([]byte) []byte {
	if t.ParseStrings == nil {
		return nil
	}
	return func(b []byte) []byte {
		s := string(b)
		for oldVal, newVal := range t.ParseStrings {
			s = strings.Replace(s, oldVal, newVal, -1)
		}
		return []byte(s)
	}
}

// NewRenderer 新建渲染接口
func (t *Config) NewRenderer(manager ...driver.Manager) driver.Driver {
	tmplDir := t.TmplDir
	if len(t.Theme) > 0 {
		tmplDir = filepath.Join(tmplDir, t.Theme)
	}
	renderer := New(t.Engine, tmplDir)
	renderer.Init()
	if len(manager) > 0 {
		renderer.SetManager(manager[0])
	}
	renderer.SetContentProcessor(t.Parser())
	if t.StaticOptions != nil {
		st := t.NewStatic()
		renderer.SetFuncMap(func() map[string]interface{} {
			return st.Register(nil)
		})
		absTmplPath, err := filepath.Abs(tmplDir)
		var absFilePath string
		if err == nil {
			absFilePath, err = filepath.Abs(t.StaticOptions.Root)
		}
		if err == nil {
			if strings.HasPrefix(absFilePath, absTmplPath) {
				//如果静态文件在模板的子文件夹时，监控模板时判断静态文件更改
				renderer.MonitorEvent(st.OnUpdate())
			}
		}
	}
	return renderer
}

func (t *Config) AddFuncSetter(set ...echo.HandlerFunc) *Config {
	if t.errorPageFuncSetter == nil {
		t.errorPageFuncSetter = make([]echo.HandlerFunc, len(DefaultOptions.SetFuncMap))
		for index, setter := range DefaultOptions.SetFuncMap {
			t.errorPageFuncSetter[index] = setter
		}
	}
	t.errorPageFuncSetter = append(t.errorPageFuncSetter, set...)
	return t
}

func (t *Config) SetFuncSetter(set ...echo.HandlerFunc) *Config {
	t.errorPageFuncSetter = set
	return t
}

func (t *Config) ApplyTo(e *echo.Echo, manager ...driver.Manager) *Config {
	if t.renderer != nil {
		t.renderer.Close()
	}
	opt := &Options{
		ErrorPages:           t.ErrorPages,
		DefaultHTTPErrorCode: t.DefaultHTTPErrorCode,
	}
	opt.SetFuncSetter(t.errorPageFuncSetter...)
	e.SetHTTPErrorHandler(HTTPErrorHandler(opt))
	var funcMapSkipper echo.Skipper
	if t.FuncMapSkipper != nil {
		funcMapSkipper = t.FuncMapSkipper
	} else {
		funcMapSkipper = DefaultFuncMapSkipper
	}
	e.Use(middleware.FuncMap(tplfunc.New(), funcMapSkipper))
	renderer := t.NewRenderer(manager...)
	if t.StaticOptions != nil {
		e.Use(middleware.Static(t.StaticOptions))
	}
	e.SetRenderer(renderer)
	t.renderer = renderer
	return t
}

func (t *Config) Renderer() driver.Driver {
	return t.renderer
}

func (t *Config) NewStatic() *resource.Static {
	return resource.NewStatic(t.StaticOptions.Path, t.StaticOptions.Root)
}

// ThemeDir 主题所在文件夹的路径
func (t *Config) ThemeDir(args ...string) string {
	if len(args) < 1 {
		return filepath.Join(t.TmplDir, t.Theme)
	}
	return filepath.Join(t.TmplDir, args[0])
}
<commit_msg>update<commit_after>package render

import (
	"path/filepath"
	"strings"

	"github.com/webx-top/echo"
	"github.com/webx-top/echo/handler/mvc/static/resource"
	"github.com/webx-top/echo/middleware"
	"github.com/webx-top/echo/middleware/render/driver"
	"github.com/webx-top/echo/middleware/tplfunc"
)

type Config struct {
	TmplDir              string
	Theme                string
	Engine               string
	Style                string
	Reload               bool
	ParseStrings         map[string]string
	ErrorPages           map[int]string
	DefaultHTTPErrorCode int
	StaticOptions        *middleware.StaticOptions
	Debug                bool
	renderer             driver.Driver
	errorPageFuncSetter  []echo.HandlerFunc
	FuncMapSkipper       echo.Skipper
}

var DefaultFuncMapSkipper = func(c echo.Context) bool {
	return c.Format() != `html` && !c.IsAjax() && !c.IsPjax()
}

func (t *Config) SetFuncMapSkipper(skipper echo.Skipper) *Config {
	t.FuncMapSkipper = skipper
	return t
}

func (t *Config) Parser() func([]byte) []byte {
	if t.ParseStrings == nil {
		return nil
	}
	return func(b []byte) []byte {
		s := string(b)
		for oldVal, newVal := range t.ParseStrings {
			s = strings.Replace(s, oldVal, newVal, -1)
		}
		return []byte(s)
	}
}

// NewRenderer 新建渲染接口
func (t *Config) NewRenderer(manager ...driver.Manager) driver.Driver {
	tmplDir := t.TmplDir
	if len(t.Theme) > 0 {
		tmplDir = filepath.Join(tmplDir, t.Theme)
	}
	renderer := New(t.Engine, tmplDir)
	renderer.Init()
	if len(manager) > 0 {
		renderer.SetManager(manager[0])
	}
	renderer.SetContentProcessor(t.Parser())
	if t.StaticOptions != nil {
		st := t.NewStatic()
		renderer.SetFuncMap(func() map[string]interface{} {
			return st.Register(nil)
		})
		absTmplPath, err := filepath.Abs(tmplDir)
		var absFilePath string
		if err == nil {
			absFilePath, err = filepath.Abs(t.StaticOptions.Root)
		}
		if err == nil {
			if strings.HasPrefix(absFilePath, absTmplPath) {
				//如果静态文件在模板的子文件夹时，监控模板时判断静态文件更改
				renderer.MonitorEvent(st.OnUpdate())
			}
		}
	}
	return renderer
}

func (t *Config) AddFuncSetter(set ...echo.HandlerFunc) *Config {
	if t.errorPageFuncSetter == nil {
		t.errorPageFuncSetter = make([]echo.HandlerFunc, len(DefaultOptions.SetFuncMap))
		for index, setter := range DefaultOptions.SetFuncMap {
			t.errorPageFuncSetter[index] = setter
		}
	}
	t.errorPageFuncSetter = append(t.errorPageFuncSetter, set...)
	return t
}

func (t *Config) SetFuncSetter(set ...echo.HandlerFunc) *Config {
	t.errorPageFuncSetter = set
	return t
}

func (t *Config) ApplyTo(e *echo.Echo, manager ...driver.Manager) *Config {
	if t.renderer != nil {
		t.renderer.Close()
	}
	opt := &Options{
		ErrorPages:           t.ErrorPages,
		DefaultHTTPErrorCode: t.DefaultHTTPErrorCode,
	}
	opt.SetFuncSetter(t.errorPageFuncSetter...)
	e.SetHTTPErrorHandler(HTTPErrorHandler(opt))
	var funcMapSkipper echo.Skipper
	if t.FuncMapSkipper != nil {
		funcMapSkipper = t.FuncMapSkipper
	} else {
		funcMapSkipper = DefaultFuncMapSkipper
	}
	e.Use(middleware.FuncMap(tplfunc.New(), funcMapSkipper))
	renderer := t.NewRenderer(manager...)
	if t.StaticOptions != nil {
		e.Use(middleware.Static(t.StaticOptions))
	}
	e.SetRenderer(renderer)
	t.renderer = renderer
	return t
}

func (t *Config) Renderer() driver.Driver {
	return t.renderer
}

func (t *Config) NewStatic() *resource.Static {
	return resource.NewStatic(t.StaticOptions.Path, t.StaticOptions.Root)
}

// ThemeDir 主题所在文件夹的路径
func (t *Config) ThemeDir(args ...string) string {
	if len(args) < 1 {
		return filepath.Join(t.TmplDir, t.Theme)
	}
	return filepath.Join(t.TmplDir, args[0])
}
<|endoftext|>


<commit_before>//Copyright 2016-2017 Beate Ottenwälder
//
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
package data_manipulation

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/ottenwbe/golook/client"
)

var _ = Describe("The query service", func() {
	It("should call the golook client", func() {
		runWithMockedGolookClient(func() {
			err := QueryFiles()
			Expect(err).To(BeNil())
			Expect(client.GolookClient.(*MockGolookClient).visitDoGetFiles).To(BeTrue())
		})
	})
})
<commit_msg>Featue: Query and Reporting lib<commit_after>//Copyright 2016-2017 Beate Ottenwälder
//
//Licensed under the Apache License, Version 2.0 (the "License");
//you may not use this file except in compliance with the License.
//You may obtain a copy of the License at
//
//http://www.apache.org/licenses/LICENSE-2.0
//
//Unless required by applicable law or agreed to in writing, software
//distributed under the License is distributed on an "AS IS" BASIS,
//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//See the License for the specific language governing permissions and
//limitations under the License.
package data_manipulation

import (
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"github.com/ottenwbe/golook/client"
)

var _ = Describe("The query service", func() {
	It("should call the golook client", func() {
		runWithMockedGolookClient(func() {
			err := QueryFiles()
			Expect(err).To(BeNil())
			Expect(client.GolookClient.(*MockGolookClient).visitDoGetFiles).To(BeTrue())
		})
	})
})
<|endoftext|>


<commit_before>package memdtool

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/pkg/errors"
)

const (
	exitCodeOK = iota
	exitCodeParseFlagErr
	exitCodeErr
)

// CLI is struct for command line tool
type CLI struct {
	OutStream, ErrStream io.Writer
}

var helpReg = regexp.MustCompile(`^--?h(?:elp)?$`)

// Run the memdtool
func (cli *CLI) Run(argv []string) int {
	log.SetOutput(cli.ErrStream)
	log.SetFlags(0)

	mode := "display"
	addr := "127.0.0.1:11211"
	if len(argv) > 0 {
		modeCandidate := argv[len(argv)-1]
		if modeCandidate == "display" || modeCandidate == "dump" {
			mode = modeCandidate
			argv = argv[:len(argv)-1]
		}
		if len(argv) > 0 {
			addr = argv[0]
			if helpReg.MatchString(addr) {
				printHelp(cli.ErrStream)
				return exitCodeOK
			}
		}
	}

	var proto = "tcp"
	if strings.Contains(addr, "/") {
		proto = "unix"
	}
	conn, err := net.Dial(proto, addr)
	if err != nil {
		log.Println(err.Error())
		return exitCodeErr
	}
	defer conn.Close()

	switch mode {
	case "display":
		return cli.display(conn)
	case "dump":
		log.Println("still not implemented")
	}
	return exitCodeErr
}

func (cli *CLI) display(conn io.ReadWriter) int {
	items, err := GetSlabStats(conn)
	if err != nil {
		log.Println(err.Error())
		return exitCodeErr
	}

	fmt.Fprint(cli.OutStream, "  #  Item_Size  Max_age   Pages   Count   Full?  Evicted Evict_Time OOM\n")
	for _, ss := range items {
		if ss.TotalPages == 0 {
			continue
		}
		size := fmt.Sprintf("%dB", ss.ChunkSize)
		if ss.ChunkSize > 1024 {
			size = fmt.Sprintf("%.1fK", float64(ss.ChunkSize)/1024.0)
		}
		full := "no"
		if ss.FreeChunksEnd == 0 {
			full = "yes"
		}
		fmt.Fprintf(cli.OutStream,
			"%3d %8s %9ds %7d %7d %7s %8d %8d %4d\n",
			ss.ID,
			size,
			ss.Age,
			ss.TotalPages,
			ss.Number,
			full,
			ss.Evicted,
			ss.EvictedTime,
			ss.Outofmemory,
		)
	}
	return exitCodeOK
}

func printHelp(w io.Writer) {
	fmt.Fprint(w, `Usage: memcached-tool <host[:port] | /path/to/socket>

       memcached-tool 127.0.0.1:11211    # shows slabs
`)
}

type SlabStat struct {
	ID             uint64
	Number         uint64 // Count?
	Age            uint64
	Evicted        uint64
	EvictedNonzero uint64
	EvictedTime    uint64
	Outofmemory    uint64
	Reclaimed      uint64
	ChunkSize      uint64
	ChunksPerPage  uint64
	TotalPages     uint64
	TotalChunks    uint64
	UsedChunks     uint64
	FreeChunks     uint64
	FreeChunksEnd  uint64
}

func GetSlabStats(conn io.ReadWriter) ([]*SlabStat, error) {
	retMap := make(map[int]*SlabStat)
	fmt.Fprint(conn, "stats items\r\n")
	scr := bufio.NewScanner(bufio.NewReader(conn))
	for scr.Scan() {
		// ex. STAT items:1:number 1
		line := scr.Text()
		if line == "END" {
			break
		}
		fields := strings.Fields(line)
		if len(fields) != 3 {
			return nil, fmt.Errorf("result of `stats items` is strange: %s", line)
		}
		fields2 := strings.Split(fields[1], ":")
		if len(fields2) != 3 {
			return nil, fmt.Errorf("result of `stats items` is strange: %s", line)
		}
		key := fields2[2]
		slabNum, _ := strconv.ParseUint(fields2[1], 10, 64)
		value, _ := strconv.ParseUint(fields[2], 10, 64)
		ss, ok := retMap[int(slabNum)]
		if !ok {
			ss = &SlabStat{ID: slabNum}
			retMap[int(slabNum)] = ss
		}
		switch key {
		case "number":
			ss.Number = value
		case "age":
			ss.Age = value
		case "evicted":
			ss.Evicted = value
		case "evicted_nonzero":
			ss.EvictedNonzero = value
		case "evicted_time":
			ss.EvictedNonzero = value
		case "outofmemory":
			ss.Outofmemory = value
		case "reclaimed":
			ss.Reclaimed = value
		}
	}
	if err := scr.Err(); err != nil {
		return nil, errors.Wrap(err, "failed to GetSlabStats while scaning stats items")
	}

	fmt.Fprint(conn, "stats slabs\r\n")
	for scr.Scan() {
		// ex. STAT 1:chunk_size 96
		line := scr.Text()
		if line == "END" {
			break
		}
		fields := strings.Fields(line)
		if len(fields) != 3 {
			return nil, fmt.Errorf("result of `stats slabs` is strange: %s", line)
		}
		fields2 := strings.Split(fields[1], ":")
		if len(fields2) != 2 {
			continue
		}
		key := fields2[1]
		slabNum, _ := strconv.ParseUint(fields2[0], 10, 64)
		value, _ := strconv.ParseUint(fields[2], 10, 64)
		ss, ok := retMap[int(slabNum)]
		if !ok {
			ss = &SlabStat{}
			retMap[int(slabNum)] = ss
		}

		switch key {
		case "chunk_size":
			ss.ChunkSize = value
		case "chunks_per_page":
			ss.ChunksPerPage = value
		case "total_pages":
			ss.TotalPages = value
		case "total_chunks":
			ss.TotalChunks = value
		case "used_chunks":
			ss.UsedChunks = value
		case "free_chunks":
			ss.FreeChunks = value
		case "free_chunks_end":
			ss.FreeChunksEnd = value
		}
	}
	if err := scr.Err(); err != nil {
		return nil, errors.Wrap(err, "failed to GetSlabStats while scaning stats slabs")
	}

	keys := make([]int, 0, len(retMap))
	for i := range retMap {
		keys = append(keys, i)
	}
	sort.Ints(keys)
	ret := make([]*SlabStat, len(keys))
	for i, v := range keys {
		ret[i] = retMap[v]
	}
	return ret, nil
}
<commit_msg>implement dump mode<commit_after>package memdtool

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/pkg/errors"
)

const (
	exitCodeOK = iota
	exitCodeParseFlagErr
	exitCodeErr
)

// CLI is struct for command line tool
type CLI struct {
	OutStream, ErrStream io.Writer
}

var helpReg = regexp.MustCompile(`^--?h(?:elp)?$`)

// Run the memdtool
func (cli *CLI) Run(argv []string) int {
	log.SetOutput(cli.ErrStream)
	log.SetFlags(0)

	mode := "display"
	addr := "127.0.0.1:11211"
	if len(argv) > 0 {
		modeCandidate := argv[len(argv)-1]
		if modeCandidate == "display" || modeCandidate == "dump" {
			mode = modeCandidate
			argv = argv[:len(argv)-1]
		}
		if len(argv) > 0 {
			addr = argv[0]
			if helpReg.MatchString(addr) {
				printHelp(cli.ErrStream)
				return exitCodeOK
			}
		}
	}

	var proto = "tcp"
	if strings.Contains(addr, "/") {
		proto = "unix"
	}
	conn, err := net.Dial(proto, addr)
	if err != nil {
		log.Println(err.Error())
		return exitCodeErr
	}
	defer conn.Close()

	switch mode {
	case "display":
		return cli.display(conn)
	case "dump":
		return cli.dump(conn)
	}
	return exitCodeErr
}

func (cli *CLI) display(conn io.ReadWriter) int {
	items, err := GetSlabStats(conn)
	if err != nil {
		log.Println(err.Error())
		return exitCodeErr
	}

	fmt.Fprint(cli.OutStream, "  #  Item_Size  Max_age   Pages   Count   Full?  Evicted Evict_Time OOM\n")
	for _, ss := range items {
		if ss.TotalPages == 0 {
			continue
		}
		size := fmt.Sprintf("%dB", ss.ChunkSize)
		if ss.ChunkSize > 1024 {
			size = fmt.Sprintf("%.1fK", float64(ss.ChunkSize)/1024.0)
		}
		full := "no"
		if ss.FreeChunksEnd == 0 {
			full = "yes"
		}
		fmt.Fprintf(cli.OutStream,
			"%3d %8s %9ds %7d %7d %7s %8d %8d %4d\n",
			ss.ID,
			size,
			ss.Age,
			ss.TotalPages,
			ss.Number,
			full,
			ss.Evicted,
			ss.EvictedTime,
			ss.Outofmemory,
		)
	}
	return exitCodeOK
}

func (cli *CLI) dump(conn io.ReadWriter) int {
	fmt.Fprint(conn, "stats items\r\n")
	slabItems := make(map[string]uint64)
	rdr := bufio.NewReader(conn)
	for {
		lineBytes, _, err := rdr.ReadLine()
		if err != nil {
			log.Println(err.Error())
			return exitCodeErr
		}
		line := string(lineBytes)
		if line == "END" {
			break
		}
		// ex. STAT items:1:number 1
		if !strings.Contains(line, ":number ") {
			continue
		}
		fields := strings.Fields(line)
		if len(fields) != 3 {
			log.Printf("result of `stats items` is strange: %s\n", line)
			return exitCodeErr
		}
		fields2 := strings.Split(fields[1], ":")
		if len(fields2) != 3 {
			log.Printf("result of `stats items` is strange: %s\n", line)
			return exitCodeErr
		}
		value, _ := strconv.ParseUint(fields[2], 10, 64)
		slabItems[fields2[1]] = value
	}

	var totalItems uint64
	for _, v := range slabItems {
		totalItems += v
	}
	fmt.Fprintf(cli.ErrStream, "Dumping memcache contents\n")
	fmt.Fprintf(cli.ErrStream, "  Number of buckets: %d\n", len(slabItems))
	fmt.Fprintf(cli.ErrStream, "  Number of items  : %d\n", totalItems)

	for k, v := range slabItems {
		fmt.Fprintf(cli.ErrStream, "Dumping bucket %s - %d total items\n", k, v)

		keyexp := make(map[string]string, int(v))
		fmt.Fprintf(conn, "stats cachedump %s %d\r\n", k, v)
		for {
			lineBytes, _, err := rdr.ReadLine()
			if err != nil {
				log.Println(err.Error())
				return exitCodeErr
			}
			line := string(lineBytes)
			if line == "END" {
				break
			}
			// return format like this
			// ITEM piyo [1 b; 1483953061 s]
			fields := strings.Fields(line)
			if len(fields) == 6 && fields[0] == "ITEM" {
				keyexp[fields[1]] = fields[4]
			}
		}

		for cachekey, exp := range keyexp {
			fmt.Fprintf(conn, "get %s\r\n", cachekey)
			for {
				lineBytes, _, err := rdr.ReadLine()
				if err != nil {
					log.Println(err.Error())
					return exitCodeErr
				}
				line := string(lineBytes)
				if line == "END" {
					break
				}
				// VALUE hoge 0 6
				// hogege
				fields := strings.Fields(line)
				if len(fields) != 4 || fields[0] != "VALUE" {
					continue
				}
				flags := fields[2]
				sizeStr := fields[3]
				size, err := strconv.Atoi(sizeStr)
				if err != nil {
					log.Println(err.Error())
					return exitCodeErr
				}
				buf := make([]byte, size)
				_, err = rdr.Read(buf)
				if err != nil {
					log.Println(err.Error())
					return exitCodeErr
				}
				fmt.Fprintf(cli.OutStream, "add %s %s %s %s\r\n%s\r\n", cachekey, flags, exp, sizeStr, string(buf))
				rdr.ReadLine()
			}
		}
	}
	return exitCodeOK
}

func printHelp(w io.Writer) {
	fmt.Fprint(w, `Usage: memcached-tool <host[:port] | /path/to/socket>

       memcached-tool 127.0.0.1:11211    # shows slabs
`)
}

type SlabStat struct {
	ID             uint64
	Number         uint64 // Count?
	Age            uint64
	Evicted        uint64
	EvictedNonzero uint64
	EvictedTime    uint64
	Outofmemory    uint64
	Reclaimed      uint64
	ChunkSize      uint64
	ChunksPerPage  uint64
	TotalPages     uint64
	TotalChunks    uint64
	UsedChunks     uint64
	FreeChunks     uint64
	FreeChunksEnd  uint64
}

func GetSlabStats(conn io.ReadWriter) ([]*SlabStat, error) {
	retMap := make(map[int]*SlabStat)
	fmt.Fprint(conn, "stats items\r\n")
	scr := bufio.NewScanner(bufio.NewReader(conn))
	for scr.Scan() {
		// ex. STAT items:1:number 1
		line := scr.Text()
		if line == "END" {
			break
		}
		fields := strings.Fields(line)
		if len(fields) != 3 {
			return nil, fmt.Errorf("result of `stats items` is strange: %s", line)
		}
		fields2 := strings.Split(fields[1], ":")
		if len(fields2) != 3 {
			return nil, fmt.Errorf("result of `stats items` is strange: %s", line)
		}
		key := fields2[2]
		slabNum, _ := strconv.ParseUint(fields2[1], 10, 64)
		value, _ := strconv.ParseUint(fields[2], 10, 64)
		ss, ok := retMap[int(slabNum)]
		if !ok {
			ss = &SlabStat{ID: slabNum}
			retMap[int(slabNum)] = ss
		}
		switch key {
		case "number":
			ss.Number = value
		case "age":
			ss.Age = value
		case "evicted":
			ss.Evicted = value
		case "evicted_nonzero":
			ss.EvictedNonzero = value
		case "evicted_time":
			ss.EvictedNonzero = value
		case "outofmemory":
			ss.Outofmemory = value
		case "reclaimed":
			ss.Reclaimed = value
		}
	}
	if err := scr.Err(); err != nil {
		return nil, errors.Wrap(err, "failed to GetSlabStats while scaning stats items")
	}

	fmt.Fprint(conn, "stats slabs\r\n")
	for scr.Scan() {
		// ex. STAT 1:chunk_size 96
		line := scr.Text()
		if line == "END" {
			break
		}
		fields := strings.Fields(line)
		if len(fields) != 3 {
			return nil, fmt.Errorf("result of `stats slabs` is strange: %s", line)
		}
		fields2 := strings.Split(fields[1], ":")
		if len(fields2) != 2 {
			continue
		}
		key := fields2[1]
		slabNum, _ := strconv.ParseUint(fields2[0], 10, 64)
		value, _ := strconv.ParseUint(fields[2], 10, 64)
		ss, ok := retMap[int(slabNum)]
		if !ok {
			ss = &SlabStat{}
			retMap[int(slabNum)] = ss
		}

		switch key {
		case "chunk_size":
			ss.ChunkSize = value
		case "chunks_per_page":
			ss.ChunksPerPage = value
		case "total_pages":
			ss.TotalPages = value
		case "total_chunks":
			ss.TotalChunks = value
		case "used_chunks":
			ss.UsedChunks = value
		case "free_chunks":
			ss.FreeChunks = value
		case "free_chunks_end":
			ss.FreeChunksEnd = value
		}
	}
	if err := scr.Err(); err != nil {
		return nil, errors.Wrap(err, "failed to GetSlabStats while scaning stats slabs")
	}

	keys := make([]int, 0, len(retMap))
	for i := range retMap {
		keys = append(keys, i)
	}
	sort.Ints(keys)
	ret := make([]*SlabStat, len(keys))
	for i, v := range keys {
		ret[i] = retMap[v]
	}
	return ret, nil
}
<|endoftext|>


<commit_before>package hoverfly

import (
	"bytes"
	"fmt"

	"github.com/boltdb/bolt"
)

// Metadata - interface to store and retrieve any metadata that is related to Hoverfly
type Metadata interface {
	Set(key, value []byte) error
	Get(key []byte) ([]byte, error)
	Delete(key []byte) error
	GetAll() ([]MetaObject, error)
	CloseDB()
}

// NewBoltDBMetadata - default metadata store
func NewBoltDBMetadata(db *bolt.DB, bucket []byte) *BoltCache {
	return &BoltCache{
		DS:             db,
		RequestsBucket: []byte(bucket),
	}
}

const MetadataBucketName = []byte("metadataBucket")

type BoltMeta struct {
	DS             *bolt.DB
	MetadataBucket []byte
}

// CloseDB - closes database
func (m *BoltMeta) CloseDB() {
	m.DS.Close()
}

// Set - saves given key and value pair to BoltDB
func (m *BoltMeta) Set(key, value []byte) error {
	err := m.DS.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists(m.MetadataBucket)
		if err != nil {
			return err
		}
		err = bucket.Put(key, value)
		if err != nil {
			return err
		}
		return nil
	})

	return err
}

// Get - gets value for given key
func (m *BoltMeta) Get(key []byte) (value []byte, err error) {
	err = m.DS.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(m.MetadataBucket)
		if bucket == nil {
			return fmt.Errorf("Bucket %q not found!", m.MetadataBucket)
		}
		var buffer bytes.Buffer
		val := bucket.Get(key)

		// If it doesn't exist then it will return nil
		if val == nil {
			return fmt.Errorf("key %q not found \n", key)
		}

		buffer.Write(val)
		value = buffer.Bytes()
		return nil
	})

	return
}

// MetaObject - container to store both keys and values of captured objects
type MetaObject struct {
	Key   []byte
	Value []byte
}

// GetAll - returns all key/value pairs
func (m *BoltMeta) GetAll() (objects []MetaObject, err error) {
	err = m.DS.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(m.MetadataBucket)
		if b == nil {
			// bucket doesn't exist
			return nil
		}
		c := b.Cursor()

		for k, v := c.First(); k != nil; k, v = c.Next() {
			obj := &MetaObject{Key: k, Value: v}
			objects = append(objects, obj)
		}
		return nil
	})
	return
}
<commit_msg>delete key function<commit_after>package hoverfly

import (
	"bytes"
	"fmt"

	"github.com/boltdb/bolt"
)

// Metadata - interface to store and retrieve any metadata that is related to Hoverfly
type Metadata interface {
	Set(key, value []byte) error
	Get(key []byte) ([]byte, error)
	Delete(key []byte) error
	GetAll() ([]MetaObject, error)
	CloseDB()
}

// NewBoltDBMetadata - default metadata store
func NewBoltDBMetadata(db *bolt.DB, bucket []byte) *BoltCache {
	return &BoltCache{
		DS:             db,
		RequestsBucket: []byte(bucket),
	}
}

const MetadataBucketName = []byte("metadataBucket")

type BoltMeta struct {
	DS             *bolt.DB
	MetadataBucket []byte
}

// CloseDB - closes database
func (m *BoltMeta) CloseDB() {
	m.DS.Close()
}

// Set - saves given key and value pair to BoltDB
func (m *BoltMeta) Set(key, value []byte) error {
	err := m.DS.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists(m.MetadataBucket)
		if err != nil {
			return err
		}
		err = bucket.Put(key, value)
		if err != nil {
			return err
		}
		return nil
	})

	return err
}

// Get - gets value for given key
func (m *BoltMeta) Get(key []byte) (value []byte, err error) {
	err = m.DS.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(m.MetadataBucket)
		if bucket == nil {
			return fmt.Errorf("Bucket %q not found!", m.MetadataBucket)
		}
		var buffer bytes.Buffer
		val := bucket.Get(key)

		// If it doesn't exist then it will return nil
		if val == nil {
			return fmt.Errorf("key %q not found \n", key)
		}

		buffer.Write(val)
		value = buffer.Bytes()
		return nil
	})

	return
}

// MetaObject - container to store both keys and values of captured objects
type MetaObject struct {
	Key   []byte
	Value []byte
}

// GetAll - returns all key/value pairs
func (m *BoltMeta) GetAll() (objects []MetaObject, err error) {
	err = m.DS.View(func(tx *bolt.Tx) error {
		b := tx.Bucket(m.MetadataBucket)
		if b == nil {
			// bucket doesn't exist
			return nil
		}
		c := b.Cursor()

		for k, v := c.First(); k != nil; k, v = c.Next() {
			obj := &MetaObject{Key: k, Value: v}
			objects = append(objects, obj)
		}
		return nil
	})
	return
}

// Delete - deletes given metadata key
func (m *BoltMeta) Delete(key []byte) error {
	err := m.DS.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists(m.MetadataBucket)
		if err != nil {
			return err
		}
		err = bucket.Delete(key)
		if err != nil {
			return err
		}
		return nil
	})

	return err
}
<|endoftext|>


<commit_before>package slack

// OutgoingMessage is used for the realtime API, and seems incomplete.
type OutgoingMessage struct {
	ID int `json:"id"`
	// channel ID
	Channel         string   `json:"channel,omitempty"`
	Text            string   `json:"text,omitempty"`
	Type            string   `json:"type,omitempty"`
	ThreadTimestamp string   `json:"thread_ts,omitempty"`
	ThreadBroadcast bool     `json:"reply_broadcast,omitempty"`
	IDs             []string `json:"ids,omitempty"`
}

// Message is an auxiliary type to allow us to have a message containing sub messages
type Message struct {
	Msg
	SubMessage *Msg `json:"message,omitempty"`
	PreviousMessage *Msg `json:"previous_message,omitempty"`
}

// Msg contains information about a slack message
type Msg struct {
	// Basic Message
	Type            string       `json:"type,omitempty"`
	Channel         string       `json:"channel,omitempty"`
	User            string       `json:"user,omitempty"`
	Text            string       `json:"text,omitempty"`
	Timestamp       string       `json:"ts,omitempty"`
	ThreadTimestamp string       `json:"thread_ts,omitempty"`
	IsStarred       bool         `json:"is_starred,omitempty"`
	PinnedTo        []string     `json:"pinned_to,omitempty"`
	Attachments     []Attachment `json:"attachments,omitempty"`
	Edited          *Edited      `json:"edited,omitempty"`
	LastRead        string       `json:"last_read,omitempty"`
	Subscribed      bool         `json:"subscribed,omitempty"`
	UnreadCount     int          `json:"unread_count,omitempty"`

	// Message Subtypes
	SubType string `json:"subtype,omitempty"`

	// Hidden Subtypes
	Hidden           bool   `json:"hidden,omitempty"`     // message_changed, message_deleted, unpinned_item
	DeletedTimestamp string `json:"deleted_ts,omitempty"` // message_deleted
	EventTimestamp   string `json:"event_ts,omitempty"`

	// bot_message (https://api.slack.com/events/message/bot_message)
	BotID    string `json:"bot_id,omitempty"`
	Username string `json:"username,omitempty"`
	Icons    *Icon  `json:"icons,omitempty"`

	// channel_join, group_join
	Inviter string `json:"inviter,omitempty"`

	// channel_topic, group_topic
	Topic string `json:"topic,omitempty"`

	// channel_purpose, group_purpose
	Purpose string `json:"purpose,omitempty"`

	// channel_name, group_name
	Name    string `json:"name,omitempty"`
	OldName string `json:"old_name,omitempty"`

	// channel_archive, group_archive
	Members []string `json:"members,omitempty"`

	// channels.replies, groups.replies, im.replies, mpim.replies
	ReplyCount   int     `json:"reply_count,omitempty"`
	Replies      []Reply `json:"replies,omitempty"`
	ParentUserId string  `json:"parent_user_id,omitempty"`

	// file_share, file_comment, file_mention
	Files []File `json:"files,omitempty"`

	// file_share
	Upload bool `json:"upload,omitempty"`

	// file_comment
	Comment *Comment `json:"comment,omitempty"`

	// pinned_item
	ItemType string `json:"item_type,omitempty"`

	// https://api.slack.com/rtm
	ReplyTo int    `json:"reply_to,omitempty"`
	Team    string `json:"team,omitempty"`

	// reactions
	Reactions []ItemReaction `json:"reactions,omitempty"`

	// slash commands and interactive messages
	ResponseType    string `json:"response_type,omitempty"`
	ReplaceOriginal bool   `json:"replace_original"`
	DeleteOriginal  bool   `json:"delete_original"`

	// Block type Message
	Blocks Blocks `json:"blocks,omitempty"`
}

const (
	// ResponseTypeInChannel in channel response for slash commands.
	ResponseTypeInChannel = "in_channel"
	// ResponseTypeEphemeral ephemeral respone for slash commands.
	ResponseTypeEphemeral = "ephemeral"
)

// Icon is used for bot messages
type Icon struct {
	IconURL   string `json:"icon_url,omitempty"`
	IconEmoji string `json:"icon_emoji,omitempty"`
}

// Edited indicates that a message has been edited.
type Edited struct {
	User      string `json:"user,omitempty"`
	Timestamp string `json:"ts,omitempty"`
}

// Reply contains information about a reply for a thread
type Reply struct {
	User      string `json:"user,omitempty"`
	Timestamp string `json:"ts,omitempty"`
}

// Event contains the event type
type Event struct {
	Type string `json:"type,omitempty"`
}

// Ping contains information about a Ping Event
type Ping struct {
	ID        int    `json:"id"`
	Type      string `json:"type"`
	Timestamp int64  `json:"timestamp"`
}

// Pong contains information about a Pong Event
type Pong struct {
	Type      string `json:"type"`
	ReplyTo   int    `json:"reply_to"`
	Timestamp int64  `json:"timestamp"`
}

// NewOutgoingMessage prepares an OutgoingMessage that the user can
// use to send a message. Use this function to properly set the
// messageID.
func (rtm *RTM) NewOutgoingMessage(text string, channelID string, options ...RTMsgOption) *OutgoingMessage {
	id := rtm.idGen.Next()
	msg := OutgoingMessage{
		ID:      id,
		Type:    "message",
		Channel: channelID,
		Text:    text,
	}
	for _, option := range options {
		option(&msg)
	}
	return &msg
}

// NewSubscribeUserPresence prepares an OutgoingMessage that the user can
// use to subscribe presence events for the specified users.
func (rtm *RTM) NewSubscribeUserPresence(ids []string) *OutgoingMessage {
	return &OutgoingMessage{
		Type: "presence_sub",
		IDs:  ids,
	}
}

// NewTypingMessage prepares an OutgoingMessage that the user can
// use to send as a typing indicator. Use this function to properly set the
// messageID.
func (rtm *RTM) NewTypingMessage(channelID string) *OutgoingMessage {
	id := rtm.idGen.Next()
	return &OutgoingMessage{
		ID:      id,
		Type:    "typing",
		Channel: channelID,
	}
}

// RTMsgOption allows configuration of various options available for sending an RTM message
type RTMsgOption func(*OutgoingMessage)

// RTMsgOptionTS sets thead timestamp of an outgoing message in order to respond to a thread
func RTMsgOptionTS(threadTimestamp string) RTMsgOption {
	return func(msg *OutgoingMessage) {
		msg.ThreadTimestamp = threadTimestamp
	}
}

// RTMsgOptionBroadcast sets broadcast reply to channel to "true"
func RTMsgOptionBroadcast() RTMsgOption {
	return func(msg *OutgoingMessage) {
		msg.ThreadBroadcast = true
	}
}
<commit_msg>Fix typo<commit_after>package slack

// OutgoingMessage is used for the realtime API, and seems incomplete.
type OutgoingMessage struct {
	ID int `json:"id"`
	// channel ID
	Channel         string   `json:"channel,omitempty"`
	Text            string   `json:"text,omitempty"`
	Type            string   `json:"type,omitempty"`
	ThreadTimestamp string   `json:"thread_ts,omitempty"`
	ThreadBroadcast bool     `json:"reply_broadcast,omitempty"`
	IDs             []string `json:"ids,omitempty"`
}

// Message is an auxiliary type to allow us to have a message containing sub messages
type Message struct {
	Msg
	SubMessage *Msg `json:"message,omitempty"`
	PreviousMessage *Msg `json:"previous_message,omitempty"`
}

// Msg contains information about a slack message
type Msg struct {
	// Basic Message
	Type            string       `json:"type,omitempty"`
	Channel         string       `json:"channel,omitempty"`
	User            string       `json:"user,omitempty"`
	Text            string       `json:"text,omitempty"`
	Timestamp       string       `json:"ts,omitempty"`
	ThreadTimestamp string       `json:"thread_ts,omitempty"`
	IsStarred       bool         `json:"is_starred,omitempty"`
	PinnedTo        []string     `json:"pinned_to,omitempty"`
	Attachments     []Attachment `json:"attachments,omitempty"`
	Edited          *Edited      `json:"edited,omitempty"`
	LastRead        string       `json:"last_read,omitempty"`
	Subscribed      bool         `json:"subscribed,omitempty"`
	UnreadCount     int          `json:"unread_count,omitempty"`

	// Message Subtypes
	SubType string `json:"subtype,omitempty"`

	// Hidden Subtypes
	Hidden           bool   `json:"hidden,omitempty"`     // message_changed, message_deleted, unpinned_item
	DeletedTimestamp string `json:"deleted_ts,omitempty"` // message_deleted
	EventTimestamp   string `json:"event_ts,omitempty"`

	// bot_message (https://api.slack.com/events/message/bot_message)
	BotID    string `json:"bot_id,omitempty"`
	Username string `json:"username,omitempty"`
	Icons    *Icon  `json:"icons,omitempty"`

	// channel_join, group_join
	Inviter string `json:"inviter,omitempty"`

	// channel_topic, group_topic
	Topic string `json:"topic,omitempty"`

	// channel_purpose, group_purpose
	Purpose string `json:"purpose,omitempty"`

	// channel_name, group_name
	Name    string `json:"name,omitempty"`
	OldName string `json:"old_name,omitempty"`

	// channel_archive, group_archive
	Members []string `json:"members,omitempty"`

	// channels.replies, groups.replies, im.replies, mpim.replies
	ReplyCount   int     `json:"reply_count,omitempty"`
	Replies      []Reply `json:"replies,omitempty"`
	ParentUserId string  `json:"parent_user_id,omitempty"`

	// file_share, file_comment, file_mention
	Files []File `json:"files,omitempty"`

	// file_share
	Upload bool `json:"upload,omitempty"`

	// file_comment
	Comment *Comment `json:"comment,omitempty"`

	// pinned_item
	ItemType string `json:"item_type,omitempty"`

	// https://api.slack.com/rtm
	ReplyTo int    `json:"reply_to,omitempty"`
	Team    string `json:"team,omitempty"`

	// reactions
	Reactions []ItemReaction `json:"reactions,omitempty"`

	// slash commands and interactive messages
	ResponseType    string `json:"response_type,omitempty"`
	ReplaceOriginal bool   `json:"replace_original"`
	DeleteOriginal  bool   `json:"delete_original"`

	// Block type Message
	Blocks Blocks `json:"blocks,omitempty"`
}

const (
	// ResponseTypeInChannel in channel response for slash commands.
	ResponseTypeInChannel = "in_channel"
	// ResponseTypeEphemeral ephemeral response for slash commands.
	ResponseTypeEphemeral = "ephemeral"
)

// Icon is used for bot messages
type Icon struct {
	IconURL   string `json:"icon_url,omitempty"`
	IconEmoji string `json:"icon_emoji,omitempty"`
}

// Edited indicates that a message has been edited.
type Edited struct {
	User      string `json:"user,omitempty"`
	Timestamp string `json:"ts,omitempty"`
}

// Reply contains information about a reply for a thread
type Reply struct {
	User      string `json:"user,omitempty"`
	Timestamp string `json:"ts,omitempty"`
}

// Event contains the event type
type Event struct {
	Type string `json:"type,omitempty"`
}

// Ping contains information about a Ping Event
type Ping struct {
	ID        int    `json:"id"`
	Type      string `json:"type"`
	Timestamp int64  `json:"timestamp"`
}

// Pong contains information about a Pong Event
type Pong struct {
	Type      string `json:"type"`
	ReplyTo   int    `json:"reply_to"`
	Timestamp int64  `json:"timestamp"`
}

// NewOutgoingMessage prepares an OutgoingMessage that the user can
// use to send a message. Use this function to properly set the
// messageID.
func (rtm *RTM) NewOutgoingMessage(text string, channelID string, options ...RTMsgOption) *OutgoingMessage {
	id := rtm.idGen.Next()
	msg := OutgoingMessage{
		ID:      id,
		Type:    "message",
		Channel: channelID,
		Text:    text,
	}
	for _, option := range options {
		option(&msg)
	}
	return &msg
}

// NewSubscribeUserPresence prepares an OutgoingMessage that the user can
// use to subscribe presence events for the specified users.
func (rtm *RTM) NewSubscribeUserPresence(ids []string) *OutgoingMessage {
	return &OutgoingMessage{
		Type: "presence_sub",
		IDs:  ids,
	}
}

// NewTypingMessage prepares an OutgoingMessage that the user can
// use to send as a typing indicator. Use this function to properly set the
// messageID.
func (rtm *RTM) NewTypingMessage(channelID string) *OutgoingMessage {
	id := rtm.idGen.Next()
	return &OutgoingMessage{
		ID:      id,
		Type:    "typing",
		Channel: channelID,
	}
}

// RTMsgOption allows configuration of various options available for sending an RTM message
type RTMsgOption func(*OutgoingMessage)

// RTMsgOptionTS sets thead timestamp of an outgoing message in order to respond to a thread
func RTMsgOptionTS(threadTimestamp string) RTMsgOption {
	return func(msg *OutgoingMessage) {
		msg.ThreadTimestamp = threadTimestamp
	}
}

// RTMsgOptionBroadcast sets broadcast reply to channel to "true"
func RTMsgOptionBroadcast() RTMsgOption {
	return func(msg *OutgoingMessage) {
		msg.ThreadBroadcast = true
	}
}
<|endoftext|>