<commit_before>package errorsx

import (
	"errors"
	"fmt"
	"runtime/debug"
	"sort"
	"strings"
)

// some common error objects to wrap
var (
	ObjectNotFound = errors.New("ObjectNotFound")
)

type kvPairsMapType map[interface{}]interface{}

type Error interface {
	Error() string
	Stack() []byte
}

type Err struct {
	err     error
	kvPairs kvPairsMapType
	stack   []byte
}

func (err *Err) Stack() []byte {
	return err.stack
}

func (err *Err) Error() string {
	var s = err.err.Error()
	var kvStrings []string
	for key, val := range err.kvPairs {
		kvStrings = append(kvStrings, fmt.Sprintf("%s=%#v", key, val))
	}
	if len(kvStrings) > 0 {
		sort.Slice(kvStrings, func(i, j int) bool {
			return kvStrings[i] < kvStrings[j]
		})
		s += fmt.Sprintf(" [%s]", strings.Join(kvStrings, ", "))
	}
	return s
}

// GoString implements the GoStringer interface,
// and so is printed with the %#v fmt directive.
// See https://golang.org/pkg/fmt/ for more details.
func (err *Err) GoString() string {
	return fmt.Sprintf("Error: %q\nStack:\n%s\n", err.Error(), err.Stack())
}

func Errorf(message string, args ...interface{}) Error {
	return &Err{
		fmt.Errorf(message, args...),
		make(kvPairsMapType),
		debug.Stack(),
	}
}

func Wrap(err error, kvPairs ...interface{}) Error {
	if err == nil {
		return nil
	}

	kvPairsMap := make(kvPairsMapType)
	for i := 0; i < len(kvPairs); i = i + 2 {
		k := kvPairs[i]

		var v interface{}
		if len(kvPairs) >= i+2 {
			v = kvPairs[i+1]
		} else {
			v = "[empty]"
		}
		kvPairsMap[k] = v
	}

	errType, ok := err.(*Err)
	if !ok {
		return &Err{
			err,
			kvPairsMap,
			debug.Stack(),
		}
	}

	// merge in kv map
	for k, v := range kvPairsMap {
		errType.kvPairs[k] = v
	}

	return errType
}

// Cause fetches the underlying cause of the error
// this should be used with errors wrapped from errors.New()
func Cause(err error) error {
	errErr, ok := err.(*Err)
	if ok {
		return Cause(errErr.err)
	}

	return err
}
<commit_msg>ErrWithStack method<commit_after>package errorsx

import (
	"errors"
	"fmt"
	"runtime/debug"
	"sort"
	"strings"
)

// some common error objects to wrap
var (
	ObjectNotFound = errors.New("ObjectNotFound")
)

type kvPairsMapType map[interface{}]interface{}

type Error interface {
	Error() string
	Stack() []byte
}

type Err struct {
	err     error
	kvPairs kvPairsMapType
	stack   []byte
}

func (err *Err) Stack() []byte {
	return err.stack
}

func (err *Err) Error() string {
	var s = err.err.Error()
	var kvStrings []string
	for key, val := range err.kvPairs {
		kvStrings = append(kvStrings, fmt.Sprintf("%s=%#v", key, val))
	}
	if len(kvStrings) > 0 {
		sort.Slice(kvStrings, func(i, j int) bool {
			return kvStrings[i] < kvStrings[j]
		})
		s += fmt.Sprintf(" [%s]", strings.Join(kvStrings, ", "))
	}
	return s
}

// ErrWithStack returns a std-lib error with the stack trace, if an error was passed in.
func ErrWithStack(err Error) error {
	if err == nil {
		return nil
	}
	return fmt.Errorf("Error: %s\nStack:\n%s\n", err.Error(), err.Stack())
}

// GoString implements the GoStringer interface,
// and so is printed with the %#v fmt directive.
// See https://golang.org/pkg/fmt/ for more details.
func (err *Err) GoString() string {
	return fmt.Sprintf("Error: %q\nStack:\n%s\n", err.Error(), err.Stack())
}

func Errorf(message string, args ...interface{}) Error {
	return &Err{
		fmt.Errorf(message, args...),
		make(kvPairsMapType),
		debug.Stack(),
	}
}

func Wrap(err error, kvPairs ...interface{}) Error {
	if err == nil {
		return nil
	}

	kvPairsMap := make(kvPairsMapType)
	for i := 0; i < len(kvPairs); i = i + 2 {
		k := kvPairs[i]

		var v interface{}
		if len(kvPairs) >= i+2 {
			v = kvPairs[i+1]
		} else {
			v = "[empty]"
		}
		kvPairsMap[k] = v
	}

	errType, ok := err.(*Err)
	if !ok {
		return &Err{
			err,
			kvPairsMap,
			debug.Stack(),
		}
	}

	// merge in kv map
	for k, v := range kvPairsMap {
		errType.kvPairs[k] = v
	}

	return errType
}

// Cause fetches the underlying cause of the error
// this should be used with errors wrapped from errors.New()
func Cause(err error) error {
	errErr, ok := err.(*Err)
	if ok {
		return Cause(errErr.err)
	}

	return err
}
<|endoftext|>


<commit_before>package example
<commit_msg>Fixed package inconsistency in the example folder<commit_after>package examples
<|endoftext|>


<commit_before>package main

import (
	"github.com/gin-gonic/gin"
	"github.com/ngerakines/ginpongo2"
	"log"
)

func main() {
	r := gin.Default()
	r.Use(ginpongo2.Pongo2())

	r.GET("/", func(c *gin.Context) {
		c.Set("template", "index.html")
		c.Set("data", map[string]interface{}{"message": "Hello World!"})
	})

	r.GET("/none", func(c *gin.Context) {
		c.Set("template", "none.html")
	})

	r.GET("/invalidTemplate", func(c *gin.Context) {
		c.Set("template", 3)
	})

	r.GET("/invalidData", func(c *gin.Context) {
		c.Set("template", "index.html")
		c.Set("data", 3)
	})

	r.GET("/emptyData", func(c *gin.Context) {
		c.Set("template", "index.html")
		c.Set("data", nil)
	})

	log.Println(r.Run(":8080"))
}
<commit_msg>fix import<commit_after>package main

import (
	"github.com/gin-gonic/gin"
	"github.com/wolfmetr/ginpongo2"
	"log"
)

func main() {
	r := gin.Default()
	r.Use(ginpongo2.Pongo2())

	r.GET("/", func(c *gin.Context) {
		c.Set("template", "index.html")
		c.Set("data", map[string]interface{}{"message": "Hello World!"})
	})

	r.GET("/none", func(c *gin.Context) {
		c.Set("template", "none.html")
	})

	r.GET("/invalidTemplate", func(c *gin.Context) {
		c.Set("template", 3)
	})

	r.GET("/invalidData", func(c *gin.Context) {
		c.Set("template", "index.html")
		c.Set("data", 3)
	})

	r.GET("/emptyData", func(c *gin.Context) {
		c.Set("template", "index.html")
		c.Set("data", nil)
	})

	log.Println(r.Run(":8080"))
}
<|endoftext|>


<commit_before>package fetch

import (
	"github.com/astaxie/beego/httplib"
	"github.com/shaalx/sstruct/log"
	"net"
	"net/http"
	"time"
)

/*根据给定的URL,fetch the data*/
func Do(url, ipaddr string) []byte {
	request := httplib.Get(url)
	// request.SetTransport(newTransport(ipaddr))
	request.Header("Host", "itunes.apple.com")
	request.Header("X-Apple-Store-Front", "143465-19,21 t:native")
	request.Header("Accept", "*/*")
	request.Header("Accept-Language", "zh-cn")
	request.Header("X-Dsid", "1458643138")
	request.Header("Connection", "keep-alive")
	request.Header("Proxy-Connection", "keep-alive")
	request.Header("Design-Agent", "AppStore/2.0 iOS/7.1.1 model/iPod5,1 build/11D201 (4; dt:81)")
	bs, err := request.Bytes()
	if log.IsError(err) {
		return nil
	}
	return bs
}

/*
* 固定IP
 */
func newTransport(ipaddr string) *http.Transport {
	transport :=
		&http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				//本地地址  ipaddr是本地外网IP
				lAddr, err := net.ResolveTCPAddr(netw, ipaddr+":0")
				if err != nil {
					return nil, err
				}
				//被请求的地址
				rAddr, err := net.ResolveTCPAddr(netw, addr)
				if err != nil {
					return nil, err
				}
				conn, err := net.DialTCP(netw, lAddr, rAddr)
				if err != nil {
					return nil, err
				}
				deadline := time.Now().Add(35 * time.Second)
				conn.SetDeadline(deadline)
				return conn, nil
			},
		}
	return transport
}
<commit_msg>fetchData<commit_after>package fetch

import (
	"github.com/shaalx/sstruct/httplib"
	"github.com/shaalx/sstruct/log"
	"net"
	"net/http"
	"time"
)

/*根据给定的URL,fetch the data*/
func Do(url, ipaddr string) []byte {
	request := httplib.Get(url)
	// request.SetTransport(newTransport(ipaddr))
	request.Header("Host", "itunes.apple.com")
	request.Header("X-Apple-Store-Front", "143465-19,21 t:native")
	request.Header("Accept", "*/*")
	request.Header("Accept-Language", "zh-cn")
	request.Header("X-Dsid", "1458643138")
	request.Header("Connection", "keep-alive")
	request.Header("Proxy-Connection", "keep-alive")
	request.Header("Design-Agent", "AppStore/2.0 iOS/7.1.1 model/iPod5,1 build/11D201 (4; dt:81)")
	bs, err := request.Bytes()
	if log.IsError(err) {
		return nil
	}
	return bs
}

/*
* 固定IP
 */
func newTransport(ipaddr string) *http.Transport {
	transport :=
		&http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				//本地地址  ipaddr是本地外网IP
				lAddr, err := net.ResolveTCPAddr(netw, ipaddr+":0")
				if err != nil {
					return nil, err
				}
				//被请求的地址
				rAddr, err := net.ResolveTCPAddr(netw, addr)
				if err != nil {
					return nil, err
				}
				conn, err := net.DialTCP(netw, lAddr, rAddr)
				if err != nil {
					return nil, err
				}
				deadline := time.Now().Add(35 * time.Second)
				conn.SetDeadline(deadline)
				return conn, nil
			},
		}
	return transport
}
<|endoftext|>


<commit_before>// Copyright 2009 The GoMPD Authors. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

// Package mpd provides the client side interface to MPD (Music Player Daemon).
// The protocol reference can be found at http://www.musicpd.org/doc/protocol/index.html
package mpd

import (
	"errors"
	"net/textproto"
	"strconv"
	"strings"
)

type Client struct {
	text *textproto.Conn
}

type Attrs map[string]string

// Dial connects to MPD listening on address addr (e.g. "127.0.0.1:6600")
// on network network (e.g. "tcp").
func Dial(network, addr string) (c *Client, err error) {
	text, err := textproto.Dial(network, addr)
	if err != nil {
		return nil, err
	}
	line, err := text.ReadLine()
	if err != nil {
		return nil, err
	}
	if line[0:6] != "OK MPD" {
		return nil, textproto.ProtocolError("no greeting")
	}
	return &Client{text: text}, nil
}

func DialAuthenticated(network, addr, password string) (c *Client, err error) {
	c, err = Dial(network, addr)
	if err == nil && password != "" {
		err = c.okCmd("password %s", password)
	}
	return c, err
}

// Close terminates the connection with MPD.
func (c *Client) Close() (err error) {
	if c.text != nil {
		c.text.PrintfLine("close")
		err = c.text.Close()
		c.text = nil
	}
	return
}

// Ping sends a no-op message to MPD. It's useful for keeping the connection alive.
func (c *Client) Ping() error {
	return c.okCmd("ping")
}

func (c *Client) readPlaylist() (pls []Attrs, err error) {
	pls = []Attrs{}

	for {
		line, err := c.text.ReadLine()
		if err != nil {
			return nil, err
		}
		if line == "OK" {
			break
		}
		if strings.HasPrefix(line, "file:") { // new song entry begins
			pls = append(pls, Attrs{})
		}
		if len(pls) == 0 {
			return nil, textproto.ProtocolError("unexpected: " + line)
		}
		z := strings.Index(line, ": ")
		if z < 0 {
			return nil, textproto.ProtocolError("can't parse line: " + line)
		}
		key := line[0:z]
		pls[len(pls)-1][key] = line[z+2:]
	}
	return pls, nil
}

func (c *Client) readAttrs() (attrs Attrs, err error) {
	attrs = make(Attrs)
	for {
		line, err := c.text.ReadLine()
		if err != nil {
			return nil, err
		}
		if line == "OK" {
			break
		}
		z := strings.Index(line, ": ")
		if z < 0 {
			return nil, textproto.ProtocolError("can't parse line: " + line)
		}
		key := line[0:z]
		attrs[key] = line[z+2:]
	}
	return
}

// CurrentSong returns information about the current song in the playlist.
func (c *Client) CurrentSong() (Attrs, error) {
	id, err := c.text.Cmd("currentsong")
	if err != nil {
		return nil, err
	}
	c.text.StartResponse(id)
	defer c.text.EndResponse(id)
	return c.readAttrs()
}

// Status returns information about the current status of MPD.
func (c *Client) Status() (Attrs, error) {
	id, err := c.text.Cmd("status")
	if err != nil {
		return nil, err
	}
	c.text.StartResponse(id)
	defer c.text.EndResponse(id)
	return c.readAttrs()
}

func (c *Client) readOKLine() (err error) {
	line, err := c.text.ReadLine()
	if err != nil {
		return
	}
	if line == "OK" {
		return nil
	}
	return textproto.ProtocolError("unexpected response: " + line)
}

func (c *Client) okCmd(format string, args ...interface{}) error {
	id, err := c.text.Cmd(format, args...)
	if err != nil {
		return err
	}
	c.text.StartResponse(id)
	defer c.text.EndResponse(id)
	return c.readOKLine()
}

//
// Playback control
//

// Next plays next song in the playlist.
func (c *Client) Next() error {
	return c.okCmd("next")
}

// Pause pauses playback if pause is true; resumes playback otherwise.
func (c *Client) Pause(pause bool) error {
	if pause {
		return c.okCmd("pause 1")
	}
	return c.okCmd("pause 0")
}

// Play starts playing the song at playlist position pos. If pos is negative,
// start playing at the current position in the playlist.
func (c *Client) Play(pos int) error {
	if pos < 0 {
		c.okCmd("play")
	}
	return c.okCmd("play %d", pos)
}

// PlayId plays the song identified by id. If id is negative, start playing
// at the currect position in playlist.
func (c *Client) PlayId(id int) error {
	if id < 0 {
		return c.okCmd("playid")
	}
	return c.okCmd("playid %d", id)
}

// Previous plays previous song in the playlist.
func (c *Client) Previous() error {
	return c.okCmd("previous")
}

// Seek seeks to the position time (in seconds) of the song at playlist position pos.
func (c *Client) Seek(pos, time int) error {
	return c.okCmd("seek %d %d", pos, time)
}

// SeekId is identical to Seek except the song is identified by it's id
// (not position in playlist).
func (c *Client) SeekId(id, time int) error {
	return c.okCmd("seekid %d %d", id, time)
}

// Stop stops playback.
func (c *Client) Stop() error {
	return c.okCmd("stop")
}

//
// Playlist related functions
//

// PlaylistInfo returns attributes for songs in the current playlist. If
// both start and end are negative, it does this for all songs in
// playlist. If end is negative but start is positive, it does it for the
// song at position start. If both start and end are positive, it does it
// for positions in range [start, end).
func (c *Client) PlaylistInfo(start, end int) (pls []Attrs, err error) {
	if start < 0 && end >= 0 {
		return nil, errors.New("negative start index")
	}
	if start >= 0 && end < 0 {
		id, err := c.text.Cmd("playlistinfo %d", start)
		if err != nil {
			return nil, err
		}
		c.text.StartResponse(id)
		defer c.text.EndResponse(id)
		return c.readPlaylist()
	}
	id, err := c.text.Cmd("playlistinfo")
	if err != nil {
		return nil, err
	}
	c.text.StartResponse(id)
	defer c.text.EndResponse(id)
	pls, err = c.readPlaylist()
	if err != nil || start < 0 || end < 0 {
		return
	}
	return pls[start:end], nil
}

// Delete deletes songs from playlist. If both start and end are positive,
// it deletes those at positions in range [start, end). If end is negative,
// it deletes the song at position start.
func (c *Client) Delete(start, end int) error {
	if start < 0 {
		return errors.New("negative start index")
	}
	if end < 0 {
		return c.okCmd("delete %d", start)
	}
	return c.okCmd("delete %d %d", start, end)
}

// DeleteId deletes the song identified by id.
func (c *Client) DeleteId(id int) error {
	return c.okCmd("deleteid %d", id)
}

// Add adds the file/directory uri to playlist. Directories add recursively.
func (c *Client) Add(uri string) error {
	return c.okCmd("add %q", uri)
}

// AddId adds the file/directory uri to playlist and returns the identity
// id of the song added. If pos is positive, the song is added to position
// pos.
func (c *Client) AddId(uri string, pos int) (int, error) {
	var id uint
	var err error
	if pos >= 0 {
		id, err = c.text.Cmd("addid %q %d", uri, pos)
	}
	id, err = c.text.Cmd("addid %q", uri)
	if err != nil {
		return -1, err
	}

	c.text.StartResponse(id)
	defer c.text.EndResponse(id)

	attrs, err := c.readAttrs()
	if err != nil {
		return -1, err
	}
	tok, ok := attrs["Id"]
	if !ok {
		return -1, textproto.ProtocolError("addid did not return Id")
	}
	return strconv.Atoi(tok)
}

// Clear clears the current playlist.
func (c *Client) Clear() error {
	return c.okCmd("clear")
}
<commit_msg>Add SetVolume()<commit_after>// Copyright 2009 The GoMPD Authors. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

// Package mpd provides the client side interface to MPD (Music Player Daemon).
// The protocol reference can be found at http://www.musicpd.org/doc/protocol/index.html
package mpd

import (
	"errors"
	"net/textproto"
	"strconv"
	"strings"
)

type Client struct {
	text *textproto.Conn
}

type Attrs map[string]string

// Dial connects to MPD listening on address addr (e.g. "127.0.0.1:6600")
// on network network (e.g. "tcp").
func Dial(network, addr string) (c *Client, err error) {
	text, err := textproto.Dial(network, addr)
	if err != nil {
		return nil, err
	}
	line, err := text.ReadLine()
	if err != nil {
		return nil, err
	}
	if line[0:6] != "OK MPD" {
		return nil, textproto.ProtocolError("no greeting")
	}
	return &Client{text: text}, nil
}

func DialAuthenticated(network, addr, password string) (c *Client, err error) {
	c, err = Dial(network, addr)
	if err == nil && password != "" {
		err = c.okCmd("password %s", password)
	}
	return c, err
}

// Close terminates the connection with MPD.
func (c *Client) Close() (err error) {
	if c.text != nil {
		c.text.PrintfLine("close")
		err = c.text.Close()
		c.text = nil
	}
	return
}

// Ping sends a no-op message to MPD. It's useful for keeping the connection alive.
func (c *Client) Ping() error {
	return c.okCmd("ping")
}

func (c *Client) readPlaylist() (pls []Attrs, err error) {
	pls = []Attrs{}

	for {
		line, err := c.text.ReadLine()
		if err != nil {
			return nil, err
		}
		if line == "OK" {
			break
		}
		if strings.HasPrefix(line, "file:") { // new song entry begins
			pls = append(pls, Attrs{})
		}
		if len(pls) == 0 {
			return nil, textproto.ProtocolError("unexpected: " + line)
		}
		z := strings.Index(line, ": ")
		if z < 0 {
			return nil, textproto.ProtocolError("can't parse line: " + line)
		}
		key := line[0:z]
		pls[len(pls)-1][key] = line[z+2:]
	}
	return pls, nil
}

func (c *Client) readAttrs() (attrs Attrs, err error) {
	attrs = make(Attrs)
	for {
		line, err := c.text.ReadLine()
		if err != nil {
			return nil, err
		}
		if line == "OK" {
			break
		}
		z := strings.Index(line, ": ")
		if z < 0 {
			return nil, textproto.ProtocolError("can't parse line: " + line)
		}
		key := line[0:z]
		attrs[key] = line[z+2:]
	}
	return
}

// CurrentSong returns information about the current song in the playlist.
func (c *Client) CurrentSong() (Attrs, error) {
	id, err := c.text.Cmd("currentsong")
	if err != nil {
		return nil, err
	}
	c.text.StartResponse(id)
	defer c.text.EndResponse(id)
	return c.readAttrs()
}

// Status returns information about the current status of MPD.
func (c *Client) Status() (Attrs, error) {
	id, err := c.text.Cmd("status")
	if err != nil {
		return nil, err
	}
	c.text.StartResponse(id)
	defer c.text.EndResponse(id)
	return c.readAttrs()
}

func (c *Client) readOKLine() (err error) {
	line, err := c.text.ReadLine()
	if err != nil {
		return
	}
	if line == "OK" {
		return nil
	}
	return textproto.ProtocolError("unexpected response: " + line)
}

func (c *Client) okCmd(format string, args ...interface{}) error {
	id, err := c.text.Cmd(format, args...)
	if err != nil {
		return err
	}
	c.text.StartResponse(id)
	defer c.text.EndResponse(id)
	return c.readOKLine()
}

//
// Playback control
//

// Next plays next song in the playlist.
func (c *Client) Next() error {
	return c.okCmd("next")
}

// Pause pauses playback if pause is true; resumes playback otherwise.
func (c *Client) Pause(pause bool) error {
	if pause {
		return c.okCmd("pause 1")
	}
	return c.okCmd("pause 0")
}

// Play starts playing the song at playlist position pos. If pos is negative,
// start playing at the current position in the playlist.
func (c *Client) Play(pos int) error {
	if pos < 0 {
		c.okCmd("play")
	}
	return c.okCmd("play %d", pos)
}

// PlayId plays the song identified by id. If id is negative, start playing
// at the currect position in playlist.
func (c *Client) PlayId(id int) error {
	if id < 0 {
		return c.okCmd("playid")
	}
	return c.okCmd("playid %d", id)
}

// Previous plays previous song in the playlist.
func (c *Client) Previous() error {
	return c.okCmd("previous")
}

// Seek seeks to the position time (in seconds) of the song at playlist position pos.
func (c *Client) Seek(pos, time int) error {
	return c.okCmd("seek %d %d", pos, time)
}

// SeekId is identical to Seek except the song is identified by it's id
// (not position in playlist).
func (c *Client) SeekId(id, time int) error {
	return c.okCmd("seekid %d %d", id, time)
}

// Stop stops playback.
func (c *Client) Stop() error {
	return c.okCmd("stop")
}

func (c *Client) SetVolume(volume int) error {
	return c.okCmd("setvol %d", volume)
}

//
// Playlist related functions
//

// PlaylistInfo returns attributes for songs in the current playlist. If
// both start and end are negative, it does this for all songs in
// playlist. If end is negative but start is positive, it does it for the
// song at position start. If both start and end are positive, it does it
// for positions in range [start, end).
func (c *Client) PlaylistInfo(start, end int) (pls []Attrs, err error) {
	if start < 0 && end >= 0 {
		return nil, errors.New("negative start index")
	}
	if start >= 0 && end < 0 {
		id, err := c.text.Cmd("playlistinfo %d", start)
		if err != nil {
			return nil, err
		}
		c.text.StartResponse(id)
		defer c.text.EndResponse(id)
		return c.readPlaylist()
	}
	id, err := c.text.Cmd("playlistinfo")
	if err != nil {
		return nil, err
	}
	c.text.StartResponse(id)
	defer c.text.EndResponse(id)
	pls, err = c.readPlaylist()
	if err != nil || start < 0 || end < 0 {
		return
	}
	return pls[start:end], nil
}

// Delete deletes songs from playlist. If both start and end are positive,
// it deletes those at positions in range [start, end). If end is negative,
// it deletes the song at position start.
func (c *Client) Delete(start, end int) error {
	if start < 0 {
		return errors.New("negative start index")
	}
	if end < 0 {
		return c.okCmd("delete %d", start)
	}
	return c.okCmd("delete %d %d", start, end)
}

// DeleteId deletes the song identified by id.
func (c *Client) DeleteId(id int) error {
	return c.okCmd("deleteid %d", id)
}

// Add adds the file/directory uri to playlist. Directories add recursively.
func (c *Client) Add(uri string) error {
	return c.okCmd("add %q", uri)
}

// AddId adds the file/directory uri to playlist and returns the identity
// id of the song added. If pos is positive, the song is added to position
// pos.
func (c *Client) AddId(uri string, pos int) (int, error) {
	var id uint
	var err error
	if pos >= 0 {
		id, err = c.text.Cmd("addid %q %d", uri, pos)
	}
	id, err = c.text.Cmd("addid %q", uri)
	if err != nil {
		return -1, err
	}

	c.text.StartResponse(id)
	defer c.text.EndResponse(id)

	attrs, err := c.readAttrs()
	if err != nil {
		return -1, err
	}
	tok, ok := attrs["Id"]
	if !ok {
		return -1, textproto.ProtocolError("addid did not return Id")
	}
	return strconv.Atoi(tok)
}

// Clear clears the current playlist.
func (c *Client) Clear() error {
	return c.okCmd("clear")
}
<|endoftext|>


<commit_before>package flake

import (
	"encoding/binary"
	"os"
	"sync"
	"sync/atomic"
	"time"
)

//  ---------------------------------------------------------------------------
//  Layout - Big Endian
//  ---------------------------------------------------------------------------
//  [0:6]   48 bits | Upper 48 bits of timestamp (milliseconds since the epoch)
//  [6:8]   16 bits | a per-interval sequence # (interval == 1 millisecond)
//  [9:14]  48 bits | a hardware id
//  [14:16] 16 bits | process ID
//  ---------------------------------------------------------------------------
//  | 0 | 1 | 2 | 3 | 4 | 5 | 6 |  7  |  8  | 9 | A | B | C | D |  E  |  F  |
//  ---------------------------------------------------------------------------
//  |           48 bits         |  16 bits  |     48 bits       |  16 bits  |
//  ---------------------------------------------------------------------------
//  |          timestamp        |  interval |    HardwareID     | ProcessID |
//  ---------------------------------------------------------------------------
//  Notes
//  ---------------------------------------------------------------------------
//  The time bits are the most signficant bits because they have the primary
//  impact on the sort order of ids. The interval/seq # is next most significant
//  as it is the tie-breaker when the time portions are equivalent.
//
//  Note that the lower 64 bits are basically random and not specifically
//  useful for ordering, although they play their party when the upper 64-bits
//  are equivalent between two ideas. Again, the ordering outcome in this
//  situation is somewhat random, but generally somewhat repeatable (hardware
//  id should be consistent and stable a vast majority of the time).
//  ---------------------------------------------------------------------------

var sequenceBits uint64 = 16
var sequenceMask = uint64(int64(-1) ^ (int64(-1) << sequenceBits))
var maxSequenceNumber = uint64(^sequenceMask)

// generator is an implementtion of Generator
type generator struct {
	epoch      int64
	hardwareID HardwareID
	processID  int
	machineID  uint64

	lastTime          int64
	lastAllocatedTime int64
	sequence          uint64

	mutex sync.Mutex
}

// NewGenerator creates an instance of generator which implements Generator
func NewGenerator(epoch int64, hardwareID HardwareID, processID int) Generator {
	// binary.BigEndian.Uint64 won't work on a []byte < len(8) so we need to
	// copy our 6-byte hardwareID into the most-signficant bits
	tempBytes := make([]byte, 8)
	copy(tempBytes[0:6], hardwareID[0:6])

	return &generator{
		epoch:      epoch,
		hardwareID: hardwareID,
		processID:  processID & 0xFFFF,
		machineID:  binary.BigEndian.Uint64(tempBytes) | uint64(processID&0xFFFF),
	}
}

// NewOvertoneEpochGenerator creates an instance of generator using the Overtone Epoch
func NewOvertoneEpochGenerator(hardwareID HardwareID) Generator {
	return NewGenerator(OvertoneEpochMs, hardwareID, os.Getpid())
}

func (gen *generator) Epoch() int64 {
	return gen.epoch
}

func (gen *generator) HardwareID() HardwareID {
	return gen.hardwareID
}

func (gen *generator) ProcessID() int {
	return gen.processID
}

func (gen *generator) LastAllocatedTime() int64 {
	// use the atomic api for both reads and writes of this value so that we do not
	// need to incur the overhead of the mutex or create additional contention
	return atomic.LoadInt64(&gen.lastAllocatedTime)
}

func (gen *generator) GenerateAsStream(count int, buffer []byte, callback func(int, []byte) error) (totalAllocated int, err error) {
	if len(buffer) < OvertFlakeIDLength {
		return 0, ErrBufferTooSmall
	}

	// while we still have ids to allocate/generate
	for count > 0 {
		var allocated uint64
		var interval int64
		var index int

		// allocate as many ids as available up to count
		allocated, interval, err = gen.allocate(count)
		if err != nil {
			return totalAllocated, err
		}

		// calculate the delta between the interval (Unix Epoch in milliseconds)
		// and the epoch being used for id generation
		delta := uint64((interval - gen.epoch) << 16)

		// for each ID that was allocated, write the bytes for the ID to
		// the results array
		for j := uint64(0); j < allocated; j++ {
			var upper = delta | (j & sequenceMask)
			binary.BigEndian.PutUint64(buffer[index:index+8], upper)
			binary.BigEndian.PutUint64(buffer[index+8:index+16], gen.machineID)
			index += 16

			// buffer is full
			if index >= len(buffer) {
				err = callback(index/16, buffer)
				if err != nil {
					return
				}

				// more were delivered so update our return value
				totalAllocated += int(index / 16)

				// back to beginning of the buffer
				index = 0
			}
		}

		// partial buffer fill
		if index > 0 {
			callback(index/16, buffer)
			if err != nil {
				return
			}

			// more were delivered so update our return value
			totalAllocated += int(index / 16)

			// back to beginning of the buffer
			index = 0
		}

		count -= int(allocated)
	}

	return
}

// Generate uses allocate to allocate as many ids as required, and writes
// each id into a contiguous []byte
func (gen *generator) Generate(count int) (results []byte, err error) {
	// allocate a buffer that will hold count IDs
	results = make([]byte, OvertFlakeIDLength*count)

	var allocated int

	// use the stream API but because our buffer can hold all allocated ids we dont need
	// to react in the callback
	allocated, err = gen.GenerateAsStream(count, results, func(allocated int, ids []byte) error {
		return nil
	})

	// we do not want to return a partial result
	if (allocated != count) || (err != nil) {
		results = nil
	}

	return
}

// allocate does all the magic of time and sequence management. It does not
// perfomm the generation of the ids, but provides the data required to do so
func (gen *generator) allocate(count int) (uint64, int64, error) {
	if uint64(count) > maxSequenceNumber {
		return 0, 0, ErrTooManyRequested
	}

	// We need to take the lock so we can manipulate the generator state
	gen.mutex.Lock()
	defer gen.mutex.Unlock()

	// current time since Unix Epoch in milliseconds
	current := timestamp()

	// Is time going backwards? Thats a problem
	if current < gen.lastTime {
		return 0, 0, ErrTimeIsMovingBackwards
	}

	if gen.lastTime != current {
		gen.lastTime = current
		gen.sequence = 0
	} else {
		// When all the ids have been allocated for this interval then we end up
		// here and we need to spin for the next cycle
		if gen.sequence == 0 {
			for current <= gen.lastTime {
				current = timestamp()
			}
		}
	}

	// allocated the request # of items, or whatever is remaining for this cycle
	var allocated uint64
	if uint64(count) > gen.sequence-sequenceMask {
		allocated = gen.sequence - sequenceMask
	} else {
		allocated = uint64(count)
	}

	// advance the sequence for the # of items allocated
	gen.sequence = (gen.sequence + allocated) & sequenceMask

	// remember the last time interval where we allocated one or more ids.
	//
	// Note that although we own the mutex, the reader uses atomic so
	// we (the writer) do the same
	atomic.StoreInt64(&gen.lastAllocatedTime, gen.lastTime)

	return allocated, current, nil
}

// timestamp returns the # of milliseconds that have passed since
// the unix epoch
func timestamp() int64 {
	return time.Now().UnixNano() / 1e6
}
<commit_msg>added WaitForTime parameter to generator<commit_after>package flake

import (
	"encoding/binary"
	"os"
	"sync"
	"sync/atomic"
	"time"
)

//  ---------------------------------------------------------------------------
//  Layout - Big Endian
//  ---------------------------------------------------------------------------
//  [0:6]   48 bits | Upper 48 bits of timestamp (milliseconds since the epoch)
//  [6:8]   16 bits | a per-interval sequence # (interval == 1 millisecond)
//  [9:14]  48 bits | a hardware id
//  [14:16] 16 bits | process ID
//  ---------------------------------------------------------------------------
//  | 0 | 1 | 2 | 3 | 4 | 5 | 6 |  7  |  8  | 9 | A | B | C | D |  E  |  F  |
//  ---------------------------------------------------------------------------
//  |           48 bits         |  16 bits  |     48 bits       |  16 bits  |
//  ---------------------------------------------------------------------------
//  |          timestamp        |  interval |    HardwareID     | ProcessID |
//  ---------------------------------------------------------------------------
//  Notes
//  ---------------------------------------------------------------------------
//  The time bits are the most signficant bits because they have the primary
//  impact on the sort order of ids. The interval/seq # is next most significant
//  as it is the tie-breaker when the time portions are equivalent.
//
//  Note that the lower 64 bits are basically random and not specifically
//  useful for ordering, although they play their party when the upper 64-bits
//  are equivalent between two ideas. Again, the ordering outcome in this
//  situation is somewhat random, but generally somewhat repeatable (hardware
//  id should be consistent and stable a vast majority of the time).
//  ---------------------------------------------------------------------------

var sequenceBits uint64 = 16
var sequenceMask = uint64(int64(-1) ^ (int64(-1) << sequenceBits))
var maxSequenceNumber = uint64(^sequenceMask)

// generator is an implementtion of Generator
type generator struct {
	epoch      int64
	hardwareID HardwareID
	processID  int
	machineID  uint64

	lastTime          int64
	lastAllocatedTime int64
	sequence          uint64

	mutex sync.Mutex
}

// NewGenerator creates an instance of generator which implements Generator
func NewGenerator(epoch int64, hardwareID HardwareID, processID int, waitForTime int64) Generator {
	// binary.BigEndian.Uint64 won't work on a []byte < len(8) so we need to
	// copy our 6-byte hardwareID into the most-signficant bits
	tempBytes := make([]byte, 8)
	copy(tempBytes[0:6], hardwareID[0:6])

	return &generator{
		epoch:      epoch,
		hardwareID: hardwareID,
		processID:  processID & 0xFFFF,
		machineID:  binary.BigEndian.Uint64(tempBytes) | uint64(processID&0xFFFF),
		lastTime:   waitForTime,
	}
}

// NewOvertoneEpochGenerator creates an instance of generator using the Overtone Epoch
func NewOvertoneEpochGenerator(hardwareID HardwareID) Generator {
	return NewGenerator(OvertoneEpochMs, hardwareID, os.Getpid(), 0)
}

func (gen *generator) Epoch() int64 {
	return gen.epoch
}

func (gen *generator) HardwareID() HardwareID {
	return gen.hardwareID
}

func (gen *generator) ProcessID() int {
	return gen.processID
}

func (gen *generator) LastAllocatedTime() int64 {
	// use the atomic api for both reads and writes of this value so that we do not
	// need to incur the overhead of the mutex or create additional contention
	return atomic.LoadInt64(&gen.lastAllocatedTime)
}

func (gen *generator) GenerateAsStream(count int, buffer []byte, callback func(int, []byte) error) (totalAllocated int, err error) {
	if len(buffer) < OvertFlakeIDLength {
		return 0, ErrBufferTooSmall
	}

	// while we still have ids to allocate/generate
	for count > 0 {
		var allocated uint64
		var interval int64
		var index int

		// allocate as many ids as available up to count
		allocated, interval, err = gen.allocate(count)
		if err != nil {
			return totalAllocated, err
		}

		// calculate the delta between the interval (Unix Epoch in milliseconds)
		// and the epoch being used for id generation
		delta := uint64((interval - gen.epoch) << 16)

		// for each ID that was allocated, write the bytes for the ID to
		// the results array
		for j := uint64(0); j < allocated; j++ {
			var upper = delta | (j & sequenceMask)
			binary.BigEndian.PutUint64(buffer[index:index+8], upper)
			binary.BigEndian.PutUint64(buffer[index+8:index+16], gen.machineID)
			index += 16

			// buffer is full
			if index >= len(buffer) {
				err = callback(index/16, buffer)
				if err != nil {
					return
				}

				// more were delivered so update our return value
				totalAllocated += int(index / 16)

				// back to beginning of the buffer
				index = 0
			}
		}

		// partial buffer fill
		if index > 0 {
			callback(index/16, buffer)
			if err != nil {
				return
			}

			// more were delivered so update our return value
			totalAllocated += int(index / 16)

			// back to beginning of the buffer
			index = 0
		}

		count -= int(allocated)
	}

	return
}

// Generate uses allocate to allocate as many ids as required, and writes
// each id into a contiguous []byte
func (gen *generator) Generate(count int) (results []byte, err error) {
	// allocate a buffer that will hold count IDs
	results = make([]byte, OvertFlakeIDLength*count)

	var allocated int

	// use the stream API but because our buffer can hold all allocated ids we dont need
	// to react in the callback
	allocated, err = gen.GenerateAsStream(count, results, func(allocated int, ids []byte) error {
		return nil
	})

	// we do not want to return a partial result
	if (allocated != count) || (err != nil) {
		results = nil
	}

	return
}

// allocate does all the magic of time and sequence management. It does not
// perfomm the generation of the ids, but provides the data required to do so
func (gen *generator) allocate(count int) (uint64, int64, error) {
	if uint64(count) > maxSequenceNumber {
		return 0, 0, ErrTooManyRequested
	}

	// We need to take the lock so we can manipulate the generator state
	gen.mutex.Lock()
	defer gen.mutex.Unlock()

	// current time since Unix Epoch in milliseconds
	current := timestamp()

	// Is time going backwards? Thats a problem
	if current < gen.lastTime {
		return 0, 0, ErrTimeIsMovingBackwards
	}

	if gen.lastTime != current {
		gen.lastTime = current
		gen.sequence = 0
	} else {
		// When all the ids have been allocated for this interval then we end up
		// here and we need to spin for the next cycle
		if gen.sequence == 0 {
			for current <= gen.lastTime {
				current = timestamp()
			}
		}
	}

	// allocated the request # of items, or whatever is remaining for this cycle
	var allocated uint64
	if uint64(count) > gen.sequence-sequenceMask {
		allocated = gen.sequence - sequenceMask
	} else {
		allocated = uint64(count)
	}

	// advance the sequence for the # of items allocated
	gen.sequence = (gen.sequence + allocated) & sequenceMask

	// remember the last time interval where we allocated one or more ids.
	//
	// Note that although we own the mutex, the reader uses atomic so
	// we (the writer) do the same
	atomic.StoreInt64(&gen.lastAllocatedTime, gen.lastTime)

	return allocated, current, nil
}

// timestamp returns the # of milliseconds that have passed since
// the unix epoch
func timestamp() int64 {
	return time.Now().UnixNano() / 1e6
}
<|endoftext|>


<commit_before>package flapper

import (
	"time"

	"github.com/JohnMurray/nbad/timewindow"
)

// Flapper is a simple struct for watching for services that are in a flapState
type Flapper struct {
	// the max amount of state-transitions that can happen before "flapping"
	max uint

	// size of sliding time-window for state-change counters in seconds
	duration uint

	// a map of servie-names to sliding window-counters
	services map[string]*timewindow.Window
}

// NewFlapper - Create a new instance of Flapper
func NewFlapper(max uint, duration uint) *Flapper {
	f := &Flapper{
		max:      max,
		duration: duration,
		services: make(map[string]*timewindow.Window),
	}
	return f
}

// NoteStateChange -
// Increment the counter for a service (or create a counter for the service if one has not
// alredy been created). (Lazily create services).
func (f *Flapper) NoteStateChange(service string) {
	if state, ok := f.services[service]; ok {
		state.Add(time.Now().Unix(), 1)
	} else {
		f.services[service] = timewindow.New(time.Now().Unix(), int(f.duration))
		f.services[service].Add(time.Now().Unix(), 1)
	}
}

// IsFlapping -
// Return boolean indicating whether or not the state is flapping or not. Note that if the
// state does not exist we always return false sine we lazily create counters.
//
// service   - the service to check against
// recompute - bool flag to recompute the time-window if it has not been updated in a while
func (f *Flapper) IsFlapping(service string, recompute bool) bool {
	if state, ok := f.services[service]; ok {
		if recompute {
			state.Add(time.Now().Unix(), 0)
		}
		return state.Total() >= int(f.max)
	}
	return false
}

// Compact -
// If a Flapper has been running for a long time, you may want to periodically clean up any
// services that do not have any data. Compaction allows us to compress our internal data-structures
// and potentially free up memory.
func (f *Flapper) Compact() {
	for service, counter := range f.services {
		if counter.Total() == 0 {
			delete(f.services, service)
		}
	}
}
<commit_msg>added package comment to 'flapper'<commit_after>// Package flapper contains simple mechanism / counter for tracking state changes and detecting potential 'flapping'
package flapper

import (
	"time"

	"github.com/JohnMurray/nbad/timewindow"
)

// Flapper is a simple struct for watching for services that are in a flapState
type Flapper struct {
	// the max amount of state-transitions that can happen before "flapping"
	max uint

	// size of sliding time-window for state-change counters in seconds
	duration uint

	// a map of servie-names to sliding window-counters
	services map[string]*timewindow.Window
}

// NewFlapper - Create a new instance of Flapper
func NewFlapper(max uint, duration uint) *Flapper {
	f := &Flapper{
		max:      max,
		duration: duration,
		services: make(map[string]*timewindow.Window),
	}
	return f
}

// NoteStateChange -
// Increment the counter for a service (or create a counter for the service if one has not
// alredy been created). (Lazily create services).
func (f *Flapper) NoteStateChange(service string) {
	if state, ok := f.services[service]; ok {
		state.Add(time.Now().Unix(), 1)
	} else {
		f.services[service] = timewindow.New(time.Now().Unix(), int(f.duration))
		f.services[service].Add(time.Now().Unix(), 1)
	}
}

// IsFlapping -
// Return boolean indicating whether or not the state is flapping or not. Note that if the
// state does not exist we always return false sine we lazily create counters.
//
// service   - the service to check against
// recompute - bool flag to recompute the time-window if it has not been updated in a while
func (f *Flapper) IsFlapping(service string, recompute bool) bool {
	if state, ok := f.services[service]; ok {
		if recompute {
			state.Add(time.Now().Unix(), 0)
		}
		return state.Total() >= int(f.max)
	}
	return false
}

// Compact -
// If a Flapper has been running for a long time, you may want to periodically clean up any
// services that do not have any data. Compaction allows us to compress our internal data-structures
// and potentially free up memory.
func (f *Flapper) Compact() {
	for service, counter := range f.services {
		if counter.Total() == 0 {
			delete(f.services, service)
		}
	}
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "github.com/nsf/gothic"

func main() {
	ir, err := gothic.NewInterpreter()
	if err != nil {
		panic(err)
	}

	entryText := ir.NewStringVar("entryText")
	ir.RegisterCallback("updateLabel", func() {
		ir.Eval(`.label configure -text {` + entryText.Get() + `}`)
		entryText.Set("")
	})

	var RGB [3]int
	ir.RegisterCallback("scaleUpdate", func(idx int, x float64) {
		if RGB[idx] == int(x) {
			return
		}
		RGB[idx] = int(x)
		col := fmt.Sprintf("%02X%02X%02X", RGB[0], RGB[1], RGB[2])
		ir.Eval(`.label configure -foreground #` + col)
	})

	ir.Eval(`ttk::button .hello -text "Press me!" -command updateLabel`)
	ir.Eval(`ttk::entry .entry -textvariable entryText`)
	ir.Eval(`ttk::label .label -text "Press a button"`)
	ir.Eval(`ttk::scale .scaleR -from 0 -to 255 -length 200 -command {scaleUpdate 0}`)
	ir.Eval(`ttk::scale .scaleG -from 0 -to 255 -length 200 -command {scaleUpdate 1}`)
	ir.Eval(`ttk::scale .scaleB -from 0 -to 255 -length 200 -command {scaleUpdate 2}`)
	ir.Eval(`pack .hello .entry .label .scaleR .scaleG .scaleB`)
	ir.MainLoop()
}
<commit_msg>Simplify colors example.<commit_after>package main

import "fmt"
import "github.com/nsf/gothic"

func main() {
	ir, err := gothic.NewInterpreter()
	if err != nil {
		panic(err)
	}

	var RGB [3]int
	ir.RegisterCallback("scaleUpdate", func(idx int, x float64) {
		if RGB[idx] == int(x) {
			return
		}
		RGB[idx] = int(x)
		col := fmt.Sprintf("%02X%02X%02X", RGB[0], RGB[1], RGB[2])
		ir.Eval(`ttk::style configure My.TFrame -background #` + col)
		ir.Eval(`.frame configure -style My.TFrame`)
	})

	ir.Eval(`
ttk::frame .frame -width 100 -height 30 -relief sunken

ttk::scale .scaleR -from 0 -to 255 -length 200 -command {scaleUpdate 0}
ttk::scale .scaleG -from 0 -to 255 -length 200 -command {scaleUpdate 1}
ttk::scale .scaleB -from 0 -to 255 -length 200 -command {scaleUpdate 2}
pack .frame -fill both
pack .scaleR .scaleG .scaleB
	`)
	ir.MainLoop()
}
<|endoftext|>


<commit_before>// Copyright 2016 The fer Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mq_test

import (
	"fmt"
	"testing"

	"github.com/sbinet-alice/fer/mq"
	_ "github.com/sbinet-alice/fer/mq/nanomsg"
	_ "github.com/sbinet-alice/fer/mq/zeromq"
)

func TestPushPullNN(t *testing.T) {
	const (
		N    = 5
		tmpl = "data-%02d"
	)

	drv, err := mq.Open("nanomsg")
	if err != nil {
		t.Fatal(err)
	}
	pull, err := drv.NewSocket(mq.Pull)
	if err != nil {
		t.Fatal(err)
	}
	defer pull.Close()

	push, err := drv.NewSocket(mq.Push)
	if err != nil {
		t.Fatal(err)
	}
	defer push.Close()

	go func() {
		err := push.Dial("tcp://localhost:5555")
		if err != nil {
			t.Fatal(err)
		}
		for i := 0; i < N; i++ {
			err = push.Send([]byte(fmt.Sprintf(tmpl, i)))
			if err != nil {
				t.Fatalf("error sending data[%d]: %v\n", i, err)
			}
		}
		err = push.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	err = pull.Listen("tcp://*:5555")
	if err != nil {
		t.Fatal(err)
	}
	for i := 0; i < N; i++ {
		msg, err := pull.Recv()
		if err != nil {
			t.Fatal(err)
		}
		if got, want := string(msg), fmt.Sprintf(tmpl, i); got != want {
			t.Errorf("push-pull[%d]: got=%q want=%q\n", i, got, want)
		}
	}
	err = pull.Close()
	if err != nil {
		t.Fatal(err)
	}
}

func TestPushPullZMQ(t *testing.T) {
	const (
		N    = 5
		tmpl = "data-%02d"
	)

	drv, err := mq.Open("zeromq")
	if err != nil {
		t.Fatal(err)
	}
	pull, err := drv.NewSocket(mq.Pull)
	if err != nil {
		t.Fatal(err)
	}
	defer pull.Close()

	push, err := drv.NewSocket(mq.Push)
	if err != nil {
		t.Fatal(err)
	}
	defer push.Close()

	go func() {
		err := push.Dial("tcp://localhost:5555")
		if err != nil {
			t.Fatal(err)
		}
		for i := 0; i < N; i++ {
			err = push.Send([]byte(fmt.Sprintf(tmpl, i)))
			if err != nil {
				t.Fatalf("error sending data[%d]: %v\n", i, err)
			}
		}
		err = push.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	err = pull.Listen("tcp://*:5555")
	if err != nil {
		t.Fatal(err)
	}
	for i := 0; i < N; i++ {
		msg, err := pull.Recv()
		if err != nil {
			t.Fatal(err)
		}
		if got, want := string(msg), fmt.Sprintf(tmpl, i); got != want {
			t.Errorf("push-pull[%d]: got=%q want=%q\n", i, got, want)
		}
	}
}
<commit_msg>mq: streamline port number for tests<commit_after>// Copyright 2016 The fer Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mq_test

import (
	"fmt"
	"testing"

	"github.com/sbinet-alice/fer/mq"
	_ "github.com/sbinet-alice/fer/mq/nanomsg"
	_ "github.com/sbinet-alice/fer/mq/zeromq"
)

func TestPushPullNN(t *testing.T) {
	const (
		N    = 5
		tmpl = "data-%02d"
		port = "6666"
	)

	drv, err := mq.Open("nanomsg")
	if err != nil {
		t.Fatal(err)
	}
	pull, err := drv.NewSocket(mq.Pull)
	if err != nil {
		t.Fatal(err)
	}
	defer pull.Close()

	push, err := drv.NewSocket(mq.Push)
	if err != nil {
		t.Fatal(err)
	}
	defer push.Close()

	go func() {
		err := push.Dial("tcp://localhost:" + port)
		if err != nil {
			t.Fatal(err)
		}
		for i := 0; i < N; i++ {
			err = push.Send([]byte(fmt.Sprintf(tmpl, i)))
			if err != nil {
				t.Fatalf("error sending data[%d]: %v\n", i, err)
			}
		}
		err = push.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	err = pull.Listen("tcp://*:" + port)
	if err != nil {
		t.Fatal(err)
	}
	for i := 0; i < N; i++ {
		msg, err := pull.Recv()
		if err != nil {
			t.Fatal(err)
		}
		if got, want := string(msg), fmt.Sprintf(tmpl, i); got != want {
			t.Errorf("push-pull[%d]: got=%q want=%q\n", i, got, want)
		}
	}
	err = pull.Close()
	if err != nil {
		t.Fatal(err)
	}
}

func TestPushPullZMQ(t *testing.T) {
	const (
		N    = 5
		tmpl = "data-%02d"
		port = "5555"
	)

	drv, err := mq.Open("zeromq")
	if err != nil {
		t.Fatal(err)
	}
	pull, err := drv.NewSocket(mq.Pull)
	if err != nil {
		t.Fatal(err)
	}
	defer pull.Close()

	push, err := drv.NewSocket(mq.Push)
	if err != nil {
		t.Fatal(err)
	}
	defer push.Close()

	go func() {
		err := push.Dial("tcp://localhost:" + port)
		if err != nil {
			t.Fatal(err)
		}
		for i := 0; i < N; i++ {
			err = push.Send([]byte(fmt.Sprintf(tmpl, i)))
			if err != nil {
				t.Fatalf("error sending data[%d]: %v\n", i, err)
			}
		}
		err = push.Close()
		if err != nil {
			t.Fatal(err)
		}
	}()

	err = pull.Listen("tcp://*:" + port)
	if err != nil {
		t.Fatal(err)
	}
	for i := 0; i < N; i++ {
		msg, err := pull.Recv()
		if err != nil {
			t.Fatal(err)
		}
		if got, want := string(msg), fmt.Sprintf(tmpl, i); got != want {
			t.Errorf("push-pull[%d]: got=%q want=%q\n", i, got, want)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"testing"
	"time"

	"github.com/golang/glog"
	"github.com/google/mtail/vm"
	"github.com/google/mtail/watcher"
)

var test_program = "/$/ { }"

func startMtail(t *testing.T, log_pathnames []string, prog_pathname string) chan bool {
	m := NewMtail()
	w, err := watcher.NewLogWatcher()
	if err != nil {
		t.Errorf("Couldn't create watcher: %s", err)
	}
	p := vm.NewProgLoader(w)
	// start server
	prog, errors := vm.Compile("test", strings.NewReader(test_program), &m.store)
	if len(errors) > 0 {
		t.Errorf("Couldn't compile program: %s", errors)
	}
	p.E.AddVm("test", prog)
	if prog_pathname != "" {
		p.LoadProgs(prog_pathname)
	}
	vm.Line_count.Set(0)
	go p.E.Run(m.lines, m.stop)
	m.StartTailing(log_pathnames)
	return m.stop
}

func TestHandleLogUpdates(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode")
	}
	// make temp dir
	workdir, err := ioutil.TempDir("", "mtail_test")
	if err != nil {
		t.Errorf("could not create temporary directory: %s", err)
	}
	defer func() {
		err := os.RemoveAll(workdir)
		if err != nil {
			t.Errorf("Could not remove temp dir: %s", err)
		}
	}()
	// touch log file
	log_filepath := path.Join(workdir, "log")
	log_file, err := os.Create(log_filepath)
	if err != nil {
		t.Errorf("could not touch log file: %s", err)
	}
	defer log_file.Close()
	pathnames := []string{log_filepath}
	stop := startMtail(t, pathnames, "")
	defer func() { stop <- true }()
	ex_lines := []string{"hi", "hi2", "hi3"}
	for i, x := range ex_lines {
		// write to log file
		log_file.WriteString(x + "\n")
		// TODO(jaq): remove slow sleep
		time.Sleep(100 * time.Millisecond)
		// check log line count increase
		expected := fmt.Sprintf("%d", i+1)
		if vm.Line_count.String() != expected {
			t.Errorf("Line count not increased\n\texpected: %s\n\treceived: %s", expected, vm.Line_count.String())
		}
	}
}

func TestHandleLogRotation(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode")
	}
	// make temp dir
	workdir, err := ioutil.TempDir("", "mtail_test")
	if err != nil {
		t.Errorf("could not create temporary directory: %s", err)
	}
	defer func() {
		err := os.RemoveAll(workdir)
		if err != nil {
			t.Errorf("Could not remove temp dir: %s", err)
		}
	}()
	log_filepath := path.Join(workdir, "log")
	// touch log file
	log_file, err := os.Create(log_filepath)
	if err != nil {
		t.Errorf("could not touch log file: %s", err)
	}
	defer log_file.Close()
	// Create a logger
	stop := make(chan bool, 1)
	hup := make(chan bool, 1)
	pathnames := []string{log_filepath}
	end := startMtail(t, pathnames, "")
	defer func() { end <- true }()

	go func() {
		log_file := log_file
		var err error
		i := 0
		running := true
		for running {
			select {
			case <-hup:
				// touch log file
				log_file, err = os.Create(log_filepath)
				if err != nil {
					t.Errorf("could not touch log file: %s", err)
				}
				defer log_file.Close()
			default:
				log_file.WriteString(fmt.Sprintf("%d\n", i))
				time.Sleep(100 * time.Millisecond)
				i++
				if i >= 10 {
					running = false
				}
			}
		}
		stop <- true
	}()
	go func() {
		for {
			select {
			case <-time.After(5 * 100 * time.Millisecond):
				err = os.Rename(log_filepath, log_filepath+".1")
				if err != nil {
					t.Errorf("could not rename log file: %s", err)
				}
				hup <- true
				return
			}
		}
	}()
	<-stop
	expected := "10"
	if vm.Line_count.String() != expected {
		t.Errorf("Line count not increased\n\texpected: %s\n\treceived: %s", expected, vm.Line_count.String())
	}
}

func TestHandleNewLogAfterStart(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode")
	}
	// make temp dir
	workdir, err := ioutil.TempDir("", "mtail_test")
	if err != nil {
		t.Errorf("could not create temporary directory: %s", err)
	}
	defer func() {
		err := os.RemoveAll(workdir)
		if err != nil {
			t.Errorf("Could not remove temp dir: %s", err)
		}
	}()
	// Start up mtail
	log_filepath := path.Join(workdir, "log")
	pathnames := []string{log_filepath}
	stop := startMtail(t, pathnames, "")
	defer func() { stop <- true }()

	// touch log file
	log_file, err := os.Create(log_filepath)
	if err != nil {
		t.Errorf("could not touch log file: %s", err)
	}
	defer log_file.Close()
	ex_lines := []string{"hi", "hi2", "hi3"}
	for _, x := range ex_lines {
		// write to log file
		log_file.WriteString(x + "\n")
		log_file.Sync()
	}
	// TODO(jaq): remove slow sleep
	time.Sleep(100 * time.Millisecond)
	// check log line count increase
	expected := fmt.Sprintf("%d", len(ex_lines))
	if vm.Line_count.String() != expected {
		t.Errorf("Line count not increased\n\texpected: %s\n\treceived: %s", expected, vm.Line_count.String())
	}
}

func TestHandleNewLogIgnored(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode")
	}
	// make temp dir
	workdir, err := ioutil.TempDir("", "mtail_test")
	if err != nil {
		t.Errorf("could not create temporary directory: %s", err)
	}
	defer func() {
		err := os.RemoveAll(workdir)
		if err != nil {
			t.Errorf("Could not remove temp dir: %s", err)
		}
	}()
	// Start mtail
	log_filepath := path.Join(workdir, "log")
	pathnames := []string{log_filepath}
	stop := startMtail(t, pathnames, "")
	defer func() { stop <- true }()

	// touch log file
	new_log_filepath := path.Join(workdir, "log1")

	log_file, err := os.Create(new_log_filepath)
	if err != nil {
		t.Errorf("could not touch log file: %s", err)
	}
	defer log_file.Close()
	expected := "0"
	if vm.Line_count.String() != expected {
		t.Errorf("Line count not increased\n\texpected: %s\n\treceived: %s", expected, vm.Line_count.String())
	}
}

func makeTempDir(t *testing.T) (workdir string) {
	var err error
	if workdir, err = ioutil.TempDir("", "mtail_test"); err != nil {
		t.Errorf("ioutil.TempDir failed: %s", err)
	}
	return
}

func removeTempDir(t *testing.T, workdir string) {
	if err := os.RemoveAll(workdir); err != nil {
		t.Errorf("os.RemoveAll failed: %s", err)
	}
}

// TODO(jaq): The sleeps in here are racy.  What can we use to sync through inotify?
func TestHandleNewProgram(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode")
	}

	workdir := makeTempDir(t)
	defer removeTempDir(t, workdir)

	stop := startMtail(t, []string{}, workdir)
	defer func() { stop <- true }()

	expected_prog_loads := "{}"
	if vm.Prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, vm.Prog_loads.String())
	}

	prog_path := path.Join(workdir, "prog.mtail")
	prog_file, err := os.Create(prog_path)
	if err != nil {
		t.Errorf("prog create failed: %s", err)
	}
	prog_file.WriteString("/$/ {}\n")
	prog_file.Close()
	glog.Infof("hi")

	// Wait for inotify
	time.Sleep(100 * time.Millisecond)
	expected_prog_loads = `{"prog.mtail": 1}`
	if vm.Prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, vm.Prog_loads.String())
	}

	bad_prog_path := path.Join(workdir, "prog.mtail.dpkg-dist")
	bad_prog_file, err := os.Create(bad_prog_path)
	if err != nil {
		t.Errorf("prog create failed: %s", err)
	}
	bad_prog_file.WriteString("/$/ {}\n")
	bad_prog_file.Close()

	time.Sleep(100 * time.Millisecond)
	expected_prog_loads = `{"prog.mtail": 1}`
	if vm.Prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, vm.Prog_loads.String())
	}
	expected_prog_errs := `{}`
	if vm.Prog_load_errors.String() != expected_prog_errs {
		t.Errorf("Prog errors not same\n\texpected: %s\n\treceived: %s", expected_prog_errs, vm.Prog_load_errors.String())
	}

	os.Rename(bad_prog_path, prog_path)
	time.Sleep(100 * time.Millisecond)
	expected_prog_loads = `{"prog.mtail": 2}`
	if vm.Prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, vm.Prog_loads.String())
	}
	expected_prog_errs = `{}`
	if vm.Prog_load_errors.String() != expected_prog_errs {
		t.Errorf("Prog errors not same\n\texpected: %s\n\treceived: %s", expected_prog_errs, vm.Prog_load_errors.String())
	}

	broken_prog_path := path.Join(workdir, "broken.mtail")
	broken_prog_file, err := os.Create(broken_prog_path)
	if err != nil {
		t.Errorf("prog create failed: %s", err)
	}
	broken_prog_file.WriteString("?\n")
	broken_prog_file.Close()

	time.Sleep(100 * time.Millisecond)

	expected_prog_loads = `{"prog.mtail": 2}`
	if vm.Prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, vm.Prog_loads.String())
	}
	expected_prog_errs = `{"broken.mtail": 1}`
	if vm.Prog_load_errors.String() != expected_prog_errs {
		t.Errorf("Prog errors not same\n\texpected: %s\n\treceived: %s", expected_prog_errs, vm.Prog_load_errors.String())
	}

}
<commit_msg>Fix test.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"
	"testing"
	"time"

	"github.com/golang/glog"
	"github.com/google/mtail/vm"
	"github.com/google/mtail/watcher"
)

var test_program = "/$/ { }"

func startMtail(t *testing.T, log_pathnames []string, prog_pathname string) chan bool {
	m := NewMtail()
	w, err := watcher.NewLogWatcher()
	if err != nil {
		t.Errorf("Couldn't create watcher: %s", err)
	}
	p := vm.NewProgLoader(w)
	// start server
	prog, errors := vm.Compile("test", strings.NewReader(test_program), &m.store)
	if len(errors) > 0 {
		t.Errorf("Couldn't compile program: %s", errors)
	}
	p.E.AddVm("test", prog)
	if prog_pathname != "" {
		p.LoadProgs(prog_pathname)
	}
	vm.Line_count.Set(0)
	go p.E.Run(m.lines, m.stop)
	m.StartTailing(log_pathnames)
	return m.stop
}

func TestHandleLogUpdates(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode")
	}
	// make temp dir
	workdir, err := ioutil.TempDir("", "mtail_test")
	if err != nil {
		t.Errorf("could not create temporary directory: %s", err)
	}
	defer func() {
		err := os.RemoveAll(workdir)
		if err != nil {
			t.Errorf("Could not remove temp dir: %s", err)
		}
	}()
	// touch log file
	log_filepath := path.Join(workdir, "log")
	log_file, err := os.Create(log_filepath)
	if err != nil {
		t.Errorf("could not touch log file: %s", err)
	}
	defer log_file.Close()
	pathnames := []string{log_filepath}
	stop := startMtail(t, pathnames, "")
	defer func() { stop <- true }()
	ex_lines := []string{"hi", "hi2", "hi3"}
	for i, x := range ex_lines {
		// write to log file
		log_file.WriteString(x + "\n")
		// TODO(jaq): remove slow sleep
		time.Sleep(100 * time.Millisecond)
		// check log line count increase
		expected := fmt.Sprintf("%d", i+1)
		if vm.Line_count.String() != expected {
			t.Errorf("Line count not increased\n\texpected: %s\n\treceived: %s", expected, vm.Line_count.String())
		}
	}
}

func TestHandleLogRotation(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode")
	}
	// make temp dir
	workdir, err := ioutil.TempDir("", "mtail_test")
	if err != nil {
		t.Errorf("could not create temporary directory: %s", err)
	}
	defer func() {
		err := os.RemoveAll(workdir)
		if err != nil {
			t.Errorf("Could not remove temp dir: %s", err)
		}
	}()
	log_filepath := path.Join(workdir, "log")
	// touch log file
	log_file, err := os.Create(log_filepath)
	if err != nil {
		t.Errorf("could not touch log file: %s", err)
	}
	defer log_file.Close()
	// Create a logger
	stop := make(chan bool, 1)
	hup := make(chan bool, 1)
	pathnames := []string{log_filepath}
	end := startMtail(t, pathnames, "")
	defer func() { end <- true }()

	go func() {
		log_file := log_file
		var err error
		i := 0
		running := true
		for running {
			select {
			case <-hup:
				// touch log file
				log_file, err = os.Create(log_filepath)
				if err != nil {
					t.Errorf("could not touch log file: %s", err)
				}
				defer log_file.Close()
			default:
				log_file.WriteString(fmt.Sprintf("%d\n", i))
				time.Sleep(100 * time.Millisecond)
				i++
				if i >= 10 {
					running = false
				}
			}
		}
		stop <- true
	}()
	go func() {
		for {
			select {
			case <-time.After(5 * 100 * time.Millisecond):
				err = os.Rename(log_filepath, log_filepath+".1")
				if err != nil {
					t.Errorf("could not rename log file: %s", err)
				}
				hup <- true
				return
			}
		}
	}()
	<-stop
	expected := "10"
	if vm.Line_count.String() != expected {
		t.Errorf("Line count not increased\n\texpected: %s\n\treceived: %s", expected, vm.Line_count.String())
	}
}

func TestHandleNewLogAfterStart(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode")
	}
	// make temp dir
	workdir, err := ioutil.TempDir("", "mtail_test")
	if err != nil {
		t.Errorf("could not create temporary directory: %s", err)
	}
	defer func() {
		err := os.RemoveAll(workdir)
		if err != nil {
			t.Errorf("Could not remove temp dir: %s", err)
		}
	}()
	// Start up mtail
	log_filepath := path.Join(workdir, "log")
	pathnames := []string{log_filepath}
	stop := startMtail(t, pathnames, "")
	defer func() { stop <- true }()

	// touch log file
	log_file, err := os.Create(log_filepath)
	if err != nil {
		t.Errorf("could not touch log file: %s", err)
	}
	defer log_file.Close()
	ex_lines := []string{"hi", "hi2", "hi3"}
	for _, x := range ex_lines {
		// write to log file
		log_file.WriteString(x + "\n")
		log_file.Sync()
	}
	// TODO(jaq): remove slow sleep
	time.Sleep(100 * time.Millisecond)
	// check log line count increase
	expected := fmt.Sprintf("%d", len(ex_lines))
	if vm.Line_count.String() != expected {
		t.Errorf("Line count not increased\n\texpected: %s\n\treceived: %s", expected, vm.Line_count.String())
	}
}

func TestHandleNewLogIgnored(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode")
	}
	// make temp dir
	workdir, err := ioutil.TempDir("", "mtail_test")
	if err != nil {
		t.Errorf("could not create temporary directory: %s", err)
	}
	defer func() {
		err := os.RemoveAll(workdir)
		if err != nil {
			t.Errorf("Could not remove temp dir: %s", err)
		}
	}()
	// Start mtail
	log_filepath := path.Join(workdir, "log")
	pathnames := []string{log_filepath}
	stop := startMtail(t, pathnames, "")
	defer func() { stop <- true }()

	// touch log file
	new_log_filepath := path.Join(workdir, "log1")

	log_file, err := os.Create(new_log_filepath)
	if err != nil {
		t.Errorf("could not touch log file: %s", err)
	}
	defer log_file.Close()
	expected := "0"
	if vm.Line_count.String() != expected {
		t.Errorf("Line count not increased\n\texpected: %s\n\treceived: %s", expected, vm.Line_count.String())
	}
}

func makeTempDir(t *testing.T) (workdir string) {
	var err error
	if workdir, err = ioutil.TempDir("", "mtail_test"); err != nil {
		t.Errorf("ioutil.TempDir failed: %s", err)
	}
	return
}

func removeTempDir(t *testing.T, workdir string) {
	if err := os.RemoveAll(workdir); err != nil {
		t.Errorf("os.RemoveAll failed: %s", err)
	}
}

// TODO(jaq): The sleeps in here are racy.  What can we use to sync through inotify?
func TestHandleNewProgram(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode")
	}

	workdir := makeTempDir(t)
	defer removeTempDir(t, workdir)

	stop := startMtail(t, []string{}, workdir)
	defer func() { stop <- true }()

	expected_prog_loads := "{}"
	if vm.Prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, vm.Prog_loads.String())
	}

	prog_path := path.Join(workdir, "prog.mtail")
	prog_file, err := os.Create(prog_path)
	if err != nil {
		t.Errorf("prog create failed: %s", err)
	}
	prog_file.WriteString("/$/ {}\n")
	prog_file.Close()
	glog.Infof("hi")

	// Wait for inotify
	time.Sleep(100 * time.Millisecond)
	expected_prog_loads = `{"prog.mtail": 1}`
	if vm.Prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, vm.Prog_loads.String())
	}

	bad_prog_path := path.Join(workdir, "prog.mtail.dpkg-dist")
	bad_prog_file, err := os.Create(bad_prog_path)
	if err != nil {
		t.Errorf("prog create failed: %s", err)
	}
	bad_prog_file.WriteString("/$/ {}\n")
	bad_prog_file.Close()

	time.Sleep(100 * time.Millisecond)
	expected_prog_loads = `{"prog.mtail": 1}`
	if vm.Prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, vm.Prog_loads.String())
	}
	expected_prog_errs := `{}`
	if vm.Prog_load_errors.String() != expected_prog_errs {
		t.Errorf("Prog errors not same\n\texpected: %s\n\treceived: %s", expected_prog_errs, vm.Prog_load_errors.String())
	}

	os.Rename(bad_prog_path, prog_path)
	time.Sleep(100 * time.Millisecond)
	expected_prog_loads = `{"prog.mtail": 1}`
	if vm.Prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, vm.Prog_loads.String())
	}
	expected_prog_errs = `{}`
	if vm.Prog_load_errors.String() != expected_prog_errs {
		t.Errorf("Prog errors not same\n\texpected: %s\n\treceived: %s", expected_prog_errs, vm.Prog_load_errors.String())
	}

	broken_prog_path := path.Join(workdir, "broken.mtail")
	broken_prog_file, err := os.Create(broken_prog_path)
	if err != nil {
		t.Errorf("prog create failed: %s", err)
	}
	broken_prog_file.WriteString("?\n")
	broken_prog_file.Close()

	time.Sleep(100 * time.Millisecond)

	expected_prog_loads = `{"prog.mtail": 1}`
	if vm.Prog_loads.String() != expected_prog_loads {
		t.Errorf("Prog loads not same\n\texpected: %s\n\treceived: %s", expected_prog_loads, vm.Prog_loads.String())
	}
	expected_prog_errs = `{"broken.mtail": 1}`
	if vm.Prog_load_errors.String() != expected_prog_errs {
		t.Errorf("Prog errors not same\n\texpected: %s\n\treceived: %s", expected_prog_errs, vm.Prog_load_errors.String())
	}

}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package localdisk

import (
	"camli/blobref"
	"camli/blobserver"
	"exec"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"strings"
	"sync"
)

var flagOpenImages = flag.Bool("showimages", false, "Show images on receiving them with eog.")

type diskStorage struct {
	root string

	hubLock sync.Mutex
	hubMap  map[blobserver.Partition]blobserver.BlobHub
}

func New(root string) (storage blobserver.Storage, err os.Error) {
	// Local disk.
	fi, staterr := os.Stat(root)
	if staterr != nil || !fi.IsDirectory() {
		err = os.NewError(fmt.Sprintf("Storage root %q doesn't exist or is not a directory.", root))
		return
	}
	storage = &diskStorage{
		root:   root,
		hubMap: make(map[blobserver.Partition]blobserver.BlobHub),
	}
	return
}

func (ds *diskStorage) GetBlobHub(partition blobserver.Partition) blobserver.BlobHub {
	ds.hubLock.Lock()
	defer ds.hubLock.Unlock()
	if hub, ok := ds.hubMap[partition]; ok {
		return hub
	}
	hub := new(blobserver.SimpleBlobHub)
	ds.hubMap[partition] = hub
	return hub
}

func (ds *diskStorage) Fetch(blob *blobref.BlobRef) (blobref.ReadSeekCloser, int64, os.Error) {
	fileName := ds.blobFileName(blob)
	stat, err := os.Stat(fileName)
	if errorIsNoEnt(err) {
		return nil, 0, err
	}
	file, err := os.Open(fileName, os.O_RDONLY, 0)
	if err != nil {
		return nil, 0, err
	}
	return file, stat.Size, nil
}

func (ds *diskStorage) Remove(partition blobserver.Partition, blobs []*blobref.BlobRef) os.Error {
	for _, blob := range blobs {
		fileName := ds.partitionBlobFileName(partition, blob)
		err := os.Remove(fileName)
		switch {
		case err == nil:
			continue
		case errorIsNoEnt(err):
			log.Printf("Deleting already-deleted file; harmless.")
			continue
		default:
			return err
		}
	}
	return nil
}

type readBlobRequest struct {
	ch      chan *blobref.SizedBlobRef
	after   string
	remain  *uint // limit countdown
	dirRoot string

	// Not used on initial request, only on recursion
	blobPrefix, pathInto string
}

type enumerateError struct {
	msg string
	err os.Error
}

func (ee *enumerateError) String() string {
	return fmt.Sprintf("Enumerate error: %s: %v", ee.msg, ee.err)
}

func readBlobs(opts readBlobRequest) os.Error {
	dirFullPath := opts.dirRoot + "/" + opts.pathInto
	dir, err := os.Open(dirFullPath, os.O_RDONLY, 0)
	if err != nil {
		return &enumerateError{"opening directory " + dirFullPath, err}
	}
	defer dir.Close()
	names, err := dir.Readdirnames(32768)
	if err != nil {
		return &enumerateError{"readdirnames of " + dirFullPath, err}
	}
	sort.SortStrings(names)
	for _, name := range names {
		if *opts.remain == 0 {
			return nil
		}

		fullPath := dirFullPath + "/" + name
		fi, err := os.Stat(fullPath)
		if err != nil {
			return &enumerateError{"stat of file " + fullPath, err}
		}

		if fi.IsDirectory() {
			var newBlobPrefix string
			if opts.blobPrefix == "" {
				newBlobPrefix = name + "-"
			} else {
				newBlobPrefix = opts.blobPrefix + name
			}
			if len(opts.after) > 0 {
				compareLen := len(newBlobPrefix)
				if len(opts.after) < compareLen {
					compareLen = len(opts.after)
				}
				if newBlobPrefix[0:compareLen] < opts.after[0:compareLen] {
					continue
				}
			}
			ropts := opts
			ropts.blobPrefix = newBlobPrefix
			ropts.pathInto = opts.pathInto + "/" + name
			readBlobs(ropts)
			continue
		}

		if fi.IsRegular() && strings.HasSuffix(name, ".dat") {
			blobName := name[0 : len(name)-4]
			if blobName <= opts.after {
				continue
			}
			blobRef := blobref.Parse(blobName)
			if blobRef != nil {
				opts.ch <- &blobref.SizedBlobRef{BlobRef: blobRef, Size: fi.Size}
				(*opts.remain)--
			}
			continue
		}
	}

	if opts.pathInto == "" {
		opts.ch <- nil
	}
	return nil
}

func (ds *diskStorage) EnumerateBlobs(dest chan *blobref.SizedBlobRef, partition blobserver.Partition, after string, limit uint) os.Error {
	dirRoot := ds.root
	if partition != "" {
		dirRoot += "/partition/" + string(partition) + "/"
	}
	limitMutable := limit
	return readBlobs(readBlobRequest{
		ch:      dest,
		dirRoot: dirRoot,
		after:   after,
		remain:  &limitMutable,
	})
}

func (ds *diskStorage) Stat(dest chan *blobref.SizedBlobRef, partition blobserver.Partition, blobs []*blobref.BlobRef, waitSeconds int) os.Error {
	var missing []*blobref.BlobRef

	// TODO: stat in parallel; keep disks busy
	for _, ref := range blobs {
		fi, err := os.Stat(ds.blobFileName(ref))
		switch {
		case err == nil && fi.IsRegular():
			dest <- &blobref.SizedBlobRef{BlobRef: ref, Size: fi.Size}
		case err != nil && errorIsNoEnt(err) && waitSeconds > 0:
			missing = append(missing, ref)
		case err != nil && !errorIsNoEnt(err):
			return err
		}
	}

	if len(missing) > 0 {
		// TODO: use waitSeconds
		log.Printf("TODO: wait for %d blobs: %#v", len(missing), missing)
	}

	return nil
}

var CorruptBlobError = os.NewError("corrupt blob; digest doesn't match")

func (ds *diskStorage) ReceiveBlob(blobRef *blobref.BlobRef, source io.Reader, mirrorPartitions []blobserver.Partition) (blobGot *blobref.SizedBlobRef, err os.Error) {
	hashedDirectory := ds.blobDirectoryName(blobRef)
	err = os.MkdirAll(hashedDirectory, 0700)
	if err != nil {
		return
	}

	var tempFile *os.File
	tempFile, err = ioutil.TempFile(hashedDirectory, BlobFileBaseName(blobRef)+".tmp")
	if err != nil {
		return
	}

	success := false // set true later
	defer func() {
		if !success {
			log.Println("Removing temp file: ", tempFile.Name())
			os.Remove(tempFile.Name())
		}
	}()

	hash := blobRef.Hash()
	var written int64
	written, err = io.Copy(io.MultiWriter(hash, tempFile), source)
	if err != nil {
		return
	}
	// TODO: fsync before close.
	if err = tempFile.Close(); err != nil {
		return
	}

	if !blobRef.HashMatches(hash) {
		err = CorruptBlobError
		return
	}

	fileName := ds.blobFileName(blobRef)
	if err = os.Rename(tempFile.Name(), fileName); err != nil {
		return
	}

	stat, err := os.Lstat(fileName)
	if err != nil {
		return
	}
	if !stat.IsRegular() || stat.Size != written {
		err = os.NewError("Written size didn't match.")
		return
	}

	for _, partition := range mirrorPartitions {
		partitionDir := ds.blobPartitionDirectoryName(partition, blobRef)
		if err = os.MkdirAll(partitionDir, 0700); err != nil {
			return
		}
		partitionFileName := ds.partitionBlobFileName(partition, blobRef)
		if err = os.Link(fileName, partitionFileName); err != nil {
			return
		}
		log.Printf("Mirrored to partition %q", partition)
	}

	blobGot = &blobref.SizedBlobRef{BlobRef: blobRef, Size: stat.Size}
	success = true

	if *flagOpenImages {
		exec.Run("/usr/bin/eog",
			[]string{"/usr/bin/eog", fileName},
			os.Environ(),
			"/",
			exec.DevNull,
			exec.DevNull,
			exec.MergeWithStdout)
	}

	hub := ds.GetBlobHub(blobserver.DefaultPartition)
	hub.NotifyBlobReceived(blobRef)
	for _, partition := range mirrorPartitions {
		hub = ds.GetBlobHub(partition)
		hub.NotifyBlobReceived(blobRef)
	}

	return
}

func BlobFileBaseName(b *blobref.BlobRef) string {
	return fmt.Sprintf("%s-%s.dat", b.HashName(), b.Digest())
}

func (ds *diskStorage) blobPartitionDirName(partitionDirSlash string, b *blobref.BlobRef) string {
	d := b.Digest()
	if len(d) < 6 {
		d = d + "______"
	}
	return fmt.Sprintf("%s/%s%s/%s/%s",
		ds.root, partitionDirSlash,
		b.HashName(), d[0:3], d[3:6])
}

func (ds *diskStorage) blobDirectoryName(b *blobref.BlobRef) string {
	return ds.blobPartitionDirName("", b)
}

func (ds *diskStorage) blobFileName(b *blobref.BlobRef) string {
	return fmt.Sprintf("%s/%s-%s.dat", ds.blobDirectoryName(b), b.HashName(), b.Digest())
}

func (ds *diskStorage) blobPartitionDirectoryName(partition blobserver.Partition, b *blobref.BlobRef) string {
	return ds.blobPartitionDirName("partition/"+string(partition)+"/", b)
}

func (ds *diskStorage) partitionBlobFileName(partition blobserver.Partition, b *blobref.BlobRef) string {
	return fmt.Sprintf("%s/%s-%s.dat", ds.blobPartitionDirectoryName(partition, b), b.HashName(), b.Digest())
}
<commit_msg>Go has fsync now.<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package localdisk

import (
	"camli/blobref"
	"camli/blobserver"
	"exec"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"sort"
	"strings"
	"sync"
)

var flagOpenImages = flag.Bool("showimages", false, "Show images on receiving them with eog.")

type diskStorage struct {
	root string

	hubLock sync.Mutex
	hubMap  map[blobserver.Partition]blobserver.BlobHub
}

func New(root string) (storage blobserver.Storage, err os.Error) {
	// Local disk.
	fi, staterr := os.Stat(root)
	if staterr != nil || !fi.IsDirectory() {
		err = os.NewError(fmt.Sprintf("Storage root %q doesn't exist or is not a directory.", root))
		return
	}
	storage = &diskStorage{
		root:   root,
		hubMap: make(map[blobserver.Partition]blobserver.BlobHub),
	}
	return
}

func (ds *diskStorage) GetBlobHub(partition blobserver.Partition) blobserver.BlobHub {
	ds.hubLock.Lock()
	defer ds.hubLock.Unlock()
	if hub, ok := ds.hubMap[partition]; ok {
		return hub
	}
	hub := new(blobserver.SimpleBlobHub)
	ds.hubMap[partition] = hub
	return hub
}

func (ds *diskStorage) Fetch(blob *blobref.BlobRef) (blobref.ReadSeekCloser, int64, os.Error) {
	fileName := ds.blobFileName(blob)
	stat, err := os.Stat(fileName)
	if errorIsNoEnt(err) {
		return nil, 0, err
	}
	file, err := os.Open(fileName, os.O_RDONLY, 0)
	if err != nil {
		return nil, 0, err
	}
	return file, stat.Size, nil
}

func (ds *diskStorage) Remove(partition blobserver.Partition, blobs []*blobref.BlobRef) os.Error {
	for _, blob := range blobs {
		fileName := ds.partitionBlobFileName(partition, blob)
		err := os.Remove(fileName)
		switch {
		case err == nil:
			continue
		case errorIsNoEnt(err):
			log.Printf("Deleting already-deleted file; harmless.")
			continue
		default:
			return err
		}
	}
	return nil
}

type readBlobRequest struct {
	ch      chan *blobref.SizedBlobRef
	after   string
	remain  *uint // limit countdown
	dirRoot string

	// Not used on initial request, only on recursion
	blobPrefix, pathInto string
}

type enumerateError struct {
	msg string
	err os.Error
}

func (ee *enumerateError) String() string {
	return fmt.Sprintf("Enumerate error: %s: %v", ee.msg, ee.err)
}

func readBlobs(opts readBlobRequest) os.Error {
	dirFullPath := opts.dirRoot + "/" + opts.pathInto
	dir, err := os.Open(dirFullPath, os.O_RDONLY, 0)
	if err != nil {
		return &enumerateError{"opening directory " + dirFullPath, err}
	}
	defer dir.Close()
	names, err := dir.Readdirnames(32768)
	if err != nil {
		return &enumerateError{"readdirnames of " + dirFullPath, err}
	}
	sort.SortStrings(names)
	for _, name := range names {
		if *opts.remain == 0 {
			return nil
		}

		fullPath := dirFullPath + "/" + name
		fi, err := os.Stat(fullPath)
		if err != nil {
			return &enumerateError{"stat of file " + fullPath, err}
		}

		if fi.IsDirectory() {
			var newBlobPrefix string
			if opts.blobPrefix == "" {
				newBlobPrefix = name + "-"
			} else {
				newBlobPrefix = opts.blobPrefix + name
			}
			if len(opts.after) > 0 {
				compareLen := len(newBlobPrefix)
				if len(opts.after) < compareLen {
					compareLen = len(opts.after)
				}
				if newBlobPrefix[0:compareLen] < opts.after[0:compareLen] {
					continue
				}
			}
			ropts := opts
			ropts.blobPrefix = newBlobPrefix
			ropts.pathInto = opts.pathInto + "/" + name
			readBlobs(ropts)
			continue
		}

		if fi.IsRegular() && strings.HasSuffix(name, ".dat") {
			blobName := name[0 : len(name)-4]
			if blobName <= opts.after {
				continue
			}
			blobRef := blobref.Parse(blobName)
			if blobRef != nil {
				opts.ch <- &blobref.SizedBlobRef{BlobRef: blobRef, Size: fi.Size}
				(*opts.remain)--
			}
			continue
		}
	}

	if opts.pathInto == "" {
		opts.ch <- nil
	}
	return nil
}

func (ds *diskStorage) EnumerateBlobs(dest chan *blobref.SizedBlobRef, partition blobserver.Partition, after string, limit uint) os.Error {
	dirRoot := ds.root
	if partition != "" {
		dirRoot += "/partition/" + string(partition) + "/"
	}
	limitMutable := limit
	return readBlobs(readBlobRequest{
		ch:      dest,
		dirRoot: dirRoot,
		after:   after,
		remain:  &limitMutable,
	})
}

func (ds *diskStorage) Stat(dest chan *blobref.SizedBlobRef, partition blobserver.Partition, blobs []*blobref.BlobRef, waitSeconds int) os.Error {
	var missing []*blobref.BlobRef

	// TODO: stat in parallel; keep disks busy
	for _, ref := range blobs {
		fi, err := os.Stat(ds.blobFileName(ref))
		switch {
		case err == nil && fi.IsRegular():
			dest <- &blobref.SizedBlobRef{BlobRef: ref, Size: fi.Size}
		case err != nil && errorIsNoEnt(err) && waitSeconds > 0:
			missing = append(missing, ref)
		case err != nil && !errorIsNoEnt(err):
			return err
		}
	}

	if len(missing) > 0 {
		// TODO: use waitSeconds
		log.Printf("TODO: wait for %d blobs: %#v", len(missing), missing)
	}

	return nil
}

var CorruptBlobError = os.NewError("corrupt blob; digest doesn't match")

func (ds *diskStorage) ReceiveBlob(blobRef *blobref.BlobRef, source io.Reader, mirrorPartitions []blobserver.Partition) (blobGot *blobref.SizedBlobRef, err os.Error) {
	hashedDirectory := ds.blobDirectoryName(blobRef)
	err = os.MkdirAll(hashedDirectory, 0700)
	if err != nil {
		return
	}

	var tempFile *os.File
	tempFile, err = ioutil.TempFile(hashedDirectory, BlobFileBaseName(blobRef)+".tmp")
	if err != nil {
		return
	}

	success := false // set true later
	defer func() {
		if !success {
			log.Println("Removing temp file: ", tempFile.Name())
			os.Remove(tempFile.Name())
		}
	}()

	hash := blobRef.Hash()
	var written int64
	written, err = io.Copy(io.MultiWriter(hash, tempFile), source)
	if err != nil {
		return
	}
	if err = tempFile.Sync(); err != nil {
		return
	}
	if err = tempFile.Close(); err != nil {
		return
	}

	if !blobRef.HashMatches(hash) {
		err = CorruptBlobError
		return
	}

	fileName := ds.blobFileName(blobRef)
	if err = os.Rename(tempFile.Name(), fileName); err != nil {
		return
	}

	stat, err := os.Lstat(fileName)
	if err != nil {
		return
	}
	if !stat.IsRegular() || stat.Size != written {
		err = os.NewError("Written size didn't match.")
		return
	}

	for _, partition := range mirrorPartitions {
		partitionDir := ds.blobPartitionDirectoryName(partition, blobRef)
		if err = os.MkdirAll(partitionDir, 0700); err != nil {
			return
		}
		partitionFileName := ds.partitionBlobFileName(partition, blobRef)
		if err = os.Link(fileName, partitionFileName); err != nil {
			return
		}
		log.Printf("Mirrored to partition %q", partition)
	}

	blobGot = &blobref.SizedBlobRef{BlobRef: blobRef, Size: stat.Size}
	success = true

	if *flagOpenImages {
		exec.Run("/usr/bin/eog",
			[]string{"/usr/bin/eog", fileName},
			os.Environ(),
			"/",
			exec.DevNull,
			exec.DevNull,
			exec.MergeWithStdout)
	}

	hub := ds.GetBlobHub(blobserver.DefaultPartition)
	hub.NotifyBlobReceived(blobRef)
	for _, partition := range mirrorPartitions {
		hub = ds.GetBlobHub(partition)
		hub.NotifyBlobReceived(blobRef)
	}

	return
}

func BlobFileBaseName(b *blobref.BlobRef) string {
	return fmt.Sprintf("%s-%s.dat", b.HashName(), b.Digest())
}

func (ds *diskStorage) blobPartitionDirName(partitionDirSlash string, b *blobref.BlobRef) string {
	d := b.Digest()
	if len(d) < 6 {
		d = d + "______"
	}
	return fmt.Sprintf("%s/%s%s/%s/%s",
		ds.root, partitionDirSlash,
		b.HashName(), d[0:3], d[3:6])
}

func (ds *diskStorage) blobDirectoryName(b *blobref.BlobRef) string {
	return ds.blobPartitionDirName("", b)
}

func (ds *diskStorage) blobFileName(b *blobref.BlobRef) string {
	return fmt.Sprintf("%s/%s-%s.dat", ds.blobDirectoryName(b), b.HashName(), b.Digest())
}

func (ds *diskStorage) blobPartitionDirectoryName(partition blobserver.Partition, b *blobref.BlobRef) string {
	return ds.blobPartitionDirName("partition/"+string(partition)+"/", b)
}

func (ds *diskStorage) partitionBlobFileName(partition blobserver.Partition, b *blobref.BlobRef) string {
	return fmt.Sprintf("%s/%s-%s.dat", ds.blobPartitionDirectoryName(partition, b), b.HashName(), b.Digest())
}
<|endoftext|>