<commit_before>package birc

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"strconv"
	"strings"
	"time"

	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/helper"
	"github.com/lrstanley/girc"
	"github.com/paulrosania/go-charset/charset"
	"github.com/saintfish/chardet"

	// We need to import the 'data' package as an implicit dependency.
	// See: https://godoc.org/github.com/paulrosania/go-charset/charset
	_ "github.com/paulrosania/go-charset/data"
)

func (b *Birc) handleCharset(msg *config.Message) error {
	if b.GetString("Charset") != "" {
		switch b.GetString("Charset") {
		case "gbk", "gb18030", "gb2312", "big5", "euc-kr", "euc-jp", "shift-jis", "iso-2022-jp":
			msg.Text = toUTF8(b.GetString("Charset"), msg.Text)
		default:
			buf := new(bytes.Buffer)
			w, err := charset.NewWriter(b.GetString("Charset"), buf)
			if err != nil {
				b.Log.Errorf("charset to utf-8 conversion failed: %s", err)
				return err
			}
			fmt.Fprint(w, msg.Text)
			w.Close()
			msg.Text = buf.String()
		}
	}
	return nil
}

// handleFiles returns true if we have handled the files, otherwise return false
func (b *Birc) handleFiles(msg *config.Message) bool {
	if msg.Extra == nil {
		return false
	}
	for _, rmsg := range helper.HandleExtra(msg, b.General) {
		b.Local <- rmsg
	}
	if len(msg.Extra["file"]) == 0 {
		return false
	}
	for _, f := range msg.Extra["file"] {
		fi := f.(config.FileInfo)
		if fi.Comment != "" {
			msg.Text += fi.Comment + " : "
		}
		if fi.URL != "" {
			msg.Text = fi.URL
			if fi.Comment != "" {
				msg.Text = fi.Comment + " : " + fi.URL
			}
		}
		b.Local <- config.Message{Text: msg.Text, Username: msg.Username, Channel: msg.Channel, Event: msg.Event}
	}
	return true
}

func (b *Birc) handleInvite(client *girc.Client, event girc.Event) {
	if len(event.Params) != 2 {
		return
	}

	channel := event.Params[1]

	b.Log.Debugf("got invite for %s", channel)

	if _, ok := b.channels[channel]; ok {
		b.i.Cmd.Join(channel)
	}
}

func (b *Birc) handleJoinPart(client *girc.Client, event girc.Event) {
	if len(event.Params) == 0 {
		b.Log.Debugf("handleJoinPart: empty Params? %#v", event)
		return
	}
	channel := strings.ToLower(event.Params[0])
	if event.Command == "KICK" && event.Params[1] == b.Nick {
		b.Log.Infof("Got kicked from %s by %s", channel, event.Source.Name)
		time.Sleep(time.Duration(b.GetInt("RejoinDelay")) * time.Second)
		b.Remote <- config.Message{Username: "system", Text: "rejoin", Channel: channel, Account: b.Account, Event: config.EventRejoinChannels}
		return
	}
	if event.Command == "QUIT" {
		if event.Source.Name == b.Nick && strings.Contains(event.Last(), "Ping timeout") {
			b.Log.Infof("%s reconnecting ..", b.Account)
			b.Remote <- config.Message{Username: "system", Text: "reconnect", Channel: channel, Account: b.Account, Event: config.EventFailure}
			return
		}
	}
	if event.Source.Name != b.Nick {
		if b.GetBool("nosendjoinpart") {
			return
		}
		msg := config.Message{Username: "system", Text: event.Source.Name + " " + strings.ToLower(event.Command) + "s", Channel: channel, Account: b.Account, Event: config.EventJoinLeave}
		if b.GetBool("verbosejoinpart") {
			b.Log.Debugf("<= Sending verbose JOIN_LEAVE event from %s to gateway", b.Account)
			msg = config.Message{Username: "system", Text: event.Source.Name + " (" + event.Source.Ident + "@" + event.Source.Host + ") " + strings.ToLower(event.Command) + "s", Channel: channel, Account: b.Account, Event: config.EventJoinLeave}
		} else {
			b.Log.Debugf("<= Sending JOIN_LEAVE event from %s to gateway", b.Account)
		}
		b.Log.Debugf("<= Message is %#v", msg)
		b.Remote <- msg
		return
	}
	b.Log.Debugf("handle %#v", event)
}

func (b *Birc) handleNewConnection(client *girc.Client, event girc.Event) {
	b.Log.Debug("Registering callbacks")
	i := b.i
	b.Nick = event.Params[0]

	i.Handlers.ClearAll()
	i.Handlers.AddBg("PRIVMSG", b.handlePrivMsg)
	i.Handlers.AddBg("CTCP_ACTION", b.handlePrivMsg)
	i.Handlers.Add(girc.RPL_TOPICWHOTIME, b.handleTopicWhoTime)
	i.Handlers.AddBg(girc.NOTICE, b.handleNotice)
	i.Handlers.AddBg("JOIN", b.handleJoinPart)
	i.Handlers.AddBg("PART", b.handleJoinPart)
	i.Handlers.AddBg("QUIT", b.handleJoinPart)
	i.Handlers.AddBg("KICK", b.handleJoinPart)
	i.Handlers.Add("INVITE", b.handleInvite)
}

func (b *Birc) handleNickServ() {
	if !b.GetBool("UseSASL") && b.GetString("NickServNick") != "" && b.GetString("NickServPassword") != "" {
		b.Log.Debugf("Sending identify to nickserv %s", b.GetString("NickServNick"))
		b.i.Cmd.Message(b.GetString("NickServNick"), "IDENTIFY "+b.GetString("NickServPassword"))
	}
	if strings.EqualFold(b.GetString("NickServNick"), "Q@CServe.quakenet.org") {
		b.Log.Debugf("Authenticating %s against %s", b.GetString("NickServUsername"), b.GetString("NickServNick"))
		b.i.Cmd.Message(b.GetString("NickServNick"), "AUTH "+b.GetString("NickServUsername")+" "+b.GetString("NickServPassword"))
	}
	// give nickserv some slack
	time.Sleep(time.Second * 5)
	b.authDone = true
}

func (b *Birc) handleNotice(client *girc.Client, event girc.Event) {
	if strings.Contains(event.String(), "This nickname is registered") && event.Source.Name == b.GetString("NickServNick") {
		b.handleNickServ()
	} else {
		b.handlePrivMsg(client, event)
	}
}

func (b *Birc) handleOther(client *girc.Client, event girc.Event) {
	if b.GetInt("DebugLevel") == 1 {
		if event.Command != "CLIENT_STATE_UPDATED" &&
			event.Command != "CLIENT_GENERAL_UPDATED" {
			b.Log.Debugf("%#v", event.String())
		}
		return
	}
	switch event.Command {
	case "372", "375", "376", "250", "251", "252", "253", "254", "255", "265", "266", "002", "003", "004", "005":
		return
	}
	b.Log.Debugf("%#v", event.String())
}

func (b *Birc) handleOtherAuth(client *girc.Client, event girc.Event) {
	b.handleNickServ()
	b.handleRunCommands()
	// we are now fully connected
	// only send on first connection
	if b.FirstConnection {
		b.connected <- nil
	}
}

func (b *Birc) handlePrivMsg(client *girc.Client, event girc.Event) {
	if b.skipPrivMsg(event) {
		return
	}

	rmsg := config.Message{
		Username: event.Source.Name,
		Channel:  strings.ToLower(event.Params[0]),
		Account:  b.Account,
		UserID:   event.Source.Ident + "@" + event.Source.Host,
	}

	b.Log.Debugf("== Receiving PRIVMSG: %s %s %#v", event.Source.Name, event.Last(), event)

	// set action event
	if event.IsAction() {
		rmsg.Event = config.EventUserAction
	}

	// set NOTICE event
	if event.Command == "NOTICE" {
		rmsg.Event = config.EventNoticeIRC
	}

	// strip action, we made an event if it was an action
	rmsg.Text += event.StripAction()

	// start detecting the charset
	mycharset := b.GetString("Charset")
	if mycharset == "" {
		// detect what were sending so that we convert it to utf-8
		detector := chardet.NewTextDetector()
		result, err := detector.DetectBest([]byte(rmsg.Text))
		if err != nil {
			b.Log.Infof("detection failed for rmsg.Text: %#v", rmsg.Text)
			return
		}
		b.Log.Debugf("detected %s confidence %#v", result.Charset, result.Confidence)
		mycharset = result.Charset
		// if we're not sure, just pick ISO-8859-1
		if result.Confidence < 80 {
			mycharset = "ISO-8859-1"
		}
	}
	switch mycharset {
	case "gbk", "gb18030", "gb2312", "big5", "euc-kr", "euc-jp", "shift-jis", "iso-2022-jp":
		rmsg.Text = toUTF8(b.GetString("Charset"), rmsg.Text)
	default:
		r, err := charset.NewReader(mycharset, strings.NewReader(rmsg.Text))
		if err != nil {
			b.Log.Errorf("charset to utf-8 conversion failed: %s", err)
			return
		}
		output, _ := ioutil.ReadAll(r)
		rmsg.Text = string(output)
	}

	b.Log.Debugf("<= Sending message from %s on %s to gateway", event.Params[0], b.Account)
	b.Remote <- rmsg
}

func (b *Birc) handleRunCommands() {
	for _, cmd := range b.GetStringSlice("RunCommands") {
		if err := b.i.Cmd.SendRaw(cmd); err != nil {
			b.Log.Errorf("RunCommands %s failed: %s", cmd, err)
		}
		time.Sleep(time.Second)
	}
}

func (b *Birc) handleTopicWhoTime(client *girc.Client, event girc.Event) {
	parts := strings.Split(event.Params[2], "!")
	t, err := strconv.ParseInt(event.Params[3], 10, 64)
	if err != nil {
		b.Log.Errorf("Invalid time stamp: %s", event.Params[3])
	}
	user := parts[0]
	if len(parts) > 1 {
		user += " [" + parts[1] + "]"
	}
	b.Log.Debugf("%s: Topic set by %s [%s]", event.Command, user, time.Unix(t, 0))
}
<commit_msg>Revert "Clear existing IRC event handlers before connecting new ones (#1795)"<commit_after>package birc

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"strconv"
	"strings"
	"time"

	"github.com/42wim/matterbridge/bridge/config"
	"github.com/42wim/matterbridge/bridge/helper"
	"github.com/lrstanley/girc"
	"github.com/paulrosania/go-charset/charset"
	"github.com/saintfish/chardet"

	// We need to import the 'data' package as an implicit dependency.
	// See: https://godoc.org/github.com/paulrosania/go-charset/charset
	_ "github.com/paulrosania/go-charset/data"
)

func (b *Birc) handleCharset(msg *config.Message) error {
	if b.GetString("Charset") != "" {
		switch b.GetString("Charset") {
		case "gbk", "gb18030", "gb2312", "big5", "euc-kr", "euc-jp", "shift-jis", "iso-2022-jp":
			msg.Text = toUTF8(b.GetString("Charset"), msg.Text)
		default:
			buf := new(bytes.Buffer)
			w, err := charset.NewWriter(b.GetString("Charset"), buf)
			if err != nil {
				b.Log.Errorf("charset to utf-8 conversion failed: %s", err)
				return err
			}
			fmt.Fprint(w, msg.Text)
			w.Close()
			msg.Text = buf.String()
		}
	}
	return nil
}

// handleFiles returns true if we have handled the files, otherwise return false
func (b *Birc) handleFiles(msg *config.Message) bool {
	if msg.Extra == nil {
		return false
	}
	for _, rmsg := range helper.HandleExtra(msg, b.General) {
		b.Local <- rmsg
	}
	if len(msg.Extra["file"]) == 0 {
		return false
	}
	for _, f := range msg.Extra["file"] {
		fi := f.(config.FileInfo)
		if fi.Comment != "" {
			msg.Text += fi.Comment + " : "
		}
		if fi.URL != "" {
			msg.Text = fi.URL
			if fi.Comment != "" {
				msg.Text = fi.Comment + " : " + fi.URL
			}
		}
		b.Local <- config.Message{Text: msg.Text, Username: msg.Username, Channel: msg.Channel, Event: msg.Event}
	}
	return true
}

func (b *Birc) handleInvite(client *girc.Client, event girc.Event) {
	if len(event.Params) != 2 {
		return
	}

	channel := event.Params[1]

	b.Log.Debugf("got invite for %s", channel)

	if _, ok := b.channels[channel]; ok {
		b.i.Cmd.Join(channel)
	}
}

func (b *Birc) handleJoinPart(client *girc.Client, event girc.Event) {
	if len(event.Params) == 0 {
		b.Log.Debugf("handleJoinPart: empty Params? %#v", event)
		return
	}
	channel := strings.ToLower(event.Params[0])
	if event.Command == "KICK" && event.Params[1] == b.Nick {
		b.Log.Infof("Got kicked from %s by %s", channel, event.Source.Name)
		time.Sleep(time.Duration(b.GetInt("RejoinDelay")) * time.Second)
		b.Remote <- config.Message{Username: "system", Text: "rejoin", Channel: channel, Account: b.Account, Event: config.EventRejoinChannels}
		return
	}
	if event.Command == "QUIT" {
		if event.Source.Name == b.Nick && strings.Contains(event.Last(), "Ping timeout") {
			b.Log.Infof("%s reconnecting ..", b.Account)
			b.Remote <- config.Message{Username: "system", Text: "reconnect", Channel: channel, Account: b.Account, Event: config.EventFailure}
			return
		}
	}
	if event.Source.Name != b.Nick {
		if b.GetBool("nosendjoinpart") {
			return
		}
		msg := config.Message{Username: "system", Text: event.Source.Name + " " + strings.ToLower(event.Command) + "s", Channel: channel, Account: b.Account, Event: config.EventJoinLeave}
		if b.GetBool("verbosejoinpart") {
			b.Log.Debugf("<= Sending verbose JOIN_LEAVE event from %s to gateway", b.Account)
			msg = config.Message{Username: "system", Text: event.Source.Name + " (" + event.Source.Ident + "@" + event.Source.Host + ") " + strings.ToLower(event.Command) + "s", Channel: channel, Account: b.Account, Event: config.EventJoinLeave}
		} else {
			b.Log.Debugf("<= Sending JOIN_LEAVE event from %s to gateway", b.Account)
		}
		b.Log.Debugf("<= Message is %#v", msg)
		b.Remote <- msg
		return
	}
	b.Log.Debugf("handle %#v", event)
}

func (b *Birc) handleNewConnection(client *girc.Client, event girc.Event) {
	b.Log.Debug("Registering callbacks")
	i := b.i
	b.Nick = event.Params[0]

	i.Handlers.AddBg("PRIVMSG", b.handlePrivMsg)
	i.Handlers.AddBg("CTCP_ACTION", b.handlePrivMsg)
	i.Handlers.Add(girc.RPL_TOPICWHOTIME, b.handleTopicWhoTime)
	i.Handlers.AddBg(girc.NOTICE, b.handleNotice)
	i.Handlers.AddBg("JOIN", b.handleJoinPart)
	i.Handlers.AddBg("PART", b.handleJoinPart)
	i.Handlers.AddBg("QUIT", b.handleJoinPart)
	i.Handlers.AddBg("KICK", b.handleJoinPart)
	i.Handlers.Add("INVITE", b.handleInvite)
}

func (b *Birc) handleNickServ() {
	if !b.GetBool("UseSASL") && b.GetString("NickServNick") != "" && b.GetString("NickServPassword") != "" {
		b.Log.Debugf("Sending identify to nickserv %s", b.GetString("NickServNick"))
		b.i.Cmd.Message(b.GetString("NickServNick"), "IDENTIFY "+b.GetString("NickServPassword"))
	}
	if strings.EqualFold(b.GetString("NickServNick"), "Q@CServe.quakenet.org") {
		b.Log.Debugf("Authenticating %s against %s", b.GetString("NickServUsername"), b.GetString("NickServNick"))
		b.i.Cmd.Message(b.GetString("NickServNick"), "AUTH "+b.GetString("NickServUsername")+" "+b.GetString("NickServPassword"))
	}
	// give nickserv some slack
	time.Sleep(time.Second * 5)
	b.authDone = true
}

func (b *Birc) handleNotice(client *girc.Client, event girc.Event) {
	if strings.Contains(event.String(), "This nickname is registered") && event.Source.Name == b.GetString("NickServNick") {
		b.handleNickServ()
	} else {
		b.handlePrivMsg(client, event)
	}
}

func (b *Birc) handleOther(client *girc.Client, event girc.Event) {
	if b.GetInt("DebugLevel") == 1 {
		if event.Command != "CLIENT_STATE_UPDATED" &&
			event.Command != "CLIENT_GENERAL_UPDATED" {
			b.Log.Debugf("%#v", event.String())
		}
		return
	}
	switch event.Command {
	case "372", "375", "376", "250", "251", "252", "253", "254", "255", "265", "266", "002", "003", "004", "005":
		return
	}
	b.Log.Debugf("%#v", event.String())
}

func (b *Birc) handleOtherAuth(client *girc.Client, event girc.Event) {
	b.handleNickServ()
	b.handleRunCommands()
	// we are now fully connected
	// only send on first connection
	if b.FirstConnection {
		b.connected <- nil
	}
}

func (b *Birc) handlePrivMsg(client *girc.Client, event girc.Event) {
	if b.skipPrivMsg(event) {
		return
	}

	rmsg := config.Message{
		Username: event.Source.Name,
		Channel:  strings.ToLower(event.Params[0]),
		Account:  b.Account,
		UserID:   event.Source.Ident + "@" + event.Source.Host,
	}

	b.Log.Debugf("== Receiving PRIVMSG: %s %s %#v", event.Source.Name, event.Last(), event)

	// set action event
	if event.IsAction() {
		rmsg.Event = config.EventUserAction
	}

	// set NOTICE event
	if event.Command == "NOTICE" {
		rmsg.Event = config.EventNoticeIRC
	}

	// strip action, we made an event if it was an action
	rmsg.Text += event.StripAction()

	// start detecting the charset
	mycharset := b.GetString("Charset")
	if mycharset == "" {
		// detect what were sending so that we convert it to utf-8
		detector := chardet.NewTextDetector()
		result, err := detector.DetectBest([]byte(rmsg.Text))
		if err != nil {
			b.Log.Infof("detection failed for rmsg.Text: %#v", rmsg.Text)
			return
		}
		b.Log.Debugf("detected %s confidence %#v", result.Charset, result.Confidence)
		mycharset = result.Charset
		// if we're not sure, just pick ISO-8859-1
		if result.Confidence < 80 {
			mycharset = "ISO-8859-1"
		}
	}
	switch mycharset {
	case "gbk", "gb18030", "gb2312", "big5", "euc-kr", "euc-jp", "shift-jis", "iso-2022-jp":
		rmsg.Text = toUTF8(b.GetString("Charset"), rmsg.Text)
	default:
		r, err := charset.NewReader(mycharset, strings.NewReader(rmsg.Text))
		if err != nil {
			b.Log.Errorf("charset to utf-8 conversion failed: %s", err)
			return
		}
		output, _ := ioutil.ReadAll(r)
		rmsg.Text = string(output)
	}

	b.Log.Debugf("<= Sending message from %s on %s to gateway", event.Params[0], b.Account)
	b.Remote <- rmsg
}

func (b *Birc) handleRunCommands() {
	for _, cmd := range b.GetStringSlice("RunCommands") {
		if err := b.i.Cmd.SendRaw(cmd); err != nil {
			b.Log.Errorf("RunCommands %s failed: %s", cmd, err)
		}
		time.Sleep(time.Second)
	}
}

func (b *Birc) handleTopicWhoTime(client *girc.Client, event girc.Event) {
	parts := strings.Split(event.Params[2], "!")
	t, err := strconv.ParseInt(event.Params[3], 10, 64)
	if err != nil {
		b.Log.Errorf("Invalid time stamp: %s", event.Params[3])
	}
	user := parts[0]
	if len(parts) > 1 {
		user += " [" + parts[1] + "]"
	}
	b.Log.Debugf("%s: Topic set by %s [%s]", event.Command, user, time.Unix(t, 0))
}
<|endoftext|>


<commit_before>package gowebdav

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
	"strings"
)

func (c *Client) req(method, path string, body io.Reader, intercept func(*http.Request)) (req *http.Response, err error) {
	// Tee the body, because if authorization fails we will need to read from it again.
	var ba bytes.Buffer
	bb := io.TeeReader(body, &ba)

	r, err := http.NewRequest(method, PathEscape(Join(c.root, path)), &ba)
	if err != nil {
		return nil, err
	}

	c.auth.Authorize(c, method, path)

	for k, vals := range c.headers {
		for _, v := range vals {
			r.Header.Add(k, v)
		}
	}

	if intercept != nil {
		intercept(r)
	}

	rs, err := c.c.Do(r)

	if rs.StatusCode == 401 && c.auth.Type() == "NoAuth" {
		if strings.Index(rs.Header.Get("Www-Authenticate"), "Digest") > -1 {
			c.auth = &DigestAuth{c.auth.User(), c.auth.Pass(), digestParts(rs)}
		} else if strings.Index(rs.Header.Get("Www-Authenticate"), "Basic") > -1 {
			c.auth = &BasicAuth{c.auth.User(), c.auth.Pass()}
		} else {
			return rs, newPathError("Authorize", c.root, rs.StatusCode)
		}
		return c.req(method, path, bb, intercept)
	} else if rs.StatusCode == 401 {
		return rs, newPathError("Authorize", c.root, rs.StatusCode)
	}

	return rs, err
}

func (c *Client) mkcol(path string) int {
	rs, err := c.req("MKCOL", path, nil, nil)
	defer rs.Body.Close()
	if err != nil {
		return 400
	}

	if rs.StatusCode == 201 || rs.StatusCode == 405 {
		return 201
	}

	return rs.StatusCode
}

func (c *Client) options(path string) (*http.Response, error) {
	return c.req("OPTIONS", path, nil, func(rq *http.Request) {
		rq.Header.Add("Depth", "0")
	})
}

func (c *Client) propfind(path string, self bool, body string, resp interface{}, parse func(resp interface{}) error) error {
	rs, err := c.req("PROPFIND", path, strings.NewReader(body), func(rq *http.Request) {
		if self {
			rq.Header.Add("Depth", "0")
		} else {
			rq.Header.Add("Depth", "1")
		}
		rq.Header.Add("Content-Type", "text/xml;charset=UTF-8")
		rq.Header.Add("Accept", "application/xml,text/xml")
		rq.Header.Add("Accept-Charset", "utf-8")
		// TODO add support for 'gzip,deflate;q=0.8,q=0.7'
		rq.Header.Add("Accept-Encoding", "")
	})
	defer rs.Body.Close()
	if err != nil {
		return err
	}

	if rs.StatusCode != 207 {
		return fmt.Errorf("%s - %s %s", rs.Status, "PROPFIND", path)
	}

	return parseXML(rs.Body, resp, parse)
}

func (c *Client) doCopyMove(method string, oldpath string, newpath string, overwrite bool) (int, io.ReadCloser) {
	rs, err := c.req(method, oldpath, nil, func(rq *http.Request) {
		rq.Header.Add("Destination", Join(c.root, newpath))
		if overwrite {
			rq.Header.Add("Overwrite", "T")
		} else {
			rq.Header.Add("Overwrite", "F")
		}
	})
	if err != nil {
		return 400, nil
	}
	return rs.StatusCode, rs.Body
}

func (c *Client) copymove(method string, oldpath string, newpath string, overwrite bool) error {
	s, data := c.doCopyMove(method, oldpath, newpath, overwrite)
	defer data.Close()

	switch s {
	case 201, 204:
		return nil

	case 207:
		// TODO handle multistat errors, worst case ...
		log(fmt.Sprintf(" TODO handle %s - %s multistatus result %s", method, oldpath, String(data)))

	case 409:
		// TODO create dst path
	}

	return newPathError(method, oldpath, s)
}

func (c *Client) put(path string, stream io.Reader) int {
	rs, err := c.req("PUT", path, stream, nil)
	defer rs.Body.Close()
	if err != nil {
		return 400
	}

	return rs.StatusCode
}
<commit_msg>quick bugfix for issue #20 (#21)<commit_after>package gowebdav

import (
	"bytes"
	"fmt"
	"io"
	"net/http"
	"strings"
)

func (c *Client) req(method, path string, body io.Reader, intercept func(*http.Request)) (req *http.Response, err error) {
	// Tee the body, because if authorization fails we will need to read from it again.
	var r *http.Request
	var ba bytes.Buffer
	bb := io.TeeReader(body, &ba)

	if body == nil {
		r, err = http.NewRequest(method, PathEscape(Join(c.root, path)), nil)
	} else {
		r, err = http.NewRequest(method, PathEscape(Join(c.root, path)), bb)
	}

	if err != nil {
		return nil, err
	}

	c.auth.Authorize(c, method, path)

	for k, vals := range c.headers {
		for _, v := range vals {
			r.Header.Add(k, v)
		}
	}

	if intercept != nil {
		intercept(r)
	}

	rs, err := c.c.Do(r)

	if rs.StatusCode == 401 && c.auth.Type() == "NoAuth" {
		if strings.Index(rs.Header.Get("Www-Authenticate"), "Digest") > -1 {
			c.auth = &DigestAuth{c.auth.User(), c.auth.Pass(), digestParts(rs)}
		} else if strings.Index(rs.Header.Get("Www-Authenticate"), "Basic") > -1 {
			c.auth = &BasicAuth{c.auth.User(), c.auth.Pass()}
		} else {
			return rs, newPathError("Authorize", c.root, rs.StatusCode)
		}

		if body == nil {
			return c.req(method, path, nil, intercept)
		} else {
			return c.req(method, path, &ba, intercept)
		}

	} else if rs.StatusCode == 401 {
		return rs, newPathError("Authorize", c.root, rs.StatusCode)
	}

	return rs, err
}

func (c *Client) mkcol(path string) int {
	rs, err := c.req("MKCOL", path, nil, nil)
	defer rs.Body.Close()
	if err != nil {
		return 400
	}

	if rs.StatusCode == 201 || rs.StatusCode == 405 {
		return 201
	}

	return rs.StatusCode
}

func (c *Client) options(path string) (*http.Response, error) {
	return c.req("OPTIONS", path, nil, func(rq *http.Request) {
		rq.Header.Add("Depth", "0")
	})
}

func (c *Client) propfind(path string, self bool, body string, resp interface{}, parse func(resp interface{}) error) error {
	rs, err := c.req("PROPFIND", path, strings.NewReader(body), func(rq *http.Request) {
		if self {
			rq.Header.Add("Depth", "0")
		} else {
			rq.Header.Add("Depth", "1")
		}
		rq.Header.Add("Content-Type", "text/xml;charset=UTF-8")
		rq.Header.Add("Accept", "application/xml,text/xml")
		rq.Header.Add("Accept-Charset", "utf-8")
		// TODO add support for 'gzip,deflate;q=0.8,q=0.7'
		rq.Header.Add("Accept-Encoding", "")
	})
	defer rs.Body.Close()
	if err != nil {
		return err
	}

	if rs.StatusCode != 207 {
		return fmt.Errorf("%s - %s %s", rs.Status, "PROPFIND", path)
	}

	return parseXML(rs.Body, resp, parse)
}

func (c *Client) doCopyMove(method string, oldpath string, newpath string, overwrite bool) (int, io.ReadCloser) {
	rs, err := c.req(method, oldpath, nil, func(rq *http.Request) {
		rq.Header.Add("Destination", Join(c.root, newpath))
		if overwrite {
			rq.Header.Add("Overwrite", "T")
		} else {
			rq.Header.Add("Overwrite", "F")
		}
	})
	if err != nil {
		return 400, nil
	}
	return rs.StatusCode, rs.Body
}

func (c *Client) copymove(method string, oldpath string, newpath string, overwrite bool) error {
	s, data := c.doCopyMove(method, oldpath, newpath, overwrite)
	defer data.Close()

	switch s {
	case 201, 204:
		return nil

	case 207:
		// TODO handle multistat errors, worst case ...
		log(fmt.Sprintf(" TODO handle %s - %s multistatus result %s", method, oldpath, String(data)))

	case 409:
		// TODO create dst path
	}

	return newPathError(method, oldpath, s)
}

func (c *Client) put(path string, stream io.Reader) int {
	rs, err := c.req("PUT", path, stream, nil)
	defer rs.Body.Close()
	if err != nil {
		return 400
	}

	return rs.StatusCode
}
<|endoftext|>


<commit_before>package codebase

import (
    "log"
    "sort"
    "time"
)

type milestoneArray struct {
    Milestones []Milestone `xml:"ticketing-milestone"`
}

type Milestone struct {
    Id          int    `xml:"id"`
    Name        string `xml:"name"`
    StartAt     string `xml:"start-at"`
    Deadline    string `xml:"deadline"`
    Status      string `xml:"status"`
    Description string `xml:"description"`
}

type byMostRecent struct {
    ms []Milestone
}

func (m byMostRecent) Len() int           { return len(m.ms) }
func (m byMostRecent) Swap(i, j int)      { m.ms[i], m.ms[j] = m.ms[j], m.ms[i] }
func (m byMostRecent) Less(i, j int) bool { return m.ms[i].toEadges() < m.ms[j].toEadges() }

func (m *Milestone) IsActive() bool {
    return m.Status == "active"
}

func (m *Milestone) toEadges() float64 {
    layout := "2006-01-02"

    start_date := time.Unix(0, 0)
    end_date := time.Unix(0, 0)
    if m.StartAt == "" {
        start_date, _ = time.Parse(layout, m.StartAt)
    }
    if m.Deadline == "" {
        end_date, _ = time.Parse(layout, m.Deadline)
    }

    to_start := time.Since(start_date).Hours()
    to_end := time.Since(end_date).Hours()

    return to_start + to_end
}

func (c *CodeBaseAPI) Milesones() (milestones []Milestone) {
    queryOpts := baseQueryOptions{}

    proxy := milestoneArray{}
    if err := c.fetchFromCodebase("milestones", &proxy, queryOpts); err != nil {
        log.Fatalln("Could not fetch milestones:", err)
    }

    milestones = proxy.Milestones
    return
}

func (c *CodeBaseAPI) CurrentMilestone() (current Milestone) {
    allMilestones := c.Milesones()

    var activeMilestones []Milestone
    for _, milestone := range allMilestones {
        if !milestone.IsActive() {
            continue
        }

        activeMilestones = append(activeMilestones, milestone)
    }

    sort.Reverse(byMostRecent{activeMilestones})
    current = activeMilestones[0]

    return
}
<commit_msg>Description of milestone that include how many days are left until deadline.<commit_after>package codebase

import (
    "errors"
    "fmt"
    "log"
    "sort"
    "strings"
    "time"
)

type milestoneArray struct {
    Milestones []Milestone `xml:"ticketing-milestone"`
}

type Milestone struct {
    Id          int    `xml:"id"`
    Name        string `xml:"name"`
    StartAt     string `xml:"start-at"`
    Deadline    string `xml:"deadline"`
    Status      string `xml:"status"`
    Description string `xml:"description"`
}

type byMostRecent struct {
    ms []Milestone
}

func (m byMostRecent) Len() int           { return len(m.ms) }
func (m byMostRecent) Swap(i, j int)      { m.ms[i], m.ms[j] = m.ms[j], m.ms[i] }
func (m byMostRecent) Less(i, j int) bool { return m.ms[i].toEadges() < m.ms[j].toEadges() }

func (m *Milestone) IsActive() bool {
    return m.Status == "active"
}

func (m *Milestone) toEadges() float64 {
    layout := "2006-01-02"

    start_date := time.Unix(0, 0)
    end_date := time.Unix(0, 0)
    if m.StartAt != "" {
        start_date, _ = time.Parse(layout, m.StartAt)
    }
    if m.Deadline != "" {
        end_date, _ = time.Parse(layout, m.Deadline)
    }

    to_start := time.Since(start_date).Hours()
    to_end := time.Since(end_date).Hours()

    return to_start + to_end
}

func (m *Milestone) DaysToEnd() (int, error) {
    layout := "2006-01-02"

    if m.Deadline == "" {
        return 0, errors.New("Deadline not available")
    }

    end_date, err := time.Parse(layout, m.Deadline)
    if err != nil {
        return 0, err
    }

    beginningOfToday := time.Now().Truncate(24 * time.Hour)
    hoursToEnd := end_date.Sub(beginningOfToday).Hours()
    return int(hoursToEnd / 24), nil
}

func (m *Milestone) DaysToEndStr() string {
    daysToEnd, err := m.DaysToEnd()
    if err != nil {
        return ""
    }

    if daysToEnd == 0 {
        return "Ends today"
    }

    ending := ""
    if daysToEnd < -1 || 1 < daysToEnd {
        ending = "s"
    }

    if daysToEnd > 0 {
        return fmt.Sprintf("Ends in %v day%v", daysToEnd, ending)
    } else {
        return fmt.Sprintf("Ended %v day%v ago", -daysToEnd, ending)
    }
}

func (c *CodeBaseAPI) Milesones() (milestones []Milestone) {
    queryOpts := baseQueryOptions{}

    proxy := milestoneArray{}
    if err := c.fetchFromCodebase("milestones", &proxy, queryOpts); err != nil {
        log.Fatalln("Could not fetch milestones:", err)
    }

    milestones = proxy.Milestones
    return
}

func (c *CodeBaseAPI) CurrentMilestone() (current Milestone) {
    allMilestones := c.Milesones()

    var activeMilestones []Milestone
    for _, milestone := range allMilestones {
        if !milestone.IsActive() {
            continue
        }

        activeMilestones = append(activeMilestones, milestone)
    }

    sort.Reverse(byMostRecent{activeMilestones})
    current = activeMilestones[0]

    return
}

func (m Milestone) String() string {
    daysToEnd := m.DaysToEndStr()
    if daysToEnd != "" {
        return fmt.Sprintf("%v (%v)", m.Name, strings.ToLower(daysToEnd))
    }

    return m.Name
}
<|endoftext|>


<commit_before>package mapstore

import (
	"reflect"
	"testing"

	"golang.org/x/net/context"

	"sync"

	"time"

	"gitlab.fg/go/stela"
)

func Test_rotateServices(t *testing.T) {
	m := MapStore{}

	var tests = []struct {
		service          stela.Service
		expectedPriority int32
	}{
		{stela.Service{
			Name:    "priority.service.fg",
			Target:  "jlu.macbook",
			Address: "127.0.0.1",
			Port:    9000,
		}, 2},
		{stela.Service{
			Name:    "priority.service.fg",
			Target:  "jlu.macbook",
			Address: "127.0.0.2",
			Port:    9000,
		}, 1},
		{stela.Service{
			Name:    "priority.service.fg",
			Target:  "jlu.macbook",
			Address: "127.0.0.3",
			Port:    9000,
		}, 0},
	}
	var expectedServices []stela.Service
	for _, test := range tests {
		// First we need to register them
		if err := m.Register(&test.service); err != nil {
			t.Fatal(err)
		}

		expectedServices = append(expectedServices, test.service)
	}

	// Loop through services registered and make sure their priorities match
	for i, s := range m.services[tests[0].service.Name] {
		ts := tests[i]
		if s.Equal(&ts.service) {
			if s.Priority != ts.expectedPriority {
				t.Fatalf("Test_rotateServices: Priorites didn't match. Expected: %v, Received: %v", ts.expectedPriority, s.Priority)
			}
		}
	}
}

// func TestRegistration(t *testing.T) {
// 	m := MapStore{}

// 	var testServices = []*stela.Service{
// 		&stela.Service{
// 			Name:    "testregister.service.fg",
// 			Target:  "jlu.macbook",
// 			Address: "127.0.0.1",
// 			Port:    9000,
// 		},
// 		&stela.Service{
// 			Name:    "testregister.service.fg",
// 			Target:  "jlu.macbook",
// 			Address: "127.0.0.2",
// 			Port:    9001,
// 		},
// 		&stela.Service{
// 			Name:    "testregister.service.fg",
// 			Target:  "jlu.macbook",
// 			Address: "127.0.0.3",
// 			Port:    9002,
// 		},
// 	}

// 	for _, s := range testServices {
// 		// Register services
// 		m.Register(s)
// 	}

// 	// Make sure the map contains the correct services
// 	var services []*stela.Service
// 	for _, s := range m.services[testServices[0].Name] {
// 		services = append(services, &s)
// 	}
// 	if len(services) != len(testServices) {
// 		t.Fatalf("TestRegistration failed to register all services. Expected %d services, received: %d", len(testServices), len(services))
// 	}
// 	equalServices(t, testServices, services)

// 	for _, s := range testServices {
// 		// Deregister services
// 		m.Deregister(s)
// 	}

// 	if len(m.services[testServices[0].Name]) != 0 {
// 		t.Fatalf("TestRegistration failed all services should be deregistered")
// 	}
// }

func TestSubscribe(t *testing.T) {
	m := MapStore{}
	c := &stela.Client{}

	var testServices = []*stela.Service{
		&stela.Service{
			Name:    "subscribe.service.fg",
			Target:  "jlu.macbook",
			Address: "127.0.0.1",
			Port:    9000,
		},
		&stela.Service{
			Name:    "subscribe.service.fg",
			Target:  "jlu.macbook",
			Address: "127.0.0.2",
			Port:    9001,
		},
		&stela.Service{
			Name:    "subscribe.service.fg",
			Target:  "jlu.macbook",
			Address: "127.0.0.3",
			Port:    9002,
		},
	}

	// Subscribe to service
	if err := m.Subscribe(testServices[0].Name, c); err != nil {
		t.Fatal(err)
	}

	var registered []*stela.Service
	var deregistered []*stela.Service

	// Listen for registered services
	wg := &sync.WaitGroup{}
	wg.Add(len(testServices))
	writeWaitChan := make(chan struct{})
	readWaitChan := make(chan struct{})
	ctx, cancel := context.WithTimeout(context.TODO(), time.Millisecond*100)
	readCount := 0
	go func() {
		close(readWaitChan)
		for {
			select {
			case s := <-c.SubscribeCh():
				switch s.Action {
				case stela.RegisterAction:
					registered = append(registered, s)
				case stela.DeregisterAction:
					deregistered = append(deregistered, s)
				}

				// Count how many times we've received a subscribed service
				// Multiple by two since we are registering an deregistering all testServices
				readCount++
				if readCount == len(testServices)*2 {
					close(writeWaitChan)
				}
			case <-ctx.Done():
				return
			}
		}
	}()

	// Block until we are ready to receive subscription information
	<-readWaitChan

	for _, s := range testServices {
		// Register services
		m.Register(s)
	}

	for _, s := range testServices {
		// Deregister services
		m.Deregister(s)
	}

	// Wait for either a timeout or all the subscribed services to be read
	select {
	case <-writeWaitChan:
		break
	case <-ctx.Done():
		if ctx.Err() != nil {
			t.Fatal("TestSubscribe timed out: ", ctx.Err())
		}
	}

	// Function to compare testServices with serviceSlice arg
	testSubscribe := func(serviceSlice []*stela.Service) {
		if len(serviceSlice) != len(testServices) {
			t.Fatalf("Subscribe didn't match testServices. Received: %d, Expected: %d", len(serviceSlice), len(testServices))
		}

		// Make sure the serviceSlice has all the testServices
		equalServices(t, testServices, serviceSlice)
	}

	// Now make sure both registered and deregistered actions were received
	testSubscribe(registered)
	testSubscribe(deregistered)

	// Stop listening for subscriptions
	cancel()
}

func TestUnsubscribe(t *testing.T) {
	serviceName := "unsubscribe.services.fg"
	m := MapStore{}
	c1 := &stela.Client{}
	c2 := &stela.Client{}

	// Subscribe
	m.Subscribe(serviceName, c1)
	m.Subscribe(serviceName, c2)

	// Unsubscribe c1
	if err := m.Unsubscribe(serviceName, c1); err != nil {
		t.Fatal("Should have Unsubscribed successfully")
	}

	// Unsubscribe c1 again with the same client should fail
	if err := m.Unsubscribe(serviceName, c1); err == nil {
		t.Fatal("Unsubscribe should have failed")
	}

	// Unsubscribe c2
	if err := m.Unsubscribe(serviceName, c2); err != nil {
		t.Fatal("Should have Unsubscribed successfully")
	}

}

func TestRegistrationAndDiscovery(t *testing.T) {
	m := MapStore{}

	var tests = []struct {
		service    *stela.Service
		shouldFail bool
	}{
		{&stela.Service{
			Name:    "testd&r.service.fg",
			Target:  "jlu.macbook",
			Address: "127.0.0.1",
			Port:    9000,
		}, false},
		// Don't allow duplicates
		{&stela.Service{
			Name:    "testd&r.service.fg",
			Target:  "jlu.macbook",
			Address: "127.0.0.1",
			Port:    9000,
		}, true},
		{&stela.Service{
			Name:    "testd&r.service.fg",
			Target:  "jlu.macbook",
			Address: "localhost",
			Port:    80,
		}, false},
		{&stela.Service{
			Name:    "",
			Target:  "",
			Address: "",
			Port:    0,
		}, true},
	}

	expectedServices := make(map[string][]stela.Service)
	for _, test := range tests {
		if err := m.Register(test.service); test.shouldFail != (err != nil) {
			t.Fatal(err)
		}

		// Store the successful services into our own expected map
		if !test.shouldFail {
			expectedServices[test.service.Name] = append([]stela.Service{*test.service}, expectedServices[test.service.Name]...)
		}
	}

	// Verify the services map is what we expect
	if !reflect.DeepEqual(expectedServices, m.services) {
		t.Fatalf("Registered services failed. Expected %v, Received %v", expectedServices, m.services)
	}

	// Now see if we can discover them
	// d := m.Discover("testd&r.service.fg")
}

// equalServices takes two slices of stela.Service and make sure they are correct
func equalServices(t *testing.T, s1, s2 []*stela.Service) {
	// Make sure the services returned were the ones sent
	total := len(s1)
	for _, rs := range s2 {
		for _, ts := range s1 {
			if rs.Equal(ts) {
				total--
			}
		}
	}

	if total != 0 {
		t.Fatalf("Services returned did not match services in slice")
	}
}
<commit_msg>Adding DiscoverOne test to mapstore_test<commit_after>package mapstore

import (
	"reflect"
	"testing"

	"golang.org/x/net/context"

	"time"

	"gitlab.fg/go/stela"
)

func Test_rotateServices(t *testing.T) {
	m := MapStore{}
	serviceName := "priority.service.fg"

	var tests = []struct {
		service          stela.Service
		expectedPriority int32
	}{
		{stela.Service{
			Name:    serviceName,
			Target:  "jlu.macbook",
			Address: "127.0.0.1",
			Port:    9000,
		}, 2},
		{stela.Service{
			Name:    serviceName,
			Target:  "jlu.macbook",
			Address: "127.0.0.2",
			Port:    9000,
		}, 1},
		{stela.Service{
			Name:    serviceName,
			Target:  "jlu.macbook",
			Address: "127.0.0.3",
			Port:    9000,
		}, 0},
	}
	var expectedServices []stela.Service
	for _, test := range tests {
		// First we need to register them
		if err := m.Register(&test.service); err != nil {
			t.Fatal(err)
		}

		expectedServices = append(expectedServices, test.service)
	}

	// Loop through services registered and make sure their priorities match
	for i, s := range m.services[serviceName] {
		ts := tests[i]
		if s.Equal(&ts.service) {
			if s.Priority != ts.expectedPriority {
				t.Fatalf("Test_rotateServices: Priorites didn't match. Expected: %v, Received: %v", ts.expectedPriority, s.Priority)
			}
		}
	}
}

func TestSubscribe(t *testing.T) {
	m := MapStore{}
	c := &stela.Client{}
	serviceName := "subscribe.service.fg"

	// Subscribe to service
	if err := m.Subscribe(serviceName, c); err != nil {
		t.Fatal(err)
	}

	var testServices = []*stela.Service{
		&stela.Service{
			Name:    serviceName,
			Target:  "jlu.macbook",
			Address: "127.0.0.1",
			Port:    9000,
		},
		&stela.Service{
			Name:    serviceName,
			Target:  "jlu.macbook",
			Address: "127.0.0.2",
			Port:    9001,
		},
		&stela.Service{
			Name:    serviceName,
			Target:  "jlu.macbook",
			Address: "127.0.0.3",
			Port:    9002,
		},
	}

	var registered []*stela.Service
	var deregistered []*stela.Service

	// Listen for registered services
	writeWaitChan := make(chan struct{})
	readWaitChan := make(chan struct{})
	ctx, cancel := context.WithTimeout(context.TODO(), time.Millisecond*100)
	readCount := 0
	go func() {
		close(readWaitChan)
		for {
			select {
			case s := <-c.SubscribeCh():
				switch s.Action {
				case stela.RegisterAction:
					registered = append(registered, s)
				case stela.DeregisterAction:
					deregistered = append(deregistered, s)
				}

				// Count how many times we've received a subscribed service
				// Multiple by two since we are registering an deregistering all testServices
				readCount++
				if readCount == len(testServices)*2 {
					close(writeWaitChan)
				}
			case <-ctx.Done():
				return
			}
		}
	}()

	// Block until we are ready to receive subscription information
	<-readWaitChan

	for _, s := range testServices {
		// Register services
		m.Register(s)
	}

	for _, s := range testServices {
		// Deregister services
		m.Deregister(s)
	}

	// Wait for either a timeout or all the subscribed services to be read
	select {
	case <-writeWaitChan:
		break
	case <-ctx.Done():
		if ctx.Err() != nil {
			t.Fatal("TestSubscribe timed out: ", ctx.Err())
		}
	}

	// Function to compare testServices with serviceSlice arg
	testSubscribe := func(serviceSlice []*stela.Service) {
		if len(serviceSlice) != len(testServices) {
			t.Fatalf("Subscribe didn't match testServices. Received: %d, Expected: %d", len(serviceSlice), len(testServices))
		}

		// Make sure the serviceSlice has all the testServices
		equalServicesPointer(t, testServices, serviceSlice)
	}

	// Now make sure both registered and deregistered actions were received
	testSubscribe(registered)
	testSubscribe(deregistered)

	// Stop listening for subscriptions
	cancel()
}

func TestUnsubscribe(t *testing.T) {
	serviceName := "unsubscribe.services.fg"
	m := MapStore{}
	c1 := &stela.Client{}
	c2 := &stela.Client{}

	// Subscribe
	m.Subscribe(serviceName, c1)
	m.Subscribe(serviceName, c2)

	// Unsubscribe c1
	if err := m.Unsubscribe(serviceName, c1); err != nil {
		t.Fatal("Should have Unsubscribed successfully")
	}

	// Unsubscribe c1 again with the same client should fail
	if err := m.Unsubscribe(serviceName, c1); err == nil {
		t.Fatal("Unsubscribe should have failed")
	}

	// Unsubscribe c2
	if err := m.Unsubscribe(serviceName, c2); err != nil {
		t.Fatal("Should have Unsubscribed successfully")
	}

}

func TestRegistrationAndDiscovery(t *testing.T) {
	m := MapStore{}
	serviceName := "testd&r.service.fg"

	var tests = []struct {
		service    *stela.Service
		shouldFail bool
	}{
		{&stela.Service{
			Name:    serviceName,
			Target:  "jlu.macbook",
			Address: "127.0.0.1",
			Port:    9000,
		}, false},
		// Don't allow duplicates
		{&stela.Service{
			Name:    serviceName,
			Target:  "jlu.macbook",
			Address: "127.0.0.1",
			Port:    9000,
		}, true},
		{&stela.Service{
			Name:    serviceName,
			Target:  "jlu.macbook",
			Address: "localhost",
			Port:    80,
		}, false},
		{&stela.Service{
			Name:    "",
			Target:  "",
			Address: "",
			Port:    0,
		}, true},
	}

	expectedServices := make(map[string][]stela.Service)
	for _, test := range tests {
		if err := m.Register(test.service); test.shouldFail != (err != nil) {
			t.Fatal(err)
		}

		// Store the successful services into our own expected map
		if !test.shouldFail {
			expectedServices[test.service.Name] = append([]stela.Service{*test.service}, expectedServices[test.service.Name]...)
		}
	}

	// Verify the services map is what we expect
	if !reflect.DeepEqual(expectedServices, m.services) {
		t.Fatalf("Registered services failed. Expected %v, Received %v", expectedServices, m.services)
	}

	// Now see if we can discover them
	services, err := m.Discover(serviceName)
	if err != nil {
		t.Fatal("Discover failed")
	}

	if len(services) != len(expectedServices[serviceName]) {
		t.Fatal("Discover failed")
	}

	// Compare the discovered services
	equalServices(t, services, expectedServices[serviceName])

	// Let's DiscoverOne
	s, err := m.DiscoverOne(serviceName)
	if err != nil {
		t.Fatal("DiscoverOne failed")
	}

	// Make sure it's one from the registered services
	found := false
	for _, rs := range m.services[serviceName] {
		if s.Equal(&rs) {
			found = true
		}
	}
	if !found {
		t.Fatal("DiscoverOne returned invalid result")
	}

	// Try to Discover/DiscoverOne a serviceName that isn't registered
	_, err = m.Discover("madeupservicename")
	if err == nil {
		t.Fatal("Discover should have failed")
	}

	_, err = m.DiscoverOne("madeupservicename")
	if err == nil {
		t.Fatal("DiscoverOne should have failed")
	}
}

// equalPointerServices takes two slices of stela.Service and make sure they are correct
func equalServicesPointer(t *testing.T, s1, s2 []*stela.Service) {
	// Make sure the services returned were the ones sent
	total := len(s1)
	for _, rs := range s2 {
		for _, ts := range s1 {
			if rs.Equal(ts) {
				total--
			}
		}
	}

	if total != 0 {
		t.Fatalf("Services returned did not match services in slice")
	}
}

// equalServices takes two slices of stela.Service and make sure they are correct
func equalServices(t *testing.T, s1, s2 []stela.Service) {
	// Make sure the services returned were the ones sent
	total := len(s1)
	for _, rs := range s2 {
		for _, ts := range s1 {
			if rs.Equal(&ts) {
				total--
			}
		}
	}

	if total != 0 {
		t.Fatalf("Services returned did not match services in slice")
	}
}
<|endoftext|>


<commit_before>package beep

import "fmt"

// Resample takes a Streamer which is assumed to stream at the old sample rate and returns a
// Streamer, which streams the data from the original Streamer, but in the new sample rate.
//
// This is, for example, useful when mixing multiple Streamer with different sample rates, either
// through a beep.Mixer, or through a speaker. Speaker has a constant sample rate. Thus, playing
// Streamer which stream at a different sample rate will lead to a changed speed and pitch of the
// playback.
//
//   sr := beep.SampleRate(48000) speaker.Init(sr, sr.N(time.Second/2))
//   speaker.Play(beep.Resample(3, format.SampleRate, sr, s))
//
// In the example, the original sample rate of the source if format.SampleRate. We want to play it
// at the speaker's native sample rate and thus we need to resample.
//
// The quality argument specifies the quality of the resampling process. Higher quality implies
// worse performance. Values below 0 or above 64 are invalid and Resample will panic. Here's a table
// for deciding which quality to pick.
//
//   quality | use case
//   --------|---------
//   1       | very high performance, on-the-fly resampling, low quality
//   3-4     | good performance, on-the-fly resampling, good quality,
//   6       | higher CPU usage, usually not suitable for on-the-fly resampling, very good quality
//   >6      | even higher CPU usage, for offline resampling, very good quality
//
// Sane quality values are usually below 16. Higher values will consume too much CPU, giving
// negligible quality improvements.
func Resample(quality int, old, new SampleRate, s Streamer) Streamer {
	if quality < 1 || 64 < quality {
		panic(fmt.Errorf("resample: invalid quality: %d", quality))
	}
	return &resample{
		s:     s,
		ratio: float64(old) / float64(new),
		first: true,
		buf1:  make([][2]float64, 512),
		buf2:  make([][2]float64, 512),
		pts:   make([]point, quality*2),
		off:   0,
		pos:   0,
	}
}

type resample struct {
	s          Streamer     // the orignal streamer
	ratio      float64      // old sample rate / new sample rate
	first      bool         // true when Stream was not called before
	buf1, buf2 [][2]float64 // buf1 contains previous buf2, new data goes into buf2, buf1 is because interpolation might require old samples
	pts        []point      // pts is for points used for interpolation
	off        int          // off is the position of the start of buf2 in the original data
	pos        int          // pos is the current position in the resampled data
}

func (r *resample) Stream(samples [][2]float64) (n int, ok bool) {
	if r.first { // if it's the first time, we need to fill buf2 with initial data, buf1 remains zeroed
		sn, _ := r.s.Stream(r.buf2)
		r.buf2 = r.buf2[:sn]
		r.first = false
	}
	// we start resampling, sample by sample
	for len(samples) > 0 {
	reload:
		for c := range samples[0] {
			// calculate the current position in the original data
			j := float64(r.pos) * r.ratio

			// find quality*2 closest samples to j and translate them to points for interpolation
			for pi := 0; pi < len(r.pts); pi++ {
				// calculate the index of one of the closest samples
				k := int(j) + pi - len(r.pts)/2 + 1

				var y float64
				switch {
				// the sample is in buf1
				case k < r.off:
					y = r.buf1[len(r.buf1)+k-r.off][c]
				// the sample is in buf2
				case k < r.off+len(r.buf2):
					y = r.buf2[k-r.off][c]
				// the sample is beyond buf2, so we need to load new data
				case k >= r.off+len(r.buf2):
					// we load into buf1
					sn, _ := r.s.Stream(r.buf1)
					r.buf1 = r.buf1[:sn]
					// this condition happens when the original Streamer got
					// drained and j is after the end of the
					// original data
					if int(j) >= r.off+len(r.buf2)+sn {
						return n, n > 0
					}
					// this condition happens when the original Streamer got
					// drained and this one of the closest samples is after the
					// end of the original data
					if k >= r.off+len(r.buf2)+sn {
						y = 0
						break
					}
					// otherwise everything is fine, we swap buffers and start
					// calculating the sample again
					r.off += len(r.buf2)
					r.buf1, r.buf2 = r.buf2, r.buf1
					goto reload
				}

				r.pts[pi] = point{float64(k), y}
			}

			// calculate the resampled sample using polynomial interpolation from the
			// quality*2 closest samples
			samples[0][c] = lagrange(r.pts, j)
		}
		samples = samples[1:]
		n++
		r.pos++
	}
	return n, true
}

func (r *resample) Err() error {
	return r.s.Err()
}

// lagrange calculates the value at x of a polynomial of order len(pts)+1 which goes through all
// points in pts
func lagrange(pts []point, x float64) (y float64) {
	y = 0.0
	for j := range pts {
		l := 1.0
		for m := range pts {
			if j == m {
				continue
			}
			l *= (x - pts[m].X) / (pts[j].X - pts[m].X)
		}
		y += pts[j].Y * l
	}
	return y
}

type point struct {
	X, Y float64
}
<commit_msg>fix and improve Resample doc<commit_after>package beep

import "fmt"

// Resample takes a Streamer which is assumed to stream at the old sample rate and returns a
// Streamer, which streams the data from the original Streamer, but in the new sample rate.
//
// This is, for example, useful when mixing multiple Streamer with different sample rates, either
// through a beep.Mixer, or through a speaker. Speaker has a constant sample rate. Thus, playing
// Streamer which stream at a different sample rate will lead to a changed speed and pitch of the
// playback.
//
//   sr := beep.SampleRate(48000) speaker.Init(sr, sr.N(time.Second/2))
//   speaker.Play(beep.Resample(3, format.SampleRate, sr, s))
//
// In the example, the original sample rate of the source if format.SampleRate. We want to play it
// at the speaker's native sample rate and thus we need to resample.
//
// The quality argument specifies the quality of the resampling process. Higher quality implies
// worse performance. Values below 1 or above 64 are invalid and Resample will panic. Here's a table
// for deciding which quality to pick.
//
//   quality | use case
//   --------|---------
//   1       | very high performance, on-the-fly resampling, low quality
//   3-4     | good performance, on-the-fly resampling, good quality,
//   6       | higher CPU usage, usually not suitable for on-the-fly resampling, very good quality
//   >6      | even higher CPU usage, for offline resampling, very good quality
//
// Sane quality values are usually below 16. Higher values will consume too much CPU, giving
// negligible quality improvements.
//
// Resample propagates errors from s.
func Resample(quality int, old, new SampleRate, s Streamer) Streamer {
	if quality < 1 || 64 < quality {
		panic(fmt.Errorf("resample: invalid quality: %d", quality))
	}
	return &resample{
		s:     s,
		ratio: float64(old) / float64(new),
		first: true,
		buf1:  make([][2]float64, 512),
		buf2:  make([][2]float64, 512),
		pts:   make([]point, quality*2),
		off:   0,
		pos:   0,
	}
}

type resample struct {
	s          Streamer     // the orignal streamer
	ratio      float64      // old sample rate / new sample rate
	first      bool         // true when Stream was not called before
	buf1, buf2 [][2]float64 // buf1 contains previous buf2, new data goes into buf2, buf1 is because interpolation might require old samples
	pts        []point      // pts is for points used for interpolation
	off        int          // off is the position of the start of buf2 in the original data
	pos        int          // pos is the current position in the resampled data
}

func (r *resample) Stream(samples [][2]float64) (n int, ok bool) {
	if r.first { // if it's the first time, we need to fill buf2 with initial data, buf1 remains zeroed
		sn, _ := r.s.Stream(r.buf2)
		r.buf2 = r.buf2[:sn]
		r.first = false
	}
	// we start resampling, sample by sample
	for len(samples) > 0 {
	reload:
		for c := range samples[0] {
			// calculate the current position in the original data
			j := float64(r.pos) * r.ratio

			// find quality*2 closest samples to j and translate them to points for interpolation
			for pi := 0; pi < len(r.pts); pi++ {
				// calculate the index of one of the closest samples
				k := int(j) + pi - len(r.pts)/2 + 1

				var y float64
				switch {
				// the sample is in buf1
				case k < r.off:
					y = r.buf1[len(r.buf1)+k-r.off][c]
				// the sample is in buf2
				case k < r.off+len(r.buf2):
					y = r.buf2[k-r.off][c]
				// the sample is beyond buf2, so we need to load new data
				case k >= r.off+len(r.buf2):
					// we load into buf1
					sn, _ := r.s.Stream(r.buf1)
					r.buf1 = r.buf1[:sn]
					// this condition happens when the original Streamer got
					// drained and j is after the end of the
					// original data
					if int(j) >= r.off+len(r.buf2)+sn {
						return n, n > 0
					}
					// this condition happens when the original Streamer got
					// drained and this one of the closest samples is after the
					// end of the original data
					if k >= r.off+len(r.buf2)+sn {
						y = 0
						break
					}
					// otherwise everything is fine, we swap buffers and start
					// calculating the sample again
					r.off += len(r.buf2)
					r.buf1, r.buf2 = r.buf2, r.buf1
					goto reload
				}

				r.pts[pi] = point{float64(k), y}
			}

			// calculate the resampled sample using polynomial interpolation from the
			// quality*2 closest samples
			samples[0][c] = lagrange(r.pts, j)
		}
		samples = samples[1:]
		n++
		r.pos++
	}
	return n, true
}

func (r *resample) Err() error {
	return r.s.Err()
}

// lagrange calculates the value at x of a polynomial of order len(pts)+1 which goes through all
// points in pts
func lagrange(pts []point, x float64) (y float64) {
	y = 0.0
	for j := range pts {
		l := 1.0
		for m := range pts {
			if j == m {
				continue
			}
			l *= (x - pts[m].X) / (pts[j].X - pts[m].X)
		}
		y += pts[j].Y * l
	}
	return y
}

type point struct {
	X, Y float64
}
<|endoftext|>


<commit_before>package lion

import (
	"net/http"

	"golang.org/x/net/context"
)

// Resource defines the minimum required methods
type Resource interface{}

// ResourceUses is an interface with the Uses() method which can be used to define global middlewares for the resource.
type ResourceUses interface {
	Uses() Middlewares
}

// GetResourceMiddlewares is an interface for defining middlewares used in Resource method
type GetResourceMiddlewares interface {
	GetMiddlewares() Middlewares
}

// PostResourceMiddlewares is an interface for defining middlewares used in Resource method
type PostResourceMiddlewares interface {
	PostMiddlewares() Middlewares
}

// PutResourceMiddlewares is an interface for defining middlewares used in Resource method
type PutResourceMiddlewares interface {
	PutMiddlewares() Middlewares
}

// DeleteResourceMiddlewares is an interface for defining middlewares used in Resource method
type DeleteResourceMiddlewares interface {
	DeleteMiddlewares() Middlewares
}

// GetResource is an interface for defining a HandlerFunc used in Resource method
type GetResource interface {
	Get(c context.Context, w http.ResponseWriter, r *http.Request)
}

// PostResource is an interface for defining a HandlerFunc used in Resource method
type PostResource interface {
	Post(c context.Context, w http.ResponseWriter, r *http.Request)
}

// PutResource is an interface for defining a HandlerFunc used in Resource method
type PutResource interface {
	Put(c context.Context, w http.ResponseWriter, r *http.Request)
}

// DeleteResource is an interface for defining a HandlerFunc used in Resource method
type DeleteResource interface {
	Delete(c context.Context, w http.ResponseWriter, r *http.Request)
}

// Resource registers a Resource with the corresponding pattern
func (r *Router) Resource(pattern string, resource Resource) {
	sub := r.Group(pattern)

	if usesRes, ok := resource.(ResourceUses); ok {
		if len(usesRes.Uses()) > 0 {
			sub.Use(usesRes.Uses()...)
		}
	}

	if res, ok := resource.(GetResource); ok {
		s := sub.Group("/")
		if mw, ok := resource.(GetResourceMiddlewares); ok {
			s.Use(mw.GetMiddlewares()...)
		}
		s.GetFunc("/", res.Get)
	}

	if res, ok := resource.(PostResource); ok {
		s := sub.Group("/")
		if mw, ok := resource.(PostResourceMiddlewares); ok {
			s.Use(mw.PostMiddlewares()...)
		}
		s.PostFunc("/", res.Post)
	}

	if res, ok := resource.(PutResource); ok {
		s := sub.Group("/")
		if mw, ok := resource.(PutResourceMiddlewares); ok {
			s.Use(mw.PutMiddlewares()...)
		}
		s.PutFunc("/", res.Put)
	}

	if res, ok := resource.(DeleteResource); ok {
		s := sub.Group("/")
		if mw, ok := resource.(DeleteResourceMiddlewares); ok {
			s.Use(mw.DeleteMiddlewares()...)
		}
		s.DeleteFunc("/", res.Delete)
	}
}
<commit_msg>Add support for HEAD, TRACE, OPTIONS, CONNECT and PATCH in resources<commit_after>package lion

import (
	"net/http"

	"golang.org/x/net/context"
)

// Resource defines the minimum required methods
type Resource interface{}

// ResourceUses is an interface with the Uses() method which can be used to define global middlewares for the resource.
type ResourceUses interface {
	Uses() Middlewares
}

// GetResourceMiddlewares is an interface for defining middlewares used in Resource method
type GetResourceMiddlewares interface {
	GetMiddlewares() Middlewares
}

// HeadResourceMiddlewares is an interface for defining middlewares used in Resource method
type HeadResourceMiddlewares interface {
	HeadMiddlewares() Middlewares
}

// PostResourceMiddlewares is an interface for defining middlewares used in Resource method
type PostResourceMiddlewares interface {
	PostMiddlewares() Middlewares
}

// PutResourceMiddlewares is an interface for defining middlewares used in Resource method
type PutResourceMiddlewares interface {
	PutMiddlewares() Middlewares
}

// DeleteResourceMiddlewares is an interface for defining middlewares used in Resource method
type DeleteResourceMiddlewares interface {
	DeleteMiddlewares() Middlewares
}

// TraceResourceMiddlewares is an interface for defining middlewares used in Resource method
type TraceResourceMiddlewares interface {
	TraceMiddlewares() Middlewares
}

// OptionsResourceMiddlewares is an interface for defining middlewares used in Resource method
type OptionsResourceMiddlewares interface {
	OptionsMiddlewares() Middlewares
}

// ConnectResourceMiddlewares is an interface for defining middlewares used in Resource method
type ConnectResourceMiddlewares interface {
	ConnectMiddlewares() Middlewares
}

// PatchResourceMiddlewares is an interface for defining middlewares used in Resource method
type PatchResourceMiddlewares interface {
	PatchMiddlewares() Middlewares
}

// GetResource is an interface for defining a HandlerFunc used in Resource method
type GetResource interface {
	Get(c context.Context, w http.ResponseWriter, r *http.Request)
}

// HeadResource is an interface for defining a HandlerFunc used in Resource method
type HeadResource interface {
	Head(c context.Context, w http.ResponseWriter, r *http.Request)
}

// PostResource is an interface for defining a HandlerFunc used in Resource method
type PostResource interface {
	Post(c context.Context, w http.ResponseWriter, r *http.Request)
}

// PutResource is an interface for defining a HandlerFunc used in Resource method
type PutResource interface {
	Put(c context.Context, w http.ResponseWriter, r *http.Request)
}

// DeleteResource is an interface for defining a HandlerFunc used in Resource method
type DeleteResource interface {
	Delete(c context.Context, w http.ResponseWriter, r *http.Request)
}

// TraceResource is an interface for defining a HandlerFunc used in Resource method
type TraceResource interface {
	Trace(c context.Context, w http.ResponseWriter, r *http.Request)
}

// OptionsResource is an interface for defining a HandlerFunc used in Resource method
type OptionsResource interface {
	Options(c context.Context, w http.ResponseWriter, r *http.Request)
}

// ConnectResource is an interface for defining a HandlerFunc used in Resource method
type ConnectResource interface {
	Connect(c context.Context, w http.ResponseWriter, r *http.Request)
}

// PatchResource is an interface for defining a HandlerFunc used in Resource method
type PatchResource interface {
	Patch(c context.Context, w http.ResponseWriter, r *http.Request)
}

// Resource registers a Resource with the corresponding pattern
func (r *Router) Resource(pattern string, resource Resource) {
	sub := r.Group(pattern)

	if usesRes, ok := resource.(ResourceUses); ok {
		if len(usesRes.Uses()) > 0 {
			sub.Use(usesRes.Uses()...)
		}
	}

	// GET
	if res, ok := resource.(GetResource); ok {
		s := sub.Group("/")
		if mw, ok := resource.(GetResourceMiddlewares); ok {
			s.Use(mw.GetMiddlewares()...)
		}
		s.GetFunc("/", res.Get)
	}

	// HEAD
	if res, ok := resource.(HeadResource); ok {
		s := sub.Group("/")
		if mw, ok := resource.(HeadResourceMiddlewares); ok {
			s.Use(mw.HeadMiddlewares()...)
		}
		s.HeadFunc("/", res.Head)
	}

	// POST
	if res, ok := resource.(PostResource); ok {
		s := sub.Group("/")
		if mw, ok := resource.(PostResourceMiddlewares); ok {
			s.Use(mw.PostMiddlewares()...)
		}
		s.PostFunc("/", res.Post)
	}

	// PUT
	if res, ok := resource.(PutResource); ok {
		s := sub.Group("/")
		if mw, ok := resource.(PutResourceMiddlewares); ok {
			s.Use(mw.PutMiddlewares()...)
		}
		s.PutFunc("/", res.Put)
	}

	// DELETE
	if res, ok := resource.(DeleteResource); ok {
		s := sub.Group("/")
		if mw, ok := resource.(DeleteResourceMiddlewares); ok {
			s.Use(mw.DeleteMiddlewares()...)
		}
		s.DeleteFunc("/", res.Delete)
	}

	// TRACE
	if res, ok := resource.(TraceResource); ok {
		s := sub.Group("/")
		if mw, ok := resource.(TraceResourceMiddlewares); ok {
			s.Use(mw.TraceMiddlewares()...)
		}
		s.TraceFunc("/", res.Trace)
	}

	// OPTIONS
	if res, ok := resource.(OptionsResource); ok {
		s := sub.Group("/")
		if mw, ok := resource.(OptionsResourceMiddlewares); ok {
			s.Use(mw.OptionsMiddlewares()...)
		}
		s.OptionsFunc("/", res.Options)
	}

	// CONNECT
	if res, ok := resource.(ConnectResource); ok {
		s := sub.Group("/")
		if mw, ok := resource.(ConnectResourceMiddlewares); ok {
			s.Use(mw.ConnectMiddlewares()...)
		}
		s.ConnectFunc("/", res.Connect)
	}

	// PATCH
	if res, ok := resource.(PatchResource); ok {
		s := sub.Group("/")
		if mw, ok := resource.(PatchResourceMiddlewares); ok {
			s.Use(mw.PatchMiddlewares()...)
		}
		s.PatchFunc("/", res.Patch)
	}
}
<|endoftext|>


<commit_before>// SPDX-License-Identifier: MIT

package web

import (
	"net/http"
	"strconv"
	"time"

	"github.com/issue9/web/server"
)

type (
	Responser = server.Responser

	object struct {
		status  int
		body    any
		headers map[string]string
	}
)

func Status(status int) Responser { return server.Status(status) }

func Object(status int, body interface{}, headers map[string]string) Responser {
	return &object{status: status, body: body, headers: headers}
}

func (o *object) Apply(ctx *Context) { ctx.Marshal(o.status, o.body, o.headers) }

func Created(v any, location string) Responser {
	if location != "" {
		return Object(http.StatusCreated, v, map[string]string{"Location": location})
	}
	return Object(http.StatusCreated, v, nil)
}

// OK 返回 200 状态码下的对象
func OK(v any) Responser { return Object(http.StatusOK, v, nil) }

func NotFound() Responser { return Status(http.StatusNotFound) }

func NoContent() Responser { return Status(http.StatusNoContent) }

func NotImplemented() Responser { return Status(http.StatusNotImplemented) }

// RetryAfter 返回 Retry-After 报头内容
//
// 一般适用于 301 和 503 报文。
//
// status 表示返回的状态码；seconds 表示秒数，如果想定义为时间格式，
// 可以采用 RetryAt 函数，两个功能是相同的，仅是时间格式上有差别。
func RetryAfter(status int, seconds uint64) Responser {
	return Object(status, nil, map[string]string{
		"Retry-After": strconv.FormatUint(seconds, 10),
	})
}

func RetryAt(status int, at time.Time) Responser {
	return Object(status, nil, map[string]string{
		"Retry-After": at.UTC().Format(http.TimeFormat),
	})
}

// Redirect 重定向至新的 URL
func Redirect(status int, url string) Responser {
	return Object(status, nil, map[string]string{"Location": url})
}
<commit_msg>refactor: object.Apply 不再忽略错误信息<commit_after>// SPDX-License-Identifier: MIT

package web

import (
	"net/http"
	"strconv"
	"time"

	"github.com/issue9/logs/v3"

	"github.com/issue9/web/server"
)

type (
	Responser = server.Responser

	object struct {
		status  int
		body    any
		headers map[string]string
	}
)

func Status(status int) Responser { return server.Status(status) }

func Object(status int, body interface{}, headers map[string]string) Responser {
	return &object{status: status, body: body, headers: headers}
}

func (o *object) Apply(ctx *Context) {
	if err := ctx.Marshal(o.status, o.body, o.headers); err != nil {
		ctx.Log(logs.LevelError, 1, err)
	}
}

func Created(v any, location string) Responser {
	if location != "" {
		return Object(http.StatusCreated, v, map[string]string{"Location": location})
	}
	return Object(http.StatusCreated, v, nil)
}

// OK 返回 200 状态码下的对象
func OK(v any) Responser { return Object(http.StatusOK, v, nil) }

func NotFound() Responser { return Status(http.StatusNotFound) }

func NoContent() Responser { return Status(http.StatusNoContent) }

func NotImplemented() Responser { return Status(http.StatusNotImplemented) }

// RetryAfter 返回 Retry-After 报头内容
//
// 一般适用于 301 和 503 报文。
//
// status 表示返回的状态码；seconds 表示秒数，如果想定义为时间格式，
// 可以采用 RetryAt 函数，两个功能是相同的，仅是时间格式上有差别。
func RetryAfter(status int, seconds uint64) Responser {
	return Object(status, nil, map[string]string{
		"Retry-After": strconv.FormatUint(seconds, 10),
	})
}

func RetryAt(status int, at time.Time) Responser {
	return Object(status, nil, map[string]string{
		"Retry-After": at.UTC().Format(http.TimeFormat),
	})
}

// Redirect 重定向至新的 URL
func Redirect(status int, url string) Responser {
	return Object(status, nil, map[string]string{"Location": url})
}
<|endoftext|>


<commit_before>package environment

import (
	"reflect"
	"sort"
	"testing"
)

func TestEnvironment_AsList(t *testing.T) {
	env := NewTaskEnivornment()
	env.SetTaskIp("127.0.0.1")
	env.SetPorts(map[string]int{"http": 80})
	env.SetMeta(map[string]string{"foo": "baz"})

	act := env.List()
	exp := []string{"NOMAD_IP=127.0.0.1", "NOMAD_PORT_http=80", "NOMAD_META_FOO=baz"}
	sort.Strings(act)
	sort.Strings(exp)
	if !reflect.DeepEqual(act, exp) {
		t.Fatalf("env.List() returned %v; want %v", act, exp)
	}
}

func TestEnvironment_ParseFromList(t *testing.T) {
	input := []string{"foo=bar", "BAZ=baM"}
	env, err := ParseFromList(input)
	if err != nil {
		t.Fatalf("ParseFromList(%#v) failed: %v", input, err)
	}

	exp := map[string]string{
		"foo": "baz",
		"BAZ": "baM",
	}
	if !reflect.DeepEqual(env, exp) {
		t.Fatalf("ParseFromList(%#v) returned %v; want %v", input, env, exp)
	}
}

func TestEnvironment_ClearEnvvars(t *testing.T) {
	env := NewTaskEnivornment()
	env.SetTaskIp("127.0.0.1")
	env.SetEnvvars(map[string]string{"foo": "baz", "bar": "bang"})

	act := env.List()
	exp := []string{"NOMAD_IP=127.0.0.1", "bar=bang", "foo=baz"}
	sort.Strings(act)
	sort.Strings(exp)
	if !reflect.DeepEqual(act, exp) {
		t.Fatalf("env.List() returned %v; want %v", act, exp)
	}

	// Clear the environent variables.
	env.ClearEnvvars()

	act = env.List()
	exp = []string{"NOMAD_IP=127.0.0.1"}
	sort.Strings(act)
	sort.Strings(exp)
	if !reflect.DeepEqual(act, exp) {
		t.Fatalf("env.List() returned %v; want %v", act, exp)
	}
}
<commit_msg>Fix broken test<commit_after>package environment

import (
	"reflect"
	"sort"
	"testing"
)

func TestEnvironment_AsList(t *testing.T) {
	env := NewTaskEnivornment()
	env.SetTaskIp("127.0.0.1")
	env.SetPorts(map[string]int{"http": 80})
	env.SetMeta(map[string]string{"foo": "baz"})

	act := env.List()
	exp := []string{"NOMAD_IP=127.0.0.1", "NOMAD_PORT_http=80", "NOMAD_META_FOO=baz"}
	sort.Strings(act)
	sort.Strings(exp)
	if !reflect.DeepEqual(act, exp) {
		t.Fatalf("env.List() returned %v; want %v", act, exp)
	}
}

func TestEnvironment_ParseFromList(t *testing.T) {
	input := []string{"foo=bar", "BAZ=baM"}
	env, err := ParseFromList(input)
	if err != nil {
		t.Fatalf("ParseFromList(%#v) failed: %v", input, err)
	}

	exp := map[string]string{
		"foo": "bar",
		"BAZ": "baM",
	}

	if len(env) != len(exp) {
		t.Fatalf("ParseFromList(%#v) has length %v; want %v", input, len(env), len(exp))
	}

	for k, v := range exp {
		if actV, ok := env[k]; !ok {
			t.Fatalf("ParseFromList(%#v) doesn't contain expected %v", input, k)
		} else if actV != v {
			t.Fatalf("ParseFromList(%#v) has incorrect value for %v; got %v; want %v", input, k, actV, v)
		}
	}
}

func TestEnvironment_ClearEnvvars(t *testing.T) {
	env := NewTaskEnivornment()
	env.SetTaskIp("127.0.0.1")
	env.SetEnvvars(map[string]string{"foo": "baz", "bar": "bang"})

	act := env.List()
	exp := []string{"NOMAD_IP=127.0.0.1", "bar=bang", "foo=baz"}
	sort.Strings(act)
	sort.Strings(exp)
	if !reflect.DeepEqual(act, exp) {
		t.Fatalf("env.List() returned %v; want %v", act, exp)
	}

	// Clear the environent variables.
	env.ClearEnvvars()

	act = env.List()
	exp = []string{"NOMAD_IP=127.0.0.1"}
	sort.Strings(act)
	sort.Strings(exp)
	if !reflect.DeepEqual(act, exp) {
		t.Fatalf("env.List() returned %v; want %v", act, exp)
	}
}
<|endoftext|>


<commit_before>// This source file is part of the Packet Guardian project.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tasks

import (
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/usi-lfkeitel/packet-guardian/src/common"
)

func init() {
	RegisterJob("Purge old web sessions", cleanUpExpiredSessions)
}

func cleanUpExpiredSessions(e *common.Environment) (string, error) {
	w := sessionWalker{n: time.Now().Add(time.Duration(-24) * time.Hour)}
	err := filepath.Walk("sessions", w.sessionDirWalker)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("Purged %d sessions", w.c), nil
}

type sessionWalker struct {
	n time.Time
	c int
}

func (s sessionWalker) sessionDirWalker(path string, info os.FileInfo, err error) error {
	if info.IsDir() {
		return filepath.SkipDir
	}
	if info.ModTime().Before(s.n) {
		s.c++
		return os.Remove(path)
	}
	return nil
}
<commit_msg>Fixes #6 - Delete sessions smaller than 100 kB<commit_after>// This source file is part of the Packet Guardian project.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tasks

import (
	"fmt"
	"os"
	"path/filepath"
	"time"

	"github.com/usi-lfkeitel/packet-guardian/src/common"
)

func init() {
	RegisterJob("Purge old web sessions", cleanUpExpiredSessions)
}

func cleanUpExpiredSessions(e *common.Environment) (string, error) {
	w := sessionWalker{n: time.Now().Add(time.Duration(-24) * time.Hour)}
	err := filepath.Walk("sessions", w.sessionDirWalker)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("Purged %d sessions", w.c), nil
}

type sessionWalker struct {
	n time.Time
	c int
}

func (s sessionWalker) sessionDirWalker(path string, info os.FileInfo, err error) error {
	if info.IsDir() {
		return filepath.SkipDir
	}
	if info.Size() < 100*1024 {
		s.c++
		return os.Remove(path)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hyper

import (
	"encoding/json"
	"net/url"
	"time"
)

// CPU usage time statistics.
type CpuUsage struct {
	// Total CPU usage.
	// Units: nanoseconds
	Total uint64 `json:"total"`

	// Per CPU/core usage of the container.
	// Unit: nanoseconds.
	PerCpu []uint64 `json:"per_cpu_usage,omitempty"`

	// Time spent in user space.
	// Unit: nanoseconds
	User uint64 `json:"user"`

	// Time spent in kernel space.
	// Unit: nanoseconds
	System uint64 `json:"system"`
}

// All CPU usage metrics are cumulative from the creation of the container
type CpuStats struct {
	Usage CpuUsage `json:"usage"`
	// Smoothed average of number of runnable threads x 1000.
	// We multiply by thousand to avoid using floats, but preserving precision.
	// Load is smoothed over the last 10 seconds. Instantaneous value can be read
	// from LoadStats.NrRunning.
	LoadAverage int32 `json:"load_average"`
}

// BlkioStatEntry is one small entity to store a piece of Blkio stats
type BlkioStatEntry struct {
	Name  string            `json:"name"`
	Major uint64            `json:"major"`
	Minor uint64            `json:"minor"`
	Stat  map[string]uint64 `json:"stat"`
}

// BlkioStats stores All IO service stats for data read and write
type BlkioStats struct {
	// number of bytes transferred to and from the block device
	IoServiceBytesRecursive []BlkioStatEntry `json:"io_service_bytes_recursive"`
	IoServicedRecursive     []BlkioStatEntry `json:"io_serviced_recursive"`
	IoQueuedRecursive       []BlkioStatEntry `json:"io_queue_recursive"`
	IoServiceTimeRecursive  []BlkioStatEntry `json:"io_service_time_recursive"`
	IoWaitTimeRecursive     []BlkioStatEntry `json:"io_wait_time_recursive"`
	IoMergedRecursive       []BlkioStatEntry `json:"io_merged_recursive"`
	IoTimeRecursive         []BlkioStatEntry `json:"io_time_recursive"`
	SectorsRecursive        []BlkioStatEntry `json:"sectors_recursive"`
}

type MemoryStats struct {
	// Current memory usage, this includes all memory regardless of when it was
	// accessed.
	// Units: Bytes.
	Usage uint64 `json:"usage"`

	// The amount of working set memory, this includes recently accessed memory,
	// dirty memory, and kernel memory. Working set is <= "usage".
	// Units: Bytes.
	WorkingSet uint64 `json:"working_set"`

	Failcnt uint64 `json:"failcnt"`

	ContainerData    MemoryStatsMemoryData `json:"container_data,omitempty"`
	HierarchicalData MemoryStatsMemoryData `json:"hierarchical_data,omitempty"`
}

type MemoryStatsMemoryData struct {
	Pgfault    uint64 `json:"pgfault"`
	Pgmajfault uint64 `json:"pgmajfault"`
}

type InterfaceStats struct {
	// The name of the interface.
	Name string `json:"name"`
	// Cumulative count of bytes received.
	RxBytes uint64 `json:"rx_bytes"`
	// Cumulative count of packets received.
	RxPackets uint64 `json:"rx_packets"`
	// Cumulative count of receive errors encountered.
	RxErrors uint64 `json:"rx_errors"`
	// Cumulative count of packets dropped while receiving.
	RxDropped uint64 `json:"rx_dropped"`
	// Cumulative count of bytes transmitted.
	TxBytes uint64 `json:"tx_bytes"`
	// Cumulative count of packets transmitted.
	TxPackets uint64 `json:"tx_packets"`
	// Cumulative count of transmit errors encountered.
	TxErrors uint64 `json:"tx_errors"`
	// Cumulative count of packets dropped while transmitting.
	TxDropped uint64 `json:"tx_dropped"`
}

type NetworkStats struct {
	Interfaces []InterfaceStats `json:"interfaces,omitempty"`
	// TCP connection stats (Established, Listen...)
	Tcp TcpStat `json:"tcp"`
	// TCP6 connection stats (Established, Listen...)
	Tcp6 TcpStat `json:"tcp6"`
}

type TcpStat struct {
	//Count of TCP connections in state "Established"
	Established uint64
	//Count of TCP connections in state "Syn_Sent"
	SynSent uint64
	//Count of TCP connections in state "Syn_Recv"
	SynRecv uint64
	//Count of TCP connections in state "Fin_Wait1"
	FinWait1 uint64
	//Count of TCP connections in state "Fin_Wait2"
	FinWait2 uint64
	//Count of TCP connections in state "Time_Wait
	TimeWait uint64
	//Count of TCP connections in state "Close"
	Close uint64
	//Count of TCP connections in state "Close_Wait"
	CloseWait uint64
	//Count of TCP connections in state "Listen_Ack"
	LastAck uint64
	//Count of TCP connections in state "Listen"
	Listen uint64
	//Count of TCP connections in state "Closing"
	Closing uint64
}

type FsStats struct {
	// The block device name associated with the filesystem.
	Device string `json:"device,omitempty"`

	// Number of bytes that can be consumed by the container on this filesystem.
	Limit uint64 `json:"capacity"`

	// Number of bytes that is consumed by the container on this filesystem.
	Usage uint64 `json:"usage"`

	// Number of bytes available for non-root user.
	Available uint64 `json:"available"`

	// Number of reads completed
	// This is the total number of reads completed successfully.
	ReadsCompleted uint64 `json:"reads_completed"`

	// Number of reads merged
	// Reads and writes which are adjacent to each other may be merged for
	// efficiency.  Thus two 4K reads may become one 8K read before it is
	// ultimately handed to the disk, and so it will be counted (and queued)
	// as only one I/O.  This field lets you know how often this was done.
	ReadsMerged uint64 `json:"reads_merged"`

	// Number of sectors read
	// This is the total number of sectors read successfully.
	SectorsRead uint64 `json:"sectors_read"`

	// Number of milliseconds spent reading
	// This is the total number of milliseconds spent by all reads (as
	// measured from __make_request() to end_that_request_last()).
	ReadTime uint64 `json:"read_time"`

	// Number of writes completed
	// This is the total number of writes completed successfully.
	WritesCompleted uint64 `json:"writes_completed"`

	// Number of writes merged
	// See the description of reads merged.
	WritesMerged uint64 `json:"writes_merged"`

	// Number of sectors written
	// This is the total number of sectors written successfully.
	SectorsWritten uint64 `json:"sectors_written"`

	// Number of milliseconds spent writing
	// This is the total number of milliseconds spent by all writes (as
	// measured from __make_request() to end_that_request_last()).
	WriteTime uint64 `json:"write_time"`

	// Number of I/Os currently in progress
	// The only field that should go to zero. Incremented as requests are
	// given to appropriate struct request_queue and decremented as they finish.
	IoInProgress uint64 `json:"io_in_progress"`

	// Number of milliseconds spent doing I/Os
	// This field increases so long as field 9 is nonzero.
	IoTime uint64 `json:"io_time"`

	// weighted number of milliseconds spent doing I/Os
	// This field is incremented at each I/O start, I/O completion, I/O
	// merge, or read of these stats by the number of I/Os in progress
	// (field 9) times the number of milliseconds spent doing I/O since the
	// last update of this field.  This can provide an easy measure of both
	// I/O completion time and the backlog that may be accumulating.
	WeightedIoTime uint64 `json:"weighted_io_time"`
}

type ContainerStats struct {
	ContainerID string       `json:"id"`
	Cpu         CpuStats     `json:"cpu,omitempty"`
	Block       BlkioStats   `json:"block,omitempty"`
	Memory      MemoryStats  `json:"memory,omitempty"`
	Network     NetworkStats `json:"network,omitempty"`
	Filesystem  []FsStats    `json:"filesystem,omitempty"`
	Timestamp   time.Time    `json:"timestamp"`
}

type PodStats struct {
	Cpu        CpuStats     `json:"cpu,omitempty"`
	Block      BlkioStats   `json:"block,omitempty"`
	Memory     MemoryStats  `json:"memory,omitempty"`
	Network    NetworkStats `json:"network,omitempty"`
	Filesystem []FsStats    `json:"filesystem,omitempty"`
	Timestamp  time.Time    `json:"timestamp"`

	ContainersStats []ContainerStats `json:"container_stats"`
}

func (client *HyperClient) GetPodStats(podID string) (*PodStats, error) {
	v := url.Values{}
	v.Set(KEY_POD_ID, podID)
	body, _, err := client.call("GET", "/pod/stats?"+v.Encode(), "", nil)
	if err != nil {
		return nil, err
	}

	var stats PodStats
	podStats := transformJson(string(body))
	err = json.Unmarshal([]byte(podStats), &stats)
	if err != nil {
		return nil, err
	}

	return &stats, nil
}
<commit_msg>update cadvisor deps<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hyper

import (
	"encoding/json"
	"net/url"
	"time"
)

// CPU usage time statistics.
type CpuUsage struct {
	// Total CPU usage.
	// Units: nanoseconds
	Total uint64 `json:"total"`

	// Per CPU/core usage of the container.
	// Unit: nanoseconds.
	PerCpu []uint64 `json:"per_cpu_usage,omitempty"`

	// Time spent in user space.
	// Unit: nanoseconds
	User uint64 `json:"user"`

	// Time spent in kernel space.
	// Unit: nanoseconds
	System uint64 `json:"system"`
}

// All CPU usage metrics are cumulative from the creation of the container
type CpuStats struct {
	Usage CpuUsage `json:"usage"`
	// Smoothed average of number of runnable threads x 1000.
	// We multiply by thousand to avoid using floats, but preserving precision.
	// Load is smoothed over the last 10 seconds. Instantaneous value can be read
	// from LoadStats.NrRunning.
	LoadAverage int32 `json:"load_average"`
}

// BlkioStatEntry is one small entity to store a piece of Blkio stats
type BlkioStatEntry struct {
	Name   string            `json:"name"`
	Type   string            `json:"type"`
	Source string            `json:"source"`
	Major  uint64            `json:"major"`
	Minor  uint64            `json:"minor"`
	Stat   map[string]uint64 `json:"stat"`
}

// BlkioStats stores All IO service stats for data read and write
type BlkioStats struct {
	// number of bytes transferred to and from the block device
	IoServiceBytesRecursive []BlkioStatEntry `json:"io_service_bytes_recursive"`
	IoServicedRecursive     []BlkioStatEntry `json:"io_serviced_recursive"`
	IoQueuedRecursive       []BlkioStatEntry `json:"io_queue_recursive"`
	IoServiceTimeRecursive  []BlkioStatEntry `json:"io_service_time_recursive"`
	IoWaitTimeRecursive     []BlkioStatEntry `json:"io_wait_time_recursive"`
	IoMergedRecursive       []BlkioStatEntry `json:"io_merged_recursive"`
	IoTimeRecursive         []BlkioStatEntry `json:"io_time_recursive"`
	SectorsRecursive        []BlkioStatEntry `json:"sectors_recursive"`
}

type MemoryStats struct {
	// Current memory usage, this includes all memory regardless of when it was
	// accessed.
	// Units: Bytes.
	Usage uint64 `json:"usage"`

	// The amount of working set memory, this includes recently accessed memory,
	// dirty memory, and kernel memory. Working set is <= "usage".
	// Units: Bytes.
	WorkingSet uint64 `json:"working_set"`

	Failcnt uint64 `json:"failcnt"`

	ContainerData    MemoryStatsMemoryData `json:"container_data,omitempty"`
	HierarchicalData MemoryStatsMemoryData `json:"hierarchical_data,omitempty"`
}

type MemoryStatsMemoryData struct {
	Pgfault    uint64 `json:"pgfault"`
	Pgmajfault uint64 `json:"pgmajfault"`
}

type InterfaceStats struct {
	// The name of the interface.
	Name string `json:"name"`
	// Cumulative count of bytes received.
	RxBytes uint64 `json:"rx_bytes"`
	// Cumulative count of packets received.
	RxPackets uint64 `json:"rx_packets"`
	// Cumulative count of receive errors encountered.
	RxErrors uint64 `json:"rx_errors"`
	// Cumulative count of packets dropped while receiving.
	RxDropped uint64 `json:"rx_dropped"`
	// Cumulative count of bytes transmitted.
	TxBytes uint64 `json:"tx_bytes"`
	// Cumulative count of packets transmitted.
	TxPackets uint64 `json:"tx_packets"`
	// Cumulative count of transmit errors encountered.
	TxErrors uint64 `json:"tx_errors"`
	// Cumulative count of packets dropped while transmitting.
	TxDropped uint64 `json:"tx_dropped"`
}

type NetworkStats struct {
	Interfaces []InterfaceStats `json:"interfaces,omitempty"`
	// TCP connection stats (Established, Listen...)
	Tcp TcpStat `json:"tcp"`
	// TCP6 connection stats (Established, Listen...)
	Tcp6 TcpStat `json:"tcp6"`
}

type TcpStat struct {
	//Count of TCP connections in state "Established"
	Established uint64
	//Count of TCP connections in state "Syn_Sent"
	SynSent uint64
	//Count of TCP connections in state "Syn_Recv"
	SynRecv uint64
	//Count of TCP connections in state "Fin_Wait1"
	FinWait1 uint64
	//Count of TCP connections in state "Fin_Wait2"
	FinWait2 uint64
	//Count of TCP connections in state "Time_Wait
	TimeWait uint64
	//Count of TCP connections in state "Close"
	Close uint64
	//Count of TCP connections in state "Close_Wait"
	CloseWait uint64
	//Count of TCP connections in state "Listen_Ack"
	LastAck uint64
	//Count of TCP connections in state "Listen"
	Listen uint64
	//Count of TCP connections in state "Closing"
	Closing uint64
}

type FsStats struct {
	// The block device name associated with the filesystem.
	Device string `json:"device,omitempty"`

	// Number of bytes that can be consumed by the container on this filesystem.
	Limit uint64 `json:"capacity"`

	// Number of bytes that is consumed by the container on this filesystem.
	Usage uint64 `json:"usage"`

	// Number of bytes available for non-root user.
	Available uint64 `json:"available"`

	// Number of reads completed
	// This is the total number of reads completed successfully.
	ReadsCompleted uint64 `json:"reads_completed"`

	// Number of reads merged
	// Reads and writes which are adjacent to each other may be merged for
	// efficiency.  Thus two 4K reads may become one 8K read before it is
	// ultimately handed to the disk, and so it will be counted (and queued)
	// as only one I/O.  This field lets you know how often this was done.
	ReadsMerged uint64 `json:"reads_merged"`

	// Number of sectors read
	// This is the total number of sectors read successfully.
	SectorsRead uint64 `json:"sectors_read"`

	// Number of milliseconds spent reading
	// This is the total number of milliseconds spent by all reads (as
	// measured from __make_request() to end_that_request_last()).
	ReadTime uint64 `json:"read_time"`

	// Number of writes completed
	// This is the total number of writes completed successfully.
	WritesCompleted uint64 `json:"writes_completed"`

	// Number of writes merged
	// See the description of reads merged.
	WritesMerged uint64 `json:"writes_merged"`

	// Number of sectors written
	// This is the total number of sectors written successfully.
	SectorsWritten uint64 `json:"sectors_written"`

	// Number of milliseconds spent writing
	// This is the total number of milliseconds spent by all writes (as
	// measured from __make_request() to end_that_request_last()).
	WriteTime uint64 `json:"write_time"`

	// Number of I/Os currently in progress
	// The only field that should go to zero. Incremented as requests are
	// given to appropriate struct request_queue and decremented as they finish.
	IoInProgress uint64 `json:"io_in_progress"`

	// Number of milliseconds spent doing I/Os
	// This field increases so long as field 9 is nonzero.
	IoTime uint64 `json:"io_time"`

	// weighted number of milliseconds spent doing I/Os
	// This field is incremented at each I/O start, I/O completion, I/O
	// merge, or read of these stats by the number of I/Os in progress
	// (field 9) times the number of milliseconds spent doing I/O since the
	// last update of this field.  This can provide an easy measure of both
	// I/O completion time and the backlog that may be accumulating.
	WeightedIoTime uint64 `json:"weighted_io_time"`
}

type ContainerStats struct {
	ContainerID string       `json:"id"`
	Cpu         CpuStats     `json:"cpu,omitempty"`
	Block       BlkioStats   `json:"block,omitempty"`
	Memory      MemoryStats  `json:"memory,omitempty"`
	Network     NetworkStats `json:"network,omitempty"`
	Filesystem  []FsStats    `json:"filesystem,omitempty"`
	Timestamp   time.Time    `json:"timestamp"`
}

type PodStats struct {
	Cpu        CpuStats     `json:"cpu,omitempty"`
	Block      BlkioStats   `json:"block,omitempty"`
	Memory     MemoryStats  `json:"memory,omitempty"`
	Network    NetworkStats `json:"network,omitempty"`
	Filesystem []FsStats    `json:"filesystem,omitempty"`
	Timestamp  time.Time    `json:"timestamp"`

	ContainersStats []ContainerStats `json:"container_stats"`
}

func (client *HyperClient) GetPodStats(podID string) (*PodStats, error) {
	v := url.Values{}
	v.Set(KEY_POD_ID, podID)
	body, _, err := client.call("GET", "/pod/stats?"+v.Encode(), "", nil)
	if err != nil {
		return nil, err
	}

	var stats PodStats
	podStats := transformJson(string(body))
	err = json.Unmarshal([]byte(podStats), &stats)
	if err != nil {
		return nil, err
	}

	return &stats, nil
}
<|endoftext|>


<commit_before>package chunk

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"time"

	"github.com/bradfitz/gomemcache/memcache"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/weaveworks/scope/common/instrument"
	"golang.org/x/net/context"
)

var (
	memcacheRequests = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "cortex",
		Name:      "memcache_requests_total",
		Help:      "Total count of chunks requested from memcache.",
	})

	memcacheHits = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "cortex",
		Name:      "memcache_hits_total",
		Help:      "Total count of chunks found in memcache.",
	})

	memcacheRequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "cortex",
		Name:      "memcache_request_duration_seconds",
		Help:      "Total time spent in seconds doing memcache requests.",
		// Memecache requests are very quick: smallest bucket is 16us, biggest is 1s
		Buckets: prometheus.ExponentialBuckets(0.000016, 4, 8),
	}, []string{"method", "status_code"})
)

func init() {
	prometheus.MustRegister(memcacheRequests)
	prometheus.MustRegister(memcacheHits)
	prometheus.MustRegister(memcacheRequestDuration)
}

// Memcache caches things
type Memcache interface {
	GetMulti(keys []string) (map[string]*memcache.Item, error)
	Set(item *memcache.Item) error
}

// Cache type caches chunks
type Cache struct {
	Memcache   Memcache
	Expiration time.Duration
}

func memcacheStatusCode(err error) string {
	// See https://godoc.org/github.com/bradfitz/gomemcache/memcache#pkg-variables
	switch err {
	case nil:
		return "200"
	case memcache.ErrCacheMiss:
		return "404"
	case memcache.ErrMalformedKey:
		return "400"
	default:
		return "500"
	}
}

func memcacheKey(userID, chunkID string) string {
	return fmt.Sprintf("%s/%s", userID, chunkID)
}

// FetchChunkData gets chunks from the chunk cache.
func (c *Cache) FetchChunkData(ctx context.Context, userID string, chunks []Chunk) (found []Chunk, missing []Chunk, err error) {
	memcacheRequests.Add(float64(len(chunks)))

	keys := make([]string, 0, len(chunks))
	for _, chunk := range chunks {
		keys = append(keys, memcacheKey(userID, chunk.ID))
	}

	var items map[string]*memcache.Item
	err = instrument.TimeRequestHistogramStatus(ctx, "Memcache.Get", memcacheRequestDuration, memcacheStatusCode, func(_ context.Context) error {
		var err error
		items, err = c.Memcache.GetMulti(keys)
		return err
	})
	if err != nil {
		return nil, chunks, err
	}

	for _, chunk := range chunks {
		item, ok := items[memcacheKey(userID, chunk.ID)]
		if !ok {
			missing = append(missing, chunk)
			continue
		}

		if err := chunk.decode(bytes.NewReader(item.Value)); err != nil {
			return nil, nil, err
		}
		found = append(found, chunk)
	}

	memcacheHits.Add(float64(len(found)))
	return found, missing, nil
}

// StoreChunkData serializes and stores a chunk in the chunk cache.
func (c *Cache) StoreChunkData(ctx context.Context, userID string, chunk *Chunk) error {
	reader, err := chunk.reader()
	if err != nil {
		return err
	}

	buf, err := ioutil.ReadAll(reader)
	if err != nil {
		return err
	}

	return instrument.TimeRequestHistogramStatus(ctx, "Memcache.Put", memcacheRequestDuration, memcacheStatusCode, func(_ context.Context) error {
		item := memcache.Item{
			Key:        memcacheKey(userID, chunk.ID),
			Value:      buf,
			Expiration: int32(c.Expiration.Seconds()),
		}
		return c.Memcache.Set(&item)
	})
}

// StoreChunks serializes and stores multiple chunks in the chunk cache.
func (c *Cache) StoreChunks(ctx context.Context, userID string, chunks []Chunk) error {
	errs := make(chan error)
	for _, chunk := range chunks {
		go func(chunk *Chunk) {
			errs <- c.StoreChunkData(ctx, userID, chunk)
		}(&chunk)
	}
	var errOut error
	for i := 0; i < len(chunks); i++ {
		if err := <-errs; err != nil {
			errOut = err
		}
	}
	return errOut
}
<commit_msg>Don't completely fail the cache lookup if we fail to decode one chunk.<commit_after>package chunk

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"time"

	"github.com/bradfitz/gomemcache/memcache"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/weaveworks/scope/common/instrument"
	"golang.org/x/net/context"
)

var (
	memcacheRequests = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "cortex",
		Name:      "memcache_requests_total",
		Help:      "Total count of chunks requested from memcache.",
	})

	memcacheHits = prometheus.NewCounter(prometheus.CounterOpts{
		Namespace: "cortex",
		Name:      "memcache_hits_total",
		Help:      "Total count of chunks found in memcache.",
	})

	memcacheRequestDuration = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace: "cortex",
		Name:      "memcache_request_duration_seconds",
		Help:      "Total time spent in seconds doing memcache requests.",
		// Memecache requests are very quick: smallest bucket is 16us, biggest is 1s
		Buckets: prometheus.ExponentialBuckets(0.000016, 4, 8),
	}, []string{"method", "status_code"})
)

func init() {
	prometheus.MustRegister(memcacheRequests)
	prometheus.MustRegister(memcacheHits)
	prometheus.MustRegister(memcacheRequestDuration)
}

// Memcache caches things
type Memcache interface {
	GetMulti(keys []string) (map[string]*memcache.Item, error)
	Set(item *memcache.Item) error
}

// Cache type caches chunks
type Cache struct {
	Memcache   Memcache
	Expiration time.Duration
}

func memcacheStatusCode(err error) string {
	// See https://godoc.org/github.com/bradfitz/gomemcache/memcache#pkg-variables
	switch err {
	case nil:
		return "200"
	case memcache.ErrCacheMiss:
		return "404"
	case memcache.ErrMalformedKey:
		return "400"
	default:
		return "500"
	}
}

func memcacheKey(userID, chunkID string) string {
	return fmt.Sprintf("%s/%s", userID, chunkID)
}

// FetchChunkData gets chunks from the chunk cache.
func (c *Cache) FetchChunkData(ctx context.Context, userID string, chunks []Chunk) (found []Chunk, missing []Chunk, err error) {
	memcacheRequests.Add(float64(len(chunks)))

	keys := make([]string, 0, len(chunks))
	for _, chunk := range chunks {
		keys = append(keys, memcacheKey(userID, chunk.ID))
	}

	var items map[string]*memcache.Item
	err = instrument.TimeRequestHistogramStatus(ctx, "Memcache.Get", memcacheRequestDuration, memcacheStatusCode, func(_ context.Context) error {
		var err error
		items, err = c.Memcache.GetMulti(keys)
		return err
	})
	if err != nil {
		return nil, chunks, err
	}

	for _, chunk := range chunks {
		item, ok := items[memcacheKey(userID, chunk.ID)]
		if !ok {
			missing = append(missing, chunk)
			continue
		}

		if err := chunk.decode(bytes.NewReader(item.Value)); err != nil {
			log.Errorf("Failed to decode chunk from cache: %v", err)
			missing = append(missing, chunk)
			continue
		}
		found = append(found, chunk)
	}

	memcacheHits.Add(float64(len(found)))
	return found, missing, nil
}

// StoreChunkData serializes and stores a chunk in the chunk cache.
func (c *Cache) StoreChunkData(ctx context.Context, userID string, chunk *Chunk) error {
	reader, err := chunk.reader()
	if err != nil {
		return err
	}

	buf, err := ioutil.ReadAll(reader)
	if err != nil {
		return err
	}

	return instrument.TimeRequestHistogramStatus(ctx, "Memcache.Put", memcacheRequestDuration, memcacheStatusCode, func(_ context.Context) error {
		item := memcache.Item{
			Key:        memcacheKey(userID, chunk.ID),
			Value:      buf,
			Expiration: int32(c.Expiration.Seconds()),
		}
		return c.Memcache.Set(&item)
	})
}

// StoreChunks serializes and stores multiple chunks in the chunk cache.
func (c *Cache) StoreChunks(ctx context.Context, userID string, chunks []Chunk) error {
	errs := make(chan error)
	for _, chunk := range chunks {
		go func(chunk *Chunk) {
			errs <- c.StoreChunkData(ctx, userID, chunk)
		}(&chunk)
	}
	var errOut error
	for i := 0; i < len(chunks); i++ {
		if err := <-errs; err != nil {
			errOut = err
		}
	}
	return errOut
}
<|endoftext|>