<commit_before>package etcd

import (
	"net/http"
	"strconv"
	"time"

	"golang.org/x/net/context"

	"github.com/Sirupsen/logrus"
	"github.com/coreos/etcd/etcdserver/stats"
	"github.com/coreos/etcd/pkg/types"
	etcdraft "github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/rafthttp"
	"github.com/relab/raft"
	"github.com/relab/raft/commonpb"
)

// Wrapper wraps an etcd/raft.Node and implements relab/raft.Raft and
// etcd/rafthttp.Raft.
type Wrapper struct {
	n         etcdraft.Node
	transport *rafthttp.Transport
	logger    logrus.FieldLogger
}

// TODO Use Peer.Context for addr?
func NewRaft(cfg *etcdraft.Config, peers []etcdraft.Peer, addrs []string, logger logrus.FieldLogger) *Wrapper {
	w := &Wrapper{logger: logger}
	w.n = etcdraft.StartNode(cfg, peers)

	ss := &stats.ServerStats{}
	ss.Initialize()

	transport := &rafthttp.Transport{
		ID:          types.ID(cfg.ID),
		ClusterID:   1,
		Raft:        w,
		ServerStats: ss,
		LeaderStats: stats.NewLeaderStats(strconv.FormatUint(cfg.ID, 10)),
		ErrorC:      make(chan error),
	}
	err := transport.Start()

	if err != nil {
		panic("start transport: " + err.Error())
	}

	for i, peer := range peers {
		w.logger.WithFields(logrus.Fields{
			"i":       i,
			"addr":    addrs[i],
			"id":      peer.ID,
			"context": string(peer.Context),
		}).Warnln("Add peer")

		transport.AddPeer(types.ID(peer.ID), []string{addrs[i]})
	}

	w.transport = transport

	go w.run()
	return w
}

func (w *Wrapper) Handler() http.Handler {
	return w.transport.Handler()
}

func (w *Wrapper) ProposeCmd(ctx context.Context, req []byte) (raft.Future, error) {
	w.logger.Warnln("ProposeCmd")

	err := w.n.Propose(ctx, req)

	if err != nil {
		return nil, err
	}

	return nil, nil
}

func (w *Wrapper) ReadCmd(context.Context, []byte) (raft.Future, error) {
	w.logger.Warnln("ProposeRead")
	return nil, nil
}

func (w *Wrapper) ProposeConf(ctx context.Context, req *commonpb.ReconfRequest) (raft.Future, error) {
	cc := raftpb.ConfChange{
		Type:    raftpb.ConfChangeType(req.ReconfType),
		ID:      req.ServerID, // ?
		NodeID:  req.ServerID,
		Context: []byte(""), // ?
	}

	err := w.n.ProposeConfChange(ctx, cc)

	w.logger.WithError(err).WithFields(logrus.Fields{
		"req": req,
		"cc":  cc,
	}).Warnln("ProposeConf")

	if err != nil {
		return nil, err
	}

	return nil, nil
}

func (w *Wrapper) run() {
	s := time.NewTicker(5 * time.Millisecond)

	for {
		select {
		case <-s.C:
			w.n.Tick()
		case rd := <-w.n.Ready():
			w.logger.WithField("rd", rd).Warnln("Ready")
			// saveToStorage(rd.State, rd.Entries, rd.Snapshot)
			if !etcdraft.IsEmptySnap(rd.Snapshot) {
				w.logger.WithField("snapshot", rd.Snapshot).Warnln("Snapshot")
				// processSnapshot(rd.Snapshot)
			}
			w.logger.WithField("messages", rd.Messages).Warnln("Sending")
			w.transport.Send(rd.Messages)
			for _, entry := range rd.CommittedEntries {
				switch entry.Type {
				case raftpb.EntryNormal:
					w.logger.WithField("entry", entry).Warnln("Committed normal entry")
					// process(entry)
				case raftpb.EntryConfChange:
					w.logger.WithField("entry", entry).Warnln("Committed conf change entry")
					var cc raftpb.ConfChange
					err := cc.Unmarshal(entry.Data)

					if err != nil {
						panic("unmarshal conf change: " + err.Error())
					}

					w.logger.WithField("cc", cc).Warnln("Applying conf change")
					w.n.ApplyConfChange(cc)
				}
			}
			w.logger.Warnln("Advance")
			w.n.Advance()
		}
	}
}

func (w *Wrapper) Process(ctx context.Context, m raftpb.Message) error {
	w.logger.WithField("m", m).Warnln("Process")
	return w.n.Step(ctx, m)
}
func (w *Wrapper) IsIDRemoved(id uint64) bool                               { return false }
func (w *Wrapper) ReportUnreachable(id uint64)                              {}
func (w *Wrapper) ReportSnapshot(id uint64, status etcdraft.SnapshotStatus) {}
<commit_msg>etcd/raft.go: Use Peer.Context for addr<commit_after>package etcd

import (
	"net/http"
	"strconv"
	"time"

	"golang.org/x/net/context"

	"github.com/Sirupsen/logrus"
	"github.com/coreos/etcd/etcdserver/stats"
	"github.com/coreos/etcd/pkg/types"
	etcdraft "github.com/coreos/etcd/raft"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/rafthttp"
	"github.com/relab/raft"
	"github.com/relab/raft/commonpb"
)

// Wrapper wraps an etcd/raft.Node and implements relab/raft.Raft and
// etcd/rafthttp.Raft.
type Wrapper struct {
	n         etcdraft.Node
	transport *rafthttp.Transport
	logger    logrus.FieldLogger
}

func NewRaft(cfg *etcdraft.Config, peers []etcdraft.Peer, logger logrus.FieldLogger) *Wrapper {
	w := &Wrapper{logger: logger}
	w.n = etcdraft.StartNode(cfg, peers)

	ss := &stats.ServerStats{}
	ss.Initialize()

	transport := &rafthttp.Transport{
		ID:          types.ID(cfg.ID),
		ClusterID:   1,
		Raft:        w,
		ServerStats: ss,
		LeaderStats: stats.NewLeaderStats(strconv.FormatUint(cfg.ID, 10)),
		ErrorC:      make(chan error),
	}
	err := transport.Start()

	if err != nil {
		panic("start transport: " + err.Error())
	}

	for i, peer := range peers {
		w.logger.WithFields(logrus.Fields{
			"i":       i,
			"id":      peer.ID,
			"context": string(peer.Context),
		}).Warnln("Add peer")

		transport.AddPeer(types.ID(peer.ID), []string{string(peer.Context)})
	}

	w.transport = transport

	go w.run()
	return w
}

func (w *Wrapper) Handler() http.Handler {
	return w.transport.Handler()
}

func (w *Wrapper) ProposeCmd(ctx context.Context, req []byte) (raft.Future, error) {
	w.logger.Warnln("ProposeCmd")

	err := w.n.Propose(ctx, req)

	if err != nil {
		return nil, err
	}

	return nil, nil
}

func (w *Wrapper) ReadCmd(context.Context, []byte) (raft.Future, error) {
	w.logger.Warnln("ProposeRead")
	return nil, nil
}

func (w *Wrapper) ProposeConf(ctx context.Context, req *commonpb.ReconfRequest) (raft.Future, error) {
	cc := raftpb.ConfChange{
		Type:    raftpb.ConfChangeType(req.ReconfType),
		ID:      req.ServerID, // ?
		NodeID:  req.ServerID,
		Context: []byte(""), // ?
	}

	err := w.n.ProposeConfChange(ctx, cc)

	w.logger.WithError(err).WithFields(logrus.Fields{
		"req": req,
		"cc":  cc,
	}).Warnln("ProposeConf")

	if err != nil {
		return nil, err
	}

	return nil, nil
}

func (w *Wrapper) run() {
	s := time.NewTicker(5 * time.Millisecond)

	for {
		select {
		case <-s.C:
			w.n.Tick()
		case rd := <-w.n.Ready():
			w.logger.WithField("rd", rd).Warnln("Ready")
			// saveToStorage(rd.State, rd.Entries, rd.Snapshot)
			if !etcdraft.IsEmptySnap(rd.Snapshot) {
				w.logger.WithField("snapshot", rd.Snapshot).Warnln("Snapshot")
				// processSnapshot(rd.Snapshot)
			}
			w.logger.WithField("messages", rd.Messages).Warnln("Sending")
			w.transport.Send(rd.Messages)
			for _, entry := range rd.CommittedEntries {
				switch entry.Type {
				case raftpb.EntryNormal:
					w.logger.WithField("entry", entry).Warnln("Committed normal entry")
					// process(entry)
				case raftpb.EntryConfChange:
					w.logger.WithField("entry", entry).Warnln("Committed conf change entry")
					var cc raftpb.ConfChange
					err := cc.Unmarshal(entry.Data)

					if err != nil {
						panic("unmarshal conf change: " + err.Error())
					}

					w.logger.WithField("cc", cc).Warnln("Applying conf change")
					w.n.ApplyConfChange(cc)
				}
			}
			w.logger.Warnln("Advance")
			w.n.Advance()
		}
	}
}

func (w *Wrapper) Process(ctx context.Context, m raftpb.Message) error {
	w.logger.WithField("m", m).Warnln("Process")
	return w.n.Step(ctx, m)
}
func (w *Wrapper) IsIDRemoved(id uint64) bool                               { return false }
func (w *Wrapper) ReportUnreachable(id uint64)                              {}
func (w *Wrapper) ReportSnapshot(id uint64, status etcdraft.SnapshotStatus) {}
<|endoftext|>


<commit_before>package proto

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"testing"
)

func TestUnmarshal(t *testing.T) {
	r := Reader{
		B: []byte{0x0, 0x0, 0x1, 0x7a, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x1, 0x0, 0x4, 0x74, 0x65, 0x73, 0x74, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x6, 0x0, 0x0, 0x1, 0x56, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2d, 0x9, 0xf8, 0xf8, 0x91, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x32, 0x30, 0x31, 0x35, 0x2d, 0x30, 0x39, 0x2d, 0x31, 0x35, 0x54, 0x32, 0x32, 0x3a, 0x31, 0x38, 0x3a, 0x30, 0x31, 0x2b, 0x30, 0x38, 0x3a, 0x30, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x2d, 0x5b, 0xc0, 0xd7, 0x36, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x32, 0x30, 0x31, 0x35, 0x2d, 0x30, 0x39, 0x2d, 0x31, 0x35, 0x54, 0x32, 0x32, 0x3a, 0x31, 0x38, 0x3a, 0x30, 0x34, 0x2b, 0x30, 0x38, 0x3a, 0x30, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x2d, 0xfd, 0xb7, 0xdc, 0x82, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x32, 0x30, 0x31, 0x35, 0x2d, 0x30, 0x39, 0x2d, 0x31, 0x35, 0x54, 0x32, 0x32, 0x3a, 0x31, 0x38, 0x3a, 0x30, 0x35, 0x2b, 0x30, 0x38, 0x3a, 0x30, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x2d, 0xe7, 0x6f, 0xfd, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x32, 0x30, 0x31, 0x35, 0x2d, 0x30, 0x39, 0x2d, 0x31, 0x35, 0x54, 0x32, 0x32, 0x3a, 0x31, 0x38, 0x3a, 0x35, 0x30, 0x2b, 0x30, 0x38, 0x3a, 0x30, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0x0, 0x0, 0x0, 0x2d, 0x41, 0x18, 0xf6, 0xf5, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x32, 0x30, 0x31, 0x35, 0x2d, 0x30, 0x39, 0x2d, 0x31, 0x35, 0x54, 0x32, 0x32, 0x3a, 0x31, 0x38, 0x3a, 0x35, 0x31, 0x2b, 0x30, 0x38, 0x3a, 0x30, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5, 0x0, 0x0, 0x0, 0x2d, 0x13, 0x20, 0xd9, 0x52, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x32, 0x30, 0x31, 0x35, 0x2d, 0x30, 0x39, 0x2d, 0x31, 0x35, 0x54, 0x32, 0x32, 0x3a, 0x31, 0x38, 0x3a, 0x35, 0x34, 0x2b, 0x30, 0x38, 0x3a, 0x30, 0x30},
	}
	fmt.Println(len(r.B))
	(&RequestOrResponse{M: &Response{ResponseMessage: &FetchResponse{}}}).Unmarshal(&r)
}

func TestUnmarshalTrunc(t *testing.T) {
	//f, _ := os.Open("truncated_response")
	f, _ := os.Open("newtrunc")
	b, _ := ioutil.ReadAll(f)
	f.Close()
	r := Reader{
		B: b,
	}
	resp := FetchResponse{}
	(&RequestOrResponse{M: &Response{ResponseMessage: &resp}}).Unmarshal(&r)
	if r.Err != nil {
		t.Fatal(r.Err)
	}
	if len(resp) != 1 ||
		len(resp[0].FetchMessageSetInPartitions) != 1 ||
		len(resp[0].FetchMessageSetInPartitions[0].MessageSet) != 2 {
		t.Fatal("fail to parse truncated message set")
	}
}

func TestUnmarshalDump(t *testing.T) {
	f, _ := os.Open("truncated_response")
	b, _ := ioutil.ReadAll(f)
	f.Close()
	o, _ := os.Create("newtrunc")
	nb := append(b[4:51], b[1048266:]...)
	size := int32(len(nb))
	nb = append([]byte{byte(size >> 24), byte(size >> 16), byte(size >> 8), byte(size)}, nb...)
	o.Write(nb)
	o.Close()
}

func toJSON(v interface{}) string {
	buf, _ := json.MarshalIndent(v, "", "    ")
	return string(buf)
}
<commit_msg>use reduced test data.<commit_after>package proto

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"testing"
)

func TestUnmarshal(t *testing.T) {
	r := Reader{
		B: []byte{0x0, 0x0, 0x1, 0x7a, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x1, 0x0, 0x4, 0x74, 0x65, 0x73, 0x74, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x6, 0x0, 0x0, 0x1, 0x56, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2d, 0x9, 0xf8, 0xf8, 0x91, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x32, 0x30, 0x31, 0x35, 0x2d, 0x30, 0x39, 0x2d, 0x31, 0x35, 0x54, 0x32, 0x32, 0x3a, 0x31, 0x38, 0x3a, 0x30, 0x31, 0x2b, 0x30, 0x38, 0x3a, 0x30, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x2d, 0x5b, 0xc0, 0xd7, 0x36, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x32, 0x30, 0x31, 0x35, 0x2d, 0x30, 0x39, 0x2d, 0x31, 0x35, 0x54, 0x32, 0x32, 0x3a, 0x31, 0x38, 0x3a, 0x30, 0x34, 0x2b, 0x30, 0x38, 0x3a, 0x30, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0x2d, 0xfd, 0xb7, 0xdc, 0x82, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x32, 0x30, 0x31, 0x35, 0x2d, 0x30, 0x39, 0x2d, 0x31, 0x35, 0x54, 0x32, 0x32, 0x3a, 0x31, 0x38, 0x3a, 0x30, 0x35, 0x2b, 0x30, 0x38, 0x3a, 0x30, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0x2d, 0xe7, 0x6f, 0xfd, 0x41, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x32, 0x30, 0x31, 0x35, 0x2d, 0x30, 0x39, 0x2d, 0x31, 0x35, 0x54, 0x32, 0x32, 0x3a, 0x31, 0x38, 0x3a, 0x35, 0x30, 0x2b, 0x30, 0x38, 0x3a, 0x30, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, 0x0, 0x0, 0x0, 0x2d, 0x41, 0x18, 0xf6, 0xf5, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x32, 0x30, 0x31, 0x35, 0x2d, 0x30, 0x39, 0x2d, 0x31, 0x35, 0x54, 0x32, 0x32, 0x3a, 0x31, 0x38, 0x3a, 0x35, 0x31, 0x2b, 0x30, 0x38, 0x3a, 0x30, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x5, 0x0, 0x0, 0x0, 0x2d, 0x13, 0x20, 0xd9, 0x52, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1f, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x32, 0x30, 0x31, 0x35, 0x2d, 0x30, 0x39, 0x2d, 0x31, 0x35, 0x54, 0x32, 0x32, 0x3a, 0x31, 0x38, 0x3a, 0x35, 0x34, 0x2b, 0x30, 0x38, 0x3a, 0x30, 0x30},
	}
	fmt.Println(len(r.B))
	(&RequestOrResponse{M: &Response{ResponseMessage: &FetchResponse{}}}).Unmarshal(&r)
}

func TestUnmarshalTrunc(t *testing.T) {
	f, _ := os.Open("truncated_response")
	b, _ := ioutil.ReadAll(f)
	f.Close()
	r := Reader{
		B: b,
	}
	resp := FetchResponse{}
	(&RequestOrResponse{M: &Response{ResponseMessage: &resp}}).Unmarshal(&r)
	if r.Err != nil {
		t.Fatal(r.Err)
	}
	if len(resp) != 1 ||
		len(resp[0].FetchMessageSetInPartitions) != 1 ||
		len(resp[0].FetchMessageSetInPartitions[0].MessageSet) != 2 {
		t.Fatal("fail to parse truncated message set")
	}
}

func toJSON(v interface{}) string {
	buf, _ := json.MarshalIndent(v, "", "    ")
	return string(buf)
}
<|endoftext|>


<commit_before>// Copyright 2017 The go-interpreter Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package exec

import "errors"

func (vm *VM) doCall(compiled compiledFunction, index int64) {
	newStack := make([]uint64, compiled.maxDepth)
	locals := make([]uint64, compiled.totalLocalVars)

	for i := compiled.args - 1; i >= 0; i-- {
		locals[i] = vm.popUint64()
	}

	// save execution context
	prevCtxt := vm.ctx

	vm.ctx = context{
		stack:   newStack,
		locals:  locals,
		code:    compiled.code,
		pc:      0,
		curFunc: index,
	}

	rtrn := vm.execCode(compiled)

	// restore execution context
	vm.ctx = prevCtxt

	if compiled.returns {
		vm.pushUint64(rtrn)
	}
}

var (
	// ErrSignatureMismatch is the error value used while trapping the VM when
	// a signature mismatch between the table entry and the type entry is found
	// in a call_indirect operation.
	ErrSignatureMismatch = errors.New("exec: signature mismatch in call_indirect")
)

func (vm *VM) call() {
	index := vm.fetchUint32()
	vm.doCall(vm.compiledFuncs[index], int64(index))
}

func (vm *VM) callIndirect() {
	index := vm.fetchUint32()
	fnExpect := vm.module.Types.Entries[index]
	_ = vm.fetchUint32() // reserved (https://github.com/WebAssembly/design/blob/27ac254c854994103c24834a994be16f74f54186/BinaryEncoding.md#call-operators-described-here)
	tableIndex := vm.popUint32()
	elemIndex := vm.module.TableIndexSpace[0][tableIndex]
	fnActual := vm.module.FunctionIndexSpace[elemIndex]

	if len(fnExpect.ParamTypes) != len(fnActual.Sig.ParamTypes) {
		panic(ErrSignatureMismatch)
	}
	if len(fnExpect.ReturnTypes) != len(fnActual.Sig.ReturnTypes) {
		panic(ErrSignatureMismatch)
	}

	for i := range fnExpect.ParamTypes {
		if fnExpect.ParamTypes[i] != fnActual.Sig.ParamTypes[i] {
			panic(ErrSignatureMismatch)
		}
	}

	for i := range fnExpect.ReturnTypes {
		if fnExpect.ReturnTypes[i] != fnActual.Sig.ReturnTypes[i] {
			panic(ErrSignatureMismatch)
		}
	}

	vm.doCall(vm.compiledFuncs[elemIndex], int64(index))
}
<commit_msg>exec: Add ErrUndefinedElementIndex<commit_after>// Copyright 2017 The go-interpreter Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package exec

import "errors"

func (vm *VM) doCall(compiled compiledFunction, index int64) {
	newStack := make([]uint64, compiled.maxDepth)
	locals := make([]uint64, compiled.totalLocalVars)

	for i := compiled.args - 1; i >= 0; i-- {
		locals[i] = vm.popUint64()
	}

	// save execution context
	prevCtxt := vm.ctx

	vm.ctx = context{
		stack:   newStack,
		locals:  locals,
		code:    compiled.code,
		pc:      0,
		curFunc: index,
	}

	rtrn := vm.execCode(compiled)

	// restore execution context
	vm.ctx = prevCtxt

	if compiled.returns {
		vm.pushUint64(rtrn)
	}
}

var (
	// ErrSignatureMismatch is the error value used while trapping the VM when
	// a signature mismatch between the table entry and the type entry is found
	// in a call_indirect operation.
	ErrSignatureMismatch = errors.New("exec: signature mismatch in call_indirect")
	// ErrUndefinedElementIndex is the error value used while trapping the VM when
	// an invalid index to the module's table space is used as an operand to
	// call_indirect
	ErrUndefinedElementIndex = errors.New("exec: undefined element index")
)

func (vm *VM) call() {
	index := vm.fetchUint32()
	vm.doCall(vm.compiledFuncs[index], int64(index))
}

func (vm *VM) callIndirect() {
	index := vm.fetchUint32()
	fnExpect := vm.module.Types.Entries[index]
	_ = vm.fetchUint32() // reserved (https://github.com/WebAssembly/design/blob/27ac254c854994103c24834a994be16f74f54186/BinaryEncoding.md#call-operators-described-here)
	tableIndex := vm.popUint32()
	if int(tableIndex) >= len(vm.module.TableIndexSpace[0]) {
		panic(ErrUndefinedElementIndex)
	}
	elemIndex := vm.module.TableIndexSpace[0][tableIndex]
	fnActual := vm.module.FunctionIndexSpace[elemIndex]

	if len(fnExpect.ParamTypes) != len(fnActual.Sig.ParamTypes) {
		panic(ErrSignatureMismatch)
	}
	if len(fnExpect.ReturnTypes) != len(fnActual.Sig.ReturnTypes) {
		panic(ErrSignatureMismatch)
	}

	for i := range fnExpect.ParamTypes {
		if fnExpect.ParamTypes[i] != fnActual.Sig.ParamTypes[i] {
			panic(ErrSignatureMismatch)
		}
	}

	for i := range fnExpect.ReturnTypes {
		if fnExpect.ReturnTypes[i] != fnActual.Sig.ReturnTypes[i] {
			panic(ErrSignatureMismatch)
		}
	}

	vm.doCall(vm.compiledFuncs[elemIndex], int64(index))
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.247"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.214", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<commit_msg>fnlb: 0.0.248 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.248"

func main() {
	// XXX (reed): normalize
	level, err := logrus.ParseLevel(os.Getenv("LOG_LEVEL"))
	if err != nil {
		level = logrus.InfoLevel
	}
	logrus.SetLevel(level)

	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.215", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory; use k8s for kuberneted")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.StringVar(&conf.MgmtListen, "mgmt-listen", ":8081", "management port to run on")
	flag.IntVar(&conf.ShutdownTimeout, "shutdown-timeout", 0, "graceful shutdown timeout")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckHealthy, "hc-healthy", 1, "threshold of success checks to declare node healthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.StringVar(&conf.Namespace, "namespace", "", "kubernetes namespace to monitor")
	flag.StringVar(&conf.LabelSelector, "label-selector", "", "kubernetes label selector to monitor")
	flag.IntVar(&conf.TargetPort, "target-port", 8080, "kubernetes port to target on selected pods")

	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	db, err := lb.NewDB(conf) // Handles case where DBurl == "k8s"
	if err != nil {
		logrus.WithError(err).Fatal("error setting up database")
	}
	defer db.Close()

	g, err := lb.NewAllGrouper(conf, db)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	servers := make([]*http.Server, 0, 1)
	handler := lb.NewProxy(k, g, r, conf)

	// a separate mgmt listener is requested? then let's create a LB traffic only server
	if conf.Listen != conf.MgmtListen {
		servers = append(servers, &http.Server{Addr: conf.Listen, Handler: handler})
		handler = lb.NullHandler()
	}

	// add mgmt endpoints to the handler
	handler = g.Wrap(handler) // add/del/list endpoints
	handler = r.Wrap(handler) // stats / dash endpoint

	servers = append(servers, &http.Server{Addr: conf.MgmtListen, Handler: handler})
	serve(servers, &conf)
}

func serve(servers []*http.Server, conf *lb.Config) {

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)

	for i := 0; i < len(servers); i++ {
		go func(idx int) {
			err := servers[idx].ListenAndServe()
			if err != nil && err != http.ErrServerClosed {
				logrus.WithFields(logrus.Fields{"server_id": idx}).WithError(err).Fatal("server error")
			} else {
				logrus.WithFields(logrus.Fields{"server_id": idx}).Info("server stopped")
			}
		}(i)
	}

	sig := <-ch
	logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")

	for i := 0; i < len(servers); i++ {

		ctx := context.Background()

		if conf.ShutdownTimeout > 0 {
			tmpCtx, cancel := context.WithTimeout(context.Background(), time.Duration(conf.ShutdownTimeout)*time.Second)
			ctx = tmpCtx
			defer cancel()
		}

		err := servers[i].Shutdown(ctx) // safe shutdown
		if err != nil {
			logrus.WithFields(logrus.Fields{"server_id": i}).WithError(err).Fatal("server shutdown error")
		} else {
			logrus.WithFields(logrus.Fields{"server_id": i}).Info("server shutdown")
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.128"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.95", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	g, err := lb.NewAllGrouper(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	h := lb.NewProxy(k, g, r, conf)
	h = g.Wrap(h) // add/del/list endpoints
	h = r.Wrap(h) // stats / dash endpoint

	err = serve(conf.Listen, h)
	if err != nil {
		logrus.WithError(err).Fatal("server error")
	}
}

func serve(addr string, handler http.Handler) error {
	server := &http.Server{Addr: addr, Handler: handler}

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)
	go func() {
		for sig := range ch {
			logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")
			server.Shutdown(context.Background()) // safe shutdown
			return
		}
	}()
	return server.ListenAndServe()
}
<commit_msg>fnlb: 0.0.129 release [skip ci]<commit_after>package main

import (
	"context"
	"crypto/tls"
	"flag"
	"net"
	"net/http"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/go-semver/semver"
	"github.com/fnproject/fn/fnlb/lb"
	"github.com/sirupsen/logrus"
)

const VERSION = "0.0.129"

func main() {
	// XXX (reed): normalize
	fnodes := flag.String("nodes", "", "comma separated list of functions nodes")
	minAPIVersion := flag.String("min-api-version", "0.0.96", "minimal node API to accept")

	var conf lb.Config
	flag.StringVar(&conf.DBurl, "db", "sqlite3://:memory:", "backend to store nodes, default to in memory")
	flag.StringVar(&conf.Listen, "listen", ":8081", "port to run on")
	flag.IntVar(&conf.HealthcheckInterval, "hc-interval", 3, "how often to check f(x) nodes, in seconds")
	flag.StringVar(&conf.HealthcheckEndpoint, "hc-path", "/version", "endpoint to determine node health")
	flag.IntVar(&conf.HealthcheckUnhealthy, "hc-unhealthy", 2, "threshold of failed checks to declare node unhealthy")
	flag.IntVar(&conf.HealthcheckTimeout, "hc-timeout", 5, "timeout of healthcheck endpoint, in seconds")
	flag.StringVar(&conf.ZipkinURL, "zipkin", "", "zipkin endpoint to send traces")
	flag.Parse()

	conf.MinAPIVersion = semver.New(*minAPIVersion)

	if len(*fnodes) > 0 {
		// starting w/o nodes is fine too
		conf.Nodes = strings.Split(*fnodes, ",")
	}

	conf.Transport = &http.Transport{
		Proxy: http.ProxyFromEnvironment,
		Dial: (&net.Dialer{
			Timeout:   10 * time.Second,
			KeepAlive: 120 * time.Second,
		}).Dial,
		MaxIdleConnsPerHost: 512,
		TLSHandshakeTimeout: 10 * time.Second,
		TLSClientConfig: &tls.Config{
			ClientSessionCache: tls.NewLRUClientSessionCache(4096),
		},
	}

	g, err := lb.NewAllGrouper(conf)
	if err != nil {
		logrus.WithError(err).Fatal("error setting up grouper")
	}

	r := lb.NewConsistentRouter(conf)
	k := func(r *http.Request) (string, error) {
		return r.URL.Path, nil
	}

	h := lb.NewProxy(k, g, r, conf)
	h = g.Wrap(h) // add/del/list endpoints
	h = r.Wrap(h) // stats / dash endpoint

	err = serve(conf.Listen, h)
	if err != nil {
		logrus.WithError(err).Fatal("server error")
	}
}

func serve(addr string, handler http.Handler) error {
	server := &http.Server{Addr: addr, Handler: handler}

	ch := make(chan os.Signal, 1)
	signal.Notify(ch, syscall.SIGQUIT, syscall.SIGINT)
	go func() {
		for sig := range ch {
			logrus.WithFields(logrus.Fields{"signal": sig}).Info("received signal")
			server.Shutdown(context.Background()) // safe shutdown
			return
		}
	}()
	return server.ListenAndServe()
}
<|endoftext|>


<commit_before>package expr

import (
	"context"

	utilctx "github.com/go-graphite/carbonapi/util/ctx"

	"github.com/ansel1/merry"
	"github.com/go-graphite/carbonapi/cmd/carbonapi/config"
	_ "github.com/go-graphite/carbonapi/expr/functions"
	"github.com/go-graphite/carbonapi/expr/helper"
	"github.com/go-graphite/carbonapi/expr/metadata"
	"github.com/go-graphite/carbonapi/expr/types"
	"github.com/go-graphite/carbonapi/pkg/parser"
	pb "github.com/go-graphite/protocol/carbonapi_v3_pb"
)

type evaluator struct{}

// FetchAndEvalExp fetch data and evalualtes expressions
func (eval evaluator) FetchAndEvalExp(ctx context.Context, exp parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
	config.Config.Limiter.Enter()
	defer config.Config.Limiter.Leave()

	multiFetchRequest := pb.MultiFetchRequest{}
	metricRequestCache := make(map[string]parser.MetricRequest)
	maxDataPoints := utilctx.GetMaxDatapoints(ctx)
	// values related to this particular `target=`
	targetValues := make(map[parser.MetricRequest][]*types.MetricData)

	for _, m := range exp.Metrics() {
		fetchRequest := pb.FetchRequest{
			Name:           m.Metric,
			PathExpression: m.Metric,
			StartTime:      m.From + from,
			StopTime:       m.Until + until,
			MaxDataPoints:  maxDataPoints,
		}
		metricRequest := parser.MetricRequest{
			Metric: fetchRequest.PathExpression,
			From:   fetchRequest.StartTime,
			Until:  fetchRequest.StopTime,
		}

		// avoid multiple requests in a function, E.g divideSeries(a.b, a.b)
		if cachedMetricRequest, ok := metricRequestCache[m.Metric]; ok &&
			cachedMetricRequest.From == metricRequest.From &&
			cachedMetricRequest.Until == metricRequest.Until {
			continue
		}

		// avoid multiple requests in a http request, E.g render?target=a.b&target=a.b
		if _, ok := values[metricRequest]; ok {
			targetValues[metricRequest] = nil
			continue
		}

		metricRequestCache[m.Metric] = metricRequest
		targetValues[metricRequest] = nil
		multiFetchRequest.Metrics = append(multiFetchRequest.Metrics, fetchRequest)
	}

	if len(multiFetchRequest.Metrics) > 0 {
		metrics, _, err := config.Config.ZipperInstance.Render(ctx, multiFetchRequest)
		// If we had only partial result, we want to do our best to actually do our job
		if err != nil && merry.HTTPCode(err) >= 400 {
			return nil, err
		}
		for _, metric := range metrics {
			metricRequest := metricRequestCache[metric.PathExpression]
			if metric.RequestStartTime != 0 && metric.RequestStopTime != 0 {
				metricRequest.From = metric.RequestStartTime
				metricRequest.Until = metric.RequestStopTime
			}
			data, ok := values[metricRequest]
			if !ok {
				data = make([]*types.MetricData, 0, 1)
			}
			values[metricRequest] = append(data, metric)
		}
	}

	for m := range targetValues {
		targetValues[m] = values[m]
	}

	if config.Config.ZipperInstance.ScaleToCommonStep() {
		targetValues = helper.ScaleValuesToCommonStep(targetValues)
	}

	return eval.Eval(ctx, exp, from, until, targetValues)
}

// Eval evalualtes expressions
func (eval evaluator) Eval(ctx context.Context, exp parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) (results []*types.MetricData, err error) {
	rewritten, targets, err := RewriteExpr(ctx, exp, from, until, values)
	if err != nil {
		return nil, err
	}
	if rewritten {
		for _, target := range targets {
			exp, _, err = parser.ParseExpr(target)
			if err != nil {
				return nil, err
			}
			result, err := eval.FetchAndEvalExp(ctx, exp, from, until, values)
			if err != nil {
				return nil, err
			}
			results = append(results, result...)
		}
		return results, nil
	}
	return EvalExpr(ctx, exp, from, until, values)
}

var _evaluator = evaluator{}

func init() {
	helper.SetEvaluator(_evaluator)
	metadata.SetEvaluator(_evaluator)
}

// FetchAndEvalExp fetch data and evalualtes expressions
func FetchAndEvalExp(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
	return _evaluator.FetchAndEvalExp(ctx, e, from, until, values)
}

// Eval is the main expression evaluator
func EvalExpr(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
	if e.IsName() {
		return values[parser.MetricRequest{Metric: e.Target(), From: from, Until: until}], nil
	} else if e.IsConst() {
		p := types.MetricData{FetchResponse: pb.FetchResponse{Name: e.Target(), Values: []float64{e.FloatValue()}}}
		return []*types.MetricData{&p}, nil
	}
	// evaluate the function

	// all functions have arguments -- check we do too
	if len(e.Args()) == 0 {
		return nil, parser.ErrMissingArgument
	}

	metadata.FunctionMD.RLock()
	f, ok := metadata.FunctionMD.Functions[e.Target()]
	metadata.FunctionMD.RUnlock()
	if ok {
		v, err := f.Do(ctx, e, from, until, values)
		if err != nil {
			err = merry.WithMessagef(err, "function=%s", e.Target())
		}
		return v, err
	}

	return nil, helper.ErrUnknownFunction(e.Target())
}

// RewriteExpr expands targets that use applyByNode into a new list of targets.
// eg:
// applyByNode(foo*, 1, "%") -> (true, ["foo1", "foo2"], nil)
// sumSeries(foo) -> (false, nil, nil)
// Assumes that applyByNode only appears as the outermost function.
func RewriteExpr(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) (bool, []string, error) {
	if e.IsFunc() {
		metadata.FunctionMD.RLock()
		f, ok := metadata.FunctionMD.RewriteFunctions[e.Target()]
		metadata.FunctionMD.RUnlock()
		if ok {
			return f.Do(ctx, e, from, until, values)
		}
	}
	return false, nil, nil
}
<commit_msg>fix fallbackSeries function<commit_after>package expr

import (
	"context"

	utilctx "github.com/go-graphite/carbonapi/util/ctx"

	"github.com/ansel1/merry"
	"github.com/go-graphite/carbonapi/cmd/carbonapi/config"
	_ "github.com/go-graphite/carbonapi/expr/functions"
	"github.com/go-graphite/carbonapi/expr/helper"
	"github.com/go-graphite/carbonapi/expr/metadata"
	"github.com/go-graphite/carbonapi/expr/types"
	"github.com/go-graphite/carbonapi/pkg/parser"
	pb "github.com/go-graphite/protocol/carbonapi_v3_pb"
)

type evaluator struct{}

// FetchAndEvalExp fetch data and evalualtes expressions
func (eval evaluator) FetchAndEvalExp(ctx context.Context, exp parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
	config.Config.Limiter.Enter()
	defer config.Config.Limiter.Leave()

	multiFetchRequest := pb.MultiFetchRequest{}
	metricRequestCache := make(map[string]parser.MetricRequest)
	maxDataPoints := utilctx.GetMaxDatapoints(ctx)
	// values related to this particular `target=`
	targetValues := make(map[parser.MetricRequest][]*types.MetricData)

	for _, m := range exp.Metrics() {
		fetchRequest := pb.FetchRequest{
			Name:           m.Metric,
			PathExpression: m.Metric,
			StartTime:      m.From + from,
			StopTime:       m.Until + until,
			MaxDataPoints:  maxDataPoints,
		}
		metricRequest := parser.MetricRequest{
			Metric: fetchRequest.PathExpression,
			From:   fetchRequest.StartTime,
			Until:  fetchRequest.StopTime,
		}

		// avoid multiple requests in a function, E.g divideSeries(a.b, a.b)
		if cachedMetricRequest, ok := metricRequestCache[m.Metric]; ok &&
			cachedMetricRequest.From == metricRequest.From &&
			cachedMetricRequest.Until == metricRequest.Until {
			continue
		}

		// avoid multiple requests in a http request, E.g render?target=a.b&target=a.b
		if _, ok := values[metricRequest]; ok {
			targetValues[metricRequest] = nil
			continue
		}

		metricRequestCache[m.Metric] = metricRequest
		targetValues[metricRequest] = nil
		multiFetchRequest.Metrics = append(multiFetchRequest.Metrics, fetchRequest)
	}

	if len(multiFetchRequest.Metrics) > 0 {
		metrics, _, err := config.Config.ZipperInstance.Render(ctx, multiFetchRequest)
		// If we had only partial result, we want to do our best to actually do our job
		if err != nil && merry.HTTPCode(err) >= 400 && exp.Target() != "fallbackSeries" {
			return nil, err
		}
		for _, metric := range metrics {
			metricRequest := metricRequestCache[metric.PathExpression]
			if metric.RequestStartTime != 0 && metric.RequestStopTime != 0 {
				metricRequest.From = metric.RequestStartTime
				metricRequest.Until = metric.RequestStopTime
			}
			data, ok := values[metricRequest]
			if !ok {
				data = make([]*types.MetricData, 0, 1)
			}
			values[metricRequest] = append(data, metric)
		}
	}

	for m := range targetValues {
		targetValues[m] = values[m]
	}

	if config.Config.ZipperInstance.ScaleToCommonStep() {
		targetValues = helper.ScaleValuesToCommonStep(targetValues)
	}

	return eval.Eval(ctx, exp, from, until, targetValues)
}

// Eval evalualtes expressions
func (eval evaluator) Eval(ctx context.Context, exp parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) (results []*types.MetricData, err error) {
	rewritten, targets, err := RewriteExpr(ctx, exp, from, until, values)
	if err != nil {
		return nil, err
	}
	if rewritten {
		for _, target := range targets {
			exp, _, err = parser.ParseExpr(target)
			if err != nil {
				return nil, err
			}
			result, err := eval.FetchAndEvalExp(ctx, exp, from, until, values)
			if err != nil {
				return nil, err
			}
			results = append(results, result...)
		}
		return results, nil
	}
	return EvalExpr(ctx, exp, from, until, values)
}

var _evaluator = evaluator{}

func init() {
	helper.SetEvaluator(_evaluator)
	metadata.SetEvaluator(_evaluator)
}

// FetchAndEvalExp fetch data and evalualtes expressions
func FetchAndEvalExp(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
	return _evaluator.FetchAndEvalExp(ctx, e, from, until, values)
}

// Eval is the main expression evaluator
func EvalExpr(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
	if e.IsName() {
		return values[parser.MetricRequest{Metric: e.Target(), From: from, Until: until}], nil
	} else if e.IsConst() {
		p := types.MetricData{FetchResponse: pb.FetchResponse{Name: e.Target(), Values: []float64{e.FloatValue()}}}
		return []*types.MetricData{&p}, nil
	}
	// evaluate the function

	// all functions have arguments -- check we do too
	if len(e.Args()) == 0 {
		return nil, parser.ErrMissingArgument
	}

	metadata.FunctionMD.RLock()
	f, ok := metadata.FunctionMD.Functions[e.Target()]
	metadata.FunctionMD.RUnlock()
	if ok {
		v, err := f.Do(ctx, e, from, until, values)
		if err != nil {
			err = merry.WithMessagef(err, "function=%s", e.Target())
		}
		return v, err
	}

	return nil, helper.ErrUnknownFunction(e.Target())
}

// RewriteExpr expands targets that use applyByNode into a new list of targets.
// eg:
// applyByNode(foo*, 1, "%") -> (true, ["foo1", "foo2"], nil)
// sumSeries(foo) -> (false, nil, nil)
// Assumes that applyByNode only appears as the outermost function.
func RewriteExpr(ctx context.Context, e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) (bool, []string, error) {
	if e.IsFunc() {
		metadata.FunctionMD.RLock()
		f, ok := metadata.FunctionMD.RewriteFunctions[e.Target()]
		metadata.FunctionMD.RUnlock()
		if ok {
			return f.Do(ctx, e, from, until, values)
		}
	}
	return false, nil, nil
}
<|endoftext|>


<commit_before>package expr

import (
	"fmt"
	"io"
	"strings"

	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/consolidation"
	"github.com/grafana/metrictank/errors"
)

type Optimizations struct {
	PreNormalization bool
	MDP              bool
}

func (o Optimizations) ApplyUserPrefs(s string) (Optimizations, error) {
	// no user override. stick to what we have
	if s == "" {
		return o, nil
	}
	// user passed an override. it's either 'none' (no optimizations) or a list of the ones that should be enabled
	o.PreNormalization = false
	o.MDP = false
	if s == "none" {
		return o, nil
	}
	prefs := strings.Split(s, ",")
	for _, pref := range prefs {
		switch pref {
		case "pn":
			o.PreNormalization = true
		case "mdp":
			o.MDP = true
		default:
			return o, fmt.Errorf("unrecognized optimization %q", pref)
		}
	}
	return o, nil
}

// Req represents a request for one/more series
type Req struct {
	Query   string // whatever was parsed as the query out of a graphite target. e.g. target=sum(foo.{b,a}r.*) -> foo.{b,a}r.* -> this will go straight to index lookup
	From    uint32
	To      uint32
	Cons    consolidation.Consolidator // can be 0 to mean undefined
	PNGroup models.PNGroup
	MDP     uint32 // if we can MDP-optimize, reflects runtime consolidation MaxDataPoints. 0 otherwise.
}

// NewReq creates a new Req. pass cons=0 to leave consolidator undefined,
// leaving up to the caller (in graphite's case, it would cause a lookup into storage-aggregation.conf)
func NewReq(query string, from, to uint32, cons consolidation.Consolidator, PNGroup models.PNGroup, MDP uint32) Req {
	return Req{
		Query:   query,
		From:    from,
		To:      to,
		Cons:    cons,
		PNGroup: PNGroup,
		MDP:     MDP,
	}
}

func NewReqFromContext(query string, c Context) Req {
	r := Req{
		Query: query,
		From:  c.from,
		To:    c.to,
		Cons:  c.consol,
	}
	if c.optimizations.PreNormalization {
		r.PNGroup = c.PNGroup
	}
	if c.optimizations.MDP {
		r.MDP = c.MDP
	}
	return r
}

// NewReqFromSeries generates a Req back from a series
// a models.Series has all the properties attached to it
// to find out which Req it came from
func NewReqFromSerie(serie models.Series) Req {
	return Req{
		Query:   serie.QueryPatt,
		From:    serie.QueryFrom,
		To:      serie.QueryTo,
		Cons:    serie.QueryCons,
		PNGroup: serie.QueryPNGroup,
		MDP:     serie.QueryMDP,
	}

}

func (r Req) ToModel() models.Req {
	return models.Req{
		Pattern:   r.Query,
		From:      r.From,
		To:        r.To,
		MaxPoints: r.MDP,
		PNGroup:   r.PNGroup,
		ConsReq:   r.Cons,
	}
}

type Plan struct {
	Reqs          []Req          // data that needs to be fetched before functions can be executed
	funcs         []GraphiteFunc // top-level funcs to execute, the head of each tree for each target
	exprs         []*expr
	MaxDataPoints uint32
	From          uint32  // global request scoped from
	To            uint32  // global request scoped to
	dataMap       DataMap // set via Run()
}

func (p Plan) Dump(w io.Writer) {
	fmt.Fprintf(w, "Plan:\n")
	fmt.Fprintf(w, "* Exprs:\n")
	for _, e := range p.exprs {
		fmt.Fprintln(w, e.Print(2))
	}
	fmt.Fprintf(w, "* Reqs:\n")

	maxQueryLen := 5
	for _, r := range p.Reqs {
		if len(r.Query) > maxQueryLen {
			maxQueryLen = len(r.Query)
		}
	}
	// ! PNGroups are pointers which can be upto 21 characters long on 64bit
	headPatt := fmt.Sprintf("%%%ds %%12s %%12s %%25s %%21s %%6s\n", maxQueryLen)
	linePatt := fmt.Sprintf("%%%ds %%12d %%12d %%25s %%21d %%6d\n", maxQueryLen)
	fmt.Fprintf(w, headPatt, "query", "from", "to", "consolidator", "PNGroup", "MDP")

	for _, r := range p.Reqs {
		fmt.Fprintf(w, linePatt, r.Query, r.From, r.To, r.Cons, r.PNGroup, r.MDP)
	}
	fmt.Fprintf(w, "MaxDataPoints: %d\n", p.MaxDataPoints)
	fmt.Fprintf(w, "From: %d\n", p.From)
	fmt.Fprintf(w, "To: %d\n", p.To)
}

// NewPlan validates the expressions and comes up with the initial (potentially non-optimal) execution plan
// which is just a list of requests and the expressions.
// traverse tree and as we go down:
// * make sure function exists
// * validation of arguments
// * allow functions to modify the Context (change data range or consolidation)
// * future version: allow functions to mark safe to pre-aggregate using consolidateBy or not
func NewPlan(exprs []*expr, from, to, mdp uint32, stable bool, optimizations Optimizations) (Plan, error) {
	plan := Plan{
		exprs:         exprs,
		MaxDataPoints: mdp,
		From:          from,
		To:            to,
	}
	for _, e := range exprs {
		context := Context{
			from:          from,
			to:            to,
			MDP:           mdp,
			PNGroup:       0, // making this explicit here for easy code grepping
			optimizations: optimizations,
		}
		fn, reqs, err := newplan(e, context, stable, plan.Reqs)
		if err != nil {
			return Plan{}, err
		}
		plan.Reqs = reqs
		plan.funcs = append(plan.funcs, fn)
	}
	return plan, nil
}

// newplan adds requests as needed for the given expr, resolving function calls as needed
func newplan(e *expr, context Context, stable bool, reqs []Req) (GraphiteFunc, []Req, error) {
	if e.etype != etFunc && e.etype != etName {
		return nil, nil, errors.NewBadRequest("request must be a function call or metric pattern")
	}
	if e.etype == etName {
		req := NewReqFromContext(e.str, context)
		reqs = append(reqs, req)
		return NewGet(req), reqs, nil
	} else if e.etype == etFunc && e.str == "seriesByTag" {
		// `seriesByTag` function requires resolving expressions to series
		// (similar to path expressions handled above). Since we need the
		// arguments of seriesByTag to do the resolution, we store the function
		// string back into the Query member of a new request to be parsed later.
		// TODO - find a way to prevent this parse/encode/parse/encode loop
		expressionStr := "seriesByTag(" + e.argsStr + ")"
		req := NewReqFromContext(expressionStr, context)
		reqs = append(reqs, req)
		return NewGet(req), reqs, nil
	}
	// here e.type is guaranteed to be etFunc
	fdef, ok := funcs[e.str]
	if !ok {
		return nil, nil, ErrUnknownFunction(e.str)
	}
	if stable && !fdef.stable {
		return nil, nil, ErrUnknownFunction(e.str)
	}

	fn := fdef.constr()
	reqs, err := newplanFunc(e, fn, context, stable, reqs)
	return fn, reqs, err
}

// newplanFunc adds requests as needed for the given expr, and validates the function input
// provided you already know the expression is a function call to the given function
func newplanFunc(e *expr, fn GraphiteFunc, context Context, stable bool, reqs []Req) ([]Req, error) {
	// first comes the interesting task of validating the arguments as specified by the function,
	// against the arguments that were parsed.

	argsExp, _ := fn.Signature()
	var err error

	// note:
	// * signature may have seriesLists in it, which means one or more args of type seriesList
	//   so it's legal to have more e.args than signature args in that case.
	// * we can't do extensive, accurate validation of the type here because what the output from a function we depend on
	//   might be dynamically typed. e.g. movingAvg returns 1..N series depending on how many it got as input

	// first validate the mandatory args
	pos := 0    // e.args[pos]     : next given arg to process
	cutoff := 0 // argsExp[cutoff] : will be first optional arg (if any)
	var argExp Arg
	for cutoff, argExp = range argsExp {
		if argExp.Optional() {
			break
		}
		if len(e.args) <= pos {
			return nil, ErrMissingArg
		}
		pos, err = e.consumeBasicArg(pos, argExp)
		if err != nil {
			return nil, err
		}
	}
	if !argExp.Optional() {
		cutoff++
	}

	// we stopped iterating the mandatory args.
	// any remaining args should be due to optional args otherwise there's too many
	// we also track here which keywords can also be used for the given optional args
	// so that those args should not be specified via their keys anymore.

	seenKwargs := make(map[string]struct{})
	for _, argOpt := range argsExp[cutoff:] {
		if len(e.args) <= pos {
			break // no more args specified. we're done.
		}
		pos, err = e.consumeBasicArg(pos, argOpt)
		if err != nil {
			return nil, err
		}
		seenKwargs[argOpt.Key()] = struct{}{}
	}
	if len(e.args) > pos {
		return nil, ErrTooManyArg
	}

	// for any provided keyword args, verify that they are what the function stipulated
	// and that they have not already been specified via their position
	for key := range e.namedArgs {
		_, ok := seenKwargs[key]
		if ok {
			return nil, ErrKwargSpecifiedTwice{key}
		}
		err = e.consumeKwarg(key, argsExp[cutoff:])
		if err != nil {
			return nil, err
		}
		seenKwargs[key] = struct{}{}
	}

	// functions now have their non-series input args set,
	// so they should now be able to specify any context alterations
	context = fn.Context(context)
	// now that we know the needed context for the data coming into
	// this function, we can set up the input arguments for the function
	// that are series
	pos = 0
	for _, argExp = range argsExp {
		if pos >= len(e.args) {
			break // no more args specified. we're done.
		}
		switch argExp.(type) {
		case ArgSeries, ArgSeriesList, ArgSeriesLists, ArgIn:
			pos, reqs, err = e.consumeSeriesArg(pos, argExp, context, stable, reqs)
			if err != nil {
				return nil, err
			}
		default:
			pos++
		}
	}
	return reqs, err
}

// Run invokes all processing as specified in the plan (expressions, from/to) against the given datamap
func (p *Plan) Run(dataMap DataMap) ([]models.Series, error) {
	var out []models.Series
	p.dataMap = dataMap
	for _, fn := range p.funcs {
		series, err := fn.Exec(p.dataMap)
		if err != nil {
			return nil, err
		}
		out = append(out, series...)
	}
	for i, o := range out {
		if p.MaxDataPoints != 0 && len(o.Datapoints) > int(p.MaxDataPoints) {
			// series may have been created by a function that didn't know which consolidation function to default to.
			// in the future maybe we can do more clever things here. e.g. perSecond maybe consolidate by max.
			if o.Consolidator == 0 {
				o.Consolidator = consolidation.Avg
			}
			out[i].Datapoints, out[i].Interval = consolidation.ConsolidateNudged(o.Datapoints, o.Interval, p.MaxDataPoints, o.Consolidator)
			out[i].Meta = out[i].Meta.CopyWithChange(func(in models.SeriesMetaProperties) models.SeriesMetaProperties {
				in.AggNumRC = consolidation.AggEvery(uint32(len(o.Datapoints)), p.MaxDataPoints)
				in.ConsolidatorRC = o.Consolidator
				return in
			})
		}
	}
	return out, nil
}

func (p Plan) Clean() {
	p.dataMap.Clean()
}
<commit_msg>deduplicate plan.Reqs<commit_after>package expr

import (
	"fmt"
	"io"
	"strings"

	"github.com/grafana/metrictank/api/models"
	"github.com/grafana/metrictank/consolidation"
	"github.com/grafana/metrictank/errors"
)

type Optimizations struct {
	PreNormalization bool
	MDP              bool
}

func (o Optimizations) ApplyUserPrefs(s string) (Optimizations, error) {
	// no user override. stick to what we have
	if s == "" {
		return o, nil
	}
	// user passed an override. it's either 'none' (no optimizations) or a list of the ones that should be enabled
	o.PreNormalization = false
	o.MDP = false
	if s == "none" {
		return o, nil
	}
	prefs := strings.Split(s, ",")
	for _, pref := range prefs {
		switch pref {
		case "pn":
			o.PreNormalization = true
		case "mdp":
			o.MDP = true
		default:
			return o, fmt.Errorf("unrecognized optimization %q", pref)
		}
	}
	return o, nil
}

// Req represents a request for one/more series
type Req struct {
	Query   string // whatever was parsed as the query out of a graphite target. e.g. target=sum(foo.{b,a}r.*) -> foo.{b,a}r.* -> this will go straight to index lookup
	From    uint32
	To      uint32
	Cons    consolidation.Consolidator // can be 0 to mean undefined
	PNGroup models.PNGroup
	MDP     uint32 // if we can MDP-optimize, reflects runtime consolidation MaxDataPoints. 0 otherwise.
}

// NewReq creates a new Req. pass cons=0 to leave consolidator undefined,
// leaving up to the caller (in graphite's case, it would cause a lookup into storage-aggregation.conf)
func NewReq(query string, from, to uint32, cons consolidation.Consolidator, PNGroup models.PNGroup, MDP uint32) Req {
	return Req{
		Query:   query,
		From:    from,
		To:      to,
		Cons:    cons,
		PNGroup: PNGroup,
		MDP:     MDP,
	}
}

func NewReqFromContext(query string, c Context) Req {
	r := Req{
		Query: query,
		From:  c.from,
		To:    c.to,
		Cons:  c.consol,
	}
	if c.optimizations.PreNormalization {
		r.PNGroup = c.PNGroup
	}
	if c.optimizations.MDP {
		r.MDP = c.MDP
	}
	return r
}

// NewReqFromSeries generates a Req back from a series
// a models.Series has all the properties attached to it
// to find out which Req it came from
func NewReqFromSerie(serie models.Series) Req {
	return Req{
		Query:   serie.QueryPatt,
		From:    serie.QueryFrom,
		To:      serie.QueryTo,
		Cons:    serie.QueryCons,
		PNGroup: serie.QueryPNGroup,
		MDP:     serie.QueryMDP,
	}

}

func (r Req) ToModel() models.Req {
	return models.Req{
		Pattern:   r.Query,
		From:      r.From,
		To:        r.To,
		MaxPoints: r.MDP,
		PNGroup:   r.PNGroup,
		ConsReq:   r.Cons,
	}
}

type Plan struct {
	Reqs          []Req          // data that needs to be fetched before functions can be executed
	funcs         []GraphiteFunc // top-level funcs to execute, the head of each tree for each target
	exprs         []*expr
	MaxDataPoints uint32
	From          uint32  // global request scoped from
	To            uint32  // global request scoped to
	dataMap       DataMap // set via Run()
}

func (p Plan) Dump(w io.Writer) {
	fmt.Fprintf(w, "Plan:\n")
	fmt.Fprintf(w, "* Exprs:\n")
	for _, e := range p.exprs {
		fmt.Fprintln(w, e.Print(2))
	}
	fmt.Fprintf(w, "* Reqs:\n")

	maxQueryLen := 5
	for _, r := range p.Reqs {
		if len(r.Query) > maxQueryLen {
			maxQueryLen = len(r.Query)
		}
	}
	// ! PNGroups are pointers which can be upto 21 characters long on 64bit
	headPatt := fmt.Sprintf("%%%ds %%12s %%12s %%25s %%21s %%6s\n", maxQueryLen)
	linePatt := fmt.Sprintf("%%%ds %%12d %%12d %%25s %%21d %%6d\n", maxQueryLen)
	fmt.Fprintf(w, headPatt, "query", "from", "to", "consolidator", "PNGroup", "MDP")

	for _, r := range p.Reqs {
		fmt.Fprintf(w, linePatt, r.Query, r.From, r.To, r.Cons, r.PNGroup, r.MDP)
	}
	fmt.Fprintf(w, "MaxDataPoints: %d\n", p.MaxDataPoints)
	fmt.Fprintf(w, "From: %d\n", p.From)
	fmt.Fprintf(w, "To: %d\n", p.To)
}

// NewPlan validates the expressions and comes up with the initial (potentially non-optimal) execution plan
// which is just a list of requests and the expressions.
// traverse tree and as we go down:
// * make sure function exists
// * validation of arguments
// * allow functions to modify the Context (change data range or consolidation)
// * future version: allow functions to mark safe to pre-aggregate using consolidateBy or not
func NewPlan(exprs []*expr, from, to, mdp uint32, stable bool, optimizations Optimizations) (Plan, error) {
	plan := Plan{
		exprs:         exprs,
		MaxDataPoints: mdp,
		From:          from,
		To:            to,
	}
	for _, e := range exprs {
		context := Context{
			from:          from,
			to:            to,
			MDP:           mdp,
			PNGroup:       0, // making this explicit here for easy code grepping
			optimizations: optimizations,
		}
		fn, reqs, err := newplan(e, context, stable, plan.Reqs)
		if err != nil {
			return Plan{}, err
		}
		plan.Reqs = reqs
		plan.funcs = append(plan.funcs, fn)
	}
	return plan, nil
}

// newplan adds requests as needed for the given expr, resolving function calls as needed
func newplan(e *expr, context Context, stable bool, reqs []Req) (GraphiteFunc, []Req, error) {

	// surpress duplicate queries such as target=foo&target=foo
	// note that unless `pre-normalization = false`,
	// this cannot surpress duplicate reqs in these cases:
	// target=foo&target=sum(foo)      // reqs are different, one has a PNGroup set
	// target=sum(foo)&target=sum(foo) // reqs get different PNGroups
	// perhaps in the future we can improve on this and
	// deduplicate the largest common (sub)expressions

	addReqIfNew := func(req Req) {
		for _, r := range reqs {
			if r == req {
				return
			}
		}
		reqs = append(reqs, req)
	}

	if e.etype != etFunc && e.etype != etName {
		return nil, nil, errors.NewBadRequest("request must be a function call or metric pattern")
	}
	if e.etype == etName {
		req := NewReqFromContext(e.str, context)
		addReqIfNew(req)
		return NewGet(req), reqs, nil
	} else if e.etype == etFunc && e.str == "seriesByTag" {
		// `seriesByTag` function requires resolving expressions to series
		// (similar to path expressions handled above). Since we need the
		// arguments of seriesByTag to do the resolution, we store the function
		// string back into the Query member of a new request to be parsed later.
		// TODO - find a way to prevent this parse/encode/parse/encode loop
		expressionStr := "seriesByTag(" + e.argsStr + ")"
		req := NewReqFromContext(expressionStr, context)
		addReqIfNew(req)
		return NewGet(req), reqs, nil
	}
	// here e.type is guaranteed to be etFunc
	fdef, ok := funcs[e.str]
	if !ok {
		return nil, nil, ErrUnknownFunction(e.str)
	}
	if stable && !fdef.stable {
		return nil, nil, ErrUnknownFunction(e.str)
	}

	fn := fdef.constr()
	reqs, err := newplanFunc(e, fn, context, stable, reqs)
	return fn, reqs, err
}

// newplanFunc adds requests as needed for the given expr, and validates the function input
// provided you already know the expression is a function call to the given function
func newplanFunc(e *expr, fn GraphiteFunc, context Context, stable bool, reqs []Req) ([]Req, error) {
	// first comes the interesting task of validating the arguments as specified by the function,
	// against the arguments that were parsed.

	argsExp, _ := fn.Signature()
	var err error

	// note:
	// * signature may have seriesLists in it, which means one or more args of type seriesList
	//   so it's legal to have more e.args than signature args in that case.
	// * we can't do extensive, accurate validation of the type here because what the output from a function we depend on
	//   might be dynamically typed. e.g. movingAvg returns 1..N series depending on how many it got as input

	// first validate the mandatory args
	pos := 0    // e.args[pos]     : next given arg to process
	cutoff := 0 // argsExp[cutoff] : will be first optional arg (if any)
	var argExp Arg
	for cutoff, argExp = range argsExp {
		if argExp.Optional() {
			break
		}
		if len(e.args) <= pos {
			return nil, ErrMissingArg
		}
		pos, err = e.consumeBasicArg(pos, argExp)
		if err != nil {
			return nil, err
		}
	}
	if !argExp.Optional() {
		cutoff++
	}

	// we stopped iterating the mandatory args.
	// any remaining args should be due to optional args otherwise there's too many
	// we also track here which keywords can also be used for the given optional args
	// so that those args should not be specified via their keys anymore.

	seenKwargs := make(map[string]struct{})
	for _, argOpt := range argsExp[cutoff:] {
		if len(e.args) <= pos {
			break // no more args specified. we're done.
		}
		pos, err = e.consumeBasicArg(pos, argOpt)
		if err != nil {
			return nil, err
		}
		seenKwargs[argOpt.Key()] = struct{}{}
	}
	if len(e.args) > pos {
		return nil, ErrTooManyArg
	}

	// for any provided keyword args, verify that they are what the function stipulated
	// and that they have not already been specified via their position
	for key := range e.namedArgs {
		_, ok := seenKwargs[key]
		if ok {
			return nil, ErrKwargSpecifiedTwice{key}
		}
		err = e.consumeKwarg(key, argsExp[cutoff:])
		if err != nil {
			return nil, err
		}
		seenKwargs[key] = struct{}{}
	}

	// functions now have their non-series input args set,
	// so they should now be able to specify any context alterations
	context = fn.Context(context)
	// now that we know the needed context for the data coming into
	// this function, we can set up the input arguments for the function
	// that are series
	pos = 0
	for _, argExp = range argsExp {
		if pos >= len(e.args) {
			break // no more args specified. we're done.
		}
		switch argExp.(type) {
		case ArgSeries, ArgSeriesList, ArgSeriesLists, ArgIn:
			pos, reqs, err = e.consumeSeriesArg(pos, argExp, context, stable, reqs)
			if err != nil {
				return nil, err
			}
		default:
			pos++
		}
	}
	return reqs, err
}

// Run invokes all processing as specified in the plan (expressions, from/to) against the given datamap
func (p *Plan) Run(dataMap DataMap) ([]models.Series, error) {
	var out []models.Series
	p.dataMap = dataMap
	for _, fn := range p.funcs {
		series, err := fn.Exec(p.dataMap)
		if err != nil {
			return nil, err
		}
		out = append(out, series...)
	}
	for i, o := range out {
		if p.MaxDataPoints != 0 && len(o.Datapoints) > int(p.MaxDataPoints) {
			// series may have been created by a function that didn't know which consolidation function to default to.
			// in the future maybe we can do more clever things here. e.g. perSecond maybe consolidate by max.
			if o.Consolidator == 0 {
				o.Consolidator = consolidation.Avg
			}
			out[i].Datapoints, out[i].Interval = consolidation.ConsolidateNudged(o.Datapoints, o.Interval, p.MaxDataPoints, o.Consolidator)
			out[i].Meta = out[i].Meta.CopyWithChange(func(in models.SeriesMetaProperties) models.SeriesMetaProperties {
				in.AggNumRC = consolidation.AggEvery(uint32(len(o.Datapoints)), p.MaxDataPoints)
				in.ConsolidatorRC = o.Consolidator
				return in
			})
		}
	}
	return out, nil
}

func (p Plan) Clean() {
	p.dataMap.Clean()
}
<|endoftext|>


<commit_before>// Copyright 2013 Marc Weistroff. All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package gogol

import (
	"strings"
)

// A formatter formats the record before being sent to a Handler
// The result of the formatting MUST be in Record.Formatted
type Formatter interface {
	Format(*Record)
}

const (
	LINE_FORMAT_SIMPLE  string = "[%datetime%] %channel%.%level_name%: %message% %extra%\n"
	LINE_FORMAT_MINIMAL string = "%channel%.%level_name%: %message%\n"
)

// A line formatter formats a Record into a line of text.
// Available formats are LINE_FORMAT_SIMPLE, LINE_FORMAT_MINIMAL or you can make your own
// with these fields:
// %datetime%: Record's creation date in the time.RFC3339Nano format
// $channel%: logger.Name
// %level_name%: Severity's name (DEBUG, WARNING, ...)
// %message%: Message text
// %extra%: All extra values, generally added by Processors
type LineFormatter struct {
	LineFormat string
}

// Instantiates a new LineFormatter with the LINE_FORMAT_MINIMAL format
func NewMinimalLineFormatter() Formatter {
	return &LineFormatter{LineFormat: LINE_FORMAT_MINIMAL}
}

// Instantiates a new LineFormatter with the LINE_FORMAT_SIMPLE format
func NewSimpleLineFormatter() Formatter {
	return &LineFormatter{LineFormat: LINE_FORMAT_SIMPLE}
}

// Format the Record r with f.LineFormat
func (f *LineFormatter) Format(r *Record) {
	replacer := strings.NewReplacer(
		"%datetime%", r.Time,
		"%channel%", r.Channel,
		"%level_name%", r.LevelName,
		"%message%", r.Message,
		"%extra%", r.Extra.String())

	r.Formatted = replacer.Replace(f.LineFormat)
}
<commit_msg>Change line formatter constant name<commit_after>// Copyright 2013 Marc Weistroff. All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package gogol

import (
	"strings"
)

// A formatter formats the record before being sent to a Handler
// The result of the formatting MUST be in Record.Formatted
type Formatter interface {
	Format(*Record)
}

const (
	LineFormatSimple  string = "[%datetime%] %channel%.%level_name%: %message% %extra%\n"
	LineFormatMinimal string = "%channel%.%level_name%: %message%\n"
)

// A line formatter formats a Record into a line of text.
// Available formats are LINE_FORMAT_SIMPLE, LINE_FORMAT_MINIMAL or you can make your own
// with these fields:
// %datetime%: Record's creation date in the time.RFC3339Nano format
// $channel%: logger.Name
// %level_name%: Severity's name (DEBUG, WARNING, ...)
// %message%: Message text
// %extra%: All extra values, generally added by Processors
type LineFormatter struct {
	LineFormat string
}

// Instantiates a new LineFormatter with the LINE_FORMAT_MINIMAL format
func NewMinimalLineFormatter() Formatter {
	return &LineFormatter{LineFormat: LineFormatMinimal}
}

// Instantiates a new LineFormatter with the LINE_FORMAT_SIMPLE format
func NewSimpleLineFormatter() Formatter {
	return &LineFormatter{LineFormat: LineFormatSimple}
}

// Format the Record r with f.LineFormat
func (f *LineFormatter) Format(r *Record) {
	replacer := strings.NewReplacer(
		"%datetime%", r.Time,
		"%channel%", r.Channel,
		"%level_name%", r.LevelName,
		"%message%", r.Message,
		"%extra%", r.Extra.String())

	r.Formatted = replacer.Replace(f.LineFormat)
}
<|endoftext|>


<commit_before>package broker

import (
	"encoding/json"
	"os"
	"sync"
	"time"

	"github.com/asim/mq/go/client"
)

var (
	Default Broker = newBroker()
)

// internal broker
type broker struct {
	exit    chan bool
	options *Options

	sync.RWMutex
	topics map[string][]chan []byte

	mtx       sync.RWMutex
	persisted map[string]bool
}

// internal message for persistence
type message struct {
	Timestamp int64  `json:"timestamp"`
	Topic     string `json:"topic"`
	Payload   []byte `json:"payload"`
}

// Broker is the message broker
type Broker interface {
	Close() error
	Publish(topic string, payload []byte) error
	Subscribe(topic string) (<-chan []byte, error)
	Unsubscribe(topic string, sub <-chan []byte) error
}

func newBroker(opts ...Option) *broker {
	options := &Options{
		Client: client.New(),
	}

	for _, o := range opts {
		o(options)
	}

	return &broker{
		exit:      make(chan bool),
		options:   options,
		topics:    make(map[string][]chan []byte),
		persisted: make(map[string]bool),
	}
}

func (b *broker) publish(payload []byte, subscribers []chan []byte) {
	n := len(subscribers)
	c := 1

	// increase concurrency if there are many subscribers
	switch {
	case n > 1000:
		c = 3
	case n > 100:
		c = 2
	}

	// publisher function
	pub := func(start int) {
		// iterate the subscribers
		for j := start; j < n; j += c {
			select {
			// push the payload to subscriber
			case subscribers[j] <- payload:
			// only wait 5 milliseconds for subscriber
			case <-time.After(time.Millisecond * 5):
			case <-b.exit:
				return
			}
		}
	}

	// concurrent publish
	for i := 0; i < c; i++ {
		go pub(i)
	}
}

func (b *broker) persist(topic string) error {
	b.mtx.Lock()
	defer b.mtx.Unlock()

	if b.persisted[topic] {
		return nil
	}

	ch, err := b.Subscribe(topic)
	if err != nil {
		return err
	}

	f, err := os.OpenFile(topic+".mq", os.O_CREATE|os.O_RDWR|os.O_APPEND, 0660)
	if err != nil {
		return err
	}

	go func() {
		var pending []byte
		newline := []byte{'\n'}
		t := time.NewTicker(time.Second)
		defer t.Stop()
		defer f.Close()

		for {
			select {
			case p := <-ch:
				b, err := json.Marshal(&message{
					Timestamp: time.Now().UnixNano(),
					Topic:     topic,
					Payload:   p,
				})
				if err != nil {
					continue
				}
				pending = append(pending, b...)
				pending = append(pending, newline...)
			case <-t.C:
				if len(pending) == 0 {
					continue
				}
				f.Write(pending)
				pending = nil
			case <-b.exit:
				return
			}
		}
	}()

	b.persisted[topic] = true

	return nil
}

func (b *broker) Close() error {
	select {
	case <-b.exit:
		return nil
	default:
		close(b.exit)
		b.Lock()
		b.topics = make(map[string][]chan []byte)
		b.Unlock()
	}
	return nil
}

func (b *broker) Publish(topic string, payload []byte) error {
	if b.options.Proxy {
		return b.options.Client.Publish(topic, payload)
	}

	b.RLock()
	subscribers, ok := b.topics[topic]
	b.RUnlock()
	if !ok {
		// persist?
		if !b.options.Persist {
			return nil
		}
		if err := b.persist(topic); err != nil {
			return err
		}
	}

	b.publish(payload, subscribers)
	return nil
}

func (b *broker) Subscribe(topic string) (<-chan []byte, error) {
	if b.options.Proxy {
		return b.options.Client.Subscribe(topic)
	}

	ch := make(chan []byte, 100)
	b.Lock()
	b.topics[topic] = append(b.topics[topic], ch)
	b.Unlock()
	return ch, nil
}

func (b *broker) Unsubscribe(topic string, sub <-chan []byte) error {
	if b.options.Proxy {
		return b.options.Client.Unsubscribe(sub)
	}

	b.RLock()
	subscribers, ok := b.topics[topic]
	b.RUnlock()

	if !ok {
		return nil
	}

	var subs []chan []byte
	for _, subscriber := range subscribers {
		if subscriber == sub {
			continue
		}
		subs = append(subs, subscriber)
	}

	b.Lock()
	b.topics[topic] = subs
	b.Unlock()
	return nil
}

func Publish(topic string, payload []byte) error {
	return Default.Publish(topic, payload)
}

func Subscribe(topic string) (<-chan []byte, error) {
	return Default.Subscribe(topic)
}

func Unsubscribe(topic string, sub <-chan []byte) error {
	return Default.Unsubscribe(topic, sub)
}

func New(opts ...Option) *broker {
	return newBroker(opts...)
}
<commit_msg>do nothing if closed<commit_after>package broker

import (
	"encoding/json"
	"errors"
	"os"
	"sync"
	"time"

	"github.com/asim/mq/go/client"
)

var (
	Default Broker = newBroker()
)

// internal broker
type broker struct {
	exit    chan bool
	options *Options

	sync.RWMutex
	topics map[string][]chan []byte

	mtx       sync.RWMutex
	persisted map[string]bool
}

// internal message for persistence
type message struct {
	Timestamp int64  `json:"timestamp"`
	Topic     string `json:"topic"`
	Payload   []byte `json:"payload"`
}

// Broker is the message broker
type Broker interface {
	Close() error
	Publish(topic string, payload []byte) error
	Subscribe(topic string) (<-chan []byte, error)
	Unsubscribe(topic string, sub <-chan []byte) error
}

func newBroker(opts ...Option) *broker {
	options := &Options{
		Client: client.New(),
	}

	for _, o := range opts {
		o(options)
	}

	return &broker{
		exit:      make(chan bool),
		options:   options,
		topics:    make(map[string][]chan []byte),
		persisted: make(map[string]bool),
	}
}

func (b *broker) publish(payload []byte, subscribers []chan []byte) {
	n := len(subscribers)
	c := 1

	// increase concurrency if there are many subscribers
	switch {
	case n > 1000:
		c = 3
	case n > 100:
		c = 2
	}

	// publisher function
	pub := func(start int) {
		// iterate the subscribers
		for j := start; j < n; j += c {
			select {
			// push the payload to subscriber
			case subscribers[j] <- payload:
			// only wait 5 milliseconds for subscriber
			case <-time.After(time.Millisecond * 5):
			case <-b.exit:
				return
			}
		}
	}

	// concurrent publish
	for i := 0; i < c; i++ {
		go pub(i)
	}
}

func (b *broker) persist(topic string) error {
	b.mtx.Lock()
	defer b.mtx.Unlock()

	if b.persisted[topic] {
		return nil
	}

	ch, err := b.Subscribe(topic)
	if err != nil {
		return err
	}

	f, err := os.OpenFile(topic+".mq", os.O_CREATE|os.O_RDWR|os.O_APPEND, 0660)
	if err != nil {
		return err
	}

	go func() {
		var pending []byte
		newline := []byte{'\n'}
		t := time.NewTicker(time.Second)
		defer t.Stop()
		defer f.Close()

		for {
			select {
			case p := <-ch:
				b, err := json.Marshal(&message{
					Timestamp: time.Now().UnixNano(),
					Topic:     topic,
					Payload:   p,
				})
				if err != nil {
					continue
				}
				pending = append(pending, b...)
				pending = append(pending, newline...)
			case <-t.C:
				if len(pending) == 0 {
					continue
				}
				f.Write(pending)
				pending = nil
			case <-b.exit:
				return
			}
		}
	}()

	b.persisted[topic] = true

	return nil
}

func (b *broker) Close() error {
	select {
	case <-b.exit:
		return nil
	default:
		close(b.exit)
		b.Lock()
		b.topics = make(map[string][]chan []byte)
		b.Unlock()
	}
	return nil
}

func (b *broker) Publish(topic string, payload []byte) error {
	select {
	case <-b.exit:
		return errors.New("broker closed")
	default:
	}

	if b.options.Proxy {
		return b.options.Client.Publish(topic, payload)
	}

	b.RLock()
	subscribers, ok := b.topics[topic]
	b.RUnlock()
	if !ok {
		// persist?
		if !b.options.Persist {
			return nil
		}
		if err := b.persist(topic); err != nil {
			return err
		}
	}

	b.publish(payload, subscribers)
	return nil
}

func (b *broker) Subscribe(topic string) (<-chan []byte, error) {
	select {
	case <-b.exit:
		return nil, errors.New("broker closed")
	default:
	}

	if b.options.Proxy {
		return b.options.Client.Subscribe(topic)
	}

	ch := make(chan []byte, 100)
	b.Lock()
	b.topics[topic] = append(b.topics[topic], ch)
	b.Unlock()
	return ch, nil
}

func (b *broker) Unsubscribe(topic string, sub <-chan []byte) error {
	select {
	case <-b.exit:
		return errors.New("broker closed")
	default:
	}

	if b.options.Proxy {
		return b.options.Client.Unsubscribe(sub)
	}

	b.RLock()
	subscribers, ok := b.topics[topic]
	b.RUnlock()

	if !ok {
		return nil
	}

	var subs []chan []byte
	for _, subscriber := range subscribers {
		if subscriber == sub {
			continue
		}
		subs = append(subs, subscriber)
	}

	b.Lock()
	b.topics[topic] = subs
	b.Unlock()
	return nil
}

func Publish(topic string, payload []byte) error {
	return Default.Publish(topic, payload)
}

func Subscribe(topic string) (<-chan []byte, error) {
	return Default.Subscribe(topic)
}

func Unsubscribe(topic string, sub <-chan []byte) error {
	return Default.Unsubscribe(topic, sub)
}

func New(opts ...Option) *broker {
	return newBroker(opts...)
}
<|endoftext|>


<commit_before>package broker

import (
	"errors"
	"net"
	"sync"
	"sync/atomic"
	"time"

	"h12.me/kpax/model"
)

type B struct {
	Addr     string
	Timeout  time.Duration
	QueueLen int

	br *broker
	mu sync.Mutex
}

func New(addr string) *B {
	b := &B{
		Addr:     addr,
		Timeout:  30 * time.Second,
		QueueLen: 1000,
	}
	return b
}

func NewDefault(addr string) model.Broker { return New(addr) }

func (b *B) Do(req model.Request, resp model.Response) error {
	var err error
	b.mu.Lock()
	if b.br == nil {
		b.br, err = b.newBroker()
	}
	b.mu.Unlock()
	if err != nil {
		return err
	}

	if err := b.br.do(req, resp); err != nil {
		b.Close()
		return err
	}
	return nil
}

func (b *B) Close() {
	b.mu.Lock()
	b.br.close()
	b.br = nil
	b.mu.Unlock()
}

type broker struct {
	timeout  time.Duration
	conn     net.Conn
	recvChan chan *brokerJob
	cid      int32
	mu       sync.Mutex
}

type brokerJob struct {
	req     model.Request
	resp    model.Response
	errChan chan error
}

func (b *B) newBroker() (*broker, error) {
	conn, err := net.DialTimeout("tcp", b.Addr, b.Timeout)
	if err != nil {
		return nil, err
	}
	br := &broker{
		timeout:  b.Timeout,
		conn:     conn,
		recvChan: make(chan *brokerJob, b.QueueLen),
	}
	go br.receiveLoop()
	return br, nil
}

func (b *broker) do(req model.Request, resp model.Response) error {
	job := &brokerJob{
		req:     req,
		resp:    resp,
		errChan: make(chan error),
	}
	if err := b.send(job); err != nil {
		return err
	}
	if job.requireAck() {
		return <-job.errChan
	}
	return nil
}

func (b *broker) send(job *brokerJob) error {
	b.mu.Lock()
	defer b.mu.Unlock()
	job.req.SetID(atomic.AddInt32(&b.cid, 1))
	if err := b.conn.SetWriteDeadline(time.Now().Add(b.timeout)); err != nil {
		return err
	}
	if err := job.req.Send(b.conn); err != nil {
		return err
	}
	if job.requireAck() {
		b.recvChan <- job
	}
	return nil
}

var (
	errCorrelationIDMismatch = errors.New("correlationID mismatch")
)

func (b *broker) receiveLoop() {
	for job := range b.recvChan {
		if err := b.conn.SetReadDeadline(time.Now().Add(b.timeout)); err != nil {
			job.errChan <- err
			continue
		}
		if err := job.resp.Receive(b.conn); err != nil {
			job.errChan <- err
			continue
		}
		if job.resp.ID() != job.req.ID() {
			job.errChan <- errCorrelationIDMismatch
			continue
		}
		job.errChan <- nil
	}
	// no need to closeConn here because when recvChan closed, the receiveLoop will do it.
	b.conn.Close()
	b.conn = nil
}

func (b *broker) close() {
	close(b.recvChan)
	b.recvChan = nil
}

func (j *brokerJob) requireAck() bool { return j.resp != nil }
<commit_msg>fix error handling logic<commit_after>package broker

import (
	"errors"
	"net"
	"sync"
	"sync/atomic"
	"time"

	"h12.me/kpax/model"
)

type B struct {
	Addr     string
	Timeout  time.Duration
	QueueLen int

	mu sync.Mutex
	br *broker
}

func New(addr string) *B {
	b := &B{
		Addr:     addr,
		Timeout:  30 * time.Second,
		QueueLen: 1000,
	}
	return b
}

func NewDefault(addr string) model.Broker { return New(addr) }

func (b *B) Do(req model.Request, resp model.Response) error {
	var (
		br  *broker
		err error
	)
	b.mu.Lock()
	if b.br == nil {
		b.br, err = b.newBroker()
	}
	br = b.br
	b.mu.Unlock()
	if err != nil {
		return err
	}

	if err := br.do(req, resp); err != nil {
		b.mu.Lock()
		if b.br == br {
			b.br = nil
		}
		b.mu.Unlock()
		br.close()
		return err
	}
	return nil
}

func (b *B) Close() {
	b.mu.Lock()
	b.br.close()
	b.br = nil
	b.mu.Unlock()
}

type broker struct {
	timeout time.Duration
	conn    net.Conn
	cid     int32

	mu       sync.Mutex
	recvChan chan *brokerJob
}

type brokerJob struct {
	req     model.Request
	resp    model.Response
	errChan chan error
}

func (b *B) newBroker() (*broker, error) {
	conn, err := net.DialTimeout("tcp", b.Addr, b.Timeout)
	if err != nil {
		return nil, err
	}
	br := &broker{
		timeout:  b.Timeout,
		conn:     conn,
		recvChan: make(chan *brokerJob, b.QueueLen),
	}
	go br.receiveLoop()
	return br, nil
}

func (b *broker) do(req model.Request, resp model.Response) error {
	job := &brokerJob{
		req:     req,
		resp:    resp,
		errChan: make(chan error),
	}
	if err := b.send(job); err != nil {
		return err
	}
	if job.requireAck() {
		return <-job.errChan
	}
	return nil
}

func (b *broker) close() {
	b.mu.Lock()
	if b.recvChan != nil {
		close(b.recvChan)
		b.recvChan = nil
	}
	b.mu.Unlock()
}

func (b *broker) send(job *brokerJob) error {
	b.mu.Lock()
	defer b.mu.Unlock()
	if b.recvChan == nil {
		return errChannelAlreadyClosed
	}
	job.req.SetID(atomic.AddInt32(&b.cid, 1))
	if err := b.conn.SetWriteDeadline(time.Now().Add(b.timeout)); err != nil {
		return err
	}
	if err := job.req.Send(b.conn); err != nil {
		return err
	}
	if job.requireAck() {
		b.recvChan <- job
	}
	return nil
}

var (
	errCorrelationIDMismatch = errors.New("correlationID mismatch")
	errChannelAlreadyClosed  = errors.New("channel already closed")
)

func (b *broker) receiveLoop() {
	for job := range b.recvChan {
		if err := b.conn.SetReadDeadline(time.Now().Add(b.timeout)); err != nil {
			job.errChan <- err
			continue
		}
		if err := job.resp.Receive(b.conn); err != nil {
			job.errChan <- err
			continue
		}
		if job.resp.ID() != job.req.ID() {
			job.errChan <- errCorrelationIDMismatch
			continue
		}
		job.errChan <- nil
	}
	// no need to closeConn here because when recvChan closed, the receiveLoop will do it.
	b.conn.Close()
	b.conn = nil
}

func (j *brokerJob) requireAck() bool { return j.resp != nil }
<|endoftext|>


<commit_before>package frank

import (
	"code.google.com/p/go.net/html"
	frankconf "github.com/breunigs/frank/config"
	irc "github.com/fluffle/goirc/client"
	rss "github.com/jteeuwen/go-pkg-rss"
	"log"
	"strconv"
	"time"
)

// how often to check the feeds (in minutes)
const checkEvery = 3

// ignore all posts that are older than X minutes
const freshness = 90

// if there’s an error reading a feed, retry after X minutes
const retryAfter = 9

// how many items to show if there have been many updates in an interval
const maxItems = 2

// reference time: Mon Jan 2 15:04:05 -0700 MST 2006
const timeFormat1 = time.RFC1123Z
const timeFormat2 = "2006-01-02T15:04:05Z"
const timeFormat3 = "2006-01-02T15:04:05-07:00"

var conn *irc.Conn

var ignoreBefore = time.Now()

var rssHttpClient = HttpClientWithTimeout()

func Rss(connection *irc.Conn) {
	conn = connection
	// this feels wrong, the missing alignment making it hard to read.
	// Does anybody have a suggestion how to make this nice in go?
	//~ go pollFeed("#i3-test", "i3", timeFormat2, "http://code.stapelberg.de/git/i3/atom/?h=next")
	go pollFeed("#i3", "i3lock", timeFormat2, "http://code.stapelberg.de/git/i3lock/atom/?h=master")
	go pollFeed("#i3", "i3status", timeFormat2, "http://code.stapelberg.de/git/i3status/atom/?h=master")
	go pollFeed("#i3", "i3website", timeFormat2, "http://code.stapelberg.de/git/i3-website/atom/?h=master")
	go pollFeed("#i3", "i3faq", timeFormat1, "https://faq.i3wm.org/feeds/rss/")

	go pollFeed("#chaos-hd", "nn-web", timeFormat3, "https://www.noname-ev.de/gitcommits.atom")
	go pollFeed("#chaos-hd", "nn-wiki", timeFormat2, "https://www.noname-ev.de/wiki/index.php?title=Special:RecentChanges&feed=atom")
	go pollFeed("#chaos-hd", "nn-planet", timeFormat2, "http://blogs.noname-ev.de/atom.xml")
	go pollFeed("#chaos-hd", "frank", timeFormat3, "https://github.com/breunigs/frank/commits/master.atom")
}

func pollFeed(channel string, feedName string, timeFormat string, uri string) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("MEGA-WTF:pkg:RSS: %v\n", r)
			time.Sleep(retryAfter * time.Minute)
			pollFeed(channel, feedName, timeFormat, uri)
		}
	}()

	if frankconf.Verbose {
		log.Printf("RSS %s: Setting up %s to post to %s \n", feedName, uri, channel)
	}

	// this will process all incoming new feed items and discard all that
	// are somehow erroneous or older than the threshold. It will directly
	// post any updates.
	itemHandler := func(feed *rss.Feed, ch *rss.Channel, newitems []*rss.Item) {
		log.Printf("RSS: %d new item(s) in %s\n", len(newitems), feedName)

		postitems := []string{}

		for _, item := range newitems {
			pubdate, err := time.Parse(timeFormat, item.PubDate)
			// ignore items with unreadable date format
			if err != nil {
				log.Printf("RSS: WTF @ reading date for %s: %s (err: %v)\n", feedName, item.PubDate, err)
				continue
			}

			// ignore items that were posted before frank booted or are older
			// than “freshness” minutes
			if ignoreBefore.After(pubdate) {
				log.Printf("RSS %s: skipping posts made before booting (posted: %s, booted: %s)\n", feedName, pubdate, ignoreBefore)
				continue
			}
			if time.Since(pubdate) >= freshness*time.Minute {
				log.Printf("RSS %s: skipping non-fresh post (posted: %s, time_ago: %s)\n", feedName, pubdate, time.Since(pubdate))
				continue
			}

			url := ""
			if len(item.Links) > 0 {
				url = item.Links[0].Href
			}

			if url != "" && isRecentUrl(url) {
				if frankconf.Verbose {
					log.Printf("RSS %s: Skipping item because saved as recent URL (URL: %s)\n", feedName, url)
				}
				continue
			}

			if url != "" {
				addRecentUrl(url)
				url = " @ " + url
			}

			author := html.UnescapeString(item.Author.Name)
			title := html.UnescapeString(item.Title)

			if author == "" {
				postitems = appendIfMiss(postitems, "::"+feedName+":: "+title+url)
			} else {
				postitems = appendIfMiss(postitems, "::"+feedName+":: "+title+url+" (by "+author+")")
			}
		}

		cnt := len(postitems)

		// hide updates if they exceed the maxItems counter. If there’s only
		// one more item in the list than specified in maxItems, all of the
		// items will be printed – otherwise that item would be replaced by
		// a useless message that it has been hidden.
		if cnt > maxItems+1 {
			cntS := strconv.Itoa(cnt)
			maxS := strconv.Itoa(maxItems)
			msg := "::" + feedName + ":: had " + cntS + " updates, showing the latest " + maxS
			conn.Privmsg(channel, msg)
			postitems = postitems[cnt-maxItems : cnt]
		}

		// newer items appear first in feeds, so reverse them here to keep
		// the order in line with how IRC wprks
		for i := len(postitems) - 1; i >= 0; i -= 1 {
			conn.Privmsg(channel, postitems[i])
			log.Printf("RSS %s: posting %s\n", feedName, postitems[i])
		}
	}

	// create the feed listener/updater
	feed := rss.New(checkEvery, true, chanHandler, itemHandler)

	// check for updates infinite loop
	for {
		if frankconf.Verbose {
			t := feed.LastUpdate().Format(time.RFC3339)
			log.Printf("RSS %s: Updating now (previous update: %s, refresh ok: %s)\n", feedName, t, feed.CanUpdate())
		}

		if err := feed.FetchClient(uri, &rssHttpClient, nil); err != nil {
			log.Printf("RSS %s: Error for %s: %s\n", feedName, uri, err)
			time.Sleep(retryAfter * time.Minute)
			continue
		}

		<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
	}
}

// unused default handler
func chanHandler(feed *rss.Feed, newchannels []*rss.Channel) {
	log.Printf("RSS: %d new channel(s) in %s\n", len(newchannels), feed.Url)
}

// append string to slice only if it’s not already present.
func appendIfMiss(slice []string, s string) []string {
	for _, elm := range slice {
		if elm == s {
			if frankconf.Verbose {
				log.Printf("RSS: Not adding “%s” because it is already present\n", s)
			}
			return slice
		}
	}
	return append(slice, s)
}

// LIFO that stores the recent posted URLs.
// Used to avoid posting entries multiple times that have been
// erroneously detected as new by the RSS library.

var recent []string = make([]string, 50)
var recentIndex = 0

func addRecentUrl(url string) {
	recent[recentIndex] = url
	recentIndex += 1
	if len(recent) == recentIndex {
		recentIndex = 0
	}
}

func isRecentUrl(url string) bool {
	for _, a := range recent {
		if url == a {
			return true
		}
	}
	return false
}
<commit_msg>remove outdated feeds<commit_after>package frank

import (
	"code.google.com/p/go.net/html"
	frankconf "github.com/breunigs/frank/config"
	irc "github.com/fluffle/goirc/client"
	rss "github.com/jteeuwen/go-pkg-rss"
	"log"
	"strconv"
	"time"
)

// how often to check the feeds (in minutes)
const checkEvery = 3

// ignore all posts that are older than X minutes
const freshness = 90

// if there’s an error reading a feed, retry after X minutes
const retryAfter = 9

// how many items to show if there have been many updates in an interval
const maxItems = 2

// reference time: Mon Jan 2 15:04:05 -0700 MST 2006
const timeFormat1 = time.RFC1123Z
const timeFormat2 = "2006-01-02T15:04:05Z"
const timeFormat3 = "2006-01-02T15:04:05-07:00"

var conn *irc.Conn

var ignoreBefore = time.Now()

var rssHttpClient = HttpClientWithTimeout()

func Rss(connection *irc.Conn) {
	conn = connection
	// this feels wrong, the missing alignment making it hard to read.
	// Does anybody have a suggestion how to make this nice in go?
	//~ go pollFeed("#i3-test", "i3", timeFormat2, "http://code.stapelberg.de/git/i3/atom/?h=next")
	go pollFeed("#i3", "i3website", timeFormat2, "http://code.stapelberg.de/git/i3-website/atom/?h=master")
	go pollFeed("#i3", "i3faq", timeFormat1, "https://faq.i3wm.org/feeds/rss/")
}

func pollFeed(channel string, feedName string, timeFormat string, uri string) {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("MEGA-WTF:pkg:RSS: %v\n", r)
			time.Sleep(retryAfter * time.Minute)
			pollFeed(channel, feedName, timeFormat, uri)
		}
	}()

	if frankconf.Verbose {
		log.Printf("RSS %s: Setting up %s to post to %s \n", feedName, uri, channel)
	}

	// this will process all incoming new feed items and discard all that
	// are somehow erroneous or older than the threshold. It will directly
	// post any updates.
	itemHandler := func(feed *rss.Feed, ch *rss.Channel, newitems []*rss.Item) {
		log.Printf("RSS: %d new item(s) in %s\n", len(newitems), feedName)

		postitems := []string{}

		for _, item := range newitems {
			pubdate, err := time.Parse(timeFormat, item.PubDate)
			// ignore items with unreadable date format
			if err != nil {
				log.Printf("RSS: WTF @ reading date for %s: %s (err: %v)\n", feedName, item.PubDate, err)
				continue
			}

			// ignore items that were posted before frank booted or are older
			// than “freshness” minutes
			if ignoreBefore.After(pubdate) {
				log.Printf("RSS %s: skipping posts made before booting (posted: %s, booted: %s)\n", feedName, pubdate, ignoreBefore)
				continue
			}
			if time.Since(pubdate) >= freshness*time.Minute {
				log.Printf("RSS %s: skipping non-fresh post (posted: %s, time_ago: %s)\n", feedName, pubdate, time.Since(pubdate))
				continue
			}

			url := ""
			if len(item.Links) > 0 {
				url = item.Links[0].Href
			}

			if url != "" && isRecentUrl(url) {
				if frankconf.Verbose {
					log.Printf("RSS %s: Skipping item because saved as recent URL (URL: %s)\n", feedName, url)
				}
				continue
			}

			if url != "" {
				addRecentUrl(url)
				url = " @ " + url
			}

			author := html.UnescapeString(item.Author.Name)
			title := html.UnescapeString(item.Title)

			if author == "" {
				postitems = appendIfMiss(postitems, "::"+feedName+":: "+title+url)
			} else {
				postitems = appendIfMiss(postitems, "::"+feedName+":: "+title+url+" (by "+author+")")
			}
		}

		cnt := len(postitems)

		// hide updates if they exceed the maxItems counter. If there’s only
		// one more item in the list than specified in maxItems, all of the
		// items will be printed – otherwise that item would be replaced by
		// a useless message that it has been hidden.
		if cnt > maxItems+1 {
			cntS := strconv.Itoa(cnt)
			maxS := strconv.Itoa(maxItems)
			msg := "::" + feedName + ":: had " + cntS + " updates, showing the latest " + maxS
			conn.Privmsg(channel, msg)
			postitems = postitems[cnt-maxItems : cnt]
		}

		// newer items appear first in feeds, so reverse them here to keep
		// the order in line with how IRC wprks
		for i := len(postitems) - 1; i >= 0; i -= 1 {
			conn.Privmsg(channel, postitems[i])
			log.Printf("RSS %s: posting %s\n", feedName, postitems[i])
		}
	}

	// create the feed listener/updater
	feed := rss.New(checkEvery, true, chanHandler, itemHandler)

	// check for updates infinite loop
	for {
		if frankconf.Verbose {
			t := feed.LastUpdate().Format(time.RFC3339)
			log.Printf("RSS %s: Updating now (previous update: %s, refresh ok: %s)\n", feedName, t, feed.CanUpdate())
		}

		if err := feed.FetchClient(uri, &rssHttpClient, nil); err != nil {
			log.Printf("RSS %s: Error for %s: %s\n", feedName, uri, err)
			time.Sleep(retryAfter * time.Minute)
			continue
		}

		<-time.After(time.Duration(feed.SecondsTillUpdate() * 1e9))
	}
}

// unused default handler
func chanHandler(feed *rss.Feed, newchannels []*rss.Channel) {
	log.Printf("RSS: %d new channel(s) in %s\n", len(newchannels), feed.Url)
}

// append string to slice only if it’s not already present.
func appendIfMiss(slice []string, s string) []string {
	for _, elm := range slice {
		if elm == s {
			if frankconf.Verbose {
				log.Printf("RSS: Not adding “%s” because it is already present\n", s)
			}
			return slice
		}
	}
	return append(slice, s)
}

// LIFO that stores the recent posted URLs.
// Used to avoid posting entries multiple times that have been
// erroneously detected as new by the RSS library.

var recent []string = make([]string, 50)
var recentIndex = 0

func addRecentUrl(url string) {
	recent[recentIndex] = url
	recentIndex += 1
	if len(recent) == recentIndex {
		recentIndex = 0
	}
}

func isRecentUrl(url string) bool {
	for _, a := range recent {
		if url == a {
			return true
		}
	}
	return false
}
<|endoftext|>