<commit_before>package tflint

import (
	"fmt"
	"log"
	"net"
	"net/rpc"

	hcl "github.com/hashicorp/hcl/v2"
	"github.com/zclconf/go-cty/cty"
	"github.com/zclconf/go-cty/cty/gocty"
)

// Client is an RPC client for plugins to query the host process for Terraform configurations
// Actually, it is an RPC client, but its details are hidden on the plugin side because it satisfies the Runner interface
type Client struct {
	rpcClient *rpc.Client
}

// NewClient returns a new Client
func NewClient(conn net.Conn) *Client {
	return &Client{rpcClient: rpc.NewClient(conn)}
}

// AttributesRequest is the interface used to communicate via RPC.
type AttributesRequest struct {
	Resource      string
	AttributeName string
}

// AttributesResponse is the interface used to communicate via RPC.
type AttributesResponse struct {
	Attributes hcl.Attributes
	Err        error
}

// WalkResourceAttributes queries the host process, receives a list of attributes that match the conditions,
// and passes each to the walker function.
func (c *Client) WalkResourceAttributes(resource, attributeName string, walker func(*hcl.Attribute) error) error {
	log.Printf("[DEBUG] Walk `%s.*.%s` attribute", resource, attributeName)

	var response AttributesResponse
	if err := c.rpcClient.Call("Plugin.Attributes", AttributesRequest{Resource: resource, AttributeName: attributeName}, &response); err != nil {
		return err
	}
	if response.Err != nil {
		return response.Err
	}

	for _, attribute := range response.Attributes {
		if err := walker(attribute); err != nil {
			return err
		}
	}

	return nil
}

// EvalExprRequest is the interface used to communicate via RPC.
type EvalExprRequest struct {
	Expr hcl.Expression
	Ret  interface{}
}

// EvalExprResponse is the interface used to communicate with RPC.
type EvalExprResponse struct {
	Val cty.Value
	Err error
}

// EvaluateExpr queries the host process for the result of evaluating the value of the passed expression
// and reflects it as the value of the second argument based on that.
func (c *Client) EvaluateExpr(expr hcl.Expression, ret interface{}) error {
	var response EvalExprResponse
	var err error

	if err := c.rpcClient.Call("Plugin.EvalExpr", EvalExprRequest{Expr: expr, Ret: ret}, &response); err != nil {
		return err
	}
	if response.Err != nil {
		return response.Err
	}

	err = gocty.FromCtyValue(response.Val, ret)
	if err != nil {
		err := &Error{
			Code:  TypeMismatchError,
			Level: ErrorLevel,
			Message: fmt.Sprintf(
				"Invalid type expression in %s:%d",
				expr.Range().Filename,
				expr.Range().Start.Line,
			),
			Cause: err,
		}
		log.Printf("[ERROR] %s", err)
		return err
	}
	return nil
}

// EmitIssueRequest is the interface used to communicate via RPC.
type EmitIssueRequest struct {
	Rule     *RuleObject
	Message  string
	Location hcl.Range
	Meta     Metadata
}

// EmitIssue emits attributes to build the issue to the host process
// Note that the passed rule need to be converted to generic objects
// because the custom structure defined in the plugin cannot be sent via RPC.
func (c *Client) EmitIssue(rule Rule, message string, location hcl.Range, meta Metadata) error {
	req := &EmitIssueRequest{
		Rule:     newObjectFromRule(rule),
		Message:  message,
		Location: location,
		Meta:     meta,
	}
	if err := c.rpcClient.Call("Plugin.EmitIssue", &req, new(interface{})); err != nil {
		return err
	}
	return nil
}

// EnsureNoError is a helper for processing when no error occurs
// This function skips processing without returning an error to the caller when the error is warning
func (*Client) EnsureNoError(err error, proc func() error) error {
	if err == nil {
		return proc()
	}

	if appErr, ok := err.(Error); ok {
		switch appErr.Level {
		case WarningLevel:
			return nil
		case ErrorLevel:
			return appErr
		default:
			panic(appErr)
		}
	} else {
		return err
	}
}
<commit_msg>Attributes are slice<commit_after>package tflint

import (
	"fmt"
	"log"
	"net"
	"net/rpc"

	hcl "github.com/hashicorp/hcl/v2"
	"github.com/zclconf/go-cty/cty"
	"github.com/zclconf/go-cty/cty/gocty"
)

// Client is an RPC client for plugins to query the host process for Terraform configurations
// Actually, it is an RPC client, but its details are hidden on the plugin side because it satisfies the Runner interface
type Client struct {
	rpcClient *rpc.Client
}

// NewClient returns a new Client
func NewClient(conn net.Conn) *Client {
	return &Client{rpcClient: rpc.NewClient(conn)}
}

// AttributesRequest is the interface used to communicate via RPC.
type AttributesRequest struct {
	Resource      string
	AttributeName string
}

// AttributesResponse is the interface used to communicate via RPC.
type AttributesResponse struct {
	Attributes []*hcl.Attribute
	Err        error
}

// WalkResourceAttributes queries the host process, receives a list of attributes that match the conditions,
// and passes each to the walker function.
func (c *Client) WalkResourceAttributes(resource, attributeName string, walker func(*hcl.Attribute) error) error {
	log.Printf("[DEBUG] Walk `%s.*.%s` attribute", resource, attributeName)

	var response AttributesResponse
	if err := c.rpcClient.Call("Plugin.Attributes", AttributesRequest{Resource: resource, AttributeName: attributeName}, &response); err != nil {
		return err
	}
	if response.Err != nil {
		return response.Err
	}

	for _, attribute := range response.Attributes {
		if err := walker(attribute); err != nil {
			return err
		}
	}

	return nil
}

// EvalExprRequest is the interface used to communicate via RPC.
type EvalExprRequest struct {
	Expr hcl.Expression
	Ret  interface{}
}

// EvalExprResponse is the interface used to communicate with RPC.
type EvalExprResponse struct {
	Val cty.Value
	Err error
}

// EvaluateExpr queries the host process for the result of evaluating the value of the passed expression
// and reflects it as the value of the second argument based on that.
func (c *Client) EvaluateExpr(expr hcl.Expression, ret interface{}) error {
	var response EvalExprResponse
	var err error

	if err := c.rpcClient.Call("Plugin.EvalExpr", EvalExprRequest{Expr: expr, Ret: ret}, &response); err != nil {
		return err
	}
	if response.Err != nil {
		return response.Err
	}

	err = gocty.FromCtyValue(response.Val, ret)
	if err != nil {
		err := &Error{
			Code:  TypeMismatchError,
			Level: ErrorLevel,
			Message: fmt.Sprintf(
				"Invalid type expression in %s:%d",
				expr.Range().Filename,
				expr.Range().Start.Line,
			),
			Cause: err,
		}
		log.Printf("[ERROR] %s", err)
		return err
	}
	return nil
}

// EmitIssueRequest is the interface used to communicate via RPC.
type EmitIssueRequest struct {
	Rule     *RuleObject
	Message  string
	Location hcl.Range
	Meta     Metadata
}

// EmitIssue emits attributes to build the issue to the host process
// Note that the passed rule need to be converted to generic objects
// because the custom structure defined in the plugin cannot be sent via RPC.
func (c *Client) EmitIssue(rule Rule, message string, location hcl.Range, meta Metadata) error {
	req := &EmitIssueRequest{
		Rule:     newObjectFromRule(rule),
		Message:  message,
		Location: location,
		Meta:     meta,
	}
	if err := c.rpcClient.Call("Plugin.EmitIssue", &req, new(interface{})); err != nil {
		return err
	}
	return nil
}

// EnsureNoError is a helper for processing when no error occurs
// This function skips processing without returning an error to the caller when the error is warning
func (*Client) EnsureNoError(err error, proc func() error) error {
	if err == nil {
		return proc()
	}

	if appErr, ok := err.(Error); ok {
		switch appErr.Level {
		case WarningLevel:
			return nil
		case ErrorLevel:
			return appErr
		default:
			panic(appErr)
		}
	} else {
		return err
	}
}
<|endoftext|>


<commit_before>package processor

import (
	"errors"
	"fmt"
	"log"
	"sync"
	"sync/atomic"
	"time"

	"golang.org/x/time/rate"

	"gopkg.in/queue.v1"
)

const consumerBackoff = time.Second
const maxBackoff = 12 * time.Hour

type Limiter interface {
	AllowRate(name string, limit rate.Limit) (delay time.Duration, allow bool)
}

type Delayer interface {
	Delay() time.Duration
}

type Stats struct {
	InFlight    uint32
	Deleting    uint32
	Processed   uint32
	Retries     uint32
	Fails       uint32
	AvgDuration time.Duration
}

type Processor struct {
	q   Queuer
	opt *Options

	handler         queue.Handler
	fallbackHandler queue.Handler

	ch chan *queue.Message
	wg sync.WaitGroup

	delLimit chan struct{}
	delCh    chan *queue.Message
	delWG    sync.WaitGroup

	stop uint32

	inFlight    uint32
	deleting    uint32
	processed   uint32
	fails       uint32
	retries     uint32
	avgDuration uint32
}

func New(q Queuer, opt *Options) *Processor {
	opt.init()

	p := &Processor{
		q:   q,
		opt: opt,

		ch: make(chan *queue.Message, opt.BufferSize),

		delLimit: make(chan struct{}, opt.Scavengers),
		delCh:    make(chan *queue.Message, opt.BufferSize),
	}
	p.SetHandler(opt.Handler)
	if opt.FallbackHandler != nil {
		p.SetFallbackHandler(opt.FallbackHandler)
	}
	return p
}

func Start(q Queuer, opt *Options) *Processor {
	p := New(q, opt)
	p.Start()
	return p
}

func (p *Processor) Start() error {
	p.wg.Add(1)
	go p.messageFetcher()

	p.startWorkers()

	p.delWG.Add(1)
	go p.messageDeleter()

	return nil
}

func (p *Processor) String() string {
	return fmt.Sprintf(
		"Processor<%s workers=%d scavengers=%d buffer=%d>",
		p.q.Name(), p.opt.Workers, p.opt.Scavengers, p.opt.BufferSize,
	)
}

func (p *Processor) Stats() *Stats {
	if p.stopped() {
		return nil
	}
	return &Stats{
		InFlight:    atomic.LoadUint32(&p.inFlight),
		Deleting:    atomic.LoadUint32(&p.deleting),
		Processed:   atomic.LoadUint32(&p.processed),
		Retries:     atomic.LoadUint32(&p.retries),
		Fails:       atomic.LoadUint32(&p.fails),
		AvgDuration: time.Duration(atomic.LoadUint32(&p.avgDuration)) * time.Millisecond,
	}
}

func (p *Processor) SetHandler(handler interface{}) {
	p.handler = queue.NewHandler(handler)
}

func (p *Processor) SetFallbackHandler(handler interface{}) {
	p.fallbackHandler = queue.NewHandler(handler)
}

func (p *Processor) AddMessage(msg *queue.Message) error {
	p.ch <- msg
	return nil
}

func (p *Processor) startWorkers() {
	p.wg.Add(p.opt.Workers)
	for i := 0; i < p.opt.Workers; i++ {
		go p.worker()
	}
}

func (p *Processor) Stop() error {
	return p.StopTimeout(30 * time.Second)
}

func (p *Processor) StopTimeout(timeout time.Duration) error {
	atomic.StoreUint32(&p.stop, 1)
	return p.waitWorkers(timeout)
}

func (p *Processor) stopped() bool {
	return atomic.LoadUint32(&p.stop) == 1
}

func (p *Processor) waitWorkers(timeout time.Duration) error {
	stopped := make(chan struct{})
	go func() {
		p.wg.Wait()

		close(p.delCh)
		p.delWG.Wait()

		close(stopped)
	}()

	select {
	case <-time.After(timeout):
		return fmt.Errorf("workers did not stop after %s seconds", timeout)
	case <-stopped:
		return nil
	}
}

func (p *Processor) ProcessAll() error {
	p.startWorkers()
	var noWork int
	for {
		isIdle := atomic.LoadUint32(&p.inFlight) == 0
		n, err := p.fetchMessages()
		if err != nil {
			return err
		}
		if n == 0 && isIdle {
			noWork++
		} else {
			noWork = 0
		}
		if noWork == 2 {
			break
		}
	}
	close(p.ch)
	return p.waitWorkers(time.Minute)
}

func (p *Processor) ProcessOne() error {
	msgs, err := p.q.ReserveN(1)
	if err != nil {
		return err
	}
	if len(msgs) == 0 {
		return errors.New("no messages in queue")
	}
	return p.Process(&msgs[0])
}

func (p *Processor) messageFetcher() {
	defer p.wg.Done()
	for {
		if p.stopped() {
			close(p.ch)
			break
		}

		_, err := p.fetchMessages()
		if err != nil {
			log.Printf("%s ReserveN failed: %s (sleeping for %s)", p.q, err, consumerBackoff)
			time.Sleep(consumerBackoff)
			continue
		}
	}
}

func (p *Processor) fetchMessages() (int, error) {
	msgs, err := p.q.ReserveN(p.opt.BufferSize)
	if err != nil {
		return 0, err
	}
	atomic.AddUint32(&p.inFlight, uint32(len(msgs)))
	for i := range msgs {
		p.ch <- &msgs[i]
	}
	return len(msgs), nil
}

func (p *Processor) messageDeleter() {
	defer p.delWG.Done()
	var msgs []*queue.Message
	for {
		var stop, timeout bool
		select {
		case msg, ok := <-p.delCh:
			if ok {
				msgs = append(msgs, msg)
			} else {
				stop = true
			}
		case <-time.After(time.Second):
			timeout = true
		}

		if (timeout && len(msgs) > 0) || len(msgs) >= 10 {
			p.delLimit <- struct{}{}
			p.delWG.Add(1)
			go p.deleteBatch(msgs)
			msgs = nil
		}

		if stop {
			break
		}
	}
}

func (p *Processor) worker() {
	defer p.wg.Done()
	for {
		if p.opt != nil && p.opt.Limiter != nil {
			delay, allow := p.opt.Limiter.AllowRate(p.q.Name(), p.opt.RateLimit)
			if !allow {
				time.Sleep(delay)
				continue
			}
		}

		msg, ok := <-p.ch
		if !ok {
			break
		}

		if msg.Delay > 0 {
			p.release(msg, nil)
			continue
		}

		p.Process(msg)
	}
}

func (p *Processor) Process(msg *queue.Message) error {
	start := time.Now()
	err := p.handler.HandleMessage(msg)
	p.updateAvgDuration(time.Since(start))

	if err == nil {
		atomic.AddUint32(&p.processed, 1)
		p.delete(msg, nil)
		return nil
	}

	if msg.ReservedCount < p.opt.Retries {
		atomic.AddUint32(&p.retries, 1)
		p.release(msg, err)
	} else {
		atomic.AddUint32(&p.fails, 1)
		p.delete(msg, err)
	}

	return err
}

func (p *Processor) release(msg *queue.Message, reason error) {
	delay := p.backoff(msg, reason)

	log.Printf("%s handler failed (retry in %s): %s", p.q, delay, reason)
	if err := p.q.Release(msg, delay); err != nil {
		log.Printf("%s Release failed: %s", p.q, err)
	}

	atomic.AddUint32(&p.inFlight, ^uint32(0))
}

func (p *Processor) backoff(msg *queue.Message, reason error) time.Duration {
	if reason != nil {
		if delayer, ok := reason.(Delayer); ok {
			return delayer.Delay()
		}
	}
	if msg.Delay > 0 {
		return msg.Delay
	}
	return exponentialBackoff(p.opt.Backoff, msg.ReservedCount)
}

func (p *Processor) delete(msg *queue.Message, reason error) {
	if reason != nil {
		log.Printf("%s handler failed: %s", p.q, reason)

		if p.fallbackHandler != nil {
			if err := p.fallbackHandler.HandleMessage(msg); err != nil {
				log.Printf("%s fallback handler failed: %s", p.q, err)
			}
		}
	}

	select {
	case p.delCh <- msg:
		atomic.AddUint32(&p.inFlight, ^uint32(0))
		atomic.AddUint32(&p.deleting, 1)
		return
	default:
	}

	if err := p.q.Delete(msg); err != nil {
		log.Printf("%s Delete failed: %s", p.q, err)
	}
	atomic.AddUint32(&p.inFlight, ^uint32(0))
}

func (p *Processor) deleteBatch(msgs []*queue.Message) {
	defer func() {
		p.delWG.Done()
		<-p.delLimit
	}()
	if err := p.q.DeleteBatch(msgs); err != nil {
		log.Printf("%s DeleteBatch failed: %s", p.q, err)
	}
	atomic.AddUint32(&p.deleting, ^uint32(len(msgs)-1))
}

func (p *Processor) updateAvgDuration(dur time.Duration) {
	const decay = float64(1) / 100

	ms := float64(dur / time.Millisecond)
	for {
		avg := atomic.LoadUint32(&p.avgDuration)
		newAvg := uint32((1-decay)*float64(avg) + decay*ms)
		if atomic.CompareAndSwapUint32(&p.avgDuration, avg, newAvg) {
			break
		}
	}
}

func exponentialBackoff(dur time.Duration, retry int) time.Duration {
	dur <<= uint(retry - 1)
	if dur > maxBackoff {
		dur = maxBackoff
	}
	return dur
}
<commit_msg>processor: start deletes in ProcessAll.<commit_after>package processor

import (
	"errors"
	"fmt"
	"log"
	"sync"
	"sync/atomic"
	"time"

	"golang.org/x/time/rate"

	"gopkg.in/queue.v1"
)

const consumerBackoff = time.Second
const maxBackoff = 12 * time.Hour

type Limiter interface {
	AllowRate(name string, limit rate.Limit) (delay time.Duration, allow bool)
}

type Delayer interface {
	Delay() time.Duration
}

type Stats struct {
	InFlight    uint32
	Deleting    uint32
	Processed   uint32
	Retries     uint32
	Fails       uint32
	AvgDuration time.Duration
}

type Processor struct {
	q   Queuer
	opt *Options

	handler         queue.Handler
	fallbackHandler queue.Handler

	ch chan *queue.Message
	wg sync.WaitGroup

	delLimit chan struct{}
	delCh    chan *queue.Message
	delWG    sync.WaitGroup

	stop uint32

	inFlight    uint32
	deleting    uint32
	processed   uint32
	fails       uint32
	retries     uint32
	avgDuration uint32
}

func New(q Queuer, opt *Options) *Processor {
	opt.init()

	p := &Processor{
		q:   q,
		opt: opt,

		ch: make(chan *queue.Message, opt.BufferSize),

		delLimit: make(chan struct{}, opt.Scavengers),
		delCh:    make(chan *queue.Message, opt.BufferSize),
	}
	p.SetHandler(opt.Handler)
	if opt.FallbackHandler != nil {
		p.SetFallbackHandler(opt.FallbackHandler)
	}
	return p
}

func Start(q Queuer, opt *Options) *Processor {
	p := New(q, opt)
	p.Start()
	return p
}

func (p *Processor) Start() error {
	p.wg.Add(1)
	go p.messageFetcher()

	p.startWorkers()

	return nil
}

func (p *Processor) String() string {
	return fmt.Sprintf(
		"Processor<%s workers=%d scavengers=%d buffer=%d>",
		p.q.Name(), p.opt.Workers, p.opt.Scavengers, p.opt.BufferSize,
	)
}

func (p *Processor) Stats() *Stats {
	if p.stopped() {
		return nil
	}
	return &Stats{
		InFlight:    atomic.LoadUint32(&p.inFlight),
		Deleting:    atomic.LoadUint32(&p.deleting),
		Processed:   atomic.LoadUint32(&p.processed),
		Retries:     atomic.LoadUint32(&p.retries),
		Fails:       atomic.LoadUint32(&p.fails),
		AvgDuration: time.Duration(atomic.LoadUint32(&p.avgDuration)) * time.Millisecond,
	}
}

func (p *Processor) SetHandler(handler interface{}) {
	p.handler = queue.NewHandler(handler)
}

func (p *Processor) SetFallbackHandler(handler interface{}) {
	p.fallbackHandler = queue.NewHandler(handler)
}

func (p *Processor) AddMessage(msg *queue.Message) error {
	p.ch <- msg
	return nil
}

func (p *Processor) startWorkers() {
	p.wg.Add(p.opt.Workers)
	for i := 0; i < p.opt.Workers; i++ {
		go p.worker()
	}

	p.delWG.Add(1)
	go p.messageDeleter()
}

func (p *Processor) Stop() error {
	return p.StopTimeout(30 * time.Second)
}

func (p *Processor) StopTimeout(timeout time.Duration) error {
	atomic.StoreUint32(&p.stop, 1)
	return p.waitWorkers(timeout)
}

func (p *Processor) stopped() bool {
	return atomic.LoadUint32(&p.stop) == 1
}

func (p *Processor) waitWorkers(timeout time.Duration) error {
	stopped := make(chan struct{})
	go func() {
		p.wg.Wait()

		close(p.delCh)
		p.delWG.Wait()

		close(stopped)
	}()

	select {
	case <-time.After(timeout):
		return fmt.Errorf("workers did not stop after %s seconds", timeout)
	case <-stopped:
		return nil
	}
}

func (p *Processor) ProcessAll() error {
	p.startWorkers()
	var noWork int
	for {
		isIdle := atomic.LoadUint32(&p.inFlight) == 0
		n, err := p.fetchMessages()
		if err != nil {
			return err
		}
		if n == 0 && isIdle {
			noWork++
		} else {
			noWork = 0
		}
		if noWork == 2 {
			break
		}
	}
	close(p.ch)
	return p.waitWorkers(time.Minute)
}

func (p *Processor) ProcessOne() error {
	msgs, err := p.q.ReserveN(1)
	if err != nil {
		return err
	}
	if len(msgs) == 0 {
		return errors.New("no messages in queue")
	}
	return p.Process(&msgs[0])
}

func (p *Processor) messageFetcher() {
	defer p.wg.Done()
	for {
		if p.stopped() {
			close(p.ch)
			break
		}

		_, err := p.fetchMessages()
		if err != nil {
			log.Printf("%s ReserveN failed: %s (sleeping for %s)", p.q, err, consumerBackoff)
			time.Sleep(consumerBackoff)
			continue
		}
	}
}

func (p *Processor) fetchMessages() (int, error) {
	msgs, err := p.q.ReserveN(p.opt.BufferSize)
	if err != nil {
		return 0, err
	}
	atomic.AddUint32(&p.inFlight, uint32(len(msgs)))
	for i := range msgs {
		p.ch <- &msgs[i]
	}
	return len(msgs), nil
}

func (p *Processor) messageDeleter() {
	defer p.delWG.Done()
	var msgs []*queue.Message
	for {
		var stop, timeout bool
		select {
		case msg, ok := <-p.delCh:
			if ok {
				msgs = append(msgs, msg)
			} else {
				stop = true
			}
		case <-time.After(time.Second):
			timeout = true
		}

		if (timeout && len(msgs) > 0) || len(msgs) >= 10 {
			p.delLimit <- struct{}{}
			p.delWG.Add(1)
			go p.deleteBatch(msgs)
			msgs = nil
		}

		if stop {
			break
		}
	}
}

func (p *Processor) worker() {
	defer p.wg.Done()
	for {
		if p.opt != nil && p.opt.Limiter != nil {
			delay, allow := p.opt.Limiter.AllowRate(p.q.Name(), p.opt.RateLimit)
			if !allow {
				time.Sleep(delay)
				continue
			}
		}

		msg, ok := <-p.ch
		if !ok {
			break
		}

		if msg.Delay > 0 {
			p.release(msg, nil)
			continue
		}

		p.Process(msg)
	}
}

func (p *Processor) Process(msg *queue.Message) error {
	start := time.Now()
	err := p.handler.HandleMessage(msg)
	p.updateAvgDuration(time.Since(start))

	if err == nil {
		atomic.AddUint32(&p.processed, 1)
		p.delete(msg, nil)
		return nil
	}

	if msg.ReservedCount < p.opt.Retries {
		atomic.AddUint32(&p.retries, 1)
		p.release(msg, err)
	} else {
		atomic.AddUint32(&p.fails, 1)
		p.delete(msg, err)
	}

	return err
}

func (p *Processor) release(msg *queue.Message, reason error) {
	delay := p.backoff(msg, reason)

	log.Printf("%s handler failed (retry in %s): %s", p.q, delay, reason)
	if err := p.q.Release(msg, delay); err != nil {
		log.Printf("%s Release failed: %s", p.q, err)
	}

	atomic.AddUint32(&p.inFlight, ^uint32(0))
}

func (p *Processor) backoff(msg *queue.Message, reason error) time.Duration {
	if reason != nil {
		if delayer, ok := reason.(Delayer); ok {
			return delayer.Delay()
		}
	}
	if msg.Delay > 0 {
		return msg.Delay
	}
	return exponentialBackoff(p.opt.Backoff, msg.ReservedCount)
}

func (p *Processor) delete(msg *queue.Message, reason error) {
	if reason != nil {
		log.Printf("%s handler failed: %s", p.q, reason)

		if p.fallbackHandler != nil {
			if err := p.fallbackHandler.HandleMessage(msg); err != nil {
				log.Printf("%s fallback handler failed: %s", p.q, err)
			}
		}
	}

	select {
	case p.delCh <- msg:
		atomic.AddUint32(&p.inFlight, ^uint32(0))
		atomic.AddUint32(&p.deleting, 1)
		return
	default:
	}

	if err := p.q.Delete(msg); err != nil {
		log.Printf("%s Delete failed: %s", p.q, err)
	}
	atomic.AddUint32(&p.inFlight, ^uint32(0))
}

func (p *Processor) deleteBatch(msgs []*queue.Message) {
	defer func() {
		p.delWG.Done()
		<-p.delLimit
	}()
	if err := p.q.DeleteBatch(msgs); err != nil {
		log.Printf("%s DeleteBatch failed: %s", p.q, err)
	}
	atomic.AddUint32(&p.deleting, ^uint32(len(msgs)-1))
}

func (p *Processor) updateAvgDuration(dur time.Duration) {
	const decay = float64(1) / 100

	ms := float64(dur / time.Millisecond)
	for {
		avg := atomic.LoadUint32(&p.avgDuration)
		newAvg := uint32((1-decay)*float64(avg) + decay*ms)
		if atomic.CompareAndSwapUint32(&p.avgDuration, avg, newAvg) {
			break
		}
	}
}

func exponentialBackoff(dur time.Duration, retry int) time.Duration {
	dur <<= uint(retry - 1)
	if dur > maxBackoff {
		dur = maxBackoff
	}
	return dur
}
<|endoftext|>


<commit_before>/*

	golden is a package designed to make it possible to compare a game to a
	golden run for testing purposes. It takes a record saved in
	storage/filesystem format and compares it.

*/
package golden

import (
	"errors"
	"github.com/jkomoros/boardgame"
	"github.com/jkomoros/boardgame/storage/filesystem/record"
	"github.com/jkomoros/boardgame/storage/memory"
	"github.com/yudai/gojsondiff"
	"github.com/yudai/gojsondiff/formatter"
	"io/ioutil"
	"path/filepath"
	"reflect"
	"strconv"
)

//Compare is the primary method in the package. It takes a game delegate and a
//filename denoting a record to compare against. delegate shiould be a fresh
//delegate not yet affiliated with a manager. It compares every version and
//move in the history (ignoring things that shouldn't be the same, like
//timestamps) and reports the first place they divrge. Any time it finds a
//move not proposed by AdminPlayerIndex it will propose that move. As long as
//your game uses state.Rand() for all randomness and is otherwise
//deterministic then everything should work.
func Compare(delegate boardgame.GameDelegate, recFilename string) error {

	manager, err := boardgame.NewGameManager(delegate, memory.NewStorageManager())

	if err != nil {
		return errors.New("Couldn't create new manager: " + err.Error())
	}

	rec, err := record.New(recFilename)

	if err != nil {
		return errors.New("Couldn't create record: " + err.Error())
	}

	return compare(manager, rec)

}

//CompareFolder is like Compare, except it will iterate through any file in
//recFolder that ends in .json. Errors if any of those files cannot be parsed
//into recs, or if no files match.
func CompareFolder(delegate boardgame.GameDelegate, recFolder string) error {
	manager, err := boardgame.NewGameManager(delegate, memory.NewStorageManager())

	if err != nil {
		return errors.New("Couldn't create new manager: " + err.Error())
	}

	infos, err := ioutil.ReadDir(recFolder)

	if err != nil {
		return errors.New("Couldn't read folder: " + err.Error())
	}

	processedRecs := 0

	for _, info := range infos {
		if info.IsDir() {
			continue
		}

		if filepath.Ext(info.Name()) != ".json" {
			continue
		}

		rec, err := record.New(filepath.Join(recFolder, info.Name()))

		if err != nil {
			return errors.New("File with name " + info.Name() + " couldn't be loaded into rec: " + err.Error())
		}

		if err := compare(manager, rec); err != nil {
			return errors.New("File named " + info.Name() + " had compare error: " + err.Error())
		}

		processedRecs++
	}

	if processedRecs < 1 {
		return errors.New("Processed 0 recs in folder")
	}

	return nil
}

func compare(manager *boardgame.GameManager, rec *record.Record) error {
	game, err := manager.RecreateGame(rec.Game())

	if err != nil {
		return errors.New("Couldn't create game: " + err.Error())
	}

	lastVerifiedVersion := 0

	for !game.Finished() {
		//Verify all new moves that have happened since the last time we
		//checked (often, fix-up moves).
		for lastVerifiedVersion < game.Version() {
			stateToCompare, err := rec.State(lastVerifiedVersion)

			if err != nil {
				return errors.New("Couldn't get " + strconv.Itoa(lastVerifiedVersion) + " state: " + err.Error())
			}

			if err := compareJsonBlobs(game.State(lastVerifiedVersion).StorageRecord(), stateToCompare); err != nil {
				return errors.New("State " + strconv.Itoa(lastVerifiedVersion) + " compared differently: " + err.Error())
			}

			if lastVerifiedVersion > 0 {

				//Version 0 has no associated move

				recMove, err := rec.Move(lastVerifiedVersion)

				if err != nil {
					return errors.New("Couldn't get move " + strconv.Itoa(lastVerifiedVersion) + " from record")
				}

				moves := game.MoveRecords(lastVerifiedVersion)

				if len(moves) < 1 {
					return errors.New("Didn't fetch historical move records for " + strconv.Itoa(lastVerifiedVersion))
				}

				//Warning: records are modified by this method
				if err := compareMoveStorageRecords(moves[len(moves)-1], recMove); err != nil {
					return errors.New("Move " + strconv.Itoa(lastVerifiedVersion) + " compared differently: " + err.Error())
				}
			}

			lastVerifiedVersion++
		}

		nextMoveRec, err := rec.Move(lastVerifiedVersion + 1)

		if err != nil {
			//We'll assume that menas that's all of the moves there are to make.
			break
		}

		if nextMoveRec.Proposer < 0 {
			return errors.New("At version " + strconv.Itoa(lastVerifiedVersion) + " the next player move to apply was not applied by a player")
		}

		nextMove, err := nextMoveRec.Inflate(game)

		if err != nil {
			return errors.New("Couldn't inflate move: " + err.Error())
		}

		if err := <-game.ProposeMove(nextMove, nextMoveRec.Proposer); err != nil {
			return errors.New("Couldn't propose next move in chain: " + err.Error())
		}

	}

	if game.Finished() != rec.Game().Finished {
		return errors.New("Game finished did not match rec")
	}

	if !reflect.DeepEqual(game.Winners(), rec.Game().Winners) {
		return errors.New("Game winners did not match")
	}

	return nil
}

var differ = gojsondiff.New()

var diffformatter = formatter.NewDeltaFormatter()

func compareJsonBlobs(one, two []byte) error {

	diff, err := differ.Compare(one, two)

	if err != nil {
		return errors.New("Couldn't diff: " + err.Error())
	}

	if diff.Modified() {

		str, err := diffformatter.Format(diff)

		if err != nil {
			return errors.New("Couldn't format diff: " + err.Error())
		}

		return errors.New("Diff: " + str)
	}

	return nil

}

//warning: modifies the records
func compareMoveStorageRecords(one, two *boardgame.MoveStorageRecord) error {

	if one == nil {
		return errors.New("One was nil")
	}

	if two == nil {
		return errors.New("Two was nil")
	}

	oneBlob := one.Blob
	twoBlob := two.Blob

	//Set the fields we know might differ to known values
	one.Blob = nil
	two.Blob = nil

	two.Timestamp = one.Timestamp

	if !reflect.DeepEqual(one, two) {
		return errors.New("Move storage records differed in base fields")
	}

	return compareJsonBlobs(oneBlob, twoBlob)

}
<commit_msg>Removed the behavior where golden.CompareFolder errors if no records match. This allows auto-generated golden_test.go to be "safe" even when no goldens are recorded. Part of #648.<commit_after>/*

	golden is a package designed to make it possible to compare a game to a
	golden run for testing purposes. It takes a record saved in
	storage/filesystem format and compares it.

*/
package golden

import (
	"errors"
	"github.com/jkomoros/boardgame"
	"github.com/jkomoros/boardgame/storage/filesystem/record"
	"github.com/jkomoros/boardgame/storage/memory"
	"github.com/yudai/gojsondiff"
	"github.com/yudai/gojsondiff/formatter"
	"io/ioutil"
	"path/filepath"
	"reflect"
	"strconv"
)

//Compare is the primary method in the package. It takes a game delegate and a
//filename denoting a record to compare against. delegate shiould be a fresh
//delegate not yet affiliated with a manager. It compares every version and
//move in the history (ignoring things that shouldn't be the same, like
//timestamps) and reports the first place they divrge. Any time it finds a
//move not proposed by AdminPlayerIndex it will propose that move. As long as
//your game uses state.Rand() for all randomness and is otherwise
//deterministic then everything should work.
func Compare(delegate boardgame.GameDelegate, recFilename string) error {

	manager, err := boardgame.NewGameManager(delegate, memory.NewStorageManager())

	if err != nil {
		return errors.New("Couldn't create new manager: " + err.Error())
	}

	rec, err := record.New(recFilename)

	if err != nil {
		return errors.New("Couldn't create record: " + err.Error())
	}

	return compare(manager, rec)

}

//CompareFolder is like Compare, except it will iterate through any file in
//recFolder that ends in .json. Errors if any of those files cannot be parsed
//into recs.
func CompareFolder(delegate boardgame.GameDelegate, recFolder string) error {
	manager, err := boardgame.NewGameManager(delegate, memory.NewStorageManager())

	if err != nil {
		return errors.New("Couldn't create new manager: " + err.Error())
	}

	infos, err := ioutil.ReadDir(recFolder)

	if err != nil {
		return errors.New("Couldn't read folder: " + err.Error())
	}

	for _, info := range infos {
		if info.IsDir() {
			continue
		}

		if filepath.Ext(info.Name()) != ".json" {
			continue
		}

		rec, err := record.New(filepath.Join(recFolder, info.Name()))

		if err != nil {
			return errors.New("File with name " + info.Name() + " couldn't be loaded into rec: " + err.Error())
		}

		if err := compare(manager, rec); err != nil {
			return errors.New("File named " + info.Name() + " had compare error: " + err.Error())
		}

	}

	return nil
}

func compare(manager *boardgame.GameManager, rec *record.Record) error {
	game, err := manager.RecreateGame(rec.Game())

	if err != nil {
		return errors.New("Couldn't create game: " + err.Error())
	}

	lastVerifiedVersion := 0

	for !game.Finished() {
		//Verify all new moves that have happened since the last time we
		//checked (often, fix-up moves).
		for lastVerifiedVersion < game.Version() {
			stateToCompare, err := rec.State(lastVerifiedVersion)

			if err != nil {
				return errors.New("Couldn't get " + strconv.Itoa(lastVerifiedVersion) + " state: " + err.Error())
			}

			if err := compareJsonBlobs(game.State(lastVerifiedVersion).StorageRecord(), stateToCompare); err != nil {
				return errors.New("State " + strconv.Itoa(lastVerifiedVersion) + " compared differently: " + err.Error())
			}

			if lastVerifiedVersion > 0 {

				//Version 0 has no associated move

				recMove, err := rec.Move(lastVerifiedVersion)

				if err != nil {
					return errors.New("Couldn't get move " + strconv.Itoa(lastVerifiedVersion) + " from record")
				}

				moves := game.MoveRecords(lastVerifiedVersion)

				if len(moves) < 1 {
					return errors.New("Didn't fetch historical move records for " + strconv.Itoa(lastVerifiedVersion))
				}

				//Warning: records are modified by this method
				if err := compareMoveStorageRecords(moves[len(moves)-1], recMove); err != nil {
					return errors.New("Move " + strconv.Itoa(lastVerifiedVersion) + " compared differently: " + err.Error())
				}
			}

			lastVerifiedVersion++
		}

		nextMoveRec, err := rec.Move(lastVerifiedVersion + 1)

		if err != nil {
			//We'll assume that menas that's all of the moves there are to make.
			break
		}

		if nextMoveRec.Proposer < 0 {
			return errors.New("At version " + strconv.Itoa(lastVerifiedVersion) + " the next player move to apply was not applied by a player")
		}

		nextMove, err := nextMoveRec.Inflate(game)

		if err != nil {
			return errors.New("Couldn't inflate move: " + err.Error())
		}

		if err := <-game.ProposeMove(nextMove, nextMoveRec.Proposer); err != nil {
			return errors.New("Couldn't propose next move in chain: " + err.Error())
		}

	}

	if game.Finished() != rec.Game().Finished {
		return errors.New("Game finished did not match rec")
	}

	if !reflect.DeepEqual(game.Winners(), rec.Game().Winners) {
		return errors.New("Game winners did not match")
	}

	return nil
}

var differ = gojsondiff.New()

var diffformatter = formatter.NewDeltaFormatter()

func compareJsonBlobs(one, two []byte) error {

	diff, err := differ.Compare(one, two)

	if err != nil {
		return errors.New("Couldn't diff: " + err.Error())
	}

	if diff.Modified() {

		str, err := diffformatter.Format(diff)

		if err != nil {
			return errors.New("Couldn't format diff: " + err.Error())
		}

		return errors.New("Diff: " + str)
	}

	return nil

}

//warning: modifies the records
func compareMoveStorageRecords(one, two *boardgame.MoveStorageRecord) error {

	if one == nil {
		return errors.New("One was nil")
	}

	if two == nil {
		return errors.New("Two was nil")
	}

	oneBlob := one.Blob
	twoBlob := two.Blob

	//Set the fields we know might differ to known values
	one.Blob = nil
	two.Blob = nil

	two.Timestamp = one.Timestamp

	if !reflect.DeepEqual(one, two) {
		return errors.New("Move storage records differed in base fields")
	}

	return compareJsonBlobs(oneBlob, twoBlob)

}
<|endoftext|>


<commit_before>/*
Copyright 2022 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tests

import (
	"testing"

	"k8s.io/apimachinery/pkg/types"

	"sigs.k8s.io/gateway-api/conformance/utils/http"
	"sigs.k8s.io/gateway-api/conformance/utils/kubernetes"
	"sigs.k8s.io/gateway-api/conformance/utils/suite"
)

func init() {
	ConformanceTests = append(ConformanceTests, HTTPRouteHeaderMatching)
}

var HTTPRouteHeaderMatching = suite.ConformanceTest{
	ShortName:   "HTTPRouteHeaderMatching",
	Description: "A single HTTPRoute with header matching for different backends",
	Manifests:   []string{"tests/httproute-header-matching.yaml"},
	Test: func(t *testing.T, suite *suite.ConformanceTestSuite) {
		ns := "gateway-conformance-infra"
		routeNN := types.NamespacedName{Name: "header-matching", Namespace: ns}
		gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns}
		gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeReady(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN)

		testCases := []http.ExpectedResponse{{
			Request:   http.Request{Path: "/", Headers: map[string]string{"Version": "one"}},
			Backend:   "infra-backend-v1",
			Namespace: ns,
		}, {
			Request:   http.Request{Path: "/", Headers: map[string]string{"Version": "two"}},
			Backend:   "infra-backend-v2",
			Namespace: ns,
		}, {
			Request:   http.Request{Path: "/", Headers: map[string]string{"Version": "two", "Color": "orange"}},
			Backend:   "infra-backend-v1",
			Namespace: ns,
		}, {
			Request:   http.Request{Path: "/", Headers: map[string]string{"Version": "two", "Color": "blue"}},
			Backend:   "infra-backend-v1",
			Namespace: ns,
		}, {
			Request:    http.Request{Path: "/", Headers: map[string]string{"Color": "orange"}},
			StatusCode: 404,
		}, {
			Request:    http.Request{Path: "/", Headers: map[string]string{"Some-Other-Header": "one"}},
			StatusCode: 404,
		}, {
			Request:   http.Request{Path: "/", Headers: map[string]string{"Color": "blue"}},
			Backend:   "infra-backend-v1",
			Namespace: ns,
		}, {
			Request:   http.Request{Path: "/", Headers: map[string]string{"Color": "green"}},
			Backend:   "infra-backend-v1",
			Namespace: ns,
		}, {
			Request:   http.Request{Path: "/", Headers: map[string]string{"Color": "red"}},
			Backend:   "infra-backend-v2",
			Namespace: ns,
		}, {
			Request:   http.Request{Path: "/", Headers: map[string]string{"Color": "yellow"}},
			Backend:   "infra-backend-v2",
			Namespace: ns,
		}, {
			Request:    http.Request{Path: "/", Headers: map[string]string{"Color": "purple"}},
			StatusCode: 404,
		}}

		for i := range testCases {
			// Declare tc here to avoid loop variable
			// reuse issues across parallel tests.
			tc := testCases[i]
			t.Run(tc.GetTestCaseName(i), func(t *testing.T) {
				t.Parallel()
				http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, tc)
			})
		}
	},
}
<commit_msg>should route to backend v2<commit_after>/*
Copyright 2022 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package tests

import (
	"testing"

	"k8s.io/apimachinery/pkg/types"

	"sigs.k8s.io/gateway-api/conformance/utils/http"
	"sigs.k8s.io/gateway-api/conformance/utils/kubernetes"
	"sigs.k8s.io/gateway-api/conformance/utils/suite"
)

func init() {
	ConformanceTests = append(ConformanceTests, HTTPRouteHeaderMatching)
}

var HTTPRouteHeaderMatching = suite.ConformanceTest{
	ShortName:   "HTTPRouteHeaderMatching",
	Description: "A single HTTPRoute with header matching for different backends",
	Manifests:   []string{"tests/httproute-header-matching.yaml"},
	Test: func(t *testing.T, suite *suite.ConformanceTestSuite) {
		ns := "gateway-conformance-infra"
		routeNN := types.NamespacedName{Name: "header-matching", Namespace: ns}
		gwNN := types.NamespacedName{Name: "same-namespace", Namespace: ns}
		gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeReady(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN)

		testCases := []http.ExpectedResponse{{
			Request:   http.Request{Path: "/", Headers: map[string]string{"Version": "one"}},
			Backend:   "infra-backend-v1",
			Namespace: ns,
		}, {
			Request:   http.Request{Path: "/", Headers: map[string]string{"Version": "two"}},
			Backend:   "infra-backend-v2",
			Namespace: ns,
		}, {
			Request:   http.Request{Path: "/", Headers: map[string]string{"Version": "two", "Color": "orange"}},
			Backend:   "infra-backend-v1",
			Namespace: ns,
		}, {
			Request:   http.Request{Path: "/", Headers: map[string]string{"Version": "two", "Color": "blue"}},
			Backend:   "infra-backend-v2",
			Namespace: ns,
		}, {
			Request:    http.Request{Path: "/", Headers: map[string]string{"Color": "orange"}},
			StatusCode: 404,
		}, {
			Request:    http.Request{Path: "/", Headers: map[string]string{"Some-Other-Header": "one"}},
			StatusCode: 404,
		}, {
			Request:   http.Request{Path: "/", Headers: map[string]string{"Color": "blue"}},
			Backend:   "infra-backend-v1",
			Namespace: ns,
		}, {
			Request:   http.Request{Path: "/", Headers: map[string]string{"Color": "green"}},
			Backend:   "infra-backend-v1",
			Namespace: ns,
		}, {
			Request:   http.Request{Path: "/", Headers: map[string]string{"Color": "red"}},
			Backend:   "infra-backend-v2",
			Namespace: ns,
		}, {
			Request:   http.Request{Path: "/", Headers: map[string]string{"Color": "yellow"}},
			Backend:   "infra-backend-v2",
			Namespace: ns,
		}, {
			Request:    http.Request{Path: "/", Headers: map[string]string{"Color": "purple"}},
			StatusCode: 404,
		}}

		for i := range testCases {
			// Declare tc here to avoid loop variable
			// reuse issues across parallel tests.
			tc := testCases[i]
			t.Run(tc.GetTestCaseName(i), func(t *testing.T) {
				t.Parallel()
				http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, tc)
			})
		}
	},
}
<|endoftext|>


<commit_before>package srnd

import (
	"log"
	"net/http"
)

type CacheInterface interface {
	RegenAll()
	RegenFrontPage()
	RegenOnModEvent(string, string, string, int)
	RegenerateBoard(group string)
	Regen(msg ArticleEntry)

	DeleteThreadMarkup(root_post_id string)
	DeleteBoardMarkup(group string)

	Start()
	Close()

	GetThreadChan() chan ArticleEntry
	GetGroupChan() chan groupRegenRequest
	GetHandler() http.Handler
}

//TODO only pass needed config
func NewCache(cache_type, host, port, user, password string, config map[string]string, db Database, store ArticleStore) CacheInterface {
	prefix := config["prefix"]
	webroot := config["webroot"]
	threads := mapGetInt(config, "regen_threads", 1)
	name := config["name"]
	attachments := mapGetInt(config, "allow_files", 1) == 1

	if cache_type == "file" {
		return NewFileCache(prefix, webroot, name, threads, attachments, db, store)
	}
	if cache_type == "null" {
		return NewNullCache(prefix, webroot, name, attachments, db, store)
	}
	if cache_type == "redis" {
		return NewRedisCache(prefix, webroot, name, threads, attachments, db, host, port, password)
	}

	log.Fatalf("invalid cache type: %s", cache_type)
	return nil
}
<commit_msg>redis cache not supported anymore<commit_after>package srnd

import (
	"log"
	"net/http"
)

type CacheInterface interface {
	RegenAll()
	RegenFrontPage()
	RegenOnModEvent(string, string, string, int)
	RegenerateBoard(group string)
	Regen(msg ArticleEntry)

	DeleteThreadMarkup(root_post_id string)
	DeleteBoardMarkup(group string)

	Start()
	Close()

	GetThreadChan() chan ArticleEntry
	GetGroupChan() chan groupRegenRequest
	GetHandler() http.Handler
}

//TODO only pass needed config
func NewCache(cache_type, host, port, user, password string, config map[string]string, db Database, store ArticleStore) CacheInterface {
	prefix := config["prefix"]
	webroot := config["webroot"]
	threads := mapGetInt(config, "regen_threads", 1)
	name := config["name"]
	attachments := mapGetInt(config, "allow_files", 1) == 1

	if cache_type == "file" {
		return NewFileCache(prefix, webroot, name, threads, attachments, db, store)
	}
	if cache_type == "null" {
		return NewNullCache(prefix, webroot, name, attachments, db, store)
	}
	if cache_type == "redis" {
		log.Fatalf("redis cache not supported, use null cache instead")
		return NewRedisCache(prefix, webroot, name, threads, attachments, db, host, port, password)
	}

	log.Fatalf("invalid cache type: %s", cache_type)
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012 Samuel Stauffer. All rights reserved.
// Use of this source code is governed by a 3-clause BSD
// license that can be found in the LICENSE file.

package thrift

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
)

const (
	DefaultMaxFrameSize = 1024 * 1024
)

type ErrFrameTooBig struct {
	Size    int
	MaxSize int
}

func (e *ErrFrameTooBig) Error() string {
	return fmt.Sprintf("thrift: frame size while reading over allowed size (%d > %d)", e.Size, e.MaxSize)
}

type Flusher interface {
	Flush() error
}

type FramedReadWriteCloser struct {
	wrapped      io.ReadWriteCloser
	maxFrameSize int
	rbuf         *bytes.Buffer
	wbuf         *bytes.Buffer
}

func NewFramedReadWriteCloser(wrapped io.ReadWriteCloser, maxFrameSize int) *FramedReadWriteCloser {
	if maxFrameSize == 0 {
		maxFrameSize = DefaultMaxFrameSize
	}
	return &FramedReadWriteCloser{
		wrapped:      wrapped,
		maxFrameSize: maxFrameSize,
		rbuf:         &bytes.Buffer{},
		wbuf:         &bytes.Buffer{},
	}
}

func (f *FramedReadWriteCloser) Read(p []byte) (int, error) {
	if err := f.fillBuffer(); err != nil {
		return 0, err
	}
	return f.rbuf.Read(p)
}

func (f *FramedReadWriteCloser) ReadByte() (byte, error) {
	if err := f.fillBuffer(); err != nil {
		return 0, err
	}
	return f.rbuf.ReadByte()
}

func (f *FramedReadWriteCloser) fillBuffer() error {
	if f.rbuf.Len() > 0 {
		return nil
	}

	f.rbuf.Reset()
	frameSize := uint32(0)
	if err := binary.Read(f.wrapped, binary.BigEndian, &frameSize); err != nil {
		return err
	}
	if int(frameSize) > f.maxFrameSize {
		return &ErrFrameTooBig{int(frameSize), f.maxFrameSize}
	}
	// TODO: Copy may return the full frame and still return an error. In that
	//       case we could return the asked for bytes to the caller (and the error).
	if _, err := io.CopyN(f.rbuf, f.wrapped, int64(frameSize)); err != nil {
		return err
	}
	return nil
}

func (f *FramedReadWriteCloser) Write(p []byte) (int, error) {
	n, err := f.wbuf.Write(p)
	if err != nil {
		return n, err
	}
	if f.wbuf.Len() > f.maxFrameSize {
		return n, &ErrFrameTooBig{f.wbuf.Len(), f.maxFrameSize}
	}
	return n, nil
}

func (f *FramedReadWriteCloser) Close() error {
	return f.wrapped.Close()
}

func (f *FramedReadWriteCloser) Flush() error {
	frameSize := uint32(f.wbuf.Len())
	if frameSize > 0 {
		if err := binary.Write(f.wrapped, binary.BigEndian, frameSize); err != nil {
			return err
		}
		_, err := io.Copy(f.wrapped, f.wbuf)
		f.wbuf.Reset()
		return err
	}
	return nil
}
<commit_msg>4 less allocations in framed transport read path<commit_after>// Copyright 2012 Samuel Stauffer. All rights reserved.
// Use of this source code is governed by a 3-clause BSD
// license that can be found in the LICENSE file.

package thrift

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"
)

const (
	DefaultMaxFrameSize = 1024 * 1024
)

type ErrFrameTooBig struct {
	Size    int
	MaxSize int
}

func (e *ErrFrameTooBig) Error() string {
	return fmt.Sprintf("thrift: frame size while reading over allowed size (%d > %d)", e.Size, e.MaxSize)
}

type Flusher interface {
	Flush() error
}

type FramedReadWriteCloser struct {
	wrapped      io.ReadWriteCloser
	maxFrameSize int
	rtmp         []byte
	rbuf         *bytes.Buffer
	wbuf         *bytes.Buffer
}

func NewFramedReadWriteCloser(wrapped io.ReadWriteCloser, maxFrameSize int) *FramedReadWriteCloser {
	if maxFrameSize == 0 {
		maxFrameSize = DefaultMaxFrameSize
	}
	return &FramedReadWriteCloser{
		wrapped:      wrapped,
		maxFrameSize: maxFrameSize,
		rtmp:         make([]byte, 4),
		rbuf:         &bytes.Buffer{},
		wbuf:         &bytes.Buffer{},
	}
}

func (f *FramedReadWriteCloser) Read(p []byte) (int, error) {
	if err := f.fillBuffer(); err != nil {
		return 0, err
	}
	return f.rbuf.Read(p)
}

func (f *FramedReadWriteCloser) ReadByte() (byte, error) {
	if err := f.fillBuffer(); err != nil {
		return 0, err
	}
	return f.rbuf.ReadByte()
}

func (f *FramedReadWriteCloser) fillBuffer() error {
	if f.rbuf.Len() > 0 {
		return nil
	}

	f.rbuf.Reset()
	if _, err := io.ReadFull(f.wrapped, f.rtmp); err != nil {
		return err
	}
	frameSize := int(binary.BigEndian.Uint32(f.rtmp))
	if frameSize > f.maxFrameSize {
		return &ErrFrameTooBig{frameSize, f.maxFrameSize}
	}
	// TODO: Copy may return the full frame and still return an error. In that
	//       case we could return the asked for bytes to the caller (and the error).
	if _, err := io.CopyN(f.rbuf, f.wrapped, int64(frameSize)); err != nil {
		return err
	}
	return nil
}

func (f *FramedReadWriteCloser) Write(p []byte) (int, error) {
	n, err := f.wbuf.Write(p)
	if err != nil {
		return n, err
	}
	if f.wbuf.Len() > f.maxFrameSize {
		return n, &ErrFrameTooBig{f.wbuf.Len(), f.maxFrameSize}
	}
	return n, nil
}

func (f *FramedReadWriteCloser) Close() error {
	return f.wrapped.Close()
}

func (f *FramedReadWriteCloser) Flush() error {
	frameSize := uint32(f.wbuf.Len())
	if frameSize > 0 {
		if err := binary.Write(f.wrapped, binary.BigEndian, frameSize); err != nil {
			return err
		}
		_, err := io.Copy(f.wrapped, f.wbuf)
		f.wbuf.Reset()
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package memlock allows multiple appengine handlers to coordinate best-effort
// mutual execution via memcache. "best-effort" here means "best-effort"...
// memcache is not reliable. However, colliding on memcache is a lot cheaper
// than, for example, colliding with datastore transactions.
package memlock

import (
	"bytes"
	"errors"
	"time"

	mc "go.chromium.org/gae/service/memcache"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/logging"
	"golang.org/x/net/context"
)

// ErrFailedToLock is returned from TryWithLock when it fails to obtain a lock
// prior to invoking the user-supplied function.
var ErrFailedToLock = errors.New("memlock: failed to obtain lock")

// ErrEmptyClientID is returned from TryWithLock when you specify an empty
// clientID.
var ErrEmptyClientID = errors.New("memlock: empty clientID")

// memlockKeyPrefix is the memcache Key prefix for all user-supplied keys.
const memlockKeyPrefix = "memlock:"

type checkOp string

// var so we can override it in the tests
var delay = time.Second

type testStopCBKeyType int

var testStopCBKey testStopCBKeyType

const (
	release checkOp = "release"
	refresh         = "refresh"
)

// memcacheLockTime is the expiration time of the memcache entry. If the lock
// is correctly released, then it will be released before this time. It's a
// var so we can override it in the tests.
var memcacheLockTime = 16 * time.Second

// TryWithLock attempts to obtains the lock once, and then invokes f if
// successful. The context provided to f will be canceled (e.g. ctx.Done() will
// be closed) if memlock detects that we've lost the lock.
//
// TryWithLock function returns ErrFailedToLock if it fails to obtain the lock,
// otherwise returns the error that f returns.
//
// `key` is the memcache key to use (i.e. the name of the lock). Clients locking
// the same data must use the same key. clientID is the unique identifier for
// this client (lock-holder). If it's empty then TryWithLock() will return
// ErrEmptyClientID.
//
// Note that the lock provided by TryWithLock is a best-effort lock... some
// other form of locking or synchronization should be used inside of f (such as
// Datastore transactions) to ensure that f is, in fact, operating exclusively.
// The purpose of TryWithLock is to have a cheap filter to prevent unnecessary
// contention on heavier synchronization primitives like transactions.
func TryWithLock(ctx context.Context, key, clientID string, f func(context.Context) error) error {
	if len(clientID) == 0 {
		return ErrEmptyClientID
	}

	log := logging.Get(
		logging.SetFields(ctx, logging.Fields{
			"key":      key,
			"clientID": clientID,
		}))

	key = memlockKeyPrefix + key
	cid := []byte(clientID)

	// checkAnd gets the current value from memcache, and then attempts to do the
	// checkOp (which can either be `refresh` or `release`). These pieces of
	// functionality are necessarially intertwined, because CAS only works with
	// the exact-same *Item which was returned from a Get.
	//
	// refresh will attempt to CAS the item with the same content to reset it's
	// timeout.
	//
	// release will attempt to CAS the item to remove it's contents (clientID).
	// another lock observing an empty clientID will know that the lock is
	// obtainable.
	checkAnd := func(op checkOp) bool {
		itm, err := mc.GetKey(ctx, key)
		if err != nil {
			log.Warningf("error getting: %s", err)
			return false
		}

		if len(itm.Value()) > 0 && !bytes.Equal(itm.Value(), cid) {
			log.Infof("lock owned by %q", string(itm.Value()))
			return false
		}

		if op == refresh {
			itm.SetValue(cid).SetExpiration(memcacheLockTime)
		} else {
			if len(itm.Value()) == 0 {
				// it's already unlocked, no need to CAS
				log.Infof("lock already released")
				return true
			}
			itm.SetValue([]byte{}).SetExpiration(delay)
		}

		if err := mc.CompareAndSwap(ctx, itm); err != nil {
			log.Warningf("failed to %s lock: %q", op, err)
			return false
		}

		return true
	}

	// Now the actual logic begins. First we 'Add' the item, which will set it if
	// it's not present in the memcache, otherwise leaves it alone.

	err := mc.Add(ctx, mc.NewItem(ctx, key).SetValue(cid).SetExpiration(memcacheLockTime))
	if err != nil {
		if err != mc.ErrNotStored {
			log.Warningf("error adding: %s", err)
		}
		if !checkAnd(refresh) {
			return ErrFailedToLock
		}
	}

	// At this point we nominally have the lock (at least for memcacheLockTime).
	finished := make(chan struct{})
	subCtx, cancelFunc := context.WithCancel(ctx)
	defer func() {
		cancelFunc()
		<-finished
	}()

	testStopCB, _ := ctx.Value(testStopCBKey).(func())

	// This goroutine checks to see if we still possess the lock, and refreshes it
	// if we do.
	go func() {
		defer func() {
			cancelFunc()
			close(finished)
		}()

		tmr := clock.NewTimer(subCtx)
		defer tmr.Stop()
		for {
			tmr.Reset(delay)

			if tr := <-tmr.GetC(); tr.Incomplete() {
				if tr.Err != context.Canceled {
					log.Debugf("context done: %s", tr.Err)
				}
				break
			}
			if !checkAnd(refresh) {
				log.Warningf("lost lock: %s", err)
				return
			}
		}

		if testStopCB != nil {
			testStopCB()
		}
		checkAnd(release)
	}()

	return f(subCtx)
}
<commit_msg>[tumble/memlock] Retry memcache calls up to 5 times<commit_after>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package memlock allows multiple appengine handlers to coordinate best-effort
// mutual execution via memcache. "best-effort" here means "best-effort"...
// memcache is not reliable. However, colliding on memcache is a lot cheaper
// than, for example, colliding with datastore transactions.
package memlock

import (
	"bytes"
	"errors"
	"time"

	mc "go.chromium.org/gae/service/memcache"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry"
	"golang.org/x/net/context"
)

// ErrFailedToLock is returned from TryWithLock when it fails to obtain a lock
// prior to invoking the user-supplied function.
var ErrFailedToLock = errors.New("memlock: failed to obtain lock")

// ErrEmptyClientID is returned from TryWithLock when you specify an empty
// clientID.
var ErrEmptyClientID = errors.New("memlock: empty clientID")

// memlockKeyPrefix is the memcache Key prefix for all user-supplied keys.
const memlockKeyPrefix = "memlock:"

type checkOp string

// var so we can override it in the tests
var delay = time.Second

type testStopCBKeyType int

var testStopCBKey testStopCBKeyType

const (
	release checkOp = "release"
	refresh         = "refresh"
)

// memcacheLockTime is the expiration time of the memcache entry. If the lock
// is correctly released, then it will be released before this time. It's a
// var so we can override it in the tests.
var memcacheLockTime = 16 * time.Second

// TryWithLock attempts to obtains the lock once, and then invokes f if
// successful. The context provided to f will be canceled (e.g. ctx.Done() will
// be closed) if memlock detects that we've lost the lock.
//
// TryWithLock function returns ErrFailedToLock if it fails to obtain the lock,
// otherwise returns the error that f returns.
//
// `key` is the memcache key to use (i.e. the name of the lock). Clients locking
// the same data must use the same key. clientID is the unique identifier for
// this client (lock-holder). If it's empty then TryWithLock() will return
// ErrEmptyClientID.
//
// Note that the lock provided by TryWithLock is a best-effort lock... some
// other form of locking or synchronization should be used inside of f (such as
// Datastore transactions) to ensure that f is, in fact, operating exclusively.
// The purpose of TryWithLock is to have a cheap filter to prevent unnecessary
// contention on heavier synchronization primitives like transactions.
func TryWithLock(ctx context.Context, key, clientID string, f func(context.Context) error) error {
	if len(clientID) == 0 {
		return ErrEmptyClientID
	}

	log := logging.Get(
		logging.SetFields(ctx, logging.Fields{
			"key":      key,
			"clientID": clientID,
		}))

	key = memlockKeyPrefix + key
	cid := []byte(clientID)

	// checkAnd gets the current value from memcache, and then attempts to do the
	// checkOp (which can either be `refresh` or `release`). These pieces of
	// functionality are necessarially intertwined, because CAS only works with
	// the exact-same *Item which was returned from a Get.
	//
	// refresh will attempt to CAS the item with the same content to reset it's
	// timeout.
	//
	// release will attempt to CAS the item to remove it's contents (clientID).
	// another lock observing an empty clientID will know that the lock is
	// obtainable.
	checkAnd := func(op checkOp) bool {
		limitedRetry := func() retry.Iterator {
			return &retry.Limited{
				Delay:   time.Second,
				Retries: 5,
			}
		}
		var itm mc.Item
		if err := retry.Retry(ctx, limitedRetry, func() (err error) {
			itm, err = mc.GetKey(ctx, key)
			return
		}, retry.LogCallback(ctx, "getting lock from memcache")); err != nil {
			log.Warningf("permanent error getting: %s", err)
			return false
		}

		if len(itm.Value()) > 0 && !bytes.Equal(itm.Value(), cid) {
			log.Infof("lock owned by %q", string(itm.Value()))
			return false
		}

		if op == refresh {
			itm.SetValue(cid).SetExpiration(memcacheLockTime)
		} else {
			if len(itm.Value()) == 0 {
				// it's already unlocked, no need to CAS
				log.Infof("lock already released")
				return true
			}
			itm.SetValue([]byte{}).SetExpiration(delay)
		}

		if err := mc.CompareAndSwap(ctx, itm); err != nil {
			log.Warningf("failed to %s lock: %q", op, err)
			return false
		}

		return true
	}

	// Now the actual logic begins. First we 'Add' the item, which will set it if
	// it's not present in the memcache, otherwise leaves it alone.

	err := mc.Add(ctx, mc.NewItem(ctx, key).SetValue(cid).SetExpiration(memcacheLockTime))
	if err != nil {
		if err != mc.ErrNotStored {
			log.Warningf("error adding: %s", err)
		}
		if !checkAnd(refresh) {
			return ErrFailedToLock
		}
	}

	// At this point we nominally have the lock (at least for memcacheLockTime).
	finished := make(chan struct{})
	subCtx, cancelFunc := context.WithCancel(ctx)
	defer func() {
		cancelFunc()
		<-finished
	}()

	testStopCB, _ := ctx.Value(testStopCBKey).(func())

	// This goroutine checks to see if we still possess the lock, and refreshes it
	// if we do.
	go func() {
		defer func() {
			cancelFunc()
			close(finished)
		}()

		tmr := clock.NewTimer(subCtx)
		defer tmr.Stop()
		for {
			tmr.Reset(delay)

			if tr := <-tmr.GetC(); tr.Incomplete() {
				if tr.Err != context.Canceled {
					log.Debugf("context done: %s", tr.Err)
				}
				break
			}
			if !checkAnd(refresh) {
				log.Warningf("lost lock: %s", err)
				return
			}
		}

		if testStopCB != nil {
			testStopCB()
		}
		checkAnd(release)
	}()

	return f(subCtx)
}
<|endoftext|>


<commit_before>// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package proggen

import (
	"fmt"
	"strings"
	"testing"

	"github.com/google/syzkaller/pkg/log"
	"github.com/google/syzkaller/prog"
	_ "github.com/google/syzkaller/sys"
	"github.com/google/syzkaller/tools/syz-trace2syz/parser"
)

const (
	OS   = "linux"
	Arch = "amd64"
)

func initializeTarget(os, arch string) *prog.Target {
	target, err := prog.GetTarget(os, arch)
	if err != nil {
		log.Fatalf("%s", err)
	}
	target.ConstMap = make(map[string]uint64)
	for _, c := range target.Consts {
		target.ConstMap[c.Name] = c.Value
	}
	return target
}

func parseSingleTrace(t *testing.T, data string) *prog.Prog {
	target := initializeTarget(OS, Arch)
	traceTree, err := parser.ParseData([]byte(data))
	if err != nil {
		t.Fatal(err)
	}
	p := genProg(traceTree.TraceMap[traceTree.RootPid], target)
	if p == nil {
		t.Fatalf("failed to parse trace")
	}
	return p
}

func TestParseTraceBasic(t *testing.T) {
	test := `open("file", 66) = 3
			 write(3, "somedata", 8) = 8`
	p := parseSingleTrace(t, test)
	expectedSeq := "open-write"
	if p.String() != expectedSeq {
		t.Fatalf("expected: %s != %s", expectedSeq, p.String())
	}
	switch a := p.Calls[1].Args[0].(type) {
	case *prog.ResultArg:
		if a.Res != p.Calls[0].Ret {
			t.Fatalf("first argument of write should equal result of open.")
		}
	default:
		t.Fatalf("expected result arg, got: %s\n", a.Type().Name())
	}
}

func TestParseVMA(t *testing.T) {
	test := `pipe({0x0, 0x1}) = 0
		 shmget(0x0, 0x1, 0x2, 0x3) = 0`
	p := parseSingleTrace(t, test)
	expectedSeq := "pipe-shmget"
	if p.String() != expectedSeq {
		t.Fatalf("expected: %s != %s", expectedSeq, p.String())
	}
}

func TestParseTraceInnerResource(t *testing.T) {
	test := `pipe([5,6]) = 0
			 write(6, "\xff\xff\xfe\xff", 4) = 4`
	p := parseSingleTrace(t, test)
	expectedSeq := "pipe-write"
	if p.String() != expectedSeq {
		t.Fatalf("Expected: %s != %s", expectedSeq, p.String())
	}
	switch a := p.Calls[1].Args[0].(type) {
	case *prog.ResultArg:
		pipeSecondFd := p.Calls[0].Args[0].(*prog.PointerArg).Res.(*prog.GroupArg).Inner[1]
		if a.Res != pipeSecondFd {
			t.Fatalf("first argument of write must match second fd from pipe")
		}
	default:
		t.Fatalf("expected result arg, got: %s\n", a.Type().Name())
	}
}

func TestNegativeResource(t *testing.T) {
	test := `socket(29, 3, 1) = 3
 			  getsockopt(-1, 132, 119, 0x200005c0, [14]) = -1 EBADF (Bad file descriptor)`

	p := parseSingleTrace(t, test)
	expectedSeq := "socket$can_raw-getsockopt$inet_sctp6_SCTP_RESET_STREAMS"
	if p.String() != expectedSeq {
		t.Fatalf("expected: %s != %s", expectedSeq, p.String())
	}
	switch a := p.Calls[1].Args[0].(type) {
	case *prog.ResultArg:
		if a.Val != ^uint64(0) {
			t.Fatalf("expected resource type to be negative, got: %d", a.Val)
		}
	default:
		t.Fatalf("expected result arg, got: %s\n", a.Type().Name())
	}
}

func TestDistinguishResourceTypes(t *testing.T) {
	test := `inotify_init() = 2
			 open("tmp", 66) = 3
			 inotify_add_watch(3, "\x2e", 0xfff) = 3
	 		 write(3, "temp", 5) = 5
			 inotify_rm_watch(2, 3) = 0`
	expectedSeq := "inotify_init-open-inotify_add_watch-write-inotify_rm_watch"
	p := parseSingleTrace(t, test)
	if p.String() != expectedSeq {
		t.Fatalf("Expected: %s != %s", expectedSeq, p.String())
	}
	write := p.Calls[len(p.Calls)-2]
	inotifyRmWatch := p.Calls[len(p.Calls)-1]
	switch a := write.Args[0].Type().(type) {
	case *prog.ResourceType:
		if a.TypeName != "fd" {
			t.Fatalf("expected first argument of write to have type fd, got: %s", a.TypeName)
		}
	default:
		t.Fatalf("first argument of write is not resource type: %s", a.Name())
	}
	switch a := inotifyRmWatch.Args[1].(type) {
	case *prog.ResultArg:
		b := a.Type().(*prog.ResourceType)
		if b.TypeName != "inotifydesc" {
			t.Fatalf("expected second argument of inotify_rm_watch to have type inoitfydesc, got: %s", b.TypeName)
		}
		if a.Res != p.Calls[2].Ret {
			t.Fatalf("inotify_rm_watch's second argument should match the result of inotify_add_watch.")
		}
	}
}

func TestSocketLevel(t *testing.T) {
	test := `socket(1, 1, 0) = 3
			 socket(1, 1 | 2048, 0) = 3
			 socket(1, 1 | 524288, 0) = 3
			 socket(1, 1 | 524288, 0) = 3`
	expectedSeq := "socket$unix-socket$unix-socket$unix-socket$unix"
	p := parseSingleTrace(t, test)
	if p.String() != expectedSeq {
		t.Fatalf("Expected: %s != %s", expectedSeq, p.String())
	}
}

func TestIdentifySockaddrStorage(t *testing.T) {
	type identifyStorageTest struct {
		test        string
		expectedSeq string
		callIdx     int
		argIdx      int
		fieldName   string
	}
	tests := []identifyStorageTest{
		{
			`open("temp", 1) = 3
			  connect(3, {sa_family=2, sin_port=37957, sin_addr=0x0}, 16) = -1`,
			"open-connect",
			1,
			1,
			"sockaddr_in",
		},
		{
			`open("temp", 1) = 3
			  connect(3, {sa_family=1, sun_path="temp"}, 110) = -1`,
			"open-connect",
			1,
			1,
			"sockaddr_un",
		},
		{
			`open("temp", 1) = 3
			  bind(5, {sa_family=16, nl_pid=0, nl_groups=00000000}, 12)  = -1`,
			"open-bind",
			1,
			1,
			"sockaddr_nl",
		},
	}

	validator := func(arg prog.Arg, field string) error {
		storagePtr := arg.(*prog.PointerArg)
		storageArg, ok := storagePtr.Res.(*prog.UnionArg)
		if !ok {
			t.Fatalf("second argument not union: %s", storagePtr.Res.Type().Name())
		}
		fieldName := storageArg.Option.Type().Name()
		if fieldName != field {
			return fmt.Errorf("incorrect storage type, expected %s != %s", field, fieldName)
		}
		return nil
	}

	for i, test := range tests {
		p := parseSingleTrace(t, test.test)
		if p.String() != test.expectedSeq {
			t.Fatalf("failed btest: %d, expected: %s != %s", i, test.expectedSeq, p.String())
		}
		err := validator(p.Calls[test.callIdx].Args[test.argIdx], test.fieldName)
		if err != nil {
			t.Fatalf("failed subtest: %d with err: %s", i, err)
		}
	}
}

func TestIdentifyIfru(t *testing.T) {
	type testIfru struct {
		test        string
		expectedSeq string
	}
	tests := []testIfru{
		{
			`socket(17, 3, 768)  = 3
			 ioctl(3, 35111, {ifr_name="\x6c\x6f", ifr_hwaddr=00:00:00:00:00:00}) = 0`,
			"socket$packet-ioctl$sock_ifreq",
		},
	}

	for i, test := range tests {
		p := parseSingleTrace(t, test.test)
		if p.String() != test.expectedSeq {
			t.Fatalf("failed subtest: %d, expected %s != %s", i, test.expectedSeq, p.String())
		}
	}
}

func TestParseVariants(t *testing.T) {
	type variantTest struct {
		test        string
		expectedSeq string
	}
	tests := []variantTest{
		{
			`socket(1, 1, 0) = 3
			  connect(3, {sa_family=1, sun_path="temp"}, 110) = -1 ENOENT (Bad file descriptor)`,
			"socket$unix-connect$unix",
		},
		{
			`socket(1, 1, 0) = 3`,
			"socket$unix",
		},
		{
			`socket(2, 1, 0) = 5
			  ioctl(5, 21537, [1]) = 0`,
			"socket$inet_tcp-ioctl$int_in",
		},
		{
			`socket(2, 1, 0) = 3
			  setsockopt(3, 1, 2, [1], 4) = 0`,
			"socket$inet_tcp-setsockopt$sock_int",
		},
		{
			`9795  socket(17, 3, 768)  = 3
			  9795  ioctl(3, 35123, {ifr_name="\x6c\x6f", }) = 0`,
			"socket$packet-ioctl$ifreq_SIOCGIFINDEX_team",
		},
		{
			`open("temp", 1) = 3
			  connect(3, {sa_family=2, sin_port=17812, sin_addr=0x0}, 16) = -1`,
			"open-connect",
		},
		{
			`ioprio_get(1, 0) = 4`,
			"ioprio_get$pid",
		},
		{
			`socket(17, 2, 768) = 3`,
			"socket$packet",
		},
	}

	for i, test := range tests {
		p := parseSingleTrace(t, test.test)
		if p.String() != test.expectedSeq {
			t.Fatalf("failed subtest: %d, expected %s != %s", i, test.expectedSeq, p.String())
		}
	}
}

func TestParseIPv4(t *testing.T) {
	type ip4test struct {
		test        string
		expectedSeq string
		ip4         uint64
	}
	tests := []ip4test{
		{
			`socket(2, 1, 0) = 3
			  connect(3, {sa_family=2, sin_port=17812, sin_addr=0x0}, 16) = 0`,
			"socket$inet_tcp-connect$inet",
			0,
		},
		{
			`socket(2, 1, 0) = 3
			  connect(3, {sa_family=2, sin_port=17812, sin_addr=0x7f000001}, 16) = 0`,
			"socket$inet_tcp-connect$inet",
			0x7f000001,
		},
	}
	testIpv4 := func(expectedIp uint64, a prog.Arg, t *testing.T) {
		sockaddr, ok := a.(*prog.PointerArg).Res.(*prog.GroupArg)
		if !ok {
			t.Fatalf("%s", a.Type().Name())
		}
		ipv4Addr, ok := sockaddr.Inner[2].(*prog.UnionArg)
		if !ok {
			t.Fatalf("expected 3rd argument to be unionArg, got %s", sockaddr.Inner[2].Type().Name())
		}
		optName := ipv4Addr.Option.Type().FieldName()
		if !strings.Contains(optName, "rand") {
			t.Fatalf("expected ip option to be random opt, got: %s", optName)
		}
		ip, ok := ipv4Addr.Option.(*prog.ConstArg)
		if !ok {
			t.Fatalf("ipv4Addr option is not IntType")
		}
		if ip.Val != expectedIp {
			t.Fatalf("parsed != expected, %d != %d", ip.Val, expectedIp)
		}
	}
	for i, test := range tests {
		p := parseSingleTrace(t, test.test)
		if p.String() != test.expectedSeq {
			t.Fatalf("failed subtest: %d, expected %s != %s", i, test.expectedSeq, p.String())
		}
		testIpv4(test.ip4, p.Calls[1].Args[1], t)
	}
}
<commit_msg>tools/syz-trace2syz/proggen: convert tests to table format<commit_after>// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package proggen

import (
	"bytes"
	"strings"
	"testing"

	"github.com/google/syzkaller/prog"
	_ "github.com/google/syzkaller/sys"
	"github.com/google/syzkaller/tools/syz-trace2syz/parser"
)

func TestParse(t *testing.T) {
	type Test struct {
		input  string
		output string
	}
	tests := []Test{
		{`
open("file", 66) = 3
write(3, "somedata", 8) = 8
`, `
r0 = open(&(0x7f0000000000)='file\x00', 0x42, 0x0)
write(r0, &(0x7f0000000040)='somedata\x00', 0x9)
`,
		}, {`
pipe([5,6]) = 0
write(6, "\xff\xff\xfe\xff", 4) = 4
`, `
pipe(&(0x7f0000000000)={0xffffffffffffffff, <r0=>0xffffffffffffffff})
write(r0, &(0x7f0000000040)="fffffeff00", 0x5)
`,
		}, {`
pipe({0x0, 0x1}) = 0
shmget(0x0, 0x1, 0x2, 0x3) = 0
`, `
pipe(&(0x7f0000000000))
shmget(0x0, 0x1, 0x2, &(0x7f0000001000/0x1)=nil)
`,
		}, {`
socket(29, 3, 1) = 3
getsockopt(-1, 132, 119, 0x200005c0, [14]) = -1 EBADF (Bad file descriptor)
`, `
socket$can_raw(0x1d, 0x3, 0x1)
getsockopt$inet_sctp6_SCTP_RESET_STREAMS(0xffffffffffffffff, 0x84, 0x77, &(0x7f0000000000), &(0x7f0000000040)=0x8)
`,
		}, {`
inotify_init() = 2
open("tmp", 66) = 3
inotify_add_watch(3, "\x2e", 0xfff) = 3
write(3, "temp", 5) = 5
inotify_rm_watch(2, 3) = 0
`, `
r0 = inotify_init()
r1 = open(&(0x7f0000000000)='tmp\x00', 0x42, 0x0)
r2 = inotify_add_watch(r1, &(0x7f0000000040)='.\x00', 0xfff)
write(r1, &(0x7f0000000080)='temp\x00', 0x5)
inotify_rm_watch(r0, r2)
`,
		}, {`
socket(1, 1, 0) = 3
socket(1, 1 | 2048, 0) = 3
socket(1, 1 | 524288, 0) = 3
socket(1, 1 | 524288, 0) = 3
`, `
socket$unix(0x1, 0x1, 0x0)
socket$unix(0x1, 0x801, 0x0)
socket$unix(0x1, 0x80001, 0x0)
socket$unix(0x1, 0x80001, 0x0)
`,
		}, {`
open("temp", 1) = 3
connect(3, {sa_family=2, sin_port=37957, sin_addr=0x0}, 16) = -1
`, `
r0 = open(&(0x7f0000000000)='temp\x00', 0x1, 0x0)
connect(r0, &(0x7f0000000040)=@in={0x2, 0x9445}, 0x80)
`,
		}, {`
open("temp", 1) = 3
connect(3, {sa_family=1, sun_path="temp"}, 110) = -1
`, `
r0 = open(&(0x7f0000000000)='temp\x00', 0x1, 0x0)
connect(r0, &(0x7f0000000040)=@un=@file={0x1, 'temp\x00'}, 0x80)
`,
		}, {`
open("temp", 1) = 3
bind(5, {sa_family=16, nl_pid=0x2, nl_groups=00000003}, 12)  = -1
`, `
open(&(0x7f0000000000)='temp\x00', 0x1, 0x0)
bind(0x5, &(0x7f0000000040)=@nl=@proc={0x10, 0x2, 0x3}, 0x80)
`,
		}, {`
socket(17, 3, 768)  = 3
ioctl(3, 35111, {ifr_name="\x6c\x6f", ifr_hwaddr=00:00:00:00:00:00}) = 0
`, `
r0 = socket$packet(0x11, 0x3, 0x300)
ioctl$sock_ifreq(r0, 0x8927, &(0x7f0000000000)={'lo\x00'})
`,
		}, {`
socket(1, 1, 0) = 3
connect(3, {sa_family=1, sun_path="temp"}, 110) = -1 ENOENT (Bad file descriptor)
`, `
r0 = socket$unix(0x1, 0x1, 0x0)
connect$unix(r0, &(0x7f0000000000)=@file={0x1, 'temp\x00'}, 0x6e)
`,
		}, {`
socket(1, 1, 0) = 3
`, `
socket$unix(0x1, 0x1, 0x0)
`,
		}, {`
socket(2, 1, 0) = 5
ioctl(5, 21537, [1]) = 0
`, `
r0 = socket$inet_tcp(0x2, 0x1, 0x0)
ioctl$int_in(r0, 0x5421, &(0x7f0000000000)=0x1)
`,
		}, {`
socket(2, 1, 0) = 3
setsockopt(3, 1, 2, [1], 4) = 0
`, `
r0 = socket$inet_tcp(0x2, 0x1, 0x0)
setsockopt$sock_int(r0, 0x1, 0x2, &(0x7f0000000000)=0x1, 0x4)
`,
		}, {`
9795  socket(17, 3, 768)  = 3
9795  ioctl(3, 35123, {ifr_name="\x6c\x6f", }) = 0
`, `
r0 = socket$packet(0x11, 0x3, 0x300)
ioctl$ifreq_SIOCGIFINDEX_team(r0, 0x8933, &(0x7f0000000000)={'lo\x00'})
`,
		}, {`
open("temp", 1) = 3
connect(3, {sa_family=2, sin_port=17812, sin_addr=0x0}, 16) = -1
`, `
r0 = open(&(0x7f0000000000)='temp\x00', 0x1, 0x0)
connect(r0, &(0x7f0000000040)=@in={0x2, 0x4594}, 0x80)
`,
		}, {`
ioprio_get(1, 0) = 4
`, `
ioprio_get$pid(0x1, 0x0)
`,
		}, {`
socket(17, 2, 768) = 3
`, `
socket$packet(0x11, 0x2, 0x300)
`,
		}, {`
socket(2, 1, 0) = 3
connect(3, {sa_family=2, sin_port=17812, sin_addr=0x0}, 16) = 0
`, `
r0 = socket$inet_tcp(0x2, 0x1, 0x0)
connect$inet(r0, &(0x7f0000000000)={0x2, 0x4594}, 0x10)
`,
		}, {`
socket(2, 1, 0) = 3
connect(3, {sa_family=2, sin_port=17812, sin_addr=0x7f000001}, 16) = 0
`, `
r0 = socket$inet_tcp(0x2, 0x1, 0x0)
connect$inet(r0, &(0x7f0000000000)={0x2, 0x4594, @rand_addr=0x7f000001}, 0x10)
`,
		},
	}
	target, err := prog.GetTarget("linux", "amd64")
	if err != nil {
		t.Fatal(err)
	}
	target.ConstMap = make(map[string]uint64)
	for _, c := range target.Consts {
		target.ConstMap[c.Name] = c.Value
	}
	for _, test := range tests {
		input := strings.TrimSpace(test.input)
		tree, err := parser.ParseData([]byte(input))
		if err != nil {
			t.Fatal(err)
		}
		p := genProg(tree.TraceMap[tree.RootPid], target)
		if p == nil {
			t.Fatalf("failed to parse trace")
		}
		got := string(bytes.TrimSpace(p.Serialize()))
		want := strings.TrimSpace(test.output)
		if want != got {
			t.Errorf("input:\n%v\n\nwant:\n%v\n\ngot:\n%v", input, want, got)
		}
	}
}
<|endoftext|>


<commit_before>package toodledo

import (
	"log"
	"os"
	"reflect"
	"testing"
)

var client = ToodleClient{AppId: os.Getenv("TOODLE_APP_ID"),
	ClientSecret: os.Getenv("TOODLE_CLIENT_SECRET"),
	AccessToken:  os.Getenv("TOODLE_ACCESS_TOKEN"),
	RefreshToken: os.Getenv("TOODLE_REFRESH_TOKEN")}

func TestAccountInfo(t *testing.T) {
	account, err := client.AccountInfo()
	if err != nil {
		t.Error(err)
		return
	}

	if account == nil {
		t.Errorf("Nil account received without error")
		return
	}

	if reflect.DeepEqual(*account, Account{}) {
		t.Errorf("Empty account received")
		return
	}
}

func TestTasks(t *testing.T) {
	taskResponse, err := client.Tasks(nil, nil, Uncompleted, 0, 0, "duedate", "duetime", "startdate", "starttime", "length", "tag", "parent")
	if err != nil {
		t.Error(err)
		return
	}

	if taskResponse == nil {
		t.Errorf("Nil account received without error")
		return
	}

	if reflect.DeepEqual(*taskResponse, TaskResponse{}) {
		t.Errorf("Empty response received for Tasks")
		return
	}

	if len(taskResponse.Tasks) == 0 {
		t.Errorf("Received empty list of tasks")
		return
	}
	for _, task := range taskResponse.Tasks {
		log.Printf("%+v", task)
	}
}

func TestRefresh(t *testing.T) {
	refreshResponse, err := client.RefreshCredentials()
	if err != nil {
		t.Error(err)
		return
	}
	if refreshResponse == nil {
		t.Errorf("Received nil response from RefreshCredentials")
		return
	}
	if reflect.DeepEqual(*refreshResponse, &RefreshResponse{}) {
		t.Errorf("Received empty RefreshResponse")
		return
	}
}
<commit_msg>Print credentials after refreshing in TestRefresh()<commit_after>package toodledo

import (
	"log"
	"os"
	"reflect"
	"testing"
)

var client = ToodleClient{AppId: os.Getenv("TOODLE_APP_ID"),
	ClientSecret: os.Getenv("TOODLE_CLIENT_SECRET"),
	AccessToken:  os.Getenv("TOODLE_ACCESS_TOKEN"),
	RefreshToken: os.Getenv("TOODLE_REFRESH_TOKEN")}

func TestAccountInfo(t *testing.T) {
	account, err := client.AccountInfo()
	if err != nil {
		t.Error(err)
		return
	}

	if account == nil {
		t.Errorf("Nil account received without error")
		return
	}

	if reflect.DeepEqual(*account, Account{}) {
		t.Errorf("Empty account received")
		return
	}
}

func TestTasks(t *testing.T) {
	taskResponse, err := client.Tasks(nil, nil, Uncompleted, 0, 0, "duedate", "duetime", "startdate", "starttime", "length", "tag", "parent")
	if err != nil {
		t.Error(err)
		return
	}

	if taskResponse == nil {
		t.Errorf("Nil account received without error")
		return
	}

	if reflect.DeepEqual(*taskResponse, TaskResponse{}) {
		t.Errorf("Empty response received for Tasks")
		return
	}

	if len(taskResponse.Tasks) == 0 {
		t.Errorf("Received empty list of tasks")
		return
	}
	for _, task := range taskResponse.Tasks {
		log.Printf("%+v", task)
	}
}

func TestRefresh(t *testing.T) {
	refreshResponse, err := client.RefreshCredentials()
	if err != nil {
		t.Error(err)
		return
	}
	if refreshResponse == nil {
		t.Errorf("Received nil response from RefreshCredentials")
		return
	}
	if reflect.DeepEqual(*refreshResponse, &RefreshResponse{}) {
		t.Errorf("Received empty RefreshResponse")
		return
	}
    log.Printf("Successfully refreshed credentials: %+v", refreshResponse)
}
<|endoftext|>


<commit_before>package solr

import (
	"fmt"
	"math"
	"net/url"
)

type Document map[string]interface{}

// Has check if a key exist in document
func (d Document) Has(k string) bool {
	_, ok := d[k]
	return ok
}

// Get returns value of a key
func (d Document) Get(k string) interface{} {
	v, _ := d[k]
	return v
}

// Set add a key/value to document
func (d Document) Set(k string, v interface{}) {
	d[k] = v
}

type Collection struct {
	Docs     []Document
	Start    int
	NumFound int
}

type SolrResult struct {
	Status       int         // status quick access to status
	Results      *Collection // results parsed documents, basically response object

	ResponseHeader map[string]interface{}
	FacetCounts map[string]interface{}
	Highlighting map[string]interface{}
	Error        map[string]interface{}

	// grouped for grouping result
	// if grouping Results will be empty
	Grouped map[string]interface{}
}

type SolrInterface struct {
	conn *Connection
}

// Return a new instance of SolrInterface
func NewSolrInterface(solrUrl, core string) (*SolrInterface, error) {
	c, err := NewConnection(solrUrl, core)
	if err != nil {
		return nil, err
	}
	return &SolrInterface{conn: c}, nil
}

// Set to new core, this is just wrapper to Connection.SetCore which mean
// it will affect all places that use this Connection instance
func (si *SolrInterface) SetCore(core string) {
	si.conn.SetCore(core)
}

// SetBasicAuth sets the request's Authorization header to use HTTP Basic Authentication with the provided username and password.
// See http://golang.org/pkg/net/http/#Request.SetBasicAuth
func (si *SolrInterface) SetBasicAuth(username, password string) {
	si.conn.SetBasicAuth(username, password)
}

// Return a new instace of Search, q is optional and one can set it later
func (si *SolrInterface) Search(q *Query) *Search {
	return NewSearch(si.conn, q)
}

// makeAddChunks splits the documents into chunks. If chunk_size is less than one it will be default to 100
func makeAddChunks(docs []Document, chunk_size int) []map[string]interface{} {
	if chunk_size < 1 {
		chunk_size = 100
	}
	docs_len := len(docs)
	num_chunk := int(math.Ceil(float64(docs_len) / float64(chunk_size)))
	doc_counter := 0
	chunks := make([]map[string]interface{}, num_chunk)
	for i := 0; i < num_chunk; i++ {
		add := make([]Document, 0, chunk_size)
		for j := 0; j < chunk_size; j++ {
			if doc_counter >= docs_len {
				break
			}
			add = append(add, docs[doc_counter])
			doc_counter++
		}
		chunks[i] = map[string]interface{}{"add": add}
	}
	return chunks
}

// Add will insert documents in batch of chunk_size. success is false as long as one chunk failed.
// The result in UpdateResponse is summery of response from all chunks
// with key chunk_%d
func (si *SolrInterface) Add(docs []Document, chunk_size int, params *url.Values) (*UpdateResponse, error) {
	result := &UpdateResponse{Success: true}
	responses := map[string]interface{}{}
	chunks := makeAddChunks(docs, chunk_size)

	for i := 0; i < len(chunks); i++ {
		res, err := si.Update(chunks[i], params)
		if err != nil {
			return nil, err
		}
		result.Success = result.Success && res.Success
		responses[fmt.Sprintf("chunk_%d", i+1)] = map[string]interface{}{
			"result":  res.Result,
			"success": res.Success,
			"total":   len(chunks[i]["add"].([]Document))}
	}
	result.Result = responses
	return result, nil
}

// Delete take data of type map and optional params which can use to specify addition parameters such as commit=true .
// Only one delete statement is supported, ie data can be { "id":"ID" } .
// If you want to delete more docs use { "query":"QUERY" } .
// Extra params can specify in params or in data such as { "query":"QUERY", "commitWithin":"500" }
func (si *SolrInterface) Delete(data map[string]interface{}, params *url.Values) (*UpdateResponse, error) {
	message := map[string]interface{}{"delete": data}
	return si.Update(message, params)
}

// DeleteAll will remove all documents and commit
func (si *SolrInterface) DeleteAll() (*UpdateResponse, error) {
	params := &url.Values{}
	params.Add("commit", "true")
	return si.Delete(map[string]interface{}{"query": "*:*"}, params)
}

// Update take data of type map and optional params which can use to specify addition parameters such as commit=true
func (si *SolrInterface) Update(data map[string]interface{}, params *url.Values) (*UpdateResponse, error) {
	if si.conn == nil {
		return nil, fmt.Errorf("No connection found for making request to solr")
	}
	return si.conn.Update(data, params)
}

// Commit the changes since the last commit
func (si *SolrInterface) Commit() (*UpdateResponse, error) {
	params := &url.Values{}
	params.Add("commit", "true")
	return si.Update(map[string]interface{}{}, params)
}

func (si *SolrInterface) Optimize(params *url.Values) (*UpdateResponse, error) {
	if params == nil {
		params = &url.Values{}
	}
	params.Set("optimize", "true")
	return si.Update(map[string]interface{}{}, params)
}

// Rollback rollbacks all add/deletes made to the index since the last commit.
// This should use with caution.
// See https://wiki.apache.org/solr/UpdateXmlMessages#A.22rollback.22
func (si *SolrInterface) Rollback() (*UpdateResponse, error) {
	return si.Update(map[string]interface{}{"rollback": map[string]interface{}{}}, nil)
}
<commit_msg>Formatting<commit_after>package solr

import (
	"fmt"
	"math"
	"net/url"
)

type Document map[string]interface{}

// Has check if a key exist in document
func (d Document) Has(k string) bool {
	_, ok := d[k]
	return ok
}

// Get returns value of a key
func (d Document) Get(k string) interface{} {
	v, _ := d[k]
	return v
}

// Set add a key/value to document
func (d Document) Set(k string, v interface{}) {
	d[k] = v
}

type Collection struct {
	Docs     []Document
	Start    int
	NumFound int
}

type SolrResult struct {
	Status         int         // status quick access to status
	Results        *Collection // results parsed documents, basically response object
	ResponseHeader map[string]interface{}
	FacetCounts    map[string]interface{}
	Highlighting   map[string]interface{}
	Error          map[string]interface{}
	// grouped for grouping result
	// if grouping Results will be empty
	Grouped map[string]interface{}
}

type SolrInterface struct {
	conn *Connection
}

// Return a new instance of SolrInterface
func NewSolrInterface(solrUrl, core string) (*SolrInterface, error) {
	c, err := NewConnection(solrUrl, core)
	if err != nil {
		return nil, err
	}
	return &SolrInterface{conn: c}, nil
}

// Set to new core, this is just wrapper to Connection.SetCore which mean
// it will affect all places that use this Connection instance
func (si *SolrInterface) SetCore(core string) {
	si.conn.SetCore(core)
}

// SetBasicAuth sets the request's Authorization header to use HTTP Basic Authentication with the provided username and password.
// See http://golang.org/pkg/net/http/#Request.SetBasicAuth
func (si *SolrInterface) SetBasicAuth(username, password string) {
	si.conn.SetBasicAuth(username, password)
}

// Return a new instace of Search, q is optional and one can set it later
func (si *SolrInterface) Search(q *Query) *Search {
	return NewSearch(si.conn, q)
}

// makeAddChunks splits the documents into chunks. If chunk_size is less than one it will be default to 100
func makeAddChunks(docs []Document, chunk_size int) []map[string]interface{} {
	if chunk_size < 1 {
		chunk_size = 100
	}
	docs_len := len(docs)
	num_chunk := int(math.Ceil(float64(docs_len) / float64(chunk_size)))
	doc_counter := 0
	chunks := make([]map[string]interface{}, num_chunk)
	for i := 0; i < num_chunk; i++ {
		add := make([]Document, 0, chunk_size)
		for j := 0; j < chunk_size; j++ {
			if doc_counter >= docs_len {
				break
			}
			add = append(add, docs[doc_counter])
			doc_counter++
		}
		chunks[i] = map[string]interface{}{"add": add}
	}
	return chunks
}

// Add will insert documents in batch of chunk_size. success is false as long as one chunk failed.
// The result in UpdateResponse is summery of response from all chunks
// with key chunk_%d
func (si *SolrInterface) Add(docs []Document, chunk_size int, params *url.Values) (*UpdateResponse, error) {
	result := &UpdateResponse{Success: true}
	responses := map[string]interface{}{}
	chunks := makeAddChunks(docs, chunk_size)

	for i := 0; i < len(chunks); i++ {
		res, err := si.Update(chunks[i], params)
		if err != nil {
			return nil, err
		}
		result.Success = result.Success && res.Success
		responses[fmt.Sprintf("chunk_%d", i+1)] = map[string]interface{}{
			"result":  res.Result,
			"success": res.Success,
			"total":   len(chunks[i]["add"].([]Document))}
	}
	result.Result = responses
	return result, nil
}

// Delete take data of type map and optional params which can use to specify addition parameters such as commit=true .
// Only one delete statement is supported, ie data can be { "id":"ID" } .
// If you want to delete more docs use { "query":"QUERY" } .
// Extra params can specify in params or in data such as { "query":"QUERY", "commitWithin":"500" }
func (si *SolrInterface) Delete(data map[string]interface{}, params *url.Values) (*UpdateResponse, error) {
	message := map[string]interface{}{"delete": data}
	return si.Update(message, params)
}

// DeleteAll will remove all documents and commit
func (si *SolrInterface) DeleteAll() (*UpdateResponse, error) {
	params := &url.Values{}
	params.Add("commit", "true")
	return si.Delete(map[string]interface{}{"query": "*:*"}, params)
}

// Update take data of type map and optional params which can use to specify addition parameters such as commit=true
func (si *SolrInterface) Update(data map[string]interface{}, params *url.Values) (*UpdateResponse, error) {
	if si.conn == nil {
		return nil, fmt.Errorf("No connection found for making request to solr")
	}
	return si.conn.Update(data, params)
}

// Commit the changes since the last commit
func (si *SolrInterface) Commit() (*UpdateResponse, error) {
	params := &url.Values{}
	params.Add("commit", "true")
	return si.Update(map[string]interface{}{}, params)
}

func (si *SolrInterface) Optimize(params *url.Values) (*UpdateResponse, error) {
	if params == nil {
		params = &url.Values{}
	}
	params.Set("optimize", "true")
	return si.Update(map[string]interface{}{}, params)
}

// Rollback rollbacks all add/deletes made to the index since the last commit.
// This should use with caution.
// See https://wiki.apache.org/solr/UpdateXmlMessages#A.22rollback.22
func (si *SolrInterface) Rollback() (*UpdateResponse, error) {
	return si.Update(map[string]interface{}{"rollback": map[string]interface{}{}}, nil)
}
<|endoftext|>