<commit_before>package object

import (
	"fmt"
	"strings"

	"github.com/Zac-Garby/pluto/ast"
	"github.com/Zac-Garby/pluto/bytecode"
	"github.com/fatih/color"
)

type (
	/* Collections and collection-likes */

	// Tuple is a tuple value, such as (1, "foo", false)
	Tuple struct {
		Value []Object
	}

	// Array is an array value, such as [1, 5, "baz", true]
	Array struct {
		Value []Object
	}

	// Map is a mapping of keys to values, such as ["x": 2, "y": 5]
	Map struct {
		Values map[string]Object
		Keys   map[string]Object
	}

	/* Others */

	// Block is an anonymous function
	Block struct {
		Params []ast.Expression
		Body   bytecode.Code
	}

	// Class is a class, containing methods and a parent
	Class struct {
		Name    string
		Parent  Object
		Methods []Object
	}

	// Instance is an instance of a class
	Instance struct {
		Base Object
		Data map[string]Object
	}
)

/* Type() methods */

// Type returns the type of this object
func (t *Tuple) Type() Type { return TupleType }

// Type returns the type of this object
func (a *Array) Type() Type { return ArrayType }

// Type returns the type of this object
func (m *Map) Type() Type { return MapType }

// Type returns the type of this object
func (b *Block) Type() Type { return BlockType }

// Type returns the type of this object
func (c *Class) Type() Type { return ClassType }

// Type returns the type of this object
func (i *Instance) Type() Type { return InstanceType }

/* Equals() methods */

// Equals checks if two objects are equal to each other
func (t *Tuple) Equals(o Object) bool {
	if other, ok := o.(*Tuple); ok {
		if len(other.Value) != len(t.Value) {
			return false
		}

		for i, elem := range t.Value {
			if !elem.Equals(other.Value[i]) {
				return false
			}
		}

		return true
	}

	return false
}

// Equals checks if two objects are equal to each other
func (a *Array) Equals(o Object) bool {
	if other, ok := o.(*Array); ok {
		if len(other.Value) != len(a.Value) {
			return false
		}

		for i, elem := range a.Value {
			if !elem.Equals(other.Value[i]) {
				return false
			}
		}

		return true
	}

	return false
}

// Equals checks if two objects are equal to each other
func (m *Map) Equals(o Object) bool {
	if other, ok := o.(*Map); ok {
		if len(other.Values) != len(m.Values) {
			return false
		}

		for k, v := range m.Values {
			if _, ok := other.Values[k]; !ok {
				return false
			}

			if !v.Equals(other.Values[k]) {
				return false
			}
		}

		return true
	}

	return false
}

// Equals checks if two objects are equal to each other
func (b *Block) Equals(o Object) bool {
	_, ok := o.(*Block)
	return ok
}

// Equals checks if two objects are equal to each other
func (c *Class) Equals(o Object) bool {
	if other, ok := o.(*Class); ok {
		return other.Name == c.Name
	}

	return false
}

// Equals checks if two objects are equal to each other
func (i *Instance) Equals(o Object) bool {
	if other, ok := o.(*Instance); ok {
		if !other.Base.Equals(i.Base) {
			return false
		}

		for k, v := range i.Data {
			if !v.Equals(other.Data[k]) {
				return false
			}
		}

		return true
	}

	return false
}

/* Stringer implementations */
func join(arr []Object, sep string) string {
	stringArr := make([]string, len(arr))

	for i, elem := range arr {
		stringArr[i] = elem.String()
	}

	return strings.Join(stringArr, ", ")
}

func (t *Tuple) String() string {
	return fmt.Sprintf("(%s)", join(t.Value, ", "))
}

func (a *Array) String() string {
	return fmt.Sprintf("[%s]", join(a.Value, ", "))
}

func (m *Map) String() string {
	stringArr := make([]string, len(m.Values))
	i := 0

	for k, v := range m.Values {
		stringArr[i] = fmt.Sprintf(
			"%s: %s",
			m.Keys[k].String(),
			v.String(),
		)

		i++
	}

	return fmt.Sprintf("[%s]", strings.Join(stringArr, ", "))
}

func (b *Block) String() string {
	return "<block>"
}

func (c *Class) String() string {
	return c.Name
}

func (i *Instance) String() string {
	if i.Base.(*Class).Name == "Error" {
		tag := color.New(color.FgRed, color.Bold).Sprint(i.Get(&String{"tag"}))
		return fmt.Sprintf("%s\t%s", tag, i.Get(&String{"msg"}))
	}

	stringMethod := i.Base.(*Class).GetMethod("string")

	if stringMethod != nil {
		panic("Implement me :)")
	}

	return fmt.Sprintf("<instance of %s>", i.Base.(*Class).Name)
}

/* Collection implementations */

// Elements returns the elements in a collection
func (t *Tuple) Elements() []Object {
	return t.Value
}

// GetIndex returns the ith element in a collection
func (t *Tuple) GetIndex(i int) Object {
	if i >= len(t.Value) || i < 0 {
		return NullObj
	}

	return t.Value[i]
}

// SetIndex sets the ith element in a collection to o
func (t *Tuple) SetIndex(i int, o Object) {
	if i >= len(t.Value) || i < 0 {
		return
	}

	t.Value[i] = o
}

// Elements returns the elements in a collection
func (a *Array) Elements() []Object {
	return a.Value
}

// GetIndex returns the ith element in a collection
func (a *Array) GetIndex(i int) Object {
	if i >= len(a.Value) || i < 0 {
		return NullObj
	}

	return a.Value[i]
}

// SetIndex sets the ith element in a collection to o
func (a *Array) SetIndex(i int, o Object) {
	if i >= len(a.Value) || i < 0 {
		return
	}

	a.Value[i] = o
}

/* Container implementations */

// Get gets an object at the given key
func (m *Map) Get(key Object) Object {
	hasher, ok := key.(Hasher)

	if !ok {
		return NullObj
	}

	if val, ok := m.Values[hasher.Hash()]; ok {
		return val
	}

	return NullObj
}

// Set sets an object at the given key
func (m *Map) Set(key, value Object) {
	if hasher, ok := key.(Hasher); ok {
		hash := hasher.Hash()
		m.Values[hash] = value
		m.Keys[hash] = key
	}
}

// Get gets an object at the given key
func (i *Instance) Get(key Object) Object {
	strkey, ok := key.(*String)

	if !ok {
		return NullObj
	}

	if val, ok := i.Data[strkey.Value]; ok {
		return val
	}

	return NullObj
}

// Set sets an object at the given key
func (i *Instance) Set(key, value Object) {
	if strkey, ok := key.(*String); ok {
		i.Data[strkey.Value] = value
	}
}

/* Other methods */

// GetMethods returns a slice of methods of a class
func (c *Class) GetMethods() []Method {
	var methods []Method

	if c.Parent != nil {
		methods = c.Parent.(*Class).GetMethods()
	}

	for _, m := range c.Methods {
		if method, ok := m.(*Method); ok {
			methods = append(methods, *method)
		}
	}

	return methods
}

// GetMethod returns a method matching the given pattern
func (c *Class) GetMethod(pattern string) *Method {
	fnPattern := strings.Split(pattern, " ")

	for _, method := range c.GetMethods() {
		methodPattern := method.Fn.Pattern

		if len(fnPattern) != len(methodPattern) {
			continue
		}

		isMatch := true
		for i, mPatItem := range methodPattern {
			_, isParam := mPatItem.(*ast.Parameter)
			_, isIdent := mPatItem.(*ast.Identifier)

			if !(fnPattern[i] == "$" && isParam || isIdent && fnPattern[i] == methodPattern[i].Token().Literal) {
				isMatch = false
			}
		}

		if isMatch {
			return &method
		}
	}

	return nil
}
<commit_msg>Add the params into Block.String()<commit_after>package object

import (
	"fmt"
	"strings"

	"github.com/Zac-Garby/pluto/ast"
	"github.com/Zac-Garby/pluto/bytecode"
	"github.com/fatih/color"
)

type (
	/* Collections and collection-likes */

	// Tuple is a tuple value, such as (1, "foo", false)
	Tuple struct {
		Value []Object
	}

	// Array is an array value, such as [1, 5, "baz", true]
	Array struct {
		Value []Object
	}

	// Map is a mapping of keys to values, such as ["x": 2, "y": 5]
	Map struct {
		Values map[string]Object
		Keys   map[string]Object
	}

	/* Others */

	// Block is an anonymous function
	Block struct {
		Params []ast.Expression
		Body   bytecode.Code
	}

	// Class is a class, containing methods and a parent
	Class struct {
		Name    string
		Parent  Object
		Methods []Object
	}

	// Instance is an instance of a class
	Instance struct {
		Base Object
		Data map[string]Object
	}
)

/* Type() methods */

// Type returns the type of this object
func (t *Tuple) Type() Type { return TupleType }

// Type returns the type of this object
func (a *Array) Type() Type { return ArrayType }

// Type returns the type of this object
func (m *Map) Type() Type { return MapType }

// Type returns the type of this object
func (b *Block) Type() Type { return BlockType }

// Type returns the type of this object
func (c *Class) Type() Type { return ClassType }

// Type returns the type of this object
func (i *Instance) Type() Type { return InstanceType }

/* Equals() methods */

// Equals checks if two objects are equal to each other
func (t *Tuple) Equals(o Object) bool {
	if other, ok := o.(*Tuple); ok {
		if len(other.Value) != len(t.Value) {
			return false
		}

		for i, elem := range t.Value {
			if !elem.Equals(other.Value[i]) {
				return false
			}
		}

		return true
	}

	return false
}

// Equals checks if two objects are equal to each other
func (a *Array) Equals(o Object) bool {
	if other, ok := o.(*Array); ok {
		if len(other.Value) != len(a.Value) {
			return false
		}

		for i, elem := range a.Value {
			if !elem.Equals(other.Value[i]) {
				return false
			}
		}

		return true
	}

	return false
}

// Equals checks if two objects are equal to each other
func (m *Map) Equals(o Object) bool {
	if other, ok := o.(*Map); ok {
		if len(other.Values) != len(m.Values) {
			return false
		}

		for k, v := range m.Values {
			if _, ok := other.Values[k]; !ok {
				return false
			}

			if !v.Equals(other.Values[k]) {
				return false
			}
		}

		return true
	}

	return false
}

// Equals checks if two objects are equal to each other
func (b *Block) Equals(o Object) bool {
	_, ok := o.(*Block)
	return ok
}

// Equals checks if two objects are equal to each other
func (c *Class) Equals(o Object) bool {
	if other, ok := o.(*Class); ok {
		return other.Name == c.Name
	}

	return false
}

// Equals checks if two objects are equal to each other
func (i *Instance) Equals(o Object) bool {
	if other, ok := o.(*Instance); ok {
		if !other.Base.Equals(i.Base) {
			return false
		}

		for k, v := range i.Data {
			if !v.Equals(other.Data[k]) {
				return false
			}
		}

		return true
	}

	return false
}

/* Stringer implementations */
func join(arr []Object, sep string) string {
	stringArr := make([]string, len(arr))

	for i, elem := range arr {
		stringArr[i] = elem.String()
	}

	return strings.Join(stringArr, ", ")
}

func (t *Tuple) String() string {
	return fmt.Sprintf("(%s)", join(t.Value, ", "))
}

func (a *Array) String() string {
	return fmt.Sprintf("[%s]", join(a.Value, ", "))
}

func (m *Map) String() string {
	stringArr := make([]string, len(m.Values))
	i := 0

	for k, v := range m.Values {
		stringArr[i] = fmt.Sprintf(
			"%s: %s",
			m.Keys[k].String(),
			v.String(),
		)

		i++
	}

	return fmt.Sprintf("[%s]", strings.Join(stringArr, ", "))
}

func (b *Block) String() string {
	if len(b.Params) == 0 {
		return "<block>"
	}

	var params []string

	for _, param := range b.Params {
		params = append(params, param.Token().Literal)
	}

	return fmt.Sprintf("<block: %s>", strings.Join(params, ", "))
}

func (c *Class) String() string {
	return c.Name
}

func (i *Instance) String() string {
	if i.Base.(*Class).Name == "Error" {
		tag := color.New(color.FgRed, color.Bold).Sprint(i.Get(&String{"tag"}))
		return fmt.Sprintf("%s\t%s", tag, i.Get(&String{"msg"}))
	}

	stringMethod := i.Base.(*Class).GetMethod("string")

	if stringMethod != nil {
		panic("Implement me :)")
	}

	return fmt.Sprintf("<instance of %s>", i.Base.(*Class).Name)
}

/* Collection implementations */

// Elements returns the elements in a collection
func (t *Tuple) Elements() []Object {
	return t.Value
}

// GetIndex returns the ith element in a collection
func (t *Tuple) GetIndex(i int) Object {
	if i >= len(t.Value) || i < 0 {
		return NullObj
	}

	return t.Value[i]
}

// SetIndex sets the ith element in a collection to o
func (t *Tuple) SetIndex(i int, o Object) {
	if i >= len(t.Value) || i < 0 {
		return
	}

	t.Value[i] = o
}

// Elements returns the elements in a collection
func (a *Array) Elements() []Object {
	return a.Value
}

// GetIndex returns the ith element in a collection
func (a *Array) GetIndex(i int) Object {
	if i >= len(a.Value) || i < 0 {
		return NullObj
	}

	return a.Value[i]
}

// SetIndex sets the ith element in a collection to o
func (a *Array) SetIndex(i int, o Object) {
	if i >= len(a.Value) || i < 0 {
		return
	}

	a.Value[i] = o
}

/* Container implementations */

// Get gets an object at the given key
func (m *Map) Get(key Object) Object {
	hasher, ok := key.(Hasher)

	if !ok {
		return NullObj
	}

	if val, ok := m.Values[hasher.Hash()]; ok {
		return val
	}

	return NullObj
}

// Set sets an object at the given key
func (m *Map) Set(key, value Object) {
	if hasher, ok := key.(Hasher); ok {
		hash := hasher.Hash()
		m.Values[hash] = value
		m.Keys[hash] = key
	}
}

// Get gets an object at the given key
func (i *Instance) Get(key Object) Object {
	strkey, ok := key.(*String)

	if !ok {
		return NullObj
	}

	if val, ok := i.Data[strkey.Value]; ok {
		return val
	}

	return NullObj
}

// Set sets an object at the given key
func (i *Instance) Set(key, value Object) {
	if strkey, ok := key.(*String); ok {
		i.Data[strkey.Value] = value
	}
}

/* Other methods */

// GetMethods returns a slice of methods of a class
func (c *Class) GetMethods() []Method {
	var methods []Method

	if c.Parent != nil {
		methods = c.Parent.(*Class).GetMethods()
	}

	for _, m := range c.Methods {
		if method, ok := m.(*Method); ok {
			methods = append(methods, *method)
		}
	}

	return methods
}

// GetMethod returns a method matching the given pattern
func (c *Class) GetMethod(pattern string) *Method {
	fnPattern := strings.Split(pattern, " ")

	for _, method := range c.GetMethods() {
		methodPattern := method.Fn.Pattern

		if len(fnPattern) != len(methodPattern) {
			continue
		}

		isMatch := true
		for i, mPatItem := range methodPattern {
			_, isParam := mPatItem.(*ast.Parameter)
			_, isIdent := mPatItem.(*ast.Identifier)

			if !(fnPattern[i] == "$" && isParam || isIdent && fnPattern[i] == methodPattern[i].Token().Literal) {
				isMatch = false
			}
		}

		if isMatch {
			return &method
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package epazote

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"sync"
	"testing"
)

func TestReportHTTPGet(t *testing.T) {
	buf.Reset()
	var wg sync.WaitGroup
	custom_s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("User-agent") != "epazote" {
			t.Error("Expecting User-agent: epazote")
		}
		if r.Method != "GET" {
			t.Errorf("Expecting Method GET got: %s", r.Method)
		}
		if r.FormValue("param") != "list" {
			t.Errorf("Expecting param = list got: %s", r.FormValue("param"))
		}
		wg.Done()
	}))
	defer custom_s.Close()

	s := &Service{
		Name: "s 1",
		URL:  "http://about.epazote.io",
		Expect: Expect{
			Status: 200,
		},
	}
	a := &Action{
		HTTP: []HTTP{
			HTTP{
				URL: fmt.Sprintf("%s/?param=list", custom_s.URL),
			},
		},
	}
	ez := &Epazote{}
	wg.Add(1)
	ez.Report(nil, s, a, nil, 1, 200, "because", "output")
	wg.Wait()
}

func TestReportHTTP0(t *testing.T) {
	buf.Reset()
	s := &Service{
		Name: "s 1",
		URL:  "http://about.epazote.io",
		Expect: Expect{
			Status: 200,
		},
	}
	a := &Action{
		HTTP: []HTTP{
			HTTP{
				URL: "http://no-call",
			},
		},
	}
	ez := &Epazote{}
	ez.debug = true
	ez.Report(nil, s, a, nil, 0, 200, "because", "output")
}

func TestReportHTTP001(t *testing.T) {
	buf.Reset()
	var wg sync.WaitGroup
	custom_s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("User-agent") != "epazote" {
			t.Error("Expecting User-agent: epazote")
		}
		if r.Method != "GET" {
			t.Errorf("Expecting Method GET got: %s", r.Method)
		}
		if r.FormValue("exit") != "0" {
			t.Errorf("Expecting exit = 0 got: %s", r.FormValue("exit"))
		}
		if r.FormValue("status") != "200" {
			t.Errorf("Expecting status = 200 got: %s", r.FormValue("status"))
		}
		wg.Done()
	}))
	defer custom_s.Close()

	s := &Service{
		Name: "s 1",
		URL:  "http://about.epazote.io",
		Expect: Expect{
			Status: 200,
		},
	}
	a := &Action{
		HTTP: []HTTP{
			HTTP{
				URL: fmt.Sprintf("%s/?exit=_exit_&status=_status_", custom_s.URL),
			},
			HTTP{
				URL: fmt.Sprintf("%s/?exit=_exit_", custom_s.URL),
			},
		},
	}
	ez := &Epazote{}
	ez.debug = true
	wg.Add(1)
	ez.Report(nil, s, a, nil, 0, 200, "because", "output")
	wg.Wait()
}

func TestReportHTTP011(t *testing.T) {
	buf.Reset()
	var wg sync.WaitGroup
	custom_s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("User-agent") != "epazote" {
			t.Error("Expecting User-agent: epazote")
		}
		if r.Method != "GET" {
			t.Errorf("Expecting Method GET got: %s", r.Method)
		}
		if r.FormValue("exit") != "1" {
			t.Errorf("Expecting exit = 1 got: %s", r.FormValue("exit"))
		}
		wg.Done()
	}))
	defer custom_s.Close()

	s := &Service{
		Name: "s 1",
		URL:  "http://about.epazote.io",
		Expect: Expect{
			Status: 200,
		},
	}
	a := &Action{
		HTTP: []HTTP{
			HTTP{
				URL: fmt.Sprintf("%s/?exit=0", custom_s.URL),
			},
			HTTP{
				URL: fmt.Sprintf("%s/?exit=1", custom_s.URL),
			},
		},
	}
	ez := &Epazote{}
	ez.debug = true
	wg.Add(1)
	ez.Report(nil, s, a, nil, 1, 200, "because", "output")
	wg.Wait()
}

func TestReportHTTPPost(t *testing.T) {
	buf.Reset()
	var wg sync.WaitGroup
	custom_s := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("User-agent") != "epazote" {
			t.Error("Expecting User-agent: epazote")
		}
		if r.Header.Get("Content-Type") != "application/x-www-form-urlencoded" {
			t.Error("Expecting Content-Type: application/x-www-form-urlencoded")
		}
		if r.Method != "POST" {
			t.Errorf("Expecting Method POST got: %s", r.Method)
		}

		body, _ := ioutil.ReadAll(r.Body)
		values, _ := url.ParseQuery(string(body))

		var expected = []struct {
			key, val string
		}{
			{"room_id", "10"},
			{"from", "Alerts"},
			{"message", "A new user signed up"},
			{"status", "200"},
		}

		for _, v := range expected {
			got := values[v.key]
			if got[0] != v.val {
				t.Errorf("Expecting %s got: %s", got[0], v.val)
			}
		}

		wg.Done()
	}))
	defer custom_s.Close()

	s := &Service{
		Name: "s 1",
		URL:  "http://about.epazote.io",
		Expect: Expect{
			Status: 200,
		},
	}
	a := &Action{
		HTTP: []HTTP{
			HTTP{
				URL:    custom_s.URL,
				Method: "post",
				Data:   "room_id=10&from=Alerts&message=A+new+user+signed+up&status=_status_",
				Header: map[string]string{
					"Content-Type": "application/x-www-form-urlencoded",
				},
			},
		},
	}
	ez := &Epazote{}
	ez.debug = true
	wg.Add(1)
	ez.Report(nil, s, a, nil, 1, 200, "because", "output")
	wg.Wait()
}

func TestReportHTTPNoURL(t *testing.T) {
	buf.Reset()
	s := &Service{
		Name: "s 1",
		URL:  "http://about.epazote.io",
		Expect: Expect{
			Status: 200,
		},
	}
	a := &Action{
		HTTP: []HTTP{
			HTTP{
				Method: "PUT",
			},
		},
	}
	ez := &Epazote{}
	ez.Report(nil, s, a, nil, 1, 200, "because", "output")
}
<commit_msg>exepct not properly working, probalby because a defer recover<commit_after><|endoftext|>


<commit_before>package pool_test

import (
	"sync"
	"testing"

	"github.com/npat-efault/varhacks/pool"
)

type S struct {
	id  int
	buf [64]byte
}

func TestPoolNil(t *testing.T) {
	p := pool.New(10, nil)
	for i := 0; i < 12; i++ {
		p.Put(&S{id: i})
	}
	for i := 0; i < 10; i++ {
		if s := p.Get(); s == nil {
			t.Fatalf("Item %d is nil", i)
		} else {
			ss := s.(*S)
			if ss.id != i {
				t.Fatalf("Item id %d != %d", ss.id, i)
			}
		}
	}
	if s := p.Get(); s != nil {
		t.Fatal("Too many items in pool")
	}
}

func TestPoolAlloc(t *testing.T) {
	p := pool.New(10, func() interface{} { return &S{} })
	for i := 0; i < 12; i++ {
		p.Put(&S{id: i})
	}
	for i := 0; i < 10; i++ {
		if s := p.Get(); s == nil {
			t.Fatalf("Item %d is nil", i)
		} else {
			ss := s.(*S)
			if ss.id != i {
				t.Fatalf("Item id %d != %d", ss.id, i)
			}
		}
	}
	if s := p.Get(); s == nil {
		t.Fatal("No new allocation")
	} else if ss := s.(*S); ss.id != 0 {
		t.Fatal("Unitialized new allocation")
	}
}

func TestPoolEmpty(t *testing.T) {
	p := pool.New(10, nil)
	for i := 0; i < 12; i++ {
		p.Put(&S{id: i})
	}
	p.Empty()
	if s := p.Get(); s != nil {
		t.Fatal("Pool not empty")
	}
}

func TestBSPoolNil(t *testing.T) {
	p := pool.NewByteSlice(10, nil)
	for i := 0; i < 12; i++ {
		b := make([]byte, 64)
		b[63] = byte(i)
		p.Put(b)
	}
	for i := 0; i < 10; i++ {
		if b := p.Get(); b == nil {
			t.Fatalf("Item %d is nil", i)
		} else if b[63] != byte(i) {
			t.Fatalf("Item %d[0] != %d", b[63], i)
		}
	}
	if b := p.Get(); b != nil {
		t.Fatal("Too many items in pool")
	}
}

func TestBSPoolAlloc(t *testing.T) {
	p := pool.NewByteSlice(10, func() []byte { return make([]byte, 64) })
	for i := 0; i < 12; i++ {
		b := make([]byte, 64)
		b[63] = byte(i)
		p.Put(b)
	}
	for i := 0; i < 10; i++ {
		if b := p.Get(); b == nil {
			t.Fatalf("Item %d is nil", i)
		} else if b[63] != byte(i) {
			t.Fatalf("Item %d[0] != %d", b[63], i)
		}
	}
	if b := p.Get(); b == nil {
		t.Fatal("No new allocation")
	} else if b[63] != 0 {
		t.Fatal("Unitialized new allocation")
	}
}

func TestBSPoolEmpty(t *testing.T) {
	p := pool.NewByteSlice(10, nil)
	for i := 0; i < 12; i++ {
		b := make([]byte, 64)
		b[63] = byte(i)
		p.Put(b)
	}
	p.Empty()
	if b := p.Get(); b != nil {
		t.Fatal("Pool not empty")
	}
}

// See allocations due to conversions from []byte to
// interface{}. Avoided by specialized ByteSlice pool.
//
// run with:
//      go test -v -memprofile=mem-bsp.out -bench='ByteSlicePool$'
//   or go test -v -memprofile=mem-p.out -bench='Pool$'
//   or go test -v -memprofile=mem-sp.out -bench='SyncPool$'
//
// see results with
//
//      go tool pprof --alloc_objects ./pool.test mem-xxx.out -text
//      ... etc ...

func BenchmarkAllocByteSlicePool(b *testing.B) {
	s := make([]byte, 10)
	p := pool.NewByteSlice(1, nil)
	for i := 0; i < b.N; i++ {
		p.Put(s)
		s = p.Get()
	}
}

func BenchmarkAllocPool(b *testing.B) {
	s := make([]byte, 10)
	p := pool.New(1, nil)
	for i := 0; i < b.N; i++ {
		p.Put(s)
		s = p.Get().([]byte)
	}
}

func BenchmarkAllocSyncPool(b *testing.B) {
	var p sync.Pool
	s := make([]byte, 10)
	for i := 0; i < b.N; i++ {
		p.Put(s)
		s = p.Get().([]byte)
	}
}
<commit_msg>Renamed repo to gohacks<commit_after>package pool_test

import (
	"sync"
	"testing"

	"github.com/npat-efault/gohacks/pool"
)

type S struct {
	id  int
	buf [64]byte
}

func TestPoolNil(t *testing.T) {
	p := pool.New(10, nil)
	for i := 0; i < 12; i++ {
		p.Put(&S{id: i})
	}
	for i := 0; i < 10; i++ {
		if s := p.Get(); s == nil {
			t.Fatalf("Item %d is nil", i)
		} else {
			ss := s.(*S)
			if ss.id != i {
				t.Fatalf("Item id %d != %d", ss.id, i)
			}
		}
	}
	if s := p.Get(); s != nil {
		t.Fatal("Too many items in pool")
	}
}

func TestPoolAlloc(t *testing.T) {
	p := pool.New(10, func() interface{} { return &S{} })
	for i := 0; i < 12; i++ {
		p.Put(&S{id: i})
	}
	for i := 0; i < 10; i++ {
		if s := p.Get(); s == nil {
			t.Fatalf("Item %d is nil", i)
		} else {
			ss := s.(*S)
			if ss.id != i {
				t.Fatalf("Item id %d != %d", ss.id, i)
			}
		}
	}
	if s := p.Get(); s == nil {
		t.Fatal("No new allocation")
	} else if ss := s.(*S); ss.id != 0 {
		t.Fatal("Unitialized new allocation")
	}
}

func TestPoolEmpty(t *testing.T) {
	p := pool.New(10, nil)
	for i := 0; i < 12; i++ {
		p.Put(&S{id: i})
	}
	p.Empty()
	if s := p.Get(); s != nil {
		t.Fatal("Pool not empty")
	}
}

func TestBSPoolNil(t *testing.T) {
	p := pool.NewByteSlice(10, nil)
	for i := 0; i < 12; i++ {
		b := make([]byte, 64)
		b[63] = byte(i)
		p.Put(b)
	}
	for i := 0; i < 10; i++ {
		if b := p.Get(); b == nil {
			t.Fatalf("Item %d is nil", i)
		} else if b[63] != byte(i) {
			t.Fatalf("Item %d[0] != %d", b[63], i)
		}
	}
	if b := p.Get(); b != nil {
		t.Fatal("Too many items in pool")
	}
}

func TestBSPoolAlloc(t *testing.T) {
	p := pool.NewByteSlice(10, func() []byte { return make([]byte, 64) })
	for i := 0; i < 12; i++ {
		b := make([]byte, 64)
		b[63] = byte(i)
		p.Put(b)
	}
	for i := 0; i < 10; i++ {
		if b := p.Get(); b == nil {
			t.Fatalf("Item %d is nil", i)
		} else if b[63] != byte(i) {
			t.Fatalf("Item %d[0] != %d", b[63], i)
		}
	}
	if b := p.Get(); b == nil {
		t.Fatal("No new allocation")
	} else if b[63] != 0 {
		t.Fatal("Unitialized new allocation")
	}
}

func TestBSPoolEmpty(t *testing.T) {
	p := pool.NewByteSlice(10, nil)
	for i := 0; i < 12; i++ {
		b := make([]byte, 64)
		b[63] = byte(i)
		p.Put(b)
	}
	p.Empty()
	if b := p.Get(); b != nil {
		t.Fatal("Pool not empty")
	}
}

// See allocations due to conversions from []byte to
// interface{}. Avoided by specialized ByteSlice pool.
//
// run with:
//      go test -v -memprofile=mem-bsp.out -bench='ByteSlicePool$'
//   or go test -v -memprofile=mem-p.out -bench='Pool$'
//   or go test -v -memprofile=mem-sp.out -bench='SyncPool$'
//
// see results with
//
//      go tool pprof --alloc_objects ./pool.test mem-xxx.out -text
//      ... etc ...

func BenchmarkAllocByteSlicePool(b *testing.B) {
	s := make([]byte, 10)
	p := pool.NewByteSlice(1, nil)
	for i := 0; i < b.N; i++ {
		p.Put(s)
		s = p.Get()
	}
}

func BenchmarkAllocPool(b *testing.B) {
	s := make([]byte, 10)
	p := pool.New(1, nil)
	for i := 0; i < b.N; i++ {
		p.Put(s)
		s = p.Get().([]byte)
	}
}

func BenchmarkAllocSyncPool(b *testing.B) {
	var p sync.Pool
	s := make([]byte, 10)
	for i := 0; i < b.N; i++ {
		p.Put(s)
		s = p.Get().([]byte)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"archive/zip"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"regexp"
	"runtime"
	"sync"
	"sync/atomic"
	"time"

	"github.com/google/syzkaller/host"
	"github.com/google/syzkaller/ipc"
	"github.com/google/syzkaller/prog"
	"github.com/google/syzkaller/sys"
)

var (
	flagCorpus   = flag.String("corpus", "", "corpus directory")
	flagExecutor = flag.String("executor", "", "path to executor binary")
	flagOutput   = flag.Bool("output", false, "print executor output to console")
	flagProcs    = flag.Int("procs", 2*runtime.NumCPU(), "number of parallel processes")
	flagLogProg  = flag.Bool("logprog", false, "print programs before execution")

	failedRe = regexp.MustCompile("runtime error: |panic: |Panic: ")

	statExec uint64
	gate     *ipc.Gate
)

const programLength = 30

func main() {
	flag.Parse()
	corpus := readCorpus()
	log.Printf("parsed %v programs", len(corpus))

	calls := buildCallList()
	prios := prog.CalculatePriorities(corpus)
	ct := prog.BuildChoiceTable(prios, calls)

	flags, timeout := ipc.DefaultFlags()
	gate = ipc.NewGate(2**flagProcs, nil)
	for pid := 0; pid < *flagProcs; pid++ {
		pid := pid
		go func() {
			env, err := ipc.MakeEnv(*flagExecutor, timeout, flags)
			if err != nil {
				failf("failed to create execution environment: %v", err)
			}
			rs := rand.NewSource(time.Now().UnixNano() + int64(pid)*1e12)
			rnd := rand.New(rs)
			for i := 0; ; i++ {
				var p *prog.Prog
				if len(corpus) == 0 || i%4 != 0 {
					p = prog.Generate(rs, programLength, ct)
					execute(pid, env, p)
					p.Mutate(rs, programLength, ct)
					execute(pid, env, p)
				} else {
					p = corpus[rnd.Intn(len(corpus))].Clone()
					p.Mutate(rs, programLength, ct)
					execute(pid, env, p)
					p.Mutate(rs, programLength, ct)
					execute(pid, env, p)
				}
			}
		}()
	}
	for range time.NewTicker(5 * time.Second).C {
		log.Printf("executed %v programs", atomic.LoadUint64(&statExec))
	}
}

var outMu sync.Mutex

func execute(pid int, env *ipc.Env, p *prog.Prog) {
	if *flagExecutor == "" {
		return
	}
	atomic.AddUint64(&statExec, 1)
	if *flagLogProg {
		ticket := gate.Enter()
		defer gate.Leave(ticket)
		outMu.Lock()
		fmt.Printf("executing program %v\n%s\n", pid, p.Serialize())
		outMu.Unlock()
	}

	output, _, _, _, _, err := env.Exec(p)
	if err != nil {
		fmt.Printf("failed to execute executor: %v\n", err)
	}
	failed := failedRe.Match(output)
	if failed {
		fmt.Printf("PROGRAM:\n%s\n", p.Serialize())
	}
	if failed || *flagOutput {
		os.Stdout.Write(output)
	}
}

func readCorpus() []*prog.Prog {
	if *flagCorpus == "" {
		return nil
	}
	zipr, err := zip.OpenReader(*flagCorpus)
	if err != nil {
		failf("failed to open bin file: %v", err)
	}
	var progs []*prog.Prog
	for _, zipf := range zipr.File {
		r, err := zipf.Open()
		if err != nil {
			failf("failed to uzip file from input archive: %v", err)
		}
		data, err := ioutil.ReadAll(r)
		if err != nil {
			failf("failed to read corpus file: %v", err)
		}
		p, err := prog.Deserialize(data)
		if err != nil {
			failf("failed to deserialize corpus program: %v", err)
		}
		progs = append(progs, p)
		r.Close()
	}
	zipr.Close()
	return progs
}

func buildCallList() map[*sys.Call]bool {
	calls, err := host.DetectSupportedSyscalls()
	if err != nil {
		log.Printf("failed to detect host supported syscalls: %v", err)
		for _, c := range sys.Calls {
			calls[c] = true
		}
	}
	for _, c := range sys.Calls {
		if !calls[c] {
			log.Printf("disabling unsupported syscall: %v", c.Name)
		}
	}
	trans := sys.TransitivelyEnabledCalls(calls)
	for c := range calls {
		if !trans[c] {
			log.Printf("disabling transitively unsupported syscall: %v", c.Name)
			delete(calls, c)
		}
	}
	return calls
}

func failf(msg string, args ...interface{}) {
	log.Fatalf(msg, args...)
}
<commit_msg>stress: print program and output on more types of bugs<commit_after>// Copyright 2015 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"archive/zip"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"regexp"
	"runtime"
	"sync"
	"sync/atomic"
	"time"

	"github.com/google/syzkaller/host"
	"github.com/google/syzkaller/ipc"
	"github.com/google/syzkaller/prog"
	"github.com/google/syzkaller/sys"
)

var (
	flagCorpus   = flag.String("corpus", "", "corpus directory")
	flagExecutor = flag.String("executor", "", "path to executor binary")
	flagOutput   = flag.Bool("output", false, "print executor output to console")
	flagProcs    = flag.Int("procs", 2*runtime.NumCPU(), "number of parallel processes")
	flagLogProg  = flag.Bool("logprog", false, "print programs before execution")

	failedRe = regexp.MustCompile("runtime error: |panic: |Panic: ")

	statExec uint64
	gate     *ipc.Gate
)

const programLength = 30

func main() {
	flag.Parse()
	corpus := readCorpus()
	log.Printf("parsed %v programs", len(corpus))

	calls := buildCallList()
	prios := prog.CalculatePriorities(corpus)
	ct := prog.BuildChoiceTable(prios, calls)

	flags, timeout := ipc.DefaultFlags()
	gate = ipc.NewGate(2**flagProcs, nil)
	for pid := 0; pid < *flagProcs; pid++ {
		pid := pid
		go func() {
			env, err := ipc.MakeEnv(*flagExecutor, timeout, flags)
			if err != nil {
				failf("failed to create execution environment: %v", err)
			}
			rs := rand.NewSource(time.Now().UnixNano() + int64(pid)*1e12)
			rnd := rand.New(rs)
			for i := 0; ; i++ {
				var p *prog.Prog
				if len(corpus) == 0 || i%4 != 0 {
					p = prog.Generate(rs, programLength, ct)
					execute(pid, env, p)
					p.Mutate(rs, programLength, ct)
					execute(pid, env, p)
				} else {
					p = corpus[rnd.Intn(len(corpus))].Clone()
					p.Mutate(rs, programLength, ct)
					execute(pid, env, p)
					p.Mutate(rs, programLength, ct)
					execute(pid, env, p)
				}
			}
		}()
	}
	for range time.NewTicker(5 * time.Second).C {
		log.Printf("executed %v programs", atomic.LoadUint64(&statExec))
	}
}

var outMu sync.Mutex

func execute(pid int, env *ipc.Env, p *prog.Prog) {
	if *flagExecutor == "" {
		return
	}
	atomic.AddUint64(&statExec, 1)
	if *flagLogProg {
		ticket := gate.Enter()
		defer gate.Leave(ticket)
		outMu.Lock()
		fmt.Printf("executing program %v\n%s\n", pid, p.Serialize())
		outMu.Unlock()
	}

	output, _, _, failed, hanged, err := env.Exec(p)
	if err != nil {
		fmt.Printf("failed to execute executor: %v\n", err)
	}
	paniced := failedRe.Match(output)
	if failed || hanged || paniced || err != nil {
		fmt.Printf("PROGRAM:\n%s\n", p.Serialize())
	}
	if failed || hanged || paniced || err != nil || *flagOutput {
		os.Stdout.Write(output)
	}
}

func readCorpus() []*prog.Prog {
	if *flagCorpus == "" {
		return nil
	}
	zipr, err := zip.OpenReader(*flagCorpus)
	if err != nil {
		failf("failed to open bin file: %v", err)
	}
	var progs []*prog.Prog
	for _, zipf := range zipr.File {
		r, err := zipf.Open()
		if err != nil {
			failf("failed to uzip file from input archive: %v", err)
		}
		data, err := ioutil.ReadAll(r)
		if err != nil {
			failf("failed to read corpus file: %v", err)
		}
		p, err := prog.Deserialize(data)
		if err != nil {
			failf("failed to deserialize corpus program: %v", err)
		}
		progs = append(progs, p)
		r.Close()
	}
	zipr.Close()
	return progs
}

func buildCallList() map[*sys.Call]bool {
	calls, err := host.DetectSupportedSyscalls()
	if err != nil {
		log.Printf("failed to detect host supported syscalls: %v", err)
		for _, c := range sys.Calls {
			calls[c] = true
		}
	}
	for _, c := range sys.Calls {
		if !calls[c] {
			log.Printf("disabling unsupported syscall: %v", c.Name)
		}
	}
	trans := sys.TransitivelyEnabledCalls(calls)
	for c := range calls {
		if !trans[c] {
			log.Printf("disabling transitively unsupported syscall: %v", c.Name)
			delete(calls, c)
		}
	}
	return calls
}

func failf(msg string, args ...interface{}) {
	log.Fatalf(msg, args...)
}
<|endoftext|>


<commit_before>// +build linux

package sysstat

import (
	"bufio"
	"encoding/csv"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/internal"
	"github.com/influxdata/telegraf/plugins/inputs"
)

var (
	firstTimestamp time.Time
	execCommand    = exec.Command // execCommand is used to mock commands in tests.
	dfltActivities = []string{"DISK"}
)

const parseInterval = 1 // parseInterval is the interval (in seconds) where the parsing of the binary file takes place.

type Sysstat struct {
	// Sadc represents the path to the sadc collector utility.
	Sadc string `toml:"sadc_path"`

	// Sadf represents the path to the sadf cmd.
	Sadf string `toml:"sadf_path"`

	// Activities is a list of activities that are passed as argument to the
	// collector utility (e.g: DISK, SNMP etc...)
	// The more activities that are added, the more data is collected.
	Activities []string

	// Options is a map of options.
	//
	// The key represents the actual option that the Sadf command is called with and
	// the value represents the description for that option.
	//
	// For example, if you have the following options map:
	//    map[string]string{"-C": "cpu", "-d": "disk"}
	// The Sadf command is run with the options -C and -d to extract cpu and
	// disk metrics from the collected binary file.
	//
	// If Group is false (see below), each metric will be prefixed with the corresponding description
	// and represents itself a measurement.
	//
	// If Group is true, metrics are grouped to a single measurement with the corresponding description as name.
	Options map[string]string

	// Group determines if metrics are grouped or not.
	Group bool

	// DeviceTags adds the possibility to add additional tags for devices.
	DeviceTags map[string][]map[string]string `toml:"device_tags"`
	tmpFile    string
	interval   int
}

func (*Sysstat) Description() string {
	return "Sysstat metrics collector"
}

var sampleConfig = `
  ## Path to the sadc command.
  #
  ## Common Defaults:
  ##   Debian/Ubuntu: /usr/lib/sysstat/sadc
  ##   Arch:          /usr/lib/sa/sadc
  ##   RHEL/CentOS:   /usr/lib64/sa/sadc
  sadc_path = "/usr/lib/sa/sadc" # required
  #
  #
  ## Path to the sadf command, if it is not in PATH
  # sadf_path = "/usr/bin/sadf"
  #
  #
  ## Activities is a list of activities, that are passed as argument to the
  ## sadc collector utility (e.g: DISK, SNMP etc...)
  ## The more activities that are added, the more data is collected.
  # activities = ["DISK"]
  #
  #
  ## Group metrics to measurements.
  ##
  ## If group is false each metric will be prefixed with a description
  ## and represents itself a measurement.
  ##
  ## If Group is true, corresponding metrics are grouped to a single measurement.
  # group = true
  #
  #
  ## Options for the sadf command. The values on the left represent the sadf
  ## options and the values on the right their description (wich are used for
  ## grouping and prefixing metrics).
  ##
  ## Run 'sar -h' or 'man sar' to find out the supported options for your
  ## sysstat version.
  [inputs.sysstat.options]
    -C = "cpu"
    -B = "paging"
    -b = "io"
    -d = "disk"             # requires DISK activity
    "-n ALL" = "network"
    "-P ALL" = "per_cpu"
    -q = "queue"
    -R = "mem"
    -r = "mem_util"
    -S = "swap_util"
    -u = "cpu_util"
    -v = "inode"
    -W = "swap"
    -w = "task"
  #  -H = "hugepages"        # only available for newer linux distributions
  #  "-I ALL" = "interrupts" # requires INT activity
  #
  #
  ## Device tags can be used to add additional tags for devices.
  ## For example the configuration below adds a tag vg with value rootvg for
  ## all metrics with sda devices.
  # [[inputs.sysstat.device_tags.sda]]
  #  vg = "rootvg"
`

func (*Sysstat) SampleConfig() string {
	return sampleConfig
}

func (s *Sysstat) Gather(acc telegraf.Accumulator) error {
	if s.interval == 0 {
		if firstTimestamp.IsZero() {
			firstTimestamp = time.Now()
		} else {
			s.interval = int(time.Since(firstTimestamp).Seconds())
		}
	}
	ts := time.Now().Add(time.Duration(s.interval) * time.Second)
	if err := s.collect(); err != nil {
		return err
	}
	var wg sync.WaitGroup
	errorChannel := make(chan error, len(s.Options)*2)
	for option := range s.Options {
		wg.Add(1)
		go func(acc telegraf.Accumulator, option string) {
			defer wg.Done()
			if err := s.parse(acc, option, ts); err != nil {
				errorChannel <- err
			}
		}(acc, option)
	}
	wg.Wait()
	close(errorChannel)

	errorStrings := []string{}
	for err := range errorChannel {
		errorStrings = append(errorStrings, err.Error())
	}

	if _, err := os.Stat(s.tmpFile); err == nil {
		if err := os.Remove(s.tmpFile); err != nil {
			errorStrings = append(errorStrings, err.Error())
		}
	}

	if len(errorStrings) == 0 {
		return nil
	}
	return errors.New(strings.Join(errorStrings, "\n"))
}

// collect collects sysstat data with the collector utility sadc.
// It runs the following command:
//     Sadc -S <Activity1> -S <Activity2> ... <collectInterval> 2 tmpFile
// The above command collects system metrics during <collectInterval> and
// saves it in binary form to tmpFile.
func (s *Sysstat) collect() error {
	options := []string{}
	for _, act := range s.Activities {
		options = append(options, "-S", act)
	}
	s.tmpFile = path.Join("/tmp", fmt.Sprintf("sysstat-%d", time.Now().Unix()))
	// collectInterval has to be smaller than the telegraf data collection interval
	collectInterval := s.interval - parseInterval

	// If true, interval is not defined yet and Gather is run for the first time.
	if collectInterval < 0 {
		collectInterval = 1 // In that case we only collect for 1 second.
	}

	options = append(options, strconv.Itoa(collectInterval), "2", s.tmpFile)
	cmd := execCommand(s.Sadc, options...)
	out, err := internal.CombinedOutputTimeout(cmd, time.Second*time.Duration(collectInterval+parseInterval))
	if err != nil {
		return fmt.Errorf("failed to run command %s: %s - %s", strings.Join(cmd.Args, " "), err, string(out))
	}
	return nil
}

// parse runs Sadf on the previously saved tmpFile:
//    Sadf -p -- -p <option> tmpFile
// and parses the output to add it to the telegraf.Accumulator acc.
func (s *Sysstat) parse(acc telegraf.Accumulator, option string, ts time.Time) error {
	cmd := execCommand(s.Sadf, s.sadfOptions(option)...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	if err := cmd.Start(); err != nil {
		return fmt.Errorf("running command '%s' failed: %s", strings.Join(cmd.Args, " "), err)
	}

	r := bufio.NewReader(stdout)
	csv := csv.NewReader(r)
	csv.Comma = '\t'
	csv.FieldsPerRecord = 6
	var measurement string
	// groupData to accumulate data when Group=true
	type groupData struct {
		tags   map[string]string
		fields map[string]interface{}
	}
	m := make(map[string]groupData)
	for {
		record, err := csv.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		device := record[3]
		value, err := strconv.ParseFloat(record[5], 64)
		if err != nil {
			return err
		}

		tags := map[string]string{}
		if device != "-" {
			tags["device"] = device
			if addTags, ok := s.DeviceTags[device]; ok {
				for _, tag := range addTags {
					for k, v := range tag {
						tags[k] = v
					}
				}

			}
		}

		if s.Group {
			measurement = s.Options[option]
			if _, ok := m[device]; !ok {
				m[device] = groupData{
					fields: make(map[string]interface{}),
					tags:   make(map[string]string),
				}
			}
			g, _ := m[device]
			if len(g.tags) == 0 {
				for k, v := range tags {
					g.tags[k] = v
				}
			}
			g.fields[escape(record[4])] = value
		} else {
			measurement = s.Options[option] + "_" + escape(record[4])
			fields := map[string]interface{}{
				"value": value,
			}
			acc.AddFields(measurement, fields, tags, ts)
		}

	}
	if s.Group {
		for _, v := range m {
			acc.AddFields(measurement, v.fields, v.tags, ts)
		}
	}
	if err := internal.WaitTimeout(cmd, time.Second*5); err != nil {
		return fmt.Errorf("command %s failed with %s",
			strings.Join(cmd.Args, " "), err)
	}
	return nil
}

// sadfOptions creates the correct options for the sadf utility.
func (s *Sysstat) sadfOptions(activityOption string) []string {
	options := []string{
		"-p",
		"--",
		"-p",
	}

	opts := strings.Split(activityOption, " ")
	options = append(options, opts...)
	options = append(options, s.tmpFile)

	return options
}

// escape removes % and / chars in field names
func escape(dirty string) string {
	var fieldEscaper = strings.NewReplacer(
		`%`, "pct_",
		`/`, "_per_",
	)
	return fieldEscaper.Replace(dirty)
}

func init() {
	s := Sysstat{
		Group:      true,
		Activities: dfltActivities,
	}
	sadf, _ := exec.LookPath("sadf")
	if len(sadf) > 0 {
		s.Sadf = sadf
	}
	inputs.Add("sysstat", func() telegraf.Input {
		return &s
	})
}
<commit_msg>fix interval rounding error<commit_after>// +build linux

package sysstat

import (
	"bufio"
	"encoding/csv"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/internal"
	"github.com/influxdata/telegraf/plugins/inputs"
)

var (
	firstTimestamp time.Time
	execCommand    = exec.Command // execCommand is used to mock commands in tests.
	dfltActivities = []string{"DISK"}
)

const parseInterval = 1 // parseInterval is the interval (in seconds) where the parsing of the binary file takes place.

type Sysstat struct {
	// Sadc represents the path to the sadc collector utility.
	Sadc string `toml:"sadc_path"`

	// Sadf represents the path to the sadf cmd.
	Sadf string `toml:"sadf_path"`

	// Activities is a list of activities that are passed as argument to the
	// collector utility (e.g: DISK, SNMP etc...)
	// The more activities that are added, the more data is collected.
	Activities []string

	// Options is a map of options.
	//
	// The key represents the actual option that the Sadf command is called with and
	// the value represents the description for that option.
	//
	// For example, if you have the following options map:
	//    map[string]string{"-C": "cpu", "-d": "disk"}
	// The Sadf command is run with the options -C and -d to extract cpu and
	// disk metrics from the collected binary file.
	//
	// If Group is false (see below), each metric will be prefixed with the corresponding description
	// and represents itself a measurement.
	//
	// If Group is true, metrics are grouped to a single measurement with the corresponding description as name.
	Options map[string]string

	// Group determines if metrics are grouped or not.
	Group bool

	// DeviceTags adds the possibility to add additional tags for devices.
	DeviceTags map[string][]map[string]string `toml:"device_tags"`
	tmpFile    string
	interval   int
}

func (*Sysstat) Description() string {
	return "Sysstat metrics collector"
}

var sampleConfig = `
  ## Path to the sadc command.
  #
  ## Common Defaults:
  ##   Debian/Ubuntu: /usr/lib/sysstat/sadc
  ##   Arch:          /usr/lib/sa/sadc
  ##   RHEL/CentOS:   /usr/lib64/sa/sadc
  sadc_path = "/usr/lib/sa/sadc" # required
  #
  #
  ## Path to the sadf command, if it is not in PATH
  # sadf_path = "/usr/bin/sadf"
  #
  #
  ## Activities is a list of activities, that are passed as argument to the
  ## sadc collector utility (e.g: DISK, SNMP etc...)
  ## The more activities that are added, the more data is collected.
  # activities = ["DISK"]
  #
  #
  ## Group metrics to measurements.
  ##
  ## If group is false each metric will be prefixed with a description
  ## and represents itself a measurement.
  ##
  ## If Group is true, corresponding metrics are grouped to a single measurement.
  # group = true
  #
  #
  ## Options for the sadf command. The values on the left represent the sadf
  ## options and the values on the right their description (wich are used for
  ## grouping and prefixing metrics).
  ##
  ## Run 'sar -h' or 'man sar' to find out the supported options for your
  ## sysstat version.
  [inputs.sysstat.options]
    -C = "cpu"
    -B = "paging"
    -b = "io"
    -d = "disk"             # requires DISK activity
    "-n ALL" = "network"
    "-P ALL" = "per_cpu"
    -q = "queue"
    -R = "mem"
    -r = "mem_util"
    -S = "swap_util"
    -u = "cpu_util"
    -v = "inode"
    -W = "swap"
    -w = "task"
  #  -H = "hugepages"        # only available for newer linux distributions
  #  "-I ALL" = "interrupts" # requires INT activity
  #
  #
  ## Device tags can be used to add additional tags for devices.
  ## For example the configuration below adds a tag vg with value rootvg for
  ## all metrics with sda devices.
  # [[inputs.sysstat.device_tags.sda]]
  #  vg = "rootvg"
`

func (*Sysstat) SampleConfig() string {
	return sampleConfig
}

func (s *Sysstat) Gather(acc telegraf.Accumulator) error {
	if s.interval == 0 {
		if firstTimestamp.IsZero() {
			firstTimestamp = time.Now()
		} else {
			s.interval = int(time.Since(firstTimestamp).Seconds() + 0.5)
		}
	}
	ts := time.Now().Add(time.Duration(s.interval) * time.Second)
	if err := s.collect(); err != nil {
		return err
	}
	var wg sync.WaitGroup
	errorChannel := make(chan error, len(s.Options)*2)
	for option := range s.Options {
		wg.Add(1)
		go func(acc telegraf.Accumulator, option string) {
			defer wg.Done()
			if err := s.parse(acc, option, ts); err != nil {
				errorChannel <- err
			}
		}(acc, option)
	}
	wg.Wait()
	close(errorChannel)

	errorStrings := []string{}
	for err := range errorChannel {
		errorStrings = append(errorStrings, err.Error())
	}

	if _, err := os.Stat(s.tmpFile); err == nil {
		if err := os.Remove(s.tmpFile); err != nil {
			errorStrings = append(errorStrings, err.Error())
		}
	}

	if len(errorStrings) == 0 {
		return nil
	}
	return errors.New(strings.Join(errorStrings, "\n"))
}

// collect collects sysstat data with the collector utility sadc.
// It runs the following command:
//     Sadc -S <Activity1> -S <Activity2> ... <collectInterval> 2 tmpFile
// The above command collects system metrics during <collectInterval> and
// saves it in binary form to tmpFile.
func (s *Sysstat) collect() error {
	options := []string{}
	for _, act := range s.Activities {
		options = append(options, "-S", act)
	}
	s.tmpFile = path.Join("/tmp", fmt.Sprintf("sysstat-%d", time.Now().Unix()))
	// collectInterval has to be smaller than the telegraf data collection interval
	collectInterval := s.interval - parseInterval

	// If true, interval is not defined yet and Gather is run for the first time.
	if collectInterval < 0 {
		collectInterval = 1 // In that case we only collect for 1 second.
	}

	options = append(options, strconv.Itoa(collectInterval), "2", s.tmpFile)
	cmd := execCommand(s.Sadc, options...)
	out, err := internal.CombinedOutputTimeout(cmd, time.Second*time.Duration(collectInterval+parseInterval))
	if err != nil {
		return fmt.Errorf("failed to run command %s: %s - %s", strings.Join(cmd.Args, " "), err, string(out))
	}
	return nil
}

// parse runs Sadf on the previously saved tmpFile:
//    Sadf -p -- -p <option> tmpFile
// and parses the output to add it to the telegraf.Accumulator acc.
func (s *Sysstat) parse(acc telegraf.Accumulator, option string, ts time.Time) error {
	cmd := execCommand(s.Sadf, s.sadfOptions(option)...)
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return err
	}
	if err := cmd.Start(); err != nil {
		return fmt.Errorf("running command '%s' failed: %s", strings.Join(cmd.Args, " "), err)
	}

	r := bufio.NewReader(stdout)
	csv := csv.NewReader(r)
	csv.Comma = '\t'
	csv.FieldsPerRecord = 6
	var measurement string
	// groupData to accumulate data when Group=true
	type groupData struct {
		tags   map[string]string
		fields map[string]interface{}
	}
	m := make(map[string]groupData)
	for {
		record, err := csv.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		device := record[3]
		value, err := strconv.ParseFloat(record[5], 64)
		if err != nil {
			return err
		}

		tags := map[string]string{}
		if device != "-" {
			tags["device"] = device
			if addTags, ok := s.DeviceTags[device]; ok {
				for _, tag := range addTags {
					for k, v := range tag {
						tags[k] = v
					}
				}

			}
		}

		if s.Group {
			measurement = s.Options[option]
			if _, ok := m[device]; !ok {
				m[device] = groupData{
					fields: make(map[string]interface{}),
					tags:   make(map[string]string),
				}
			}
			g, _ := m[device]
			if len(g.tags) == 0 {
				for k, v := range tags {
					g.tags[k] = v
				}
			}
			g.fields[escape(record[4])] = value
		} else {
			measurement = s.Options[option] + "_" + escape(record[4])
			fields := map[string]interface{}{
				"value": value,
			}
			acc.AddFields(measurement, fields, tags, ts)
		}

	}
	if s.Group {
		for _, v := range m {
			acc.AddFields(measurement, v.fields, v.tags, ts)
		}
	}
	if err := internal.WaitTimeout(cmd, time.Second*5); err != nil {
		return fmt.Errorf("command %s failed with %s",
			strings.Join(cmd.Args, " "), err)
	}
	return nil
}

// sadfOptions creates the correct options for the sadf utility.
func (s *Sysstat) sadfOptions(activityOption string) []string {
	options := []string{
		"-p",
		"--",
		"-p",
	}

	opts := strings.Split(activityOption, " ")
	options = append(options, opts...)
	options = append(options, s.tmpFile)

	return options
}

// escape removes % and / chars in field names
func escape(dirty string) string {
	var fieldEscaper = strings.NewReplacer(
		`%`, "pct_",
		`/`, "_per_",
	)
	return fieldEscaper.Replace(dirty)
}

func init() {
	s := Sysstat{
		Group:      true,
		Activities: dfltActivities,
	}
	sadf, _ := exec.LookPath("sadf")
	if len(sadf) > 0 {
		s.Sadf = sadf
	}
	inputs.Add("sysstat", func() telegraf.Input {
		return &s
	})
}
<|endoftext|>


<commit_before>package wal

import (
	"bytes"
	"encoding/binary"
	"io"
	"sync/atomic"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/errors"
	"golang.org/x/crypto/blake2b"
)

// Update defines a single update that can be sent to the WAL and saved
// atomically. Updates are sent to the wal in groups of one or more, and upon
// being signaled, will be saved to disk in a high-integrity, all- or-nothing
// fasion that follows ACID principles.
//
// The name and version are optional, however best-practice code will make use
// of these fields.
//
// When using the Update, it recommended that you typecast the Update type to
// another type which has methods on it for creating and applying the Update +
// instructions, including any special handling based on the version.
type Update struct {
	// The name of the update type. When the WAL is loaded after an unclean
	// shutdown, any un-committed changes will be passed as Updates back to the
	// caller instantiating the WAL. The caller should determine what code to
	// run on the the update based on the name and version.
	Name string

	// The version of the update type. Update types and implementations can be
	// tweaked over time, and a version field allows for easy compatibility
	// between releases, even if an upgrade occurs between an unclean shutdown
	// and a reload.
	Version string

	// The marshalled data directing the update. The data is an opaque set of
	// instructions to follow that implement and idempotent change to a set of
	// persistent files. A series of unclean shutdowns in rapid succession could
	// mean that these instructions get followed multiple times, which means
	// idempotent instructions are required.
	Instructions []byte
}

// Transaction defines a series of updates that are to be performed atomically.
// In the event of an unclean shutdown, either all updates will have been saved
// together at full integrity, or none of the updates in the transaction will
// have been saved at all.
//
// While multiple transactions can be safely open at the same time, multiple
// methods should not be called on the transaction at the same time - the WAL
// is thread-safe, but the transactions are not.
//
// A Transaction is created by calling NewTransaction. Afterwards, the
// transactions SignalSetupComplete has to be called which returns a channel
// that is closed once the transaction is committed. Finally
// SignalUpdatesApplied needs to be called after the transaction was committed
// to signal a successfull transaction and free used pages.
type Transaction struct {
	// setupComplete, commitComplete, and releaseComplete signal the progress of
	// the transaction, and should be set to 'true' in order.
	//
	// When setupComplete is set to true, it means that the creater of the
	// transaction is ready for the transaction to be committed.
	//
	// When commitComplete is set to true, it means that the WAL has
	// successfully and fully committed the transaction.
	//
	// releaseComplete is set to true when the caller has fully applied the
	// transaction, meaning the transaction can be over-written safely in the
	// WAL, and the on-disk pages can be reclaimed for future transactions.
	setupComplete   bool
	commitComplete  bool
	releaseComplete bool

	// firstPage is the first page of the transaction. It is the last page that
	// is written when finalizing a commit and when releasing a transaction.
	firstPage *page

	// Updates defines the set of updates that compose the transaction.
	Updates []Update

	// The wal that was used to create the transaction
	wal *WAL
	// An internal channel to signal that the transaction was initialized and
	// can be committed
	initComplete chan error
}

// checksum calculates the checksum of a transaction excluding the checksum
// field of each page
func (t Transaction) checksum() (c checksum) {
	h, _ := blake2b.New256(nil)
	buf := make([]byte, pageSize)
	for page := t.firstPage; page != nil; page = page.nextPage {
		b := page.appendTo(buf[:0])
		h.Write(b[checksumSize:]) // exclude checksum
	}
	copy(c[:], h.Sum(buf[:0]))
	return
}

// commit commits a transaction by setting the correct status and checksum
func (t *Transaction) commit(done chan error) {
	// Signal completion of the commit
	defer close(done)

	// Make sure that the initialization of the transaction finished
	err := <-t.initComplete
	if err != nil {
		done <- err
		return
	}

	// set the status of the first page first
	t.firstPage.pageStatus = pageStatusComitted

	// Set the transaction number of the first page and increase the transactionCounter of the wal
	t.firstPage.transactionNumber = atomic.AddUint64(&t.wal.atomicNextTxnNum, 1) - 1

	// calculate the checksum
	t.firstPage.transactionChecksum = t.checksum()

	// Finalize the commit by writing the first page with the updated status if
	// there have been no errors so far.
	if t.wal.deps.disrupt("CommitFail") {
		// Disk failure causes the commit to fail
		done <- errors.New("Write failed on purpose")
		return
	}
	if _, err := t.wal.logFile.WriteAt(t.firstPage.appendTo(nil), int64(t.firstPage.offset)); err != nil {
		done <- build.ExtendErr("Writing the first page failed", err)
		return
	}

	t.commitComplete = true
	t.wal.fSync()
}

// marshalUpdates marshals the updates of a transaction
func marshalUpdates(updates []Update) ([]byte, error) {
	buffer := new(bytes.Buffer)

	for _, update := range updates {
		// Marshal name
		name := []byte(update.Name)
		err1 := binary.Write(buffer, binary.LittleEndian, uint64(len(name)))
		_, err2 := buffer.Write(name)

		// Marshal version
		version := []byte(update.Version)
		err3 := binary.Write(buffer, binary.LittleEndian, uint64(len(version)))
		_, err4 := buffer.Write(version)

		// Append instructions
		err5 := binary.Write(buffer, binary.LittleEndian, uint64(len(update.Instructions)))
		_, err6 := buffer.Write(update.Instructions)

		if err1 != nil || err2 != nil || err3 != nil || err4 != nil || err5 != nil || err6 != nil {
			return nil, errors.Compose(errors.New("Failed to marshal updates"), err1, err2, err3, err4, err5, err6)
		}
	}

	return buffer.Bytes(), nil
}

// unmarshalUpdates unmarshals the updates of a transaction
func unmarshalUpdates(data []byte) ([]Update, error) {
	buffer := bytes.NewBuffer(data)
	updates := make([]Update, 0)

	for {
		update := Update{}

		// Unmarshal name
		var nameLength uint64
		err1 := binary.Read(buffer, binary.LittleEndian, &nameLength)
		if err1 == io.EOF {
			// End of buffer reached
			break
		}

		name := make([]byte, nameLength)
		_, err2 := buffer.Read(name)

		// Unmarshal version
		var versionLength uint64
		err3 := binary.Read(buffer, binary.LittleEndian, &versionLength)

		version := make([]byte, versionLength)
		_, err4 := buffer.Read(version)

		// Unmarshal instructions
		var instructionsLength uint64
		err5 := binary.Read(buffer, binary.LittleEndian, &instructionsLength)

		instructions := make([]byte, instructionsLength)
		_, err6 := buffer.Read(instructions)

		// Check if any errors occured
		if err1 != nil || err2 != nil || err3 != nil || err4 != nil || err5 != nil || err6 != nil {
			return nil, errors.Compose(errors.New("Failed to unmarshal updates"), err1, err2, err3, err4, err5, err6)
		}

		update.Name = string(name)
		update.Version = string(version)
		update.Instructions = instructions

		updates = append(updates, update)
	}

	return updates, nil
}

// threadedInitTransaction reserves pages of the wal, marshalls the
// transactions's updates into a payload and splits the payload equally among
// the pages. Once finished those pages are written to disk and the transaction
// is committed.
func initTransaction(t *Transaction) {
	defer close(t.initComplete)

	// Marshal all the updates to get their total length on disk
	data, err := marshalUpdates(t.Updates)
	if err != nil {
		t.initComplete <- build.ExtendErr("could not marshal update", err)
		return
	}

	// Get the pages from the wal and set the first page's status
	pages := t.wal.managedReservePages(data)
	pages[0].pageStatus = pageStatusWritten

	// Set the first page of the transaction
	t.firstPage = &pages[0]

	// write the pages to disk
	if err := t.writeToFile(); err != nil {
		t.initComplete <- build.ExtendErr("Couldn't write the page to file", err)
		return
	}
}

// validateChecksum checks if a transaction has been corrupted by computing a hash
// and comparing it to the one in the firstPage of the transaction
func (t Transaction) validateChecksum() error {
	if t.firstPage == nil {
		return errors.New("firstPage is nil")
	} else if t.checksum() != t.firstPage.transactionChecksum {
		return errors.New("checksum not valid")
	}
	return nil
}

// SignalUpdatesApplied  informs the WAL that it is safe to free the used pages to reuse them in a new transaction
func (t *Transaction) SignalUpdatesApplied() error {
	if !t.setupComplete || !t.commitComplete || t.releaseComplete {
		return errors.New("misuse of transaction - call each of the signaling methods exactly once, in serial, in order")
	}
	t.releaseComplete = true

	// Set the page status to applied
	t.firstPage.pageStatus = pageStatusApplied

	// Write the page to disk
	var err error
	if t.wal.deps.disrupt("ReleaseFail") {
		// Disk failure causes the commit to fail
		err = errors.New("Write failed on purpose")
	} else {
		_, err = t.wal.logFile.WriteAt(t.firstPage.appendTo(nil), int64(t.firstPage.offset))
	}
	if err != nil {
		return build.ExtendErr("Couldn't write the page to file", err)
	}
	t.wal.fSync()

	// Update the wal's available pages
	t.wal.mu.Lock()
	for page := t.firstPage; page != nil; page = page.nextPage {
		// Append the index of the freed page
		t.wal.availablePages = append(t.wal.availablePages, page.offset)
	}
	t.wal.mu.Unlock()

	// Decrease the number of active transactions
	if atomic.AddInt64(&t.wal.atomicUnfinishedTxns, -1) < 0 {
		panic("Sanity check failed. atomicUnfinishedTxns should never be negative")
	}

	return nil
}

// append is a helper function to append updates to a transaction on which
// SignalSetupComplete hasn't been called yet
func (t *Transaction) append(updates []Update, done chan error) {
	defer close(done)

	// If there is nothing to append we are done
	if len(updates) == 0 {
		return
	}

	// Make sure that the initialization finished
	err := <-t.initComplete
	if err != nil {
		done <- err
		return
	}

	// Marshal the data
	data, err := marshalUpdates(updates)
	if err != nil {
		done <- errors.Extend(errors.New("failed to marshal new updates"), err)
		return
	}

	// Find last page to which we append and count the pages
	lastPage := t.firstPage
	for lastPage.nextPage != nil {
		lastPage = lastPage.nextPage
	}

	// Write as much data to the last page as possible
	lenDiff := maxPayloadSize - len(lastPage.payload)
	if len(data) <= lenDiff {
		lastPage.payload = append(lastPage.payload, data...)
		data = nil
	} else {
		lastPage.payload = append(lastPage.payload, data[:lenDiff]...)
		data = data[lenDiff:]
	}

	// If there is no more data to write we are done
	if data == nil {
		return
	}

	// Get enough pages for the remaining data
	pages := t.wal.managedReservePages(data)
	lastPage.nextPage = &pages[0]

	// Write the new pages to disk and sync them
	buf := make([]byte, pageSize)
	for _, page := range pages {
		b := page.appendTo(buf[:0])
		if _, err := t.wal.logFile.WriteAt(b, int64(page.offset)); err != nil {
			done <- build.ExtendErr("Writing the page to disk failed", err)
			return
		}
	}
	if err := t.wal.logFile.Sync(); err != nil {
		done <- build.ExtendErr("Syncing the WAL to disk failed", err)
		return
	}

	// Link the new pages to the last one and sync the last page
	b := lastPage.appendTo(buf[:0])
	if _, err := t.wal.logFile.WriteAt(b, int64(lastPage.offset)); err != nil {
		done <- build.ExtendErr("Writing the last page to disk failed", err)
		return
	}
	if err := t.wal.logFile.Sync(); err != nil {
		done <- build.ExtendErr("Syncing the last page to disk failed", err)
		return
	}

	// Append the updates to the transaction
	t.Updates = append(t.Updates, updates...)
}

// Append appends additional updates to a transaction
func (t *Transaction) Append(updates []Update) <-chan error {
	done := make(chan error, 1)

	if t.setupComplete || t.commitComplete || t.releaseComplete {
		done <- errors.New("misuse of trnasaction - can't append to transaction once it is committed/released")
		return done
	}

	go t.append(updates, done)
	return done
}

// SignalSetupComplete will signal to the WAL that any required setup has
// completed, and that the WAL can safely commit to the transaction being
// applied atomically.
func (t *Transaction) SignalSetupComplete() <-chan error {
	done := make(chan error, 1)

	if t.setupComplete || t.commitComplete || t.releaseComplete {
		done <- errors.New("misuse of transaction - call each of the signaling methods exactly ones, in serial, in order")
		return done
	}
	t.setupComplete = true

	// Commit the transaction non-blocking
	go t.commit(done)
	return done
}

// NewTransaction creates a transaction from a set of updates
func (w *WAL) NewTransaction(updates []Update) (*Transaction, error) {
	if !w.recoveryComplete {
		return nil, errors.New("can't call NewTransaction before recovery is complete")
	}
	// Check that there are updates for the transaction to process.
	if len(updates) == 0 {
		return nil, errors.New("cannot create a transaction without updates")
	}

	// Create new transaction
	newTransaction := Transaction{
		Updates:      updates,
		wal:          w,
		initComplete: make(chan error),
	}

	// Initialize the transaction by splitting up the payload among free pages
	// and writing them to disk.
	go initTransaction(&newTransaction)

	// Increase the number of active transaction
	atomic.AddInt64(&w.atomicUnfinishedTxns, 1)

	return &newTransaction, nil
}

// writeToFile writes all the pages of the transaction to disk
func (t *Transaction) writeToFile() error {
	buf := make([]byte, pageSize)
	for page := t.firstPage; page != nil; page = page.nextPage {
		b := page.appendTo(buf[:0])
		if _, err := t.wal.logFile.WriteAt(b, int64(page.offset)); err != nil {
			return build.ExtendErr("Writing the page to disk failed", err)
		}
	}
	return nil
}
<commit_msg>change sync calls in append to the wallets fSync<commit_after>package wal

import (
	"bytes"
	"encoding/binary"
	"io"
	"sync/atomic"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/errors"
	"golang.org/x/crypto/blake2b"
)

// Update defines a single update that can be sent to the WAL and saved
// atomically. Updates are sent to the wal in groups of one or more, and upon
// being signaled, will be saved to disk in a high-integrity, all- or-nothing
// fasion that follows ACID principles.
//
// The name and version are optional, however best-practice code will make use
// of these fields.
//
// When using the Update, it recommended that you typecast the Update type to
// another type which has methods on it for creating and applying the Update +
// instructions, including any special handling based on the version.
type Update struct {
	// The name of the update type. When the WAL is loaded after an unclean
	// shutdown, any un-committed changes will be passed as Updates back to the
	// caller instantiating the WAL. The caller should determine what code to
	// run on the the update based on the name and version.
	Name string

	// The version of the update type. Update types and implementations can be
	// tweaked over time, and a version field allows for easy compatibility
	// between releases, even if an upgrade occurs between an unclean shutdown
	// and a reload.
	Version string

	// The marshalled data directing the update. The data is an opaque set of
	// instructions to follow that implement and idempotent change to a set of
	// persistent files. A series of unclean shutdowns in rapid succession could
	// mean that these instructions get followed multiple times, which means
	// idempotent instructions are required.
	Instructions []byte
}

// Transaction defines a series of updates that are to be performed atomically.
// In the event of an unclean shutdown, either all updates will have been saved
// together at full integrity, or none of the updates in the transaction will
// have been saved at all.
//
// While multiple transactions can be safely open at the same time, multiple
// methods should not be called on the transaction at the same time - the WAL
// is thread-safe, but the transactions are not.
//
// A Transaction is created by calling NewTransaction. Afterwards, the
// transactions SignalSetupComplete has to be called which returns a channel
// that is closed once the transaction is committed. Finally
// SignalUpdatesApplied needs to be called after the transaction was committed
// to signal a successfull transaction and free used pages.
type Transaction struct {
	// setupComplete, commitComplete, and releaseComplete signal the progress of
	// the transaction, and should be set to 'true' in order.
	//
	// When setupComplete is set to true, it means that the creater of the
	// transaction is ready for the transaction to be committed.
	//
	// When commitComplete is set to true, it means that the WAL has
	// successfully and fully committed the transaction.
	//
	// releaseComplete is set to true when the caller has fully applied the
	// transaction, meaning the transaction can be over-written safely in the
	// WAL, and the on-disk pages can be reclaimed for future transactions.
	setupComplete   bool
	commitComplete  bool
	releaseComplete bool

	// firstPage is the first page of the transaction. It is the last page that
	// is written when finalizing a commit and when releasing a transaction.
	firstPage *page

	// Updates defines the set of updates that compose the transaction.
	Updates []Update

	// The wal that was used to create the transaction
	wal *WAL
	// An internal channel to signal that the transaction was initialized and
	// can be committed
	initComplete chan error
}

// checksum calculates the checksum of a transaction excluding the checksum
// field of each page
func (t Transaction) checksum() (c checksum) {
	h, _ := blake2b.New256(nil)
	buf := make([]byte, pageSize)
	for page := t.firstPage; page != nil; page = page.nextPage {
		b := page.appendTo(buf[:0])
		h.Write(b[checksumSize:]) // exclude checksum
	}
	copy(c[:], h.Sum(buf[:0]))
	return
}

// commit commits a transaction by setting the correct status and checksum
func (t *Transaction) commit(done chan error) {
	// Signal completion of the commit
	defer close(done)

	// Make sure that the initialization of the transaction finished
	err := <-t.initComplete
	if err != nil {
		done <- err
		return
	}

	// set the status of the first page first
	t.firstPage.pageStatus = pageStatusComitted

	// Set the transaction number of the first page and increase the transactionCounter of the wal
	t.firstPage.transactionNumber = atomic.AddUint64(&t.wal.atomicNextTxnNum, 1) - 1

	// calculate the checksum
	t.firstPage.transactionChecksum = t.checksum()

	// Finalize the commit by writing the first page with the updated status if
	// there have been no errors so far.
	if t.wal.deps.disrupt("CommitFail") {
		// Disk failure causes the commit to fail
		done <- errors.New("Write failed on purpose")
		return
	}
	if _, err := t.wal.logFile.WriteAt(t.firstPage.appendTo(nil), int64(t.firstPage.offset)); err != nil {
		done <- build.ExtendErr("Writing the first page failed", err)
		return
	}

	t.commitComplete = true
	t.wal.fSync()
}

// marshalUpdates marshals the updates of a transaction
func marshalUpdates(updates []Update) ([]byte, error) {
	buffer := new(bytes.Buffer)

	for _, update := range updates {
		// Marshal name
		name := []byte(update.Name)
		err1 := binary.Write(buffer, binary.LittleEndian, uint64(len(name)))
		_, err2 := buffer.Write(name)

		// Marshal version
		version := []byte(update.Version)
		err3 := binary.Write(buffer, binary.LittleEndian, uint64(len(version)))
		_, err4 := buffer.Write(version)

		// Append instructions
		err5 := binary.Write(buffer, binary.LittleEndian, uint64(len(update.Instructions)))
		_, err6 := buffer.Write(update.Instructions)

		if err1 != nil || err2 != nil || err3 != nil || err4 != nil || err5 != nil || err6 != nil {
			return nil, errors.Compose(errors.New("Failed to marshal updates"), err1, err2, err3, err4, err5, err6)
		}
	}

	return buffer.Bytes(), nil
}

// unmarshalUpdates unmarshals the updates of a transaction
func unmarshalUpdates(data []byte) ([]Update, error) {
	buffer := bytes.NewBuffer(data)
	updates := make([]Update, 0)

	for {
		update := Update{}

		// Unmarshal name
		var nameLength uint64
		err1 := binary.Read(buffer, binary.LittleEndian, &nameLength)
		if err1 == io.EOF {
			// End of buffer reached
			break
		}

		name := make([]byte, nameLength)
		_, err2 := buffer.Read(name)

		// Unmarshal version
		var versionLength uint64
		err3 := binary.Read(buffer, binary.LittleEndian, &versionLength)

		version := make([]byte, versionLength)
		_, err4 := buffer.Read(version)

		// Unmarshal instructions
		var instructionsLength uint64
		err5 := binary.Read(buffer, binary.LittleEndian, &instructionsLength)

		instructions := make([]byte, instructionsLength)
		_, err6 := buffer.Read(instructions)

		// Check if any errors occured
		if err1 != nil || err2 != nil || err3 != nil || err4 != nil || err5 != nil || err6 != nil {
			return nil, errors.Compose(errors.New("Failed to unmarshal updates"), err1, err2, err3, err4, err5, err6)
		}

		update.Name = string(name)
		update.Version = string(version)
		update.Instructions = instructions

		updates = append(updates, update)
	}

	return updates, nil
}

// threadedInitTransaction reserves pages of the wal, marshalls the
// transactions's updates into a payload and splits the payload equally among
// the pages. Once finished those pages are written to disk and the transaction
// is committed.
func initTransaction(t *Transaction) {
	defer close(t.initComplete)

	// Marshal all the updates to get their total length on disk
	data, err := marshalUpdates(t.Updates)
	if err != nil {
		t.initComplete <- build.ExtendErr("could not marshal update", err)
		return
	}

	// Get the pages from the wal and set the first page's status
	pages := t.wal.managedReservePages(data)
	pages[0].pageStatus = pageStatusWritten

	// Set the first page of the transaction
	t.firstPage = &pages[0]

	// write the pages to disk
	if err := t.writeToFile(); err != nil {
		t.initComplete <- build.ExtendErr("Couldn't write the page to file", err)
		return
	}
}

// validateChecksum checks if a transaction has been corrupted by computing a hash
// and comparing it to the one in the firstPage of the transaction
func (t Transaction) validateChecksum() error {
	if t.firstPage == nil {
		return errors.New("firstPage is nil")
	} else if t.checksum() != t.firstPage.transactionChecksum {
		return errors.New("checksum not valid")
	}
	return nil
}

// SignalUpdatesApplied  informs the WAL that it is safe to free the used pages to reuse them in a new transaction
func (t *Transaction) SignalUpdatesApplied() error {
	if !t.setupComplete || !t.commitComplete || t.releaseComplete {
		return errors.New("misuse of transaction - call each of the signaling methods exactly once, in serial, in order")
	}
	t.releaseComplete = true

	// Set the page status to applied
	t.firstPage.pageStatus = pageStatusApplied

	// Write the page to disk
	var err error
	if t.wal.deps.disrupt("ReleaseFail") {
		// Disk failure causes the commit to fail
		err = errors.New("Write failed on purpose")
	} else {
		_, err = t.wal.logFile.WriteAt(t.firstPage.appendTo(nil), int64(t.firstPage.offset))
	}
	if err != nil {
		return build.ExtendErr("Couldn't write the page to file", err)
	}
	t.wal.fSync()

	// Update the wal's available pages
	t.wal.mu.Lock()
	for page := t.firstPage; page != nil; page = page.nextPage {
		// Append the index of the freed page
		t.wal.availablePages = append(t.wal.availablePages, page.offset)
	}
	t.wal.mu.Unlock()

	// Decrease the number of active transactions
	if atomic.AddInt64(&t.wal.atomicUnfinishedTxns, -1) < 0 {
		panic("Sanity check failed. atomicUnfinishedTxns should never be negative")
	}

	return nil
}

// append is a helper function to append updates to a transaction on which
// SignalSetupComplete hasn't been called yet
func (t *Transaction) append(updates []Update, done chan error) {
	defer close(done)

	// If there is nothing to append we are done
	if len(updates) == 0 {
		return
	}

	// Make sure that the initialization finished
	err := <-t.initComplete
	if err != nil {
		done <- err
		return
	}

	// Marshal the data
	data, err := marshalUpdates(updates)
	if err != nil {
		done <- errors.Extend(errors.New("failed to marshal new updates"), err)
		return
	}

	// Find last page to which we append and count the pages
	lastPage := t.firstPage
	for lastPage.nextPage != nil {
		lastPage = lastPage.nextPage
	}

	// Write as much data to the last page as possible
	lenDiff := maxPayloadSize - len(lastPage.payload)
	if len(data) <= lenDiff {
		lastPage.payload = append(lastPage.payload, data...)
		data = nil
	} else {
		lastPage.payload = append(lastPage.payload, data[:lenDiff]...)
		data = data[lenDiff:]
	}

	// If there is no more data to write we are done
	if data == nil {
		return
	}

	// Get enough pages for the remaining data
	pages := t.wal.managedReservePages(data)
	lastPage.nextPage = &pages[0]

	// Write the new pages to disk and sync them
	buf := make([]byte, pageSize)
	for _, page := range pages {
		b := page.appendTo(buf[:0])
		if _, err := t.wal.logFile.WriteAt(b, int64(page.offset)); err != nil {
			done <- build.ExtendErr("Writing the page to disk failed", err)
			return
		}
	}
	t.wal.fSync()

	// Link the new pages to the last one and sync the last page
	b := lastPage.appendTo(buf[:0])
	if _, err := t.wal.logFile.WriteAt(b, int64(lastPage.offset)); err != nil {
		done <- build.ExtendErr("Writing the last page to disk failed", err)
		return
	}
	t.wal.fSync()

	// Append the updates to the transaction
	t.Updates = append(t.Updates, updates...)
}

// Append appends additional updates to a transaction
func (t *Transaction) Append(updates []Update) <-chan error {
	done := make(chan error, 1)

	if t.setupComplete || t.commitComplete || t.releaseComplete {
		done <- errors.New("misuse of trnasaction - can't append to transaction once it is committed/released")
		return done
	}

	go t.append(updates, done)
	return done
}

// SignalSetupComplete will signal to the WAL that any required setup has
// completed, and that the WAL can safely commit to the transaction being
// applied atomically.
func (t *Transaction) SignalSetupComplete() <-chan error {
	done := make(chan error, 1)

	if t.setupComplete || t.commitComplete || t.releaseComplete {
		done <- errors.New("misuse of transaction - call each of the signaling methods exactly ones, in serial, in order")
		return done
	}
	t.setupComplete = true

	// Commit the transaction non-blocking
	go t.commit(done)
	return done
}

// NewTransaction creates a transaction from a set of updates
func (w *WAL) NewTransaction(updates []Update) (*Transaction, error) {
	if !w.recoveryComplete {
		return nil, errors.New("can't call NewTransaction before recovery is complete")
	}
	// Check that there are updates for the transaction to process.
	if len(updates) == 0 {
		return nil, errors.New("cannot create a transaction without updates")
	}

	// Create new transaction
	newTransaction := Transaction{
		Updates:      updates,
		wal:          w,
		initComplete: make(chan error),
	}

	// Initialize the transaction by splitting up the payload among free pages
	// and writing them to disk.
	go initTransaction(&newTransaction)

	// Increase the number of active transaction
	atomic.AddInt64(&w.atomicUnfinishedTxns, 1)

	return &newTransaction, nil
}

// writeToFile writes all the pages of the transaction to disk
func (t *Transaction) writeToFile() error {
	buf := make([]byte, pageSize)
	for page := t.firstPage; page != nil; page = page.nextPage {
		b := page.appendTo(buf[:0])
		if _, err := t.wal.logFile.WriteAt(b, int64(page.offset)); err != nil {
			return build.ExtendErr("Writing the page to disk failed", err)
		}
	}
	return nil
}
<|endoftext|>