<commit_before>package mppuma

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"

	mp "github.com/mackerelio/go-mackerel-plugin"
)

var graphdefGC = map[string]mp.Graphs{
	"gc.count": {
		Label: "Puma GC Count",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "total", Label: "Total GC count", Stacked: false},
			{Name: "minor", Label: "Minor GC count", Stacked: true},
			{Name: "major", Label: "Major GC count", Stacked: true},
		},
	},
	"gc.heap_slot": {
		Label: "Puma GC Heap Slot",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "available_slots", Label: "Heap available slots", Stacked: false},
			{Name: "live_slots", Label: "Heap live slots", Stacked: true},
			{Name: "free_slots", Label: "Heap free slots", Stacked: true},
			{Name: "final_slots", Label: "Heap final slots", Stacked: false},
			{Name: "marked_slots", Label: "Heap marked slots", Stacked: false},
		},
	},
	"gc.old_objects": {
		Label: "Puma GC Old Objects",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "old_count", Label: "Old Objects", Stacked: false},
			{Name: "old_limit", Label: "Old Objects Limit", Stacked: true},
		},
	},
	"gc.old_malloc": {
		Label: "Puma GC Old Malloc Increase",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "old_malloc_bytes", Label: "Old Malloc Increase Bytes", Stacked: false},
			{Name: "old_malloc_limit", Label: "Old Malloc Increase Bytes Limit", Stacked: true},
		},
	},
}

type response map[string]float64

// GCStats is convered from /gc-stats json
type GCStats struct {
	// Ruby2.0
	Count                json.Number `json:"count"`
	HeapFinalNum         json.Number `json:"heap_final_num"`
	HeapFreeNum          json.Number `json:"heap_free_num"`
	HeapIncrement        json.Number `json:"heap_increment"`
	HeapLength           json.Number `json:"heap_length"`
	HeapLiveNum          json.Number `json:"heap_live_num"`
	HeapUsed             json.Number `json:"heap_used"`
	TotalAllocatedObject json.Number `json:"total_allocated_object"`
	TotalFreedObject     json.Number `json:"total_freed_object"`
	// Added since Ruby2.1
	HeapLiveSlot               json.Number `json:"heap_live_slot"`
	HeapFreeSlot               json.Number `json:"heap_free_slot"`
	HeapFinalSlot              json.Number `json:"heap_final_slot"`
	HeapSweptSlot              json.Number `json:"heap_swept_slot"`
	HeapEdenPageLength         json.Number `json:"heap_eden_page_length"`
	HeapTombPageLength         json.Number `json:"heap_tomb_page_length"`
	MallocIncrease             json.Number `json:"malloc_increase"`
	MallocLimit                json.Number `json:"malloc_limit"`
	MinorGcCount               json.Number `json:"minor_gc_count"`
	MajorGcCount               json.Number `json:"major_gc_count"`
	RememberedShadyObject      json.Number `json:"remembered_shady_object"`
	RememberedShadyObjectLimit json.Number `json:"remembered_shady_object_limit"`
	OldObject                  json.Number `json:"old_object"`
	OldObjectLimit             json.Number `json:"old_object_limit"`
	OldmallocIncrease          json.Number `json:"oldmalloc_increase"`
	OldmallocLimit             json.Number `json:"oldmalloc_limit"`
	// Added since Ruby2.2
	HeapAllocatedPages                  json.Number `json:"heap_allocated_pages"`
	HeapSortedLength                    json.Number `json:"heap_sorted_length"`
	HeapAllocatablePages                json.Number `json:"heap_allocatable_pages"`
	HeapAvailableSlots                  json.Number `json:"heap_available_slots"`
	HeapLiveSlots                       json.Number `json:"heap_live_slots"`
	HeapFreeSlots                       json.Number `json:"heap_free_slots"`
	HeapFinalSlots                      json.Number `json:"heap_final_slots"`
	HeapMarkedSlots                     json.Number `json:"heap_marked_slots"`
	HeapSweptSlots                      json.Number `json:"heap_swept_slots"`
	HeapEdenPages                       json.Number `json:"heap_eden_pages"`
	HeapTombPages                       json.Number `json:"heap_tomb_pages"`
	TotalAllocatedPages                 json.Number `json:"total_allocated_pages"`
	TotalFreedPages                     json.Number `json:"total_freed_pages"`
	TotalAllocatedObjects               json.Number `json:"total_allocated_objects"`
	TotalFreedObjects                   json.Number `json:"total_freed_objects"`
	MallocIncreaseBytes                 json.Number `json:"malloc_increase_bytes"`
	MallocIncreaseBytesLimit            json.Number `json:"malloc_increase_bytes_limit"`
	RememberedWbUnprotectedObjects      json.Number `json:"remembered_wb_unprotected_objects"`
	RememberedWbUnprotectedObjectsLimit json.Number `json:"remembered_wb_unprotected_objects_limit"`
	OldObjects                          json.Number `json:"old_objects"`
	OldObjectsLimit                     json.Number `json:"old_objects_limit"`
	OldmallocIncreaseBytes              json.Number `json:"oldmalloc_increase_bytes"`
	OldmallocIncreaseBytesLimit         json.Number `json:"oldmalloc_increase_bytes_limit"`
	// Ruby2.3 is same as Ruby2.2
	// Ruby2.4 is almost same Ruby2.3 (deletes heap_swept_slots)
}

// Fetch /gc-stats
func (p PumaPlugin) getGCStatsAPI() (*GCStats, error) {

	var gcStats GCStats

	uri := fmt.Sprintf("http://%s:%s/%s?token=%s", p.Host, p.Port, "gc-stats", p.Token)
	resp, err := http.Get(uri)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return nil, errors.New(resp.Status)
	}
	if err := json.NewDecoder(resp.Body).Decode(&gcStats); err != nil {
		return nil, err
	}

	return &gcStats, nil
}

func (p PumaPlugin) fetchGCStatsMetrics(gcStats *GCStats) (map[string]float64, error) {
	ret := make(map[string]float64)

	// gc.count
	ret["total"], _ = gcStats.Count.Float64()

	if gcStats.MinorGcCount.String() != "" {
		ret["minor"], _ = gcStats.MinorGcCount.Float64()
	}
	if gcStats.MajorGcCount.String() != "" {
		ret["major"], _ = gcStats.MajorGcCount.Float64()
	}

	// gc.heap_slot
	if gcStats.HeapAvailableSlots.String() != "" {
		ret["available_slots"], _ = gcStats.HeapAvailableSlots.Float64()
	}
	if gcStats.HeapLiveSlots.String() != "" {
		ret["live_slots"], _ = gcStats.HeapLiveSlots.Float64()
	}
	if gcStats.HeapFreeSlots.String() != "" {
		ret["free_slots"], _ = gcStats.HeapFreeSlots.Float64()
	}
	if gcStats.HeapFinalSlots.String() != "" {
		ret["final_slots"], _ = gcStats.HeapFinalSlots.Float64()
	}
	if gcStats.HeapMarkedSlots.String() != "" {
		ret["marked_slots"], _ = gcStats.HeapMarkedSlots.Float64()
	}

	// old
	if gcStats.OldObjects.String() != "" {
		ret["old_count"], _ = gcStats.OldObjects.Float64()
	}
	if gcStats.OldObjectsLimit.String() != "" {
		ret["old_limit"], _ = gcStats.OldObjectsLimit.Float64()
	}
	if gcStats.OldmallocIncreaseBytes.String() != "" {
		ret["old_malloc_bytes"], _ = gcStats.OldmallocIncreaseBytes.Float64()
	}
	if gcStats.OldmallocIncreaseBytesLimit.String() != "" {
		ret["old_malloc_limit"], _ = gcStats.OldmallocIncreaseBytesLimit.Float64()
	}

	return ret, nil

}
<commit_msg>Pass test<commit_after>package mppuma

import (
	"encoding/json"
	"errors"
	"fmt"
	"net/http"

	mp "github.com/mackerelio/go-mackerel-plugin"
)

var graphdefGC = map[string]mp.Graphs{
	"gc.count": {
		Label: "Puma GC Count",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "total", Label: "Total GC count", Stacked: false},
			{Name: "minor", Label: "Minor GC count", Stacked: true},
			{Name: "major", Label: "Major GC count", Stacked: true},
		},
	},
	"gc.heap_slot": {
		Label: "Puma GC Heap Slot",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "available_slots", Label: "Heap available slots", Stacked: false},
			{Name: "live_slots", Label: "Heap live slots", Stacked: true},
			{Name: "free_slots", Label: "Heap free slots", Stacked: true},
			{Name: "final_slots", Label: "Heap final slots", Stacked: false},
			{Name: "marked_slots", Label: "Heap marked slots", Stacked: false},
		},
	},
	"gc.old_objects": {
		Label: "Puma GC Old Objects",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "old_count", Label: "Old Objects", Stacked: false},
			{Name: "old_limit", Label: "Old Objects Limit", Stacked: true},
		},
	},
	"gc.old_malloc": {
		Label: "Puma GC Old Malloc Increase",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "old_malloc_bytes", Label: "Old Malloc Increase Bytes", Stacked: false},
			{Name: "old_malloc_limit", Label: "Old Malloc Increase Bytes Limit", Stacked: true},
		},
	},
}

type response map[string]float64

// GCStats is convered from /gc-stats json
type GCStats struct {
	// Ruby2.0
	Count                json.Number `json:"count"`
	HeapFinalNum         json.Number `json:"heap_final_num"`
	HeapFreeNum          json.Number `json:"heap_free_num"`
	HeapIncrement        json.Number `json:"heap_increment"`
	HeapLength           json.Number `json:"heap_length"`
	HeapLiveNum          json.Number `json:"heap_live_num"`
	HeapUsed             json.Number `json:"heap_used"`
	TotalAllocatedObject json.Number `json:"total_allocated_object"`
	TotalFreedObject     json.Number `json:"total_freed_object"`
	// Added since Ruby2.1
	HeapLiveSlot               json.Number `json:"heap_live_slot"`
	HeapFreeSlot               json.Number `json:"heap_free_slot"`
	HeapFinalSlot              json.Number `json:"heap_final_slot"`
	HeapSweptSlot              json.Number `json:"heap_swept_slot"`
	HeapEdenPageLength         json.Number `json:"heap_eden_page_length"`
	HeapTombPageLength         json.Number `json:"heap_tomb_page_length"`
	MallocIncrease             json.Number `json:"malloc_increase"`
	MallocLimit                json.Number `json:"malloc_limit"`
	MinorGcCount               json.Number `json:"minor_gc_count"`
	MajorGcCount               json.Number `json:"major_gc_count"`
	RememberedShadyObject      json.Number `json:"remembered_shady_object"`
	RememberedShadyObjectLimit json.Number `json:"remembered_shady_object_limit"`
	OldObject                  json.Number `json:"old_object"`
	OldObjectLimit             json.Number `json:"old_object_limit"`
	OldmallocIncrease          json.Number `json:"oldmalloc_increase"`
	OldmallocLimit             json.Number `json:"oldmalloc_limit"`
	// Added since Ruby2.2
	HeapAllocatedPages                  json.Number `json:"heap_allocated_pages"`
	HeapSortedLength                    json.Number `json:"heap_sorted_length"`
	HeapAllocatablePages                json.Number `json:"heap_allocatable_pages"`
	HeapAvailableSlots                  json.Number `json:"heap_available_slots"`
	HeapLiveSlots                       json.Number `json:"heap_live_slots"`
	HeapFreeSlots                       json.Number `json:"heap_free_slots"`
	HeapFinalSlots                      json.Number `json:"heap_final_slots"`
	HeapMarkedSlots                     json.Number `json:"heap_marked_slots"`
	HeapSweptSlots                      json.Number `json:"heap_swept_slots"`
	HeapEdenPages                       json.Number `json:"heap_eden_pages"`
	HeapTombPages                       json.Number `json:"heap_tomb_pages"`
	TotalAllocatedPages                 json.Number `json:"total_allocated_pages"`
	TotalFreedPages                     json.Number `json:"total_freed_pages"`
	TotalAllocatedObjects               json.Number `json:"total_allocated_objects"`
	TotalFreedObjects                   json.Number `json:"total_freed_objects"`
	MallocIncreaseBytes                 json.Number `json:"malloc_increase_bytes"`
	MallocIncreaseBytesLimit            json.Number `json:"malloc_increase_bytes_limit"`
	RememberedWbUnprotectedObjects      json.Number `json:"remembered_wb_unprotected_objects"`
	RememberedWbUnprotectedObjectsLimit json.Number `json:"remembered_wb_unprotected_objects_limit"`
	OldObjects                          json.Number `json:"old_objects"`
	OldObjectsLimit                     json.Number `json:"old_objects_limit"`
	OldmallocIncreaseBytes              json.Number `json:"oldmalloc_increase_bytes"`
	OldmallocIncreaseBytesLimit         json.Number `json:"oldmalloc_increase_bytes_limit"`
	// Ruby2.3 is same as Ruby2.2
	// Ruby2.4 is almost same Ruby2.3 (deletes heap_swept_slots)
}

// Fetch /gc-stats
func (p PumaPlugin) getGCStatsAPI() (*GCStats, error) {

	var gcStats GCStats

	uri := fmt.Sprintf("http://%s:%s/%s?token=%s", p.Host, p.Port, "gc-stats", p.Token)
	resp, err := http.Get(uri)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return nil, errors.New(resp.Status)
	}
	if err := json.NewDecoder(resp.Body).Decode(&gcStats); err != nil {
		return nil, err
	}

	return &gcStats, nil
}

func (p PumaPlugin) fetchGCStatsMetrics(gcStats *GCStats) (map[string]float64, error) {
	ret := make(map[string]float64)

	// gc.count
	ret["total"], _ = gcStats.Count.Float64()

	if gcStats.MinorGcCount.String() != "" {
		ret["minor"], _ = gcStats.MinorGcCount.Float64()
	}
	if gcStats.MajorGcCount.String() != "" {
		ret["major"], _ = gcStats.MajorGcCount.Float64()
	}

	// gc.heap_slot
	if gcStats.HeapAvailableSlots.String() != "" {
		ret["available_slots"], _ = gcStats.HeapAvailableSlots.Float64()
	}

	if gcStats.HeapLiveNum.String() != "" {
		ret["live_slots"], _ = gcStats.HeapLiveNum.Float64()
	}
	if gcStats.HeapLiveSlots.String() != "" {
		ret["live_slots"], _ = gcStats.HeapLiveSlots.Float64()
	}

	if gcStats.HeapFreeNum.String() != "" {
		ret["free_slots"], _ = gcStats.HeapFreeNum.Float64()
	}
	if gcStats.HeapFreeSlots.String() != "" {
		ret["free_slots"], _ = gcStats.HeapFreeSlots.Float64()
	}

	if gcStats.HeapFinalNum.String() != "" {
		ret["final_slots"], _ = gcStats.HeapFinalNum.Float64()
	}
	if gcStats.HeapFinalSlots.String() != "" {
		ret["final_slots"], _ = gcStats.HeapFinalSlots.Float64()
	}

	if gcStats.HeapMarkedSlots.String() != "" {
		ret["marked_slots"], _ = gcStats.HeapMarkedSlots.Float64()
	}

	// old
	if gcStats.OldObjects.String() != "" {
		ret["old_count"], _ = gcStats.OldObjects.Float64()
	}
	if gcStats.OldObjectsLimit.String() != "" {
		ret["old_limit"], _ = gcStats.OldObjectsLimit.Float64()
	}
	if gcStats.OldmallocIncreaseBytes.String() != "" {
		ret["old_malloc_bytes"], _ = gcStats.OldmallocIncreaseBytes.Float64()
	}
	if gcStats.OldmallocIncreaseBytesLimit.String() != "" {
		ret["old_malloc_limit"], _ = gcStats.OldmallocIncreaseBytesLimit.Float64()
	}

	return ret, nil

}
<|endoftext|>


<commit_before>package thrift

import (
	"bytes"
	"fmt"
	"reflect"
	"testing"
)

type TestStruct2 struct {
	Str    string `thrift:"1"`
	Binary []byte `thrift:"2"`
}

func (t *TestStruct2) String() string {
	return fmt.Sprintf("{Str:%s Binary:%+v}", t.Str, t.Binary)
}

type TestStruct struct {
	String  string            `thrift:"1"`
	Int     *int              `thrift:"2"`
	List    []string          `thrift:"3"`
	Map     map[string]string `thrift:"4"`
	Struct  *TestStruct2      `thrift:"5"`
	List2   []*string         `thrift:"6"`
	Struct2 TestStruct2       `thrift:"7"`
	Binary  []byte            `thrift:"8"`
	Set     []string          `thrift:"9,set"`
}

type TestStructRequiredOptional struct {
	RequiredPtr *string `thrift:"1,required"`
	Required    string  `thrift:"2,required"`
	OptionalPtr *string `thrift:"3"`
	Optional    string  `thrift:"4"`
}

type TestEmptyStruct struct{}

func TestKeepEmpty(t *testing.T) {
	buf := &bytes.Buffer{}

	s := struct {
		Str1 string `thrift:"1"`
	}{}
	err := EncodeStruct(buf, BinaryProtocol, s)
	if err != nil {
		t.Fatal(err)
	}
	if buf.Len() != 1 || buf.Bytes()[0] != 0 {
		t.Fatal("missing keepempty should mean empty fields are not serialized")
	}

	buf.Reset()
	s2 := struct {
		Str1 string `thrift:"1,keepempty"`
	}{}
	err = EncodeStruct(buf, BinaryProtocol, s2)
	if err != nil {
		t.Fatal(err)
	}
	if buf.Len() != 8 {
		t.Fatal("keepempty should cause empty fields to be serialized")
	}
}

func TestEncodeRequired(t *testing.T) {
	buf := &bytes.Buffer{}

	s := struct {
		Str1 string `thrift:"1,required"`
	}{}
	err := EncodeStruct(buf, BinaryProtocol, s)
	if err != nil {
		t.Fatal(err)
	}
	if buf.Len() != 8 {
		t.Fatal("Non-pointer required fields that aren't 'keepempty' should be serialized empty")
	}

	buf.Reset()
	s2 := struct {
		Str1 *string `thrift:"1,required"`
	}{}
	err = EncodeStruct(buf, BinaryProtocol, s2)
	_, ok := err.(*MissingRequiredField)
	if !ok {
		t.Fatalf("Missing required field should throw MissingRequiredField instead of %+v", err)
	}
}

func TestBasics(t *testing.T) {
	i := 123
	str := "bar"
	ts2 := TestStruct2{"qwerty", []byte{1, 2, 3}}
	s := &TestStruct{
		"test",
		&i,
		[]string{"a", "b"},
		map[string]string{"a": "b", "1": "2"},
		&ts2,
		[]*string{&str},
		ts2,
		[]byte{1, 2, 3},
		[]string{"a", "b"},
	}
	buf := &bytes.Buffer{}

	err := EncodeStruct(buf, BinaryProtocol, s)
	if err != nil {
		t.Fatal(err)
	}

	s2 := &TestStruct{}
	err = DecodeStruct(buf, BinaryProtocol, s2)
	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(s, s2) {
		t.Fatalf("encdec doesn't match: %+v != %+v", s, s2)
	}
}

func TestEncodeRequiredFields(t *testing.T) {
	buf := &bytes.Buffer{}

	// encode nil pointer required field

	s := &TestStructRequiredOptional{nil, "", nil, ""}
	err := EncodeStruct(buf, BinaryProtocol, s)
	if err == nil {
		t.Fatal("Expected MissingRequiredField exception")
	}
	e, ok := err.(*MissingRequiredField)
	if !ok {
		t.Fatalf("Expected MissingRequiredField exception instead %+v", err)
	}
	if e.StructName != "TestStructRequiredOptional" || e.FieldName != "RequiredPtr" {
		t.Fatalf("Expected MissingRequiredField{'TestStructRequiredOptional', 'RequiredPtr'} instead %+v", e)
	}

	// encode empty non-pointer required field

	str := "foo"
	s = &TestStructRequiredOptional{&str, "", nil, ""}
	err = EncodeStruct(buf, BinaryProtocol, s)
	if err != nil {
		t.Fatal("Empty non-pointer required fields shouldn't return an error")
	}
}

func TestDecodeRequiredFields(t *testing.T) {
	buf := &bytes.Buffer{}

	s := &TestEmptyStruct{}
	err := EncodeStruct(buf, BinaryProtocol, s)
	if err != nil {
		t.Fatal("Failed to encode empty struct")
	}

	s2 := &TestStructRequiredOptional{}
	err = DecodeStruct(buf, BinaryProtocol, s2)
	if err == nil {
		t.Fatal("Expected MissingRequiredField exception")
	}
	e, ok := err.(*MissingRequiredField)
	if !ok {
		t.Fatalf("Expected MissingRequiredField exception instead %+v", err)
	}
	if e.StructName != "TestStructRequiredOptional" || e.FieldName != "RequiredPtr" {
		t.Fatalf("Expected MissingRequiredField{'TestStructRequiredOptional', 'RequiredPtr'} instead %+v", e)
	}
}

func TestDecodeUnknownFields(t *testing.T) {
	buf := &bytes.Buffer{}

	str := "foo"
	s := &TestStructRequiredOptional{&str, str, &str, str}
	err := EncodeStruct(buf, BinaryProtocol, s)
	if err != nil {
		t.Fatal("Failed to encode TestStructRequiredOptional struct")
	}

	s2 := &TestEmptyStruct{}
	err = DecodeStruct(buf, BinaryProtocol, s2)
	if err != nil {
		t.Fatalf("Unknown fields during decode weren't ignored: %+v", err)
	}
}

func BenchmarkEncodeEmptyStruct(b *testing.B) {
	buf := nullWriter(0)
	st := &struct{}{}
	for i := 0; i < b.N; i++ {
		EncodeStruct(buf, BinaryProtocol, st)
	}
}

func BenchmarkDecodeEmptyStruct(b *testing.B) {
	b.StopTimer()
	buf1 := &bytes.Buffer{}
	st := &struct{}{}
	EncodeStruct(buf1, BinaryProtocol, st)
	buf := bytes.NewBuffer(bytes.Repeat(buf1.Bytes(), b.N))
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		DecodeStruct(buf, BinaryProtocol, st)
	}
}

func BenchmarkEncodeSimpleStruct(b *testing.B) {
	buf := nullWriter(0)
	st := &struct {
		Str string `thrift:"1,required"`
		Int int32  `thrift:"2,required"`
	}{
		Str: "test",
		Int: 123,
	}
	for i := 0; i < b.N; i++ {
		EncodeStruct(buf, BinaryProtocol, st)
	}
}
<commit_msg>Add benchmark for decoding simple structs<commit_after>package thrift

import (
	"bytes"
	"fmt"
	"reflect"
	"testing"
)

type TestStruct2 struct {
	Str    string `thrift:"1"`
	Binary []byte `thrift:"2"`
}

func (t *TestStruct2) String() string {
	return fmt.Sprintf("{Str:%s Binary:%+v}", t.Str, t.Binary)
}

type TestStruct struct {
	String  string            `thrift:"1"`
	Int     *int              `thrift:"2"`
	List    []string          `thrift:"3"`
	Map     map[string]string `thrift:"4"`
	Struct  *TestStruct2      `thrift:"5"`
	List2   []*string         `thrift:"6"`
	Struct2 TestStruct2       `thrift:"7"`
	Binary  []byte            `thrift:"8"`
	Set     []string          `thrift:"9,set"`
}

type TestStructRequiredOptional struct {
	RequiredPtr *string `thrift:"1,required"`
	Required    string  `thrift:"2,required"`
	OptionalPtr *string `thrift:"3"`
	Optional    string  `thrift:"4"`
}

type TestEmptyStruct struct{}

func TestKeepEmpty(t *testing.T) {
	buf := &bytes.Buffer{}

	s := struct {
		Str1 string `thrift:"1"`
	}{}
	err := EncodeStruct(buf, BinaryProtocol, s)
	if err != nil {
		t.Fatal(err)
	}
	if buf.Len() != 1 || buf.Bytes()[0] != 0 {
		t.Fatal("missing keepempty should mean empty fields are not serialized")
	}

	buf.Reset()
	s2 := struct {
		Str1 string `thrift:"1,keepempty"`
	}{}
	err = EncodeStruct(buf, BinaryProtocol, s2)
	if err != nil {
		t.Fatal(err)
	}
	if buf.Len() != 8 {
		t.Fatal("keepempty should cause empty fields to be serialized")
	}
}

func TestEncodeRequired(t *testing.T) {
	buf := &bytes.Buffer{}

	s := struct {
		Str1 string `thrift:"1,required"`
	}{}
	err := EncodeStruct(buf, BinaryProtocol, s)
	if err != nil {
		t.Fatal(err)
	}
	if buf.Len() != 8 {
		t.Fatal("Non-pointer required fields that aren't 'keepempty' should be serialized empty")
	}

	buf.Reset()
	s2 := struct {
		Str1 *string `thrift:"1,required"`
	}{}
	err = EncodeStruct(buf, BinaryProtocol, s2)
	_, ok := err.(*MissingRequiredField)
	if !ok {
		t.Fatalf("Missing required field should throw MissingRequiredField instead of %+v", err)
	}
}

func TestBasics(t *testing.T) {
	i := 123
	str := "bar"
	ts2 := TestStruct2{"qwerty", []byte{1, 2, 3}}
	s := &TestStruct{
		"test",
		&i,
		[]string{"a", "b"},
		map[string]string{"a": "b", "1": "2"},
		&ts2,
		[]*string{&str},
		ts2,
		[]byte{1, 2, 3},
		[]string{"a", "b"},
	}
	buf := &bytes.Buffer{}

	err := EncodeStruct(buf, BinaryProtocol, s)
	if err != nil {
		t.Fatal(err)
	}

	s2 := &TestStruct{}
	err = DecodeStruct(buf, BinaryProtocol, s2)
	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(s, s2) {
		t.Fatalf("encdec doesn't match: %+v != %+v", s, s2)
	}
}

func TestEncodeRequiredFields(t *testing.T) {
	buf := &bytes.Buffer{}

	// encode nil pointer required field

	s := &TestStructRequiredOptional{nil, "", nil, ""}
	err := EncodeStruct(buf, BinaryProtocol, s)
	if err == nil {
		t.Fatal("Expected MissingRequiredField exception")
	}
	e, ok := err.(*MissingRequiredField)
	if !ok {
		t.Fatalf("Expected MissingRequiredField exception instead %+v", err)
	}
	if e.StructName != "TestStructRequiredOptional" || e.FieldName != "RequiredPtr" {
		t.Fatalf("Expected MissingRequiredField{'TestStructRequiredOptional', 'RequiredPtr'} instead %+v", e)
	}

	// encode empty non-pointer required field

	str := "foo"
	s = &TestStructRequiredOptional{&str, "", nil, ""}
	err = EncodeStruct(buf, BinaryProtocol, s)
	if err != nil {
		t.Fatal("Empty non-pointer required fields shouldn't return an error")
	}
}

func TestDecodeRequiredFields(t *testing.T) {
	buf := &bytes.Buffer{}

	s := &TestEmptyStruct{}
	err := EncodeStruct(buf, BinaryProtocol, s)
	if err != nil {
		t.Fatal("Failed to encode empty struct")
	}

	s2 := &TestStructRequiredOptional{}
	err = DecodeStruct(buf, BinaryProtocol, s2)
	if err == nil {
		t.Fatal("Expected MissingRequiredField exception")
	}
	e, ok := err.(*MissingRequiredField)
	if !ok {
		t.Fatalf("Expected MissingRequiredField exception instead %+v", err)
	}
	if e.StructName != "TestStructRequiredOptional" || e.FieldName != "RequiredPtr" {
		t.Fatalf("Expected MissingRequiredField{'TestStructRequiredOptional', 'RequiredPtr'} instead %+v", e)
	}
}

func TestDecodeUnknownFields(t *testing.T) {
	buf := &bytes.Buffer{}

	str := "foo"
	s := &TestStructRequiredOptional{&str, str, &str, str}
	err := EncodeStruct(buf, BinaryProtocol, s)
	if err != nil {
		t.Fatal("Failed to encode TestStructRequiredOptional struct")
	}

	s2 := &TestEmptyStruct{}
	err = DecodeStruct(buf, BinaryProtocol, s2)
	if err != nil {
		t.Fatalf("Unknown fields during decode weren't ignored: %+v", err)
	}
}

func BenchmarkEncodeEmptyStruct(b *testing.B) {
	buf := nullWriter(0)
	st := &struct{}{}
	for i := 0; i < b.N; i++ {
		EncodeStruct(buf, BinaryProtocol, st)
	}
}

func BenchmarkDecodeEmptyStruct(b *testing.B) {
	b.StopTimer()
	buf1 := &bytes.Buffer{}
	st := &struct{}{}
	EncodeStruct(buf1, BinaryProtocol, st)
	buf := bytes.NewBuffer(bytes.Repeat(buf1.Bytes(), b.N))
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		DecodeStruct(buf, BinaryProtocol, st)
	}
}

func BenchmarkEncodeSimpleStruct(b *testing.B) {
	buf := nullWriter(0)
	st := &struct {
		Str string `thrift:"1,required"`
		Int int32  `thrift:"2,required"`
	}{
		Str: "test",
		Int: 123,
	}
	for i := 0; i < b.N; i++ {
		EncodeStruct(buf, BinaryProtocol, st)
	}
}

func BenchmarkDecodeSimpleStruct(b *testing.B) {
	b.StopTimer()
	buf1 := &bytes.Buffer{}
	st := &struct {
		Str string `thrift:"1,required"`
		Int int32  `thrift:"2,required"`
	}{
		Str: "test",
		Int: 123,
	}
	buf := bytes.NewBuffer(bytes.Repeat(buf1.Bytes(), b.N))
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		DecodeStruct(buf, BinaryProtocol, st)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Orinoco Payments
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ens

import (
	"bytes"
	"compress/zlib"
	"errors"
	"io"
	"io/ioutil"
	"math/big"
	"strings"

	"github.com/ethereum/go-ethereum/accounts"
	"github.com/ethereum/go-ethereum/accounts/abi/bind"
	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/ethclient"
	etherutils "github.com/orinocopay/go-etherutils"
	"github.com/orinocopay/go-etherutils/ens/resolvercontract"
)

var zeroHash = make([]byte, 32)

// UnknownAddress is the address to which unknown entries resolve
var UnknownAddress = common.HexToAddress("00")

// PublicResolver obtains the public resolver for a chain
func PublicResolver(client *ethclient.Client) (address common.Address, err error) {
	address, err = resolverAddress(client, "resolver.eth")

	return
}

func resolverAddress(client *ethclient.Client, name string) (address common.Address, err error) {
	nameHash := NameHash(name)

	registryContract, err := RegistryContract(client)
	if err != nil {
		return
	}

	// Check that this name is owned
	ownerAddress, err := registryContract.Owner(nil, nameHash)
	if err != nil {
		return
	}
	if bytes.Compare(ownerAddress.Bytes(), UnknownAddress.Bytes()) == 0 {
		err = errors.New("unregistered name")
		return
	}

	// Obtain the resolver address for this name
	address, err = registryContract.Resolver(nil, nameHash)
	if err != nil {
		return
	}
	if bytes.Compare(address.Bytes(), UnknownAddress.Bytes()) == 0 {
		err = errors.New("no resolver")
		return
	}

	return
}

// Resolve resolves an ENS name in to an Etheruem address
// This will return an error if the name is not found or otherwise 0
func Resolve(client *ethclient.Client, input string) (address common.Address, err error) {
	if strings.HasSuffix(input, ".eth") {
		return resolveName(client, input)
	}
	address = common.HexToAddress(input)
	if address == UnknownAddress {
		err = errors.New("could not parse address")
	}

	return
}

func resolveName(client *ethclient.Client, input string) (address common.Address, err error) {
	var nameHash [32]byte
	nameHash = NameHash(input)
	if bytes.Compare(nameHash[:], zeroHash) == 0 {
		err = errors.New("Bad name")
	} else {
		address, err = resolveHash(client, input)
	}
	return
}

func resolveHash(client *ethclient.Client, name string) (address common.Address, err error) {
	contract, err := ResolverContract(client, name)
	if err != nil {
		return UnknownAddress, err
	}

	// Resolve the name
	address, err = contract.Addr(nil, NameHash(name))
	if err != nil {
		return UnknownAddress, err
	}
	if bytes.Compare(address.Bytes(), UnknownAddress.Bytes()) == 0 {
		return UnknownAddress, errors.New("no address")
	}

	return
}

// CreateResolverSession creates a session suitable for multiple calls
func CreateResolverSession(chainID *big.Int, wallet *accounts.Wallet, account *accounts.Account, passphrase string, contract *resolvercontract.ResolverContract, gasPrice *big.Int) *resolvercontract.ResolverContractSession {
	// Create a signer
	signer := etherutils.AccountSigner(chainID, wallet, account, passphrase)

	// Return our session
	session := &resolvercontract.ResolverContractSession{
		Contract: contract,
		CallOpts: bind.CallOpts{
			Pending: true,
		},
		TransactOpts: bind.TransactOpts{
			From:     account.Address,
			Signer:   signer,
			GasPrice: gasPrice,
		},
	}

	return session
}

// SetResolution sets the address to which a name resolves
func SetResolution(session *resolvercontract.ResolverContractSession, name string, resolutionAddress *common.Address) (tx *types.Transaction, err error) {
	session.TransactOpts.GasPrice = big.NewInt(40000)
	tx, err = session.SetAddr(NameHash(name), *resolutionAddress)
	return
}

// SetAbi sets the ABI associated with a name
func SetAbi(session *resolvercontract.ResolverContractSession, name string, abi string, contentType *big.Int) (tx *types.Transaction, err error) {
	var data []byte
	if contentType.Cmp(big.NewInt(1)) == 0 {
		// Uncompressed JSON
		data = []byte(abi)
	} else if contentType.Cmp(big.NewInt(2)) == 0 {
		// Zlib-compressed JSON
		var b bytes.Buffer
		w := zlib.NewWriter(&b)
		w.Write([]byte(abi))
		w.Close()
		data = b.Bytes()
	} else {
		err = errors.New("Unsupported content type")
	}
	// Gas cost is around 50000 + 64 per byte; add 10000 headroom to be safe
	session.TransactOpts.GasLimit = big.NewInt(int64(600000 + len(data)*64))
	tx, err = session.SetABI(NameHash(name), contentType, data)

	return
}

// Abi returns the ABI associated with a name
func Abi(resolver *resolvercontract.ResolverContract, name string) (abi string, err error) {
	var result struct {
		ContentType *big.Int
		Data        []byte
	}
	contentTypes := big.NewInt(3)
	result, err = resolver.ABI(nil, NameHash(name), contentTypes)
	if err == nil {
		if result.ContentType.Cmp(big.NewInt(1)) == 0 {
			// Uncompressed JSON
			abi = string(result.Data)
		} else if result.ContentType.Cmp(big.NewInt(2)) == 0 {
			// Zlib-compressed JSON
			b := bytes.NewReader(result.Data)
			var z io.ReadCloser
			z, err = zlib.NewReader(b)
			if err != nil {
				return
			}
			defer z.Close()
			var uncompressed []byte
			uncompressed, err = ioutil.ReadAll(z)
			abi = string(uncompressed)
		}
	}
	return
}

// ResolverContractByAddress instantiates the resolver contract at aspecific address
func ResolverContractByAddress(client *ethclient.Client, resolverAddress common.Address) (resolver *resolvercontract.ResolverContract, err error) {
	// Instantiate the resolver contract
	resolver, err = resolvercontract.NewResolverContract(resolverAddress, client)

	return
}

// ResolverContract obtains the resolver contract for a name
func ResolverContract(client *ethclient.Client, name string) (resolver *resolvercontract.ResolverContract, err error) {
	resolverAddress, err := resolverAddress(client, name)
	if err != nil {
		return
	}
	if bytes.Compare(resolverAddress.Bytes(), UnknownAddress.Bytes()) == 0 {
		err = errors.New("no resolver")
		return
	}

	resolver, err = ResolverContractByAddress(client, resolverAddress)
	return
}
<commit_msg>Remove hard-coded gas price<commit_after>// Copyright 2017 Orinoco Payments
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ens

import (
	"bytes"
	"compress/zlib"
	"errors"
	"io"
	"io/ioutil"
	"math/big"
	"strings"

	"github.com/ethereum/go-ethereum/accounts"
	"github.com/ethereum/go-ethereum/accounts/abi/bind"
	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/ethclient"
	etherutils "github.com/orinocopay/go-etherutils"
	"github.com/orinocopay/go-etherutils/ens/resolvercontract"
)

var zeroHash = make([]byte, 32)

// UnknownAddress is the address to which unknown entries resolve
var UnknownAddress = common.HexToAddress("00")

// PublicResolver obtains the public resolver for a chain
func PublicResolver(client *ethclient.Client) (address common.Address, err error) {
	address, err = resolverAddress(client, "resolver.eth")

	return
}

func resolverAddress(client *ethclient.Client, name string) (address common.Address, err error) {
	nameHash := NameHash(name)

	registryContract, err := RegistryContract(client)
	if err != nil {
		return
	}

	// Check that this name is owned
	ownerAddress, err := registryContract.Owner(nil, nameHash)
	if err != nil {
		return
	}
	if bytes.Compare(ownerAddress.Bytes(), UnknownAddress.Bytes()) == 0 {
		err = errors.New("unregistered name")
		return
	}

	// Obtain the resolver address for this name
	address, err = registryContract.Resolver(nil, nameHash)
	if err != nil {
		return
	}
	if bytes.Compare(address.Bytes(), UnknownAddress.Bytes()) == 0 {
		err = errors.New("no resolver")
		return
	}

	return
}

// Resolve resolves an ENS name in to an Etheruem address
// This will return an error if the name is not found or otherwise 0
func Resolve(client *ethclient.Client, input string) (address common.Address, err error) {
	if strings.HasSuffix(input, ".eth") {
		return resolveName(client, input)
	}
	address = common.HexToAddress(input)
	if address == UnknownAddress {
		err = errors.New("could not parse address")
	}

	return
}

func resolveName(client *ethclient.Client, input string) (address common.Address, err error) {
	var nameHash [32]byte
	nameHash = NameHash(input)
	if bytes.Compare(nameHash[:], zeroHash) == 0 {
		err = errors.New("Bad name")
	} else {
		address, err = resolveHash(client, input)
	}
	return
}

func resolveHash(client *ethclient.Client, name string) (address common.Address, err error) {
	contract, err := ResolverContract(client, name)
	if err != nil {
		return UnknownAddress, err
	}

	// Resolve the name
	address, err = contract.Addr(nil, NameHash(name))
	if err != nil {
		return UnknownAddress, err
	}
	if bytes.Compare(address.Bytes(), UnknownAddress.Bytes()) == 0 {
		return UnknownAddress, errors.New("no address")
	}

	return
}

// CreateResolverSession creates a session suitable for multiple calls
func CreateResolverSession(chainID *big.Int, wallet *accounts.Wallet, account *accounts.Account, passphrase string, contract *resolvercontract.ResolverContract, gasPrice *big.Int) *resolvercontract.ResolverContractSession {
	// Create a signer
	signer := etherutils.AccountSigner(chainID, wallet, account, passphrase)

	// Return our session
	session := &resolvercontract.ResolverContractSession{
		Contract: contract,
		CallOpts: bind.CallOpts{
			Pending: true,
		},
		TransactOpts: bind.TransactOpts{
			From:     account.Address,
			Signer:   signer,
			GasPrice: gasPrice,
		},
	}

	return session
}

// SetResolution sets the address to which a name resolves
func SetResolution(session *resolvercontract.ResolverContractSession, name string, resolutionAddress *common.Address) (tx *types.Transaction, err error) {
	tx, err = session.SetAddr(NameHash(name), *resolutionAddress)
	return
}

// SetAbi sets the ABI associated with a name
func SetAbi(session *resolvercontract.ResolverContractSession, name string, abi string, contentType *big.Int) (tx *types.Transaction, err error) {
	var data []byte
	if contentType.Cmp(big.NewInt(1)) == 0 {
		// Uncompressed JSON
		data = []byte(abi)
	} else if contentType.Cmp(big.NewInt(2)) == 0 {
		// Zlib-compressed JSON
		var b bytes.Buffer
		w := zlib.NewWriter(&b)
		w.Write([]byte(abi))
		w.Close()
		data = b.Bytes()
	} else {
		err = errors.New("Unsupported content type")
	}
	// Gas cost is around 50000 + 64 per byte; add 10000 headroom to be safe
	//session.TransactOpts.GasLimit = big.NewInt(int64(600000 + len(data)*64))
	tx, err = session.SetABI(NameHash(name), contentType, data)

	return
}

// Abi returns the ABI associated with a name
func Abi(resolver *resolvercontract.ResolverContract, name string) (abi string, err error) {
	var result struct {
		ContentType *big.Int
		Data        []byte
	}
	contentTypes := big.NewInt(3)
	result, err = resolver.ABI(nil, NameHash(name), contentTypes)
	if err == nil {
		if result.ContentType.Cmp(big.NewInt(1)) == 0 {
			// Uncompressed JSON
			abi = string(result.Data)
		} else if result.ContentType.Cmp(big.NewInt(2)) == 0 {
			// Zlib-compressed JSON
			b := bytes.NewReader(result.Data)
			var z io.ReadCloser
			z, err = zlib.NewReader(b)
			if err != nil {
				return
			}
			defer z.Close()
			var uncompressed []byte
			uncompressed, err = ioutil.ReadAll(z)
			abi = string(uncompressed)
		}
	}
	return
}

// ResolverContractByAddress instantiates the resolver contract at aspecific address
func ResolverContractByAddress(client *ethclient.Client, resolverAddress common.Address) (resolver *resolvercontract.ResolverContract, err error) {
	// Instantiate the resolver contract
	resolver, err = resolvercontract.NewResolverContract(resolverAddress, client)

	return
}

// ResolverContract obtains the resolver contract for a name
func ResolverContract(client *ethclient.Client, name string) (resolver *resolvercontract.ResolverContract, err error) {
	resolverAddress, err := resolverAddress(client, name)
	if err != nil {
		return
	}
	if bytes.Compare(resolverAddress.Bytes(), UnknownAddress.Bytes()) == 0 {
		err = errors.New("no resolver")
		return
	}

	resolver, err = ResolverContractByAddress(client, resolverAddress)
	return
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package eval

import (
	"bytes"
	"context"
	"fmt"
	"sort"
	"strings"
	"sync"

	"golang.org/x/sync/errgroup"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/sync/parallel"

	evalpb "go.chromium.org/luci/rts/presubmit/eval/proto"
)

// Safety is result of algorithm safety evaluation.
// A safe algorithm does not let bad CLs pass CQ.
type Safety struct {
	// TotalRejections is the total number of analyzed rejections.
	TotalRejections int

	// EligibleRejections is the number of rejections eligible for safety
	// evaluation.
	EligibleRejections int

	// LostRejections are the rejections that would not be preserved
	// by the candidate algorithm, i.e. the bad patchsets would land.
	// The candidate RTS algorithm did not select any of the failed tests
	// in these rejections.
	//
	// Ideally this slice is empty.
	LostRejections []*evalpb.Rejection
}

func (r *evalRun) evaluateSafety(ctx context.Context, rejectionC chan *evalpb.Rejection) (*Safety, error) {
	eg, ctx := errgroup.WithContext(ctx)

	var ret Safety
	var mu sync.Mutex
	for i := 0; i < r.Concurrency; i++ {
		eg.Go(func() error {
			buf := &bytes.Buffer{}
			p := newPrinter(buf)
			for {
				select {
				case <-ctx.Done():
					return ctx.Err()

				case rej, ok := <-rejectionC:
					if !ok {
						return nil
					}

					eligible, wouldReject, err := r.processRejection(ctx, rej)

					mu.Lock()
					ret.TotalRejections++
					if err == nil && eligible {
						ret.EligibleRejections++
						if !wouldReject {
							ret.LostRejections = append(ret.LostRejections, rej)
						}
					}
					mu.Unlock()
					if err != nil {
						return errors.Annotate(err, "failed to process rejection %q", rej).Err()
					}

					if !wouldReject {
						buf.Reset()
						printLostRejection(p, rej)
						logging.Infof(ctx, "%s", buf.Bytes())
					}
					return nil
				}
			}
		})
	}
	if err := eg.Wait(); err != nil {
		return nil, err
	}

	return &ret, nil
}

func (r *evalRun) processRejection(ctx context.Context, rej *evalpb.Rejection) (eligible, wouldReject bool, err error) {
	// TODO(crbug.com/1112125): add support for CL stacks.
	// This call returns only files modified in the particular patchset and
	// ignores possible parent CLs that were also tested.

	// TODO(crbug.com/1112125): skip the patchset if it has a ton of failed tests.
	// Most RTS algorithms would reject such a patchset, so it represents noise.

	files, err := r.changedFiles(ctx, rej.Patchsets...)
	switch {
	case len(files) == 0:
		// The CL is deleted  => not eligible.
		err = nil
		return

	case err != nil:
		err = errors.Annotate(err, "failed to read changed files of %s", rej).Err()
		return
	}

	// Compare the prediction to facts.
	in := Input{ChangedFiles: files}
	for _, tv := range rej.FailedTestVariants {
		in.TestVariant = tv
		var out Output
		if out, err = r.Algorithm(ctx, in); err != nil {
			err = errors.Annotate(err, "RTS algorithm failed").Err()
			return
		}
		eligible = true

		if out.ShouldRun {
			// At least one failed test would run => the bad patchset would be rejected.
			wouldReject = true
			return
		}
	}
	return
}

// changedFiles retrieves changed files of all patchsets in the rejection.
func (r *evalRun) changedFiles(ctx context.Context, patchsets ...*evalpb.GerritPatchset) ([]*SourceFile, error) {
	var ret []*SourceFile
	var mu sync.Mutex
	err := parallel.FanOutIn(func(workC chan<- func() error) {
		for _, ps := range patchsets {
			ps := ps
			workC <- func() error {
				changedFiles, err := r.gerrit.ChangedFiles(ctx, ps)
				if err != nil {
					return err
				}

				repo := fmt.Sprintf("https://%s/%s", ps.Change.Host, strings.TrimSuffix(ps.Change.Project, ".git"))

				mu.Lock()
				defer mu.Unlock()
				for _, path := range changedFiles {
					ret = append(ret, &SourceFile{
						Repo: repo,
						Path: "//" + path,
					})
				}
				return nil
			}
		}
	})
	if err != nil {
		return nil, err
	}
	return ret, err
}

// printLostRejection prints a rejection that wouldn't be preserved by the
// candidate RTS algorithm.
func printLostRejection(p *printer, rej *evalpb.Rejection) error {
	pf := p.printf

	pf("Lost rejection:\n")
	p.Level++

	// Print patchsets.
	if len(rej.Patchsets) == 1 {
		pf("%s\n", psURL(rej.Patchsets[0]))
	} else {
		pf("- patchsets:\n")
		p.Level++
		for _, ps := range rej.Patchsets {
			pf("%s\n", psURL(ps))
		}
		p.Level--
	}

	printTestVariants(p, rej.FailedTestVariants)

	p.Level--
	return p.err
}

// printTestVariants prints tests grouped by variant.
func printTestVariants(p *printer, testVariants []*evalpb.TestVariant) {
	pf := p.printf

	// Group by variant.
	byVariant := map[string][]*evalpb.TestVariant{}
	var keys []string
	for _, tv := range testVariants {
		key := variantString(tv.Variant)
		tests, ok := byVariant[key]
		byVariant[key] = append(tests, tv)
		if !ok {
			keys = append(keys, key)
		}
	}
	sort.Strings(keys)

	// Print tests grouped by variant.
	pf("Failed and not selected tests:\n")
	p.Level++
	for _, key := range keys {
		pf("- ")
		if key == "" {
			pf("<empty test variant>\n")
		} else {
			pf("%s\n", key)
		}

		ts := byVariant[key]
		sort.Slice(ts, func(i, j int) bool {
			return ts[i].Id < ts[j].Id
		})

		p.Level++
		for _, t := range ts {
			p.printf("- %s\n", t.Id)
			if t.FileName != "" {
				p.printf("  in %s\n", t.FileName)
			}
		}
		p.Level--
	}
	p.Level--
}
<commit_msg>[rts] Fix logging of lost patchsets<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package eval

import (
	"bytes"
	"context"
	"fmt"
	"sort"
	"strings"
	"sync"

	"golang.org/x/sync/errgroup"

	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/sync/parallel"

	evalpb "go.chromium.org/luci/rts/presubmit/eval/proto"
)

// Safety is result of algorithm safety evaluation.
// A safe algorithm does not let bad CLs pass CQ.
type Safety struct {
	// TotalRejections is the total number of analyzed rejections.
	TotalRejections int

	// EligibleRejections is the number of rejections eligible for safety
	// evaluation.
	EligibleRejections int

	// LostRejections are the rejections that would not be preserved
	// by the candidate algorithm, i.e. the bad patchsets would land.
	// The candidate RTS algorithm did not select any of the failed tests
	// in these rejections.
	//
	// Ideally this slice is empty.
	LostRejections []*evalpb.Rejection
}

func (r *evalRun) evaluateSafety(ctx context.Context, rejectionC chan *evalpb.Rejection) (*Safety, error) {
	eg, ctx := errgroup.WithContext(ctx)

	var ret Safety
	var mu sync.Mutex
	for i := 0; i < r.Concurrency; i++ {
		eg.Go(func() error {
			buf := &bytes.Buffer{}
			p := newPrinter(buf)
			for {
				select {
				case <-ctx.Done():
					return ctx.Err()

				case rej, ok := <-rejectionC:
					if !ok {
						return nil
					}

					eligible, wouldReject, err := r.processRejection(ctx, rej)

					mu.Lock()
					ret.TotalRejections++
					if err == nil && eligible {
						ret.EligibleRejections++
						if !wouldReject {
							ret.LostRejections = append(ret.LostRejections, rej)
						}
					}
					mu.Unlock()
					if err != nil {
						return errors.Annotate(err, "failed to process rejection %q", rej).Err()
					}

					if !wouldReject {
						buf.Reset()
						printLostRejection(p, rej)
						logging.Infof(ctx, "%s", buf.Bytes())
					}
				}
			}
		})
	}
	if err := eg.Wait(); err != nil {
		return nil, err
	}

	return &ret, nil
}

func (r *evalRun) processRejection(ctx context.Context, rej *evalpb.Rejection) (eligible, wouldReject bool, err error) {
	// TODO(crbug.com/1112125): add support for CL stacks.
	// This call returns only files modified in the particular patchset and
	// ignores possible parent CLs that were also tested.

	// TODO(crbug.com/1112125): skip the patchset if it has a ton of failed tests.
	// Most RTS algorithms would reject such a patchset, so it represents noise.

	files, err := r.changedFiles(ctx, rej.Patchsets...)
	switch {
	case len(files) == 0:
		// The CL is deleted  => not eligible.
		err = nil
		return

	case err != nil:
		err = errors.Annotate(err, "failed to read changed files of %s", rej).Err()
		return
	}

	// Compare the prediction to facts.
	in := Input{ChangedFiles: files}
	for _, tv := range rej.FailedTestVariants {
		in.TestVariant = tv
		var out Output
		if out, err = r.Algorithm(ctx, in); err != nil {
			err = errors.Annotate(err, "RTS algorithm failed").Err()
			return
		}
		eligible = true

		if out.ShouldRun {
			// At least one failed test would run => the bad patchset would be rejected.
			wouldReject = true
			return
		}
	}
	return
}

// changedFiles retrieves changed files of all patchsets in the rejection.
func (r *evalRun) changedFiles(ctx context.Context, patchsets ...*evalpb.GerritPatchset) ([]*SourceFile, error) {
	var ret []*SourceFile
	var mu sync.Mutex
	err := parallel.FanOutIn(func(workC chan<- func() error) {
		for _, ps := range patchsets {
			ps := ps
			workC <- func() error {
				changedFiles, err := r.gerrit.ChangedFiles(ctx, ps)
				if err != nil {
					return err
				}

				repo := fmt.Sprintf("https://%s/%s", ps.Change.Host, strings.TrimSuffix(ps.Change.Project, ".git"))

				mu.Lock()
				defer mu.Unlock()
				for _, path := range changedFiles {
					ret = append(ret, &SourceFile{
						Repo: repo,
						Path: "//" + path,
					})
				}
				return nil
			}
		}
	})
	if err != nil {
		return nil, err
	}
	return ret, err
}

// printLostRejection prints a rejection that wouldn't be preserved by the
// candidate RTS algorithm.
func printLostRejection(p *printer, rej *evalpb.Rejection) error {
	pf := p.printf

	pf("Lost rejection:\n")
	p.Level++

	// Print patchsets.
	if len(rej.Patchsets) == 1 {
		pf("%s\n", psURL(rej.Patchsets[0]))
	} else {
		pf("- patchsets:\n")
		p.Level++
		for _, ps := range rej.Patchsets {
			pf("%s\n", psURL(ps))
		}
		p.Level--
	}

	printTestVariants(p, rej.FailedTestVariants)

	p.Level--
	return p.err
}

// printTestVariants prints tests grouped by variant.
func printTestVariants(p *printer, testVariants []*evalpb.TestVariant) {
	pf := p.printf

	// Group by variant.
	byVariant := map[string][]*evalpb.TestVariant{}
	var keys []string
	for _, tv := range testVariants {
		key := variantString(tv.Variant)
		tests, ok := byVariant[key]
		byVariant[key] = append(tests, tv)
		if !ok {
			keys = append(keys, key)
		}
	}
	sort.Strings(keys)

	// Print tests grouped by variant.
	pf("Failed and not selected tests:\n")
	p.Level++
	for _, key := range keys {
		pf("- ")
		if key == "" {
			pf("<empty test variant>\n")
		} else {
			pf("%s\n", key)
		}

		ts := byVariant[key]
		sort.Slice(ts, func(i, j int) bool {
			return ts[i].Id < ts[j].Id
		})

		p.Level++
		for _, t := range ts {
			p.printf("- %s\n", t.Id)
			if t.FileName != "" {
				p.printf("  in %s\n", t.FileName)
			}
		}
		p.Level--
	}
	p.Level--
}
<|endoftext|>


<commit_before>package filer

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/util"
	"io"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
)

type MetaAggregator struct {
	filers         []string
	grpcDialOption grpc.DialOption
	MetaLogBuffer  *log_buffer.LogBuffer
	// notifying clients
	ListenersLock sync.Mutex
	ListenersCond *sync.Cond
}

// MetaAggregator only aggregates data "on the fly". The logs are not re-persisted to disk.
// The old data comes from what each LocalMetadata persisted on disk.
func NewMetaAggregator(filers []string, grpcDialOption grpc.DialOption) *MetaAggregator {
	t := &MetaAggregator{
		filers:         filers,
		grpcDialOption: grpcDialOption,
	}
	t.ListenersCond = sync.NewCond(&t.ListenersLock)
	t.MetaLogBuffer = log_buffer.NewLogBuffer(LogFlushInterval, nil, func() {
		t.ListenersCond.Broadcast()
	})
	return t
}

func (ma *MetaAggregator) StartLoopSubscribe(f *Filer, self string) {
	for _, filer := range ma.filers {
		go ma.subscribeToOneFiler(f, self, filer)
	}
}

func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self string, peer string) {

	/*
		Each filer reads the "filer.store.id", which is the store's signature when filer starts.

		When reading from other filers' local meta changes:
		* if the received change does not contain signature from self, apply the change to current filer store.

		Upon connecting to other filers, need to remember their signature and their offsets.

	*/

	var maybeReplicateMetadataChange func(*filer_pb.SubscribeMetadataResponse)
	lastPersistTime := time.Now()
	lastTsNs := time.Now().Add(-LogFlushInterval).UnixNano()

	peerSignature, err := ma.readFilerStoreSignature(peer)
	for err != nil {
		glog.V(0).Infof("connecting to peer filer %s: %v", peer, err)
		time.Sleep(1357 * time.Millisecond)
		peerSignature, err = ma.readFilerStoreSignature(peer)
	}

	if peerSignature != f.Signature {
		if prevTsNs, err := ma.readOffset(f, file, peerSignature); err == nil {
			lastTsNs = prevTsNs
		}

		glog.V(0).Infof("follow peer: %v, last %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
		maybeReplicateMetadataChange = func(event *filer_pb.SubscribeMetadataResponse) {
			if err := Replay(f.Store.ActualStore, event); err != nil {
				glog.Errorf("failed to reply metadata change from %v: %v", peer, err)
				return
			}
			if lastPersistTime.Add(time.Minute).Before(time.Now()) {
				if err := ma.updateOffset(f, peer, peerSignature, event.TsNs); err == nil {
					if event.TsNs < time.Now().Add(-2*time.Minute).UnixNano() {
						glog.V(0).Infof("sync with %s progressed to: %v", peer, time.Unix(0, event.TsNs))
					}
					lastPersistTime = time.Now()
				} else {
					glog.V(0).Infof("failed to update offset for %v: %v", peer, err)
				}
			}
		}
	}

	processEventFn := func(event *filer_pb.SubscribeMetadataResponse) error {
		data, err := proto.Marshal(event)
		if err != nil {
			glog.Errorf("failed to marshal subscribed filer_pb.SubscribeMetadataResponse %+v: %v", event, err)
			return err
		}
		dir := event.Directory
		// println("received meta change", dir, "size", len(data))
		ma.MetaLogBuffer.AddToBuffer([]byte(dir), data, event.TsNs)
		if maybeReplicateMetadataChange != nil {
			maybeReplicateMetadataChange(event)
		}
		return nil
	}

	for {
		err := pb.WithFilerClient(peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
			stream, err := client.SubscribeLocalMetadata(context.Background(), &filer_pb.SubscribeMetadataRequest{
				ClientName: "filer:" + self,
				PathPrefix: "/",
				SinceNs:    lastTsNs,
			})
			if err != nil {
				return fmt.Errorf("subscribe: %v", err)
			}

			for {
				resp, listenErr := stream.Recv()
				if listenErr == io.EOF {
					return nil
				}
				if listenErr != nil {
					return listenErr
				}

				if err := processEventFn(resp); err != nil {
					return fmt.Errorf("process %v: %v", resp, err)
				}
				lastTsNs = resp.TsNs
			}
		})
		if err != nil {
			glog.V(0).Infof("subscribing remote %s meta change: %v", peer, err)
			time.Sleep(1733 * time.Millisecond)
		}
	}
}

func (ma *MetaAggregator) readFilerStoreSignature(peer string) (sig int32, err error) {
	err = pb.WithFilerClient(peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
		if err != nil {
			return err
		}
		sig = resp.Signature
		return nil
	})
	return
}

const(
	MetaOffsetPrefix = "Meta"
)

func (ma *MetaAggregator) readOffset(f *Filer, peer string, peerSignature int32) (lastTsNs int64, err error) {

	key := []byte(MetaOffsetPrefix+"xxxx")
	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))

	value, err := f.Store.KvGet(context.Background(), key)

	if err == ErrKvNotFound {
		glog.Warningf("readOffset %s not found", peer)
		return 0, nil
	}

	if err != nil {
		return 0, fmt.Errorf("readOffset %s : %v", peer, err)
	}

	lastTsNs = int64(util.BytesToUint64(value))

	glog.V(0).Infof("readOffset %s : %d", peer, lastTsNs)

	return
}

func (ma *MetaAggregator) updateOffset(f *Filer, peer string, peerSignature int32, lastTsNs int64) (err error) {

	key := []byte(MetaOffsetPrefix+"xxxx")
	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))

	value := make([]byte, 8)
	util.Uint64toBytes(value, uint64(lastTsNs))

	err = f.Store.KvPut(context.Background(), key, value)

	if err != nil {
		return fmt.Errorf("updateOffset %s : %v", peer, err)
	}

	glog.V(4).Infof("updateOffset %s : %d", peer, lastTsNs)

	return
}
<commit_msg>fix compilation<commit_after>package filer

import (
	"context"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/util"
	"io"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
)

type MetaAggregator struct {
	filers         []string
	grpcDialOption grpc.DialOption
	MetaLogBuffer  *log_buffer.LogBuffer
	// notifying clients
	ListenersLock sync.Mutex
	ListenersCond *sync.Cond
}

// MetaAggregator only aggregates data "on the fly". The logs are not re-persisted to disk.
// The old data comes from what each LocalMetadata persisted on disk.
func NewMetaAggregator(filers []string, grpcDialOption grpc.DialOption) *MetaAggregator {
	t := &MetaAggregator{
		filers:         filers,
		grpcDialOption: grpcDialOption,
	}
	t.ListenersCond = sync.NewCond(&t.ListenersLock)
	t.MetaLogBuffer = log_buffer.NewLogBuffer(LogFlushInterval, nil, func() {
		t.ListenersCond.Broadcast()
	})
	return t
}

func (ma *MetaAggregator) StartLoopSubscribe(f *Filer, self string) {
	for _, filer := range ma.filers {
		go ma.subscribeToOneFiler(f, self, filer)
	}
}

func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self string, peer string) {

	/*
		Each filer reads the "filer.store.id", which is the store's signature when filer starts.

		When reading from other filers' local meta changes:
		* if the received change does not contain signature from self, apply the change to current filer store.

		Upon connecting to other filers, need to remember their signature and their offsets.

	*/

	var maybeReplicateMetadataChange func(*filer_pb.SubscribeMetadataResponse)
	lastPersistTime := time.Now()
	lastTsNs := time.Now().Add(-LogFlushInterval).UnixNano()

	peerSignature, err := ma.readFilerStoreSignature(peer)
	for err != nil {
		glog.V(0).Infof("connecting to peer filer %s: %v", peer, err)
		time.Sleep(1357 * time.Millisecond)
		peerSignature, err = ma.readFilerStoreSignature(peer)
	}

	if peerSignature != f.Signature {
		if prevTsNs, err := ma.readOffset(f, peer, peerSignature); err == nil {
			lastTsNs = prevTsNs
		}

		glog.V(0).Infof("follow peer: %v, last %v (%d)", peer, time.Unix(0, lastTsNs), lastTsNs)
		maybeReplicateMetadataChange = func(event *filer_pb.SubscribeMetadataResponse) {
			if err := Replay(f.Store.ActualStore, event); err != nil {
				glog.Errorf("failed to reply metadata change from %v: %v", peer, err)
				return
			}
			if lastPersistTime.Add(time.Minute).Before(time.Now()) {
				if err := ma.updateOffset(f, peer, peerSignature, event.TsNs); err == nil {
					if event.TsNs < time.Now().Add(-2*time.Minute).UnixNano() {
						glog.V(0).Infof("sync with %s progressed to: %v", peer, time.Unix(0, event.TsNs))
					}
					lastPersistTime = time.Now()
				} else {
					glog.V(0).Infof("failed to update offset for %v: %v", peer, err)
				}
			}
		}
	}

	processEventFn := func(event *filer_pb.SubscribeMetadataResponse) error {
		data, err := proto.Marshal(event)
		if err != nil {
			glog.Errorf("failed to marshal subscribed filer_pb.SubscribeMetadataResponse %+v: %v", event, err)
			return err
		}
		dir := event.Directory
		// println("received meta change", dir, "size", len(data))
		ma.MetaLogBuffer.AddToBuffer([]byte(dir), data, event.TsNs)
		if maybeReplicateMetadataChange != nil {
			maybeReplicateMetadataChange(event)
		}
		return nil
	}

	for {
		err := pb.WithFilerClient(peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
			stream, err := client.SubscribeLocalMetadata(context.Background(), &filer_pb.SubscribeMetadataRequest{
				ClientName: "filer:" + self,
				PathPrefix: "/",
				SinceNs:    lastTsNs,
			})
			if err != nil {
				return fmt.Errorf("subscribe: %v", err)
			}

			for {
				resp, listenErr := stream.Recv()
				if listenErr == io.EOF {
					return nil
				}
				if listenErr != nil {
					return listenErr
				}

				if err := processEventFn(resp); err != nil {
					return fmt.Errorf("process %v: %v", resp, err)
				}
				lastTsNs = resp.TsNs
			}
		})
		if err != nil {
			glog.V(0).Infof("subscribing remote %s meta change: %v", peer, err)
			time.Sleep(1733 * time.Millisecond)
		}
	}
}

func (ma *MetaAggregator) readFilerStoreSignature(peer string) (sig int32, err error) {
	err = pb.WithFilerClient(peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
		if err != nil {
			return err
		}
		sig = resp.Signature
		return nil
	})
	return
}

const(
	MetaOffsetPrefix = "Meta"
)

func (ma *MetaAggregator) readOffset(f *Filer, peer string, peerSignature int32) (lastTsNs int64, err error) {

	key := []byte(MetaOffsetPrefix+"xxxx")
	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))

	value, err := f.Store.KvGet(context.Background(), key)

	if err == ErrKvNotFound {
		glog.Warningf("readOffset %s not found", peer)
		return 0, nil
	}

	if err != nil {
		return 0, fmt.Errorf("readOffset %s : %v", peer, err)
	}

	lastTsNs = int64(util.BytesToUint64(value))

	glog.V(0).Infof("readOffset %s : %d", peer, lastTsNs)

	return
}

func (ma *MetaAggregator) updateOffset(f *Filer, peer string, peerSignature int32, lastTsNs int64) (err error) {

	key := []byte(MetaOffsetPrefix+"xxxx")
	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))

	value := make([]byte, 8)
	util.Uint64toBytes(value, uint64(lastTsNs))

	err = f.Store.KvPut(context.Background(), key, value)

	if err != nil {
		return fmt.Errorf("updateOffset %s : %v", peer, err)
	}

	glog.V(4).Infof("updateOffset %s : %d", peer, lastTsNs)

	return
}
<|endoftext|>


<commit_before>package errors

import (
	"bytes"
	"fmt"
	"net/http"
	"text/template"

	"github.com/mdouchement/lss/utils"
)

// HTTPCoder interface is implemented by application errors
type HTTPCoder interface {
	// HTTPCode return the HTTP status code for the given error
	HTTPCode() int
}

// M is metadata structure
type M map[string]interface{}

// Error describe all errors occurred when a license is asked.
type Error struct {
	Status     int          `json:"status"`
	StatusText string       `json:"status_text"`
	Errors     []InnerError `json:"errors"`
}

type InnerErrors []InnerError

type InnerError struct {
	Code     string `json:"code"`
	Kind     string `json:"type"`
	Metadata M      `json:"metadata,omitempty"`
}

var codeList = map[string]map[string]string{
	"controllers-copy": {
		"code":        "500-000",
		"status":      "500",
		"status_text": "Internal Server Error",
	},
	"engines-open": {
		"code":        "500-001",
		"status":      "500",
		"status_text": "Internal Server Error",
	},
	"engines-create": {
		"code":        "500-002",
		"status":      "500",
		"status_text": "Internal Server Error",
	},
	"engines-delete": {
		"code":        "500-003",
		"status":      "500",
		"status_text": "Internal Server Error",
	},
	"controllers-invalid_path": {
		"code":        "422-000",
		"status":      "422",
		"status_text": "Unprocessable Entity",
		"reason":      "Path `{{.path}}` is not valid.",
	},
}

func StatusCode(err error) int {
	if hc, ok := err.(HTTPCoder); ok {
		return hc.HTTPCode()
	}
	return http.StatusInternalServerError
}

func (e *InnerError) Error() string {
	return fmt.Sprintf("%s-%s: %s", e.Kind, e.Code, e.Metadata["reason"])
}

// Error contacts all InnerError in a single string.
func (e *Error) Error() string {
	var errf bytes.Buffer
	errf.WriteString("[")
	for i, err := range e.Errors {
		errf.WriteString("\"")
		errf.WriteString(err.Error())
		if i < len(e.Errors)-1 {
			errf.WriteString("\",")
		} else {
			errf.WriteString("\"]")
		}
	}
	return errf.String()
}

func (e *Error) HTTPCode() int {
	return e.Status
}

func code(key string) string {
	return codeList[key]["code"]
}

func statusText(key string) string {
	return codeList[key]["status_text"]
}

func status(key string) int {
	return utils.MustAtoi(codeList[key]["status"])
}

func appendReasonTo(key string, metadata M) M {
	if reasonTemplate, ok := codeList[key]["reason"]; ok {
		t := template.Must(template.New("reason").Parse(reasonTemplate))
		reason := &bytes.Buffer{}
		t.Execute(reason, metadata)
		metadata["reason"] = reason.String()
	}
	return metadata
}
<commit_msg>Improved documentation<commit_after>package errors

import (
	"bytes"
	"fmt"
	"net/http"
	"text/template"

	"github.com/mdouchement/lss/utils"
)

// HTTPCoder interface is implemented by application errors
type HTTPCoder interface {
	// HTTPCode return the HTTP status code for the given error
	HTTPCode() int
}

// M is metadata structure
type M map[string]interface{}

// Error describe all errors occurred when a license is asked.
type Error struct {
	Status     int          `json:"status"`
	StatusText string       `json:"status_text"`
	Errors     []InnerError `json:"errors"`
}

// InnerErrors holds all errors raised during a request.
type InnerErrors []InnerError

// InnerError details an error occurred during a request.
type InnerError struct {
	Code     string `json:"code"`
	Kind     string `json:"type"`
	Metadata M      `json:"metadata,omitempty"`
}

var codeList = map[string]map[string]string{
	"controllers-copy": {
		"code":        "500-000",
		"status":      "500",
		"status_text": "Internal Server Error",
	},
	"engines-open": {
		"code":        "500-001",
		"status":      "500",
		"status_text": "Internal Server Error",
	},
	"engines-create": {
		"code":        "500-002",
		"status":      "500",
		"status_text": "Internal Server Error",
	},
	"engines-delete": {
		"code":        "500-003",
		"status":      "500",
		"status_text": "Internal Server Error",
	},
	"controllers-invalid_path": {
		"code":        "422-000",
		"status":      "422",
		"status_text": "Unprocessable Entity",
		"reason":      "Path `{{.path}}` is not valid.",
	},
}

// StatusCode returns the HTTP status code from the given err.
func StatusCode(err error) int {
	if hc, ok := err.(HTTPCoder); ok {
		return hc.HTTPCode()
	}
	return http.StatusInternalServerError
}

// Error returns a well formated string of the current error.
func (e *InnerError) Error() string {
	return fmt.Sprintf("%s-%s: %s", e.Kind, e.Code, e.Metadata["reason"])
}

// Error concats all InnerError in a single string.
func (e *Error) Error() string {
	var errf bytes.Buffer
	errf.WriteString("[")
	for i, err := range e.Errors {
		errf.WriteString("\"")
		errf.WriteString(err.Error())
		if i < len(e.Errors)-1 {
			errf.WriteString("\",")
		} else {
			errf.WriteString("\"]")
		}
	}
	return errf.String()
}

// HTTPCode returns the HHTP status code of the current error.
func (e *Error) HTTPCode() int {
	return e.Status
}

func code(key string) string {
	return codeList[key]["code"]
}

func statusText(key string) string {
	return codeList[key]["status_text"]
}

func status(key string) int {
	return utils.MustAtoi(codeList[key]["status"])
}

func appendReasonTo(key string, metadata M) M {
	if reasonTemplate, ok := codeList[key]["reason"]; ok {
		t := template.Must(template.New("reason").Parse(reasonTemplate))
		reason := &bytes.Buffer{}
		t.Execute(reason, metadata)
		metadata["reason"] = reason.String()
	}
	return metadata
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gif

import (
	"bytes"
	"compress/lzw"
	"image"
	"image/color"
	"reflect"
	"testing"
)

// header, palette and trailer are parts of a valid 2x1 GIF image.
const (
	headerStr = "GIF89a" +
		"\x02\x00\x01\x00" + // width=2, height=1
		"\x80\x00\x00" // headerFields=(a color table of 2 pixels), backgroundIndex, aspect
	paletteStr = "\x10\x20\x30\x40\x50\x60" // the color table, also known as a palette
	trailerStr = "\x3b"
)

// lzwEncode returns an LZW encoding (with 2-bit literals) of n zeroes.
func lzwEncode(n int) []byte {
	b := &bytes.Buffer{}
	w := lzw.NewWriter(b, lzw.LSB, 2)
	w.Write(make([]byte, n))
	w.Close()
	return b.Bytes()
}

func TestDecode(t *testing.T) {
	testCases := []struct {
		nPix    int  // The number of pixels in the image data.
		extra   bool // Whether to write an extra block after the LZW-encoded data.
		wantErr error
	}{
		{0, false, errNotEnough},
		{1, false, errNotEnough},
		{2, false, nil},
		{2, true, errTooMuch},
		{3, false, errTooMuch},
	}
	for _, tc := range testCases {
		b := &bytes.Buffer{}
		b.WriteString(headerStr)
		b.WriteString(paletteStr)
		// Write an image with bounds 2x1 but tc.nPix pixels. If tc.nPix != 2
		// then this should result in an invalid GIF image. First, write a
		// magic 0x2c (image descriptor) byte, bounds=(0,0)-(2,1), a flags
		// byte, and 2-bit LZW literals.
		b.WriteString("\x2c\x00\x00\x00\x00\x02\x00\x01\x00\x00\x02")
		if tc.nPix > 0 {
			enc := lzwEncode(tc.nPix)
			if len(enc) > 0xff {
				t.Errorf("nPix=%d, extra=%t: compressed length %d is too large", tc.nPix, tc.extra, len(enc))
				continue
			}
			b.WriteByte(byte(len(enc)))
			b.Write(enc)
		}
		if tc.extra {
			b.WriteString("\x01\x02") // A 1-byte payload with an 0x02 byte.
		}
		b.WriteByte(0x00) // An empty block signifies the end of the image data.
		b.WriteString(trailerStr)

		got, err := Decode(b)
		if err != tc.wantErr {
			t.Errorf("nPix=%d, extra=%t\ngot  %v\nwant %v", tc.nPix, tc.extra, err, tc.wantErr)
		}

		if tc.wantErr != nil {
			continue
		}
		want := &image.Paletted{
			Pix:    []uint8{0, 0},
			Stride: 2,
			Rect:   image.Rect(0, 0, 2, 1),
			Palette: color.Palette{
				color.RGBA{0x10, 0x20, 0x30, 0xff},
				color.RGBA{0x40, 0x50, 0x60, 0xff},
			},
		}
		if !reflect.DeepEqual(got, want) {
			t.Errorf("nPix=%d, extra=%t\ngot  %v\nwant %v", tc.nPix, tc.extra, got, want)
		}
	}
}

func TestTransparentIndex(t *testing.T) {
	b := &bytes.Buffer{}
	b.WriteString(headerStr)
	b.WriteString(paletteStr)
	for transparentIndex := 0; transparentIndex < 3; transparentIndex++ {
		if transparentIndex < 2 {
			// Write the graphic control for the transparent index.
			b.WriteString("\x21\xf9\x00\x01\x00\x00")
			b.WriteByte(byte(transparentIndex))
			b.WriteByte(0)
		}
		// Write an image with bounds 2x1, as per TestDecode.
		b.WriteString("\x2c\x00\x00\x00\x00\x02\x00\x01\x00\x00\x02")
		enc := lzwEncode(2)
		if len(enc) > 0xff {
			t.Fatalf("compressed length %d is too large", len(enc))
		}
		b.WriteByte(byte(len(enc)))
		b.Write(enc)
		b.WriteByte(0x00)
	}
	b.WriteString(trailerStr)

	g, err := DecodeAll(b)
	if err != nil {
		t.Fatalf("DecodeAll: %v", err)
	}
	c0 := color.RGBA{paletteStr[0], paletteStr[1], paletteStr[2], 0xff}
	c1 := color.RGBA{paletteStr[3], paletteStr[4], paletteStr[5], 0xff}
	cz := color.RGBA{}
	wants := []color.Palette{
		{cz, c1},
		{c0, cz},
		{c0, c1},
	}
	if len(g.Image) != len(wants) {
		t.Fatalf("got %d images, want %d", len(g.Image), len(wants))
	}
	for i, want := range wants {
		got := g.Image[i].Palette
		if !reflect.DeepEqual(got, want) {
			t.Errorf("palette #%d:\ngot  %v\nwant %v", i, got, want)
		}
	}
}

// testGIF is a simple GIF that we can modify to test different scenarios.
var testGIF = []byte{
	'G', 'I', 'F', '8', '9', 'a',
	1, 0, 1, 0, // w=1, h=1 (6)
	128, 0, 0, // headerFields, bg, aspect (10)
	0, 0, 0, 1, 1, 1, // color table and graphics control (13)
	0x21, 0xf9, 0x04, 0x00, 0x00, 0x00, 0xff, 0x00, // (19)
	// frame 1 (0,0 - 1,1)
	0x2c,
	0x00, 0x00, 0x00, 0x00,
	0x01, 0x00, 0x01, 0x00, // (32)
	0x00,
	0x02, 0x02, 0x4c, 0x01, 0x00, // lzw pixels
	// trailer
	0x3b,
}

func try(t *testing.T, b []byte, want string) {
	_, err := DecodeAll(bytes.NewReader(b))
	var got string
	if err != nil {
		got = err.Error()
	}
	if got != want {
		t.Fatalf("got %v, want %v", got, want)
	}
}

func TestBounds(t *testing.T) {
	// Make a local copy of testGIF.
	gif := make([]byte, len(testGIF))
	copy(gif, testGIF)
	// Make the bounds too big, just by one.
	gif[32] = 2
	want := "gif: frame bounds larger than image bounds"
	try(t, gif, want)

	// Make the bounds too small; does not trigger bounds
	// check, but now there's too much data.
	gif[32] = 0
	want = "gif: too much image data"
	try(t, gif, want)
	gif[32] = 1

	// Make the bounds really big, expect an error.
	want = "gif: frame bounds larger than image bounds"
	for i := 0; i < 4; i++ {
		gif[32+i] = 0xff
	}
	try(t, gif, want)
}

func TestNoPalette(t *testing.T) {
	// https://go-review.googlesource.com/#/c/11227/
	// changed the lzw encoder to reject input bytes that are too large,
	// so that this test code no longer generates the right invalid GIF.
	// TODO(nigeltao): re-enable this test somehow.
	return

	b := &bytes.Buffer{}

	// Manufacture a GIF with no palette, so any pixel at all
	// will be invalid.
	b.WriteString(headerStr[:len(headerStr)-3])
	b.WriteString("\x00\x00\x00") // No global palette.

	// Image descriptor: 2x1, no local palette.
	b.WriteString("\x2c\x00\x00\x00\x00\x02\x00\x01\x00\x00\x02")

	// Encode the pixels: neither is in range, because there is no palette.
	pix := []byte{0, 128}
	enc := &bytes.Buffer{}
	w := lzw.NewWriter(enc, lzw.LSB, 2)
	w.Write(pix)
	w.Close()
	b.WriteByte(byte(len(enc.Bytes())))
	b.Write(enc.Bytes())
	b.WriteByte(0x00) // An empty block signifies the end of the image data.

	b.WriteString(trailerStr)

	try(t, b.Bytes(), "gif: no color table")
}

func TestPixelOutsidePaletteRange(t *testing.T) {
	// https://go-review.googlesource.com/#/c/11227/
	// changed the lzw encoder to reject input bytes that are too large,
	// so that this test code no longer generates the right invalid GIF.
	// TODO(nigeltao): re-enable this test somehow.
	return

	for _, pval := range []byte{0, 1, 2, 3, 255} {
		b := &bytes.Buffer{}

		// Manufacture a GIF with a 2 color palette.
		b.WriteString(headerStr)
		b.WriteString(paletteStr)

		// Image descriptor: 2x1, no local palette.
		b.WriteString("\x2c\x00\x00\x00\x00\x02\x00\x01\x00\x00\x02")

		// Encode the pixels; some pvals trigger the expected error.
		pix := []byte{pval, pval}
		enc := &bytes.Buffer{}
		w := lzw.NewWriter(enc, lzw.LSB, 2)
		w.Write(pix)
		w.Close()
		b.WriteByte(byte(len(enc.Bytes())))
		b.Write(enc.Bytes())
		b.WriteByte(0x00) // An empty block signifies the end of the image data.

		b.WriteString(trailerStr)

		// No error expected, unless the pixels are beyond the 2 color palette.
		want := ""
		if pval >= 2 {
			want = "gif: invalid pixel value"
		}
		try(t, b.Bytes(), want)
	}
}
<commit_msg>image/gif: re-enable some invalid-palette tests.<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gif

import (
	"bytes"
	"compress/lzw"
	"image"
	"image/color"
	"reflect"
	"testing"
)

// header, palette and trailer are parts of a valid 2x1 GIF image.
const (
	headerStr = "GIF89a" +
		"\x02\x00\x01\x00" + // width=2, height=1
		"\x80\x00\x00" // headerFields=(a color table of 2 pixels), backgroundIndex, aspect
	paletteStr = "\x10\x20\x30\x40\x50\x60" // the color table, also known as a palette
	trailerStr = "\x3b"
)

// lzwEncode returns an LZW encoding (with 2-bit literals) of n zeroes.
func lzwEncode(n int) []byte {
	b := &bytes.Buffer{}
	w := lzw.NewWriter(b, lzw.LSB, 2)
	w.Write(make([]byte, n))
	w.Close()
	return b.Bytes()
}

func TestDecode(t *testing.T) {
	testCases := []struct {
		nPix    int  // The number of pixels in the image data.
		extra   bool // Whether to write an extra block after the LZW-encoded data.
		wantErr error
	}{
		{0, false, errNotEnough},
		{1, false, errNotEnough},
		{2, false, nil},
		{2, true, errTooMuch},
		{3, false, errTooMuch},
	}
	for _, tc := range testCases {
		b := &bytes.Buffer{}
		b.WriteString(headerStr)
		b.WriteString(paletteStr)
		// Write an image with bounds 2x1 but tc.nPix pixels. If tc.nPix != 2
		// then this should result in an invalid GIF image. First, write a
		// magic 0x2c (image descriptor) byte, bounds=(0,0)-(2,1), a flags
		// byte, and 2-bit LZW literals.
		b.WriteString("\x2c\x00\x00\x00\x00\x02\x00\x01\x00\x00\x02")
		if tc.nPix > 0 {
			enc := lzwEncode(tc.nPix)
			if len(enc) > 0xff {
				t.Errorf("nPix=%d, extra=%t: compressed length %d is too large", tc.nPix, tc.extra, len(enc))
				continue
			}
			b.WriteByte(byte(len(enc)))
			b.Write(enc)
		}
		if tc.extra {
			b.WriteString("\x01\x02") // A 1-byte payload with an 0x02 byte.
		}
		b.WriteByte(0x00) // An empty block signifies the end of the image data.
		b.WriteString(trailerStr)

		got, err := Decode(b)
		if err != tc.wantErr {
			t.Errorf("nPix=%d, extra=%t\ngot  %v\nwant %v", tc.nPix, tc.extra, err, tc.wantErr)
		}

		if tc.wantErr != nil {
			continue
		}
		want := &image.Paletted{
			Pix:    []uint8{0, 0},
			Stride: 2,
			Rect:   image.Rect(0, 0, 2, 1),
			Palette: color.Palette{
				color.RGBA{0x10, 0x20, 0x30, 0xff},
				color.RGBA{0x40, 0x50, 0x60, 0xff},
			},
		}
		if !reflect.DeepEqual(got, want) {
			t.Errorf("nPix=%d, extra=%t\ngot  %v\nwant %v", tc.nPix, tc.extra, got, want)
		}
	}
}

func TestTransparentIndex(t *testing.T) {
	b := &bytes.Buffer{}
	b.WriteString(headerStr)
	b.WriteString(paletteStr)
	for transparentIndex := 0; transparentIndex < 3; transparentIndex++ {
		if transparentIndex < 2 {
			// Write the graphic control for the transparent index.
			b.WriteString("\x21\xf9\x00\x01\x00\x00")
			b.WriteByte(byte(transparentIndex))
			b.WriteByte(0)
		}
		// Write an image with bounds 2x1, as per TestDecode.
		b.WriteString("\x2c\x00\x00\x00\x00\x02\x00\x01\x00\x00\x02")
		enc := lzwEncode(2)
		if len(enc) > 0xff {
			t.Fatalf("compressed length %d is too large", len(enc))
		}
		b.WriteByte(byte(len(enc)))
		b.Write(enc)
		b.WriteByte(0x00)
	}
	b.WriteString(trailerStr)

	g, err := DecodeAll(b)
	if err != nil {
		t.Fatalf("DecodeAll: %v", err)
	}
	c0 := color.RGBA{paletteStr[0], paletteStr[1], paletteStr[2], 0xff}
	c1 := color.RGBA{paletteStr[3], paletteStr[4], paletteStr[5], 0xff}
	cz := color.RGBA{}
	wants := []color.Palette{
		{cz, c1},
		{c0, cz},
		{c0, c1},
	}
	if len(g.Image) != len(wants) {
		t.Fatalf("got %d images, want %d", len(g.Image), len(wants))
	}
	for i, want := range wants {
		got := g.Image[i].Palette
		if !reflect.DeepEqual(got, want) {
			t.Errorf("palette #%d:\ngot  %v\nwant %v", i, got, want)
		}
	}
}

// testGIF is a simple GIF that we can modify to test different scenarios.
var testGIF = []byte{
	'G', 'I', 'F', '8', '9', 'a',
	1, 0, 1, 0, // w=1, h=1 (6)
	128, 0, 0, // headerFields, bg, aspect (10)
	0, 0, 0, 1, 1, 1, // color table and graphics control (13)
	0x21, 0xf9, 0x04, 0x00, 0x00, 0x00, 0xff, 0x00, // (19)
	// frame 1 (0,0 - 1,1)
	0x2c,
	0x00, 0x00, 0x00, 0x00,
	0x01, 0x00, 0x01, 0x00, // (32)
	0x00,
	0x02, 0x02, 0x4c, 0x01, 0x00, // lzw pixels
	// trailer
	0x3b,
}

func try(t *testing.T, b []byte, want string) {
	_, err := DecodeAll(bytes.NewReader(b))
	var got string
	if err != nil {
		got = err.Error()
	}
	if got != want {
		t.Fatalf("got %v, want %v", got, want)
	}
}

func TestBounds(t *testing.T) {
	// Make a local copy of testGIF.
	gif := make([]byte, len(testGIF))
	copy(gif, testGIF)
	// Make the bounds too big, just by one.
	gif[32] = 2
	want := "gif: frame bounds larger than image bounds"
	try(t, gif, want)

	// Make the bounds too small; does not trigger bounds
	// check, but now there's too much data.
	gif[32] = 0
	want = "gif: too much image data"
	try(t, gif, want)
	gif[32] = 1

	// Make the bounds really big, expect an error.
	want = "gif: frame bounds larger than image bounds"
	for i := 0; i < 4; i++ {
		gif[32+i] = 0xff
	}
	try(t, gif, want)
}

func TestNoPalette(t *testing.T) {
	b := &bytes.Buffer{}

	// Manufacture a GIF with no palette, so any pixel at all
	// will be invalid.
	b.WriteString(headerStr[:len(headerStr)-3])
	b.WriteString("\x00\x00\x00") // No global palette.

	// Image descriptor: 2x1, no local palette.
	b.WriteString("\x2c\x00\x00\x00\x00\x02\x00\x01\x00\x00\x02")

	// Encode the pixels: neither is in range, because there is no palette.
	pix := []byte{0, 3}
	enc := &bytes.Buffer{}
	w := lzw.NewWriter(enc, lzw.LSB, 2)
	if _, err := w.Write(pix); err != nil {
		t.Fatalf("Write: %v", err)
	}
	if err := w.Close(); err != nil {
		t.Fatalf("Close: %v", err)
	}
	b.WriteByte(byte(len(enc.Bytes())))
	b.Write(enc.Bytes())
	b.WriteByte(0x00) // An empty block signifies the end of the image data.

	b.WriteString(trailerStr)

	try(t, b.Bytes(), "gif: no color table")
}

func TestPixelOutsidePaletteRange(t *testing.T) {
	for _, pval := range []byte{0, 1, 2, 3} {
		b := &bytes.Buffer{}

		// Manufacture a GIF with a 2 color palette.
		b.WriteString(headerStr)
		b.WriteString(paletteStr)

		// Image descriptor: 2x1, no local palette.
		b.WriteString("\x2c\x00\x00\x00\x00\x02\x00\x01\x00\x00\x02")

		// Encode the pixels; some pvals trigger the expected error.
		pix := []byte{pval, pval}
		enc := &bytes.Buffer{}
		w := lzw.NewWriter(enc, lzw.LSB, 2)
		if _, err := w.Write(pix); err != nil {
			t.Fatalf("Write: %v", err)
		}
		if err := w.Close(); err != nil {
			t.Fatalf("Close: %v", err)
		}
		b.WriteByte(byte(len(enc.Bytes())))
		b.Write(enc.Bytes())
		b.WriteByte(0x00) // An empty block signifies the end of the image data.

		b.WriteString(trailerStr)

		// No error expected, unless the pixels are beyond the 2 color palette.
		want := ""
		if pval >= 2 {
			want = "gif: invalid pixel value"
		}
		try(t, b.Bytes(), want)
	}
}
<|endoftext|>


<commit_before>package ast

import (
	"fmt"
	"github.com/gtfierro/hod/lang/token"
	"github.com/gtfierro/hod/turtle"
	"github.com/kr/pretty"
	"strings"
)

var debug = false

func SetDebug() {
	debug = true
}
func ClearDebug() {
	debug = false
}

type Query struct {
	Select    SelectClause
	From      FromClause
	Count     bool
	Where     WhereClause
	Variables []string
}

func (q Query) CopyWithNewTerms(terms []Triple) Query {
	newq := Query{
		Select:    q.Select,
		From:      q.From,
		Variables: q.Variables,
	}
	newq.Where.Terms = make([]Triple, len(terms))
	copy(newq.Where.Terms, terms)
	return newq
}

func (q Query) Copy() *Query {
	return &Query{
		Select:    q.Select,
		From:      q.From,
		Variables: q.Variables,
		Where:     q.Where,
		Count:     q.Count,
	}
}

func NewQuery(selectclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		Select: selectclause.(SelectClause),
		Count:  count,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	return q, nil
}

func NewQueryMulti(selectclause, fromclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		From:   fromclause.(FromClause),
		Select: selectclause.(SelectClause),
		Count:  count,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	return q, nil
}

func (q *Query) PopulateVars() {
	vars := make(map[string]int)
	// get all variables
	for _, triple := range q.Where.Terms {
		AddIfVar(triple.Subject, vars)
		AddIfVar(triple.Object, vars)
		for _, path := range triple.Predicates {
			AddIfVar(path.Predicate, vars)
		}
	}
	if q.Where.GraphGroup != nil {
		VarsFromGroup(*q.Where.GraphGroup, vars)
	}
	q.Variables = []string{} // clear
	for varname := range vars {
		q.Variables = append(q.Variables, varname)
	}
}

func (q Query) IterTriples(f func(t Triple) Triple) {
	for idx, triple := range q.Where.Terms {
		q.Where.Terms[idx] = f(triple)
	}
	if q.Where.GraphGroup != nil {
		q.Where.GraphGroup.IterTriples(f)
	}
}

func AddIfVar(uri turtle.URI, m map[string]int) {
	if uri.IsVariable() {
		m[uri.String()] = 1
	}
}

func VarsFromGroup(group GraphGroup, m map[string]int) {
	for _, triple := range group.Terms {
		AddIfVar(triple.Subject, m)
		AddIfVar(triple.Object, m)
		for _, path := range triple.Predicates {
			AddIfVar(path.Predicate, m)
		}
	}
	for _, union := range group.Unions {
		VarsFromGroup(union, m)
	}
}

func (grp GraphGroup) Expand() [][]Triple {
	var terms = make([]Triple, len(grp.Terms))
	copy(terms, grp.Terms)
	var groups [][]Triple

	if len(grp.Unions) > 0 {
		for _, union := range grp.Unions {
			for _, subgroup := range union.Expand() {
				groups = append(groups, append(terms, subgroup...))
			}
		}
	} else {
		groups = append(groups, terms)
	}
	return groups
}

func (grp GraphGroup) Iter(f func(t turtle.URI)) {
	for _, triple := range grp.Terms {
		f(triple.Subject)
		f(triple.Object)
		for _, path := range triple.Predicates {
			f(path.Predicate)
		}
	}
	for _, union := range grp.Unions {
		union.Iter(f)
	}
}

func (grp *GraphGroup) IterTriples(f func(t Triple) Triple) {
	for idx, triple := range grp.Terms {
		grp.Terms[idx] = f(triple)
	}
	for _, union := range grp.Unions {
		union.IterTriples(f)
	}
}

type SelectClause struct {
	Vars    []string
	AllVars bool
}

func NewAllSelectClause() (SelectClause, error) {
	return SelectClause{AllVars: true}, nil
}

func NewSelectClause(varlist interface{}) (SelectClause, error) {
	return SelectClause{Vars: varlist.([]string)}, nil
}

type FromClause struct {
	Databases []string
	AllDBs    bool
}

func (f FromClause) String() string {
	if f.AllDBs {
		return "*"
	}
	return strings.Join(f.Databases, " ")
}

func NewAllFromClause() (FromClause, error) {
	return FromClause{AllDBs: true}, nil
}

func NewFromClause(dblist interface{}) (FromClause, error) {
	return FromClause{Databases: dblist.([]string)}, nil
}

func (from FromClause) Empty() bool {
	return len(from.Databases) == 0 && !from.AllDBs
}

type WhereClause struct {
	Terms      []Triple
	GraphGroup *GraphGroup
}

func NewWhereClause(triples interface{}) (WhereClause, error) {
	return WhereClause{
		Terms: triples.([]Triple),
	}, nil
}

func NewWhereClauseWithGraphGroup(triples, group interface{}) (WhereClause, error) {
	g := group.(GraphGroup)
	return WhereClause{
		Terms:      triples.([]Triple),
		GraphGroup: &g,
	}, nil
}

func NewWhereClauseGraphGroup(group interface{}) (WhereClause, error) {
	g := group.(GraphGroup)
	return WhereClause{
		GraphGroup: &g,
	}, nil
}

type GraphGroup struct {
	Terms  []Triple
	Unions []GraphGroup
}

func GraphGroupFromTriples(triples interface{}) (GraphGroup, error) {
	return GraphGroup{
		Terms: triples.([]Triple),
	}, nil
}

func GraphGroupUnion(left, right interface{}) (GraphGroup, error) {
	return GraphGroup{
		Unions: []GraphGroup{left.(GraphGroup), right.(GraphGroup)},
	}, nil
}

func AddTriplesToGraphGroup(left, triples interface{}) (GraphGroup, error) {
	return GraphGroup{
		Terms:  append(left.(GraphGroup).Terms, triples.([]Triple)...),
		Unions: left.(GraphGroup).Unions,
	}, nil
}

func MergeGraphGroups(left, right interface{}) (GraphGroup, error) {
	return GraphGroup{
		Terms:  append(left.(GraphGroup).Terms, right.(GraphGroup).Terms...),
		Unions: append(left.(GraphGroup).Unions, right.(GraphGroup).Unions...),
	}, nil
}

type Triple struct {
	Subject    turtle.URI
	Predicates []PathPattern
	Object     turtle.URI
}

func (t Triple) String() string {
	s := "<" + t.Subject.String() + "|"
	for _, pp := range t.Predicates {
		s += " " + pp.String()
	}
	return s + " | " + t.Object.String() + ">"
}

func NewTriple(subject, predicates, object interface{}) (Triple, error) {
	return Triple{
		Subject:    subject.(turtle.URI),
		Predicates: predicates.([]PathPattern),
		Object:     object.(turtle.URI),
	}, nil
}

func NewTripleBlock(triple interface{}) ([]Triple, error) {
	return []Triple{triple.(Triple)}, nil
}

func AppendTripleBlock(block, triple interface{}) ([]Triple, error) {
	return append(block.([]Triple), triple.(Triple)), nil
}

func NewURI(value interface{}) (turtle.URI, error) {
	return turtle.ParseURI(value.(string)), nil
}

func NewVarList(_var interface{}) ([]string, error) {
	return []string{_var.(string)}, nil
}

func AppendVar(varlist, _var interface{}) ([]string, error) {
	return append(varlist.([]string), _var.(string)), nil
}

func NewStringList(_str interface{}) ([]string, error) {
	return []string{_str.(string)}, nil
}

func AppendString(strlist, _str interface{}) ([]string, error) {
	return append(strlist.([]string), _str.(string)), nil
}

func ParseString(_var interface{}) (string, error) {
	return string(_var.(*token.Token).Lit), nil
}

func ParseQuotedString(_var interface{}) (string, error) {
	return string(_var.(*token.Token).Lit), nil
}

func NewPathSequence(_pred interface{}) ([]PathPattern, error) {
	return []PathPattern{_pred.(PathPattern)}, nil
}

func AppendPathSequence(_seq, _pred interface{}) ([]PathPattern, error) {
	return append(_seq.([]PathPattern), _pred.(PathPattern)), nil
}

func NewPathPattern(_pred interface{}) (PathPattern, error) {
	pred, _ := ParseString(_pred)
	return PathPattern{
		Predicate: turtle.ParseURI(pred),
		Pattern:   PATTERN_SINGLE,
	}, nil
}

func AddPathMod(_pred, _mod interface{}) (PathPattern, error) {
	return PathPattern{
		Predicate: _pred.(PathPattern).Predicate,
		Pattern:   _mod.(Pattern),
	}, nil
}

type PathPattern struct {
	Predicate turtle.URI
	Pattern   Pattern
}

func PathFromVar(_var interface{}) ([]PathPattern, error) {
	return []PathPattern{
		PathPattern{
			Predicate: turtle.ParseURI(_var.(string)),
			Pattern:   PATTERN_SINGLE,
		},
	}, nil
}

func (pp PathPattern) String() string {
	return pp.Predicate.String() + pp.Pattern.String()
}

type Pattern uint

const (
	PATTERN_SINGLE = iota + 1
	PATTERN_ZERO_ONE
	PATTERN_ONE_PLUS
	PATTERN_ZERO_PLUS
)

func (p Pattern) String() string {
	switch p {
	case PATTERN_SINGLE:
		return ""
	case PATTERN_ZERO_ONE:
		return "?"
	case PATTERN_ONE_PLUS:
		return "+"
	case PATTERN_ZERO_PLUS:
		return "*"
	}
	return "unknown"
}
<commit_msg>support a for rdf:type<commit_after>package ast

import (
	"fmt"
	"github.com/gtfierro/hod/lang/token"
	"github.com/gtfierro/hod/turtle"
	"github.com/kr/pretty"
	"strings"
)

var debug = false

func SetDebug() {
	debug = true
}
func ClearDebug() {
	debug = false
}

type Query struct {
	Select    SelectClause
	From      FromClause
	Count     bool
	Where     WhereClause
	Variables []string
}

func (q Query) CopyWithNewTerms(terms []Triple) Query {
	newq := Query{
		Select:    q.Select,
		From:      q.From,
		Variables: q.Variables,
	}
	newq.Where.Terms = make([]Triple, len(terms))
	copy(newq.Where.Terms, terms)
	return newq
}

func (q Query) Copy() *Query {
	return &Query{
		Select:    q.Select,
		From:      q.From,
		Variables: q.Variables,
		Where:     q.Where,
		Count:     q.Count,
	}
}

func NewQuery(selectclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		Select: selectclause.(SelectClause),
		Count:  count,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	return q, nil
}

func NewQueryMulti(selectclause, fromclause, whereclause interface{}, count bool) (Query, error) {
	if debug {
		fmt.Printf("%# v", pretty.Formatter(whereclause.(WhereClause)))
	}
	q := Query{
		Where:  whereclause.(WhereClause),
		From:   fromclause.(FromClause),
		Select: selectclause.(SelectClause),
		Count:  count,
	}
	if q.From.Empty() {
		q.From.AllDBs = true
	}
	return q, nil
}

func (q *Query) PopulateVars() {
	vars := make(map[string]int)
	// get all variables
	for _, triple := range q.Where.Terms {
		AddIfVar(triple.Subject, vars)
		AddIfVar(triple.Object, vars)
		for _, path := range triple.Predicates {
			AddIfVar(path.Predicate, vars)
		}
	}
	if q.Where.GraphGroup != nil {
		VarsFromGroup(*q.Where.GraphGroup, vars)
	}
	q.Variables = []string{} // clear
	for varname := range vars {
		q.Variables = append(q.Variables, varname)
	}
}

func (q Query) IterTriples(f func(t Triple) Triple) {
	for idx, triple := range q.Where.Terms {
		q.Where.Terms[idx] = f(triple)
	}
	if q.Where.GraphGroup != nil {
		q.Where.GraphGroup.IterTriples(f)
	}
}

func AddIfVar(uri turtle.URI, m map[string]int) {
	if uri.IsVariable() {
		m[uri.String()] = 1
	}
}

func VarsFromGroup(group GraphGroup, m map[string]int) {
	for _, triple := range group.Terms {
		AddIfVar(triple.Subject, m)
		AddIfVar(triple.Object, m)
		for _, path := range triple.Predicates {
			AddIfVar(path.Predicate, m)
		}
	}
	for _, union := range group.Unions {
		VarsFromGroup(union, m)
	}
}

func (grp GraphGroup) Expand() [][]Triple {
	var terms = make([]Triple, len(grp.Terms))
	copy(terms, grp.Terms)
	var groups [][]Triple

	if len(grp.Unions) > 0 {
		for _, union := range grp.Unions {
			for _, subgroup := range union.Expand() {
				groups = append(groups, append(terms, subgroup...))
			}
		}
	} else {
		groups = append(groups, terms)
	}
	return groups
}

func (grp GraphGroup) Iter(f func(t turtle.URI)) {
	for _, triple := range grp.Terms {
		f(triple.Subject)
		f(triple.Object)
		for _, path := range triple.Predicates {
			f(path.Predicate)
		}
	}
	for _, union := range grp.Unions {
		union.Iter(f)
	}
}

func (grp *GraphGroup) IterTriples(f func(t Triple) Triple) {
	for idx, triple := range grp.Terms {
		grp.Terms[idx] = f(triple)
	}
	for _, union := range grp.Unions {
		union.IterTriples(f)
	}
}

type SelectClause struct {
	Vars    []string
	AllVars bool
}

func NewAllSelectClause() (SelectClause, error) {
	return SelectClause{AllVars: true}, nil
}

func NewSelectClause(varlist interface{}) (SelectClause, error) {
	return SelectClause{Vars: varlist.([]string)}, nil
}

type FromClause struct {
	Databases []string
	AllDBs    bool
}

func (f FromClause) String() string {
	if f.AllDBs {
		return "*"
	}
	return strings.Join(f.Databases, " ")
}

func NewAllFromClause() (FromClause, error) {
	return FromClause{AllDBs: true}, nil
}

func NewFromClause(dblist interface{}) (FromClause, error) {
	return FromClause{Databases: dblist.([]string)}, nil
}

func (from FromClause) Empty() bool {
	return len(from.Databases) == 0 && !from.AllDBs
}

type WhereClause struct {
	Terms      []Triple
	GraphGroup *GraphGroup
}

func NewWhereClause(triples interface{}) (WhereClause, error) {
	return WhereClause{
		Terms: triples.([]Triple),
	}, nil
}

func NewWhereClauseWithGraphGroup(triples, group interface{}) (WhereClause, error) {
	g := group.(GraphGroup)
	return WhereClause{
		Terms:      triples.([]Triple),
		GraphGroup: &g,
	}, nil
}

func NewWhereClauseGraphGroup(group interface{}) (WhereClause, error) {
	g := group.(GraphGroup)
	return WhereClause{
		GraphGroup: &g,
	}, nil
}

type GraphGroup struct {
	Terms  []Triple
	Unions []GraphGroup
}

func GraphGroupFromTriples(triples interface{}) (GraphGroup, error) {
	return GraphGroup{
		Terms: triples.([]Triple),
	}, nil
}

func GraphGroupUnion(left, right interface{}) (GraphGroup, error) {
	return GraphGroup{
		Unions: []GraphGroup{left.(GraphGroup), right.(GraphGroup)},
	}, nil
}

func AddTriplesToGraphGroup(left, triples interface{}) (GraphGroup, error) {
	return GraphGroup{
		Terms:  append(left.(GraphGroup).Terms, triples.([]Triple)...),
		Unions: left.(GraphGroup).Unions,
	}, nil
}

func MergeGraphGroups(left, right interface{}) (GraphGroup, error) {
	return GraphGroup{
		Terms:  append(left.(GraphGroup).Terms, right.(GraphGroup).Terms...),
		Unions: append(left.(GraphGroup).Unions, right.(GraphGroup).Unions...),
	}, nil
}

type Triple struct {
	Subject    turtle.URI
	Predicates []PathPattern
	Object     turtle.URI
}

func (t Triple) String() string {
	s := "<" + t.Subject.String() + "|"
	for _, pp := range t.Predicates {
		s += " " + pp.String()
	}
	return s + " | " + t.Object.String() + ">"
}

func NewTriple(subject, predicates, object interface{}) (Triple, error) {
	return Triple{
		Subject:    subject.(turtle.URI),
		Predicates: predicates.([]PathPattern),
		Object:     object.(turtle.URI),
	}, nil
}

func NewTripleBlock(triple interface{}) ([]Triple, error) {
	return []Triple{triple.(Triple)}, nil
}

func AppendTripleBlock(block, triple interface{}) ([]Triple, error) {
	return append(block.([]Triple), triple.(Triple)), nil
}

func NewURI(value interface{}) (turtle.URI, error) {
	return turtle.ParseURI(value.(string)), nil
}

func NewVarList(_var interface{}) ([]string, error) {
	return []string{_var.(string)}, nil
}

func AppendVar(varlist, _var interface{}) ([]string, error) {
	return append(varlist.([]string), _var.(string)), nil
}

func NewStringList(_str interface{}) ([]string, error) {
	return []string{_str.(string)}, nil
}

func AppendString(strlist, _str interface{}) ([]string, error) {
	return append(strlist.([]string), _str.(string)), nil
}

func ParseString(_var interface{}) (string, error) {
	return string(_var.(*token.Token).Lit), nil
}

func ParseQuotedString(_var interface{}) (string, error) {
	return string(_var.(*token.Token).Lit), nil
}

func NewPathSequence(_pred interface{}) ([]PathPattern, error) {
	return []PathPattern{_pred.(PathPattern)}, nil
}

func AppendPathSequence(_seq, _pred interface{}) ([]PathPattern, error) {
	return append(_seq.([]PathPattern), _pred.(PathPattern)), nil
}

func NewPathPattern(_pred interface{}) (PathPattern, error) {
	pred, _ := ParseString(_pred)
	if pred == "a" {
		pred = "rdf:type"
	}
	return PathPattern{
		Predicate: turtle.ParseURI(pred),
		Pattern:   PATTERN_SINGLE,
	}, nil
}

func AddPathMod(_pred, _mod interface{}) (PathPattern, error) {
	return PathPattern{
		Predicate: _pred.(PathPattern).Predicate,
		Pattern:   _mod.(Pattern),
	}, nil
}

type PathPattern struct {
	Predicate turtle.URI
	Pattern   Pattern
}

func PathFromVar(_var interface{}) ([]PathPattern, error) {
	return []PathPattern{
		PathPattern{
			Predicate: turtle.ParseURI(_var.(string)),
			Pattern:   PATTERN_SINGLE,
		},
	}, nil
}

func (pp PathPattern) String() string {
	return pp.Predicate.String() + pp.Pattern.String()
}

type Pattern uint

const (
	PATTERN_SINGLE = iota + 1
	PATTERN_ZERO_ONE
	PATTERN_ONE_PLUS
	PATTERN_ZERO_PLUS
)

func (p Pattern) String() string {
	switch p {
	case PATTERN_SINGLE:
		return ""
	case PATTERN_ZERO_ONE:
		return "?"
	case PATTERN_ONE_PLUS:
		return "+"
	case PATTERN_ZERO_PLUS:
		return "*"
	}
	return "unknown"
}
<|endoftext|>