<commit_before>package experiment_test

import (
	"context"
	"errors"
	"fmt"
	"sync"
	"testing"
	"time"

	"github.com/jelmersnoeck/experiment"
)

func TestRun_Sequential(t *testing.T) {
	exp, pub := testExperiment()

	t.Run("it should record a success", func(t *testing.T) {
		pub.fnc = func(o experiment.Observation) {
			if o.Name == "correct" && !o.Success {
				t.Errorf("Expected a success, got mismatch")
			}
		}

		exp.Run()
	})

	t.Run("it should record a mismatch", func(t *testing.T) {
		pub.fnc = func(o experiment.Observation) {
			if o.Name == "mismatch" && o.Success {
				t.Errorf("Expected a mismatch, got success")
			}
		}

		exp.Run()
	})

	t.Run("it should record an error", func(t *testing.T) {
		pub.fnc = func(o experiment.Observation) {
			if o.Name == "error" && o.Error == nil {
				t.Errorf("Expected an error, got none")
			}
		}

		exp.Run()
	})

	t.Run("it should record the panic", func(t *testing.T) {
		pub.fnc = func(o experiment.Observation) {
			if o.Name == "panic" && o.Panic == nil {
				t.Errorf("Expected a panic, did not record one")
			}
		}

		exp.Run()
	})

	t.Run("it should record the clean", func(t *testing.T) {
		pub.fnc = func(o experiment.Observation) {
			if o.Panic == nil && o.Error == nil && o.Success {
				cleaned := fmt.Sprintf("Cleaned %s", o.Value.(string))
				if o.CleanValue.(string) != cleaned {
					t.Errorf("Expected value to be '%s', got '%s'", cleaned, o.CleanValue.(string))
				}
			}
		}

		exp.Run()
	})

	t.Run("it should return the correct value", func(t *testing.T) {
		val, err := exp.Run()
		if val.(string) != "control" {
			t.Errorf("Expected value to be 'control', got '%s'", val.(string))
		}

		if err != nil {
			t.Errorf("Expected error to be 'nil', got '%s'", err.Error())
		}
	})
}

func TestRun_Before(t *testing.T) {
	exp := experiment.New()
	exp.Force(true)

	exp.Control(func() (interface{}, error) {
		return "", nil
	})

	t.Run("with an error", func(t *testing.T) {
		expected := errors.New("before error")
		exp.Before(func() error {
			return expected
		})

		_, err := exp.Run()
		if expected != err {
			t.Errorf("Expected error '%v', got '%v'", expected, err)
		}
	})

	t.Run("without an error", func(t *testing.T) {
		var ran bool
		exp.Before(func() error {
			ran = true
			return nil
		})

		exp.Run()
		if !ran {
			t.Errorf("Expected before to have run, did not")
		}
	})
}

func TestRun_Ignore(t *testing.T) {
	exp, pub := testExperiment(experiment.WithPercentage(100))
	exp.Ignore(true)

	var count int
	var lock sync.Mutex
	pub.fnc = func(_ experiment.Observation) {
		lock.Lock()
		defer lock.Unlock()
		count++
	}

	exp.Run()

	if count != 0 {
		t.Errorf("Expected '0' observations, got '%d'", count)
	}
}

func TestRun_Concurrent(t *testing.T) {
	ctx, _ := context.WithTimeout(context.Background(), time.Second)
	exp, pub := testExperiment(experiment.WithConcurrency())
	expected := 5
	eChan := make(chan bool)

	var count int
	var lock sync.Mutex
	pub.fnc = func(_ experiment.Observation) {
		lock.Lock()
		defer lock.Unlock()
		count++
		eChan <- true
	}

	exp.Run()

	for i := 0; i < expected; i++ {
		select {
		case <-eChan:
		case <-ctx.Done():
			t.Errorf("Expected to run the experiment within one second")
			break
		}
	}

	if count != expected {
		t.Errorf("Expected '%d' observations, got '%d'", expected, count)
	}
}

func testExperiment(cfg ...experiment.ConfigFunc) (*experiment.Experiment, *testPublisher) {
	pub := &testPublisher{}

	config := []experiment.ConfigFunc{
		experiment.WithPublisher(pub),
	}
	config = append(config, cfg...)

	exp := experiment.New(config...)
	exp.Force(true)

	exp.Control(func() (interface{}, error) {
		return "control", nil
	})

	exp.Candidate("correct", func() (interface{}, error) {
		return "control", nil
	})

	exp.Candidate("mismatch", func() (interface{}, error) {
		return "mismatch", nil
	})

	exp.Candidate("error", func() (interface{}, error) {
		return nil, errors.New("errored")
	})

	exp.Candidate("panic", func() (interface{}, error) {
		panic("candidate")
	})

	exp.Compare(func(control interface{}, candidate interface{}) bool {
		return control.(string) == candidate.(string)
	})

	exp.Clean(func(c interface{}) interface{} {
		return fmt.Sprintf("Cleaned %s", c.(string))
	})

	return exp, pub

}

type testPublisher struct {
	fnc func(experiment.Observation)
}

func (t *testPublisher) Publish(o experiment.Observation) {
	if t.fnc != nil {
		t.fnc(o)
	}
}
<commit_msg>Always call cancel.<commit_after>package experiment_test

import (
	"context"
	"errors"
	"fmt"
	"sync"
	"testing"
	"time"

	"github.com/jelmersnoeck/experiment"
)

func TestRun_Sequential(t *testing.T) {
	exp, pub := testExperiment()

	t.Run("it should record a success", func(t *testing.T) {
		pub.fnc = func(o experiment.Observation) {
			if o.Name == "correct" && !o.Success {
				t.Errorf("Expected a success, got mismatch")
			}
		}

		exp.Run()
	})

	t.Run("it should record a mismatch", func(t *testing.T) {
		pub.fnc = func(o experiment.Observation) {
			if o.Name == "mismatch" && o.Success {
				t.Errorf("Expected a mismatch, got success")
			}
		}

		exp.Run()
	})

	t.Run("it should record an error", func(t *testing.T) {
		pub.fnc = func(o experiment.Observation) {
			if o.Name == "error" && o.Error == nil {
				t.Errorf("Expected an error, got none")
			}
		}

		exp.Run()
	})

	t.Run("it should record the panic", func(t *testing.T) {
		pub.fnc = func(o experiment.Observation) {
			if o.Name == "panic" && o.Panic == nil {
				t.Errorf("Expected a panic, did not record one")
			}
		}

		exp.Run()
	})

	t.Run("it should record the clean", func(t *testing.T) {
		pub.fnc = func(o experiment.Observation) {
			if o.Panic == nil && o.Error == nil && o.Success {
				cleaned := fmt.Sprintf("Cleaned %s", o.Value.(string))
				if o.CleanValue.(string) != cleaned {
					t.Errorf("Expected value to be '%s', got '%s'", cleaned, o.CleanValue.(string))
				}
			}
		}

		exp.Run()
	})

	t.Run("it should return the correct value", func(t *testing.T) {
		val, err := exp.Run()
		if val.(string) != "control" {
			t.Errorf("Expected value to be 'control', got '%s'", val.(string))
		}

		if err != nil {
			t.Errorf("Expected error to be 'nil', got '%s'", err.Error())
		}
	})
}

func TestRun_Before(t *testing.T) {
	exp := experiment.New()
	exp.Force(true)

	exp.Control(func() (interface{}, error) {
		return "", nil
	})

	t.Run("with an error", func(t *testing.T) {
		expected := errors.New("before error")
		exp.Before(func() error {
			return expected
		})

		_, err := exp.Run()
		if expected != err {
			t.Errorf("Expected error '%v', got '%v'", expected, err)
		}
	})

	t.Run("without an error", func(t *testing.T) {
		var ran bool
		exp.Before(func() error {
			ran = true
			return nil
		})

		exp.Run()
		if !ran {
			t.Errorf("Expected before to have run, did not")
		}
	})
}

func TestRun_Ignore(t *testing.T) {
	exp, pub := testExperiment(experiment.WithPercentage(100))
	exp.Ignore(true)

	var count int
	var lock sync.Mutex
	pub.fnc = func(_ experiment.Observation) {
		lock.Lock()
		defer lock.Unlock()
		count++
	}

	exp.Run()

	if count != 0 {
		t.Errorf("Expected '0' observations, got '%d'", count)
	}
}

func TestRun_Concurrent(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
	defer cancel()
	exp, pub := testExperiment(experiment.WithConcurrency())
	expected := 5
	eChan := make(chan bool)

	var count int
	var lock sync.Mutex
	pub.fnc = func(_ experiment.Observation) {
		lock.Lock()
		defer lock.Unlock()
		count++
		eChan <- true
	}

	exp.Run()

	for i := 0; i < expected; i++ {
		select {
		case <-eChan:
		case <-ctx.Done():
			t.Errorf("Expected to run the experiment within one second")
			break
		}
	}

	if count != expected {
		t.Errorf("Expected '%d' observations, got '%d'", expected, count)
	}
}

func testExperiment(cfg ...experiment.ConfigFunc) (*experiment.Experiment, *testPublisher) {
	pub := &testPublisher{}

	config := []experiment.ConfigFunc{
		experiment.WithPublisher(pub),
	}
	config = append(config, cfg...)

	exp := experiment.New(config...)
	exp.Force(true)

	exp.Control(func() (interface{}, error) {
		return "control", nil
	})

	exp.Candidate("correct", func() (interface{}, error) {
		return "control", nil
	})

	exp.Candidate("mismatch", func() (interface{}, error) {
		return "mismatch", nil
	})

	exp.Candidate("error", func() (interface{}, error) {
		return nil, errors.New("errored")
	})

	exp.Candidate("panic", func() (interface{}, error) {
		panic("candidate")
	})

	exp.Compare(func(control interface{}, candidate interface{}) bool {
		return control.(string) == candidate.(string)
	})

	exp.Clean(func(c interface{}) interface{} {
		return fmt.Sprintf("Cleaned %s", c.(string))
	})

	return exp, pub

}

type testPublisher struct {
	fnc func(experiment.Observation)
}

func (t *testPublisher) Publish(o experiment.Observation) {
	if t.fnc != nil {
		t.fnc(o)
	}
}
<|endoftext|>


<commit_before>// Package filters provides a data-record filtering mechanism and basic implementations
// for typical use cases. It is intended as a complement to the formats sister package,
// useful for automating unique record extraction from a data file.
//
// The current filter set includes:
//    "require"        => RequireFilter
//    "excludes"       => ExcludeFilter
//    "null_fields"    => NullFilter
//    "split_fields"   => SplitFieldFilter
//    "date_formats"   => DateFormatFilter
//
// A loose naming convention of adding "s" on the end implies that the filter is applied
// independently for each field of the record. Thus the missing "s" on the RequireFilter
// means that all fields listed are required simultaneously.
package filters

import (
	"fmt"
	"strings"

	"github.com/pbnjay/strptime"
)

// Filter defines an interface that manipulates fields from one record into a new slice of
// records (most often 1-to-1). These manipulations can have optional parameters provided
// by Setup to control them.
type Filter interface {
	// Setup defines the part strings used to apply this filter to new records.
	Setup(parts map[interface{}]string) error
	// Apply takes an input record and applies the Filter to create 0 or more records.
	Apply(fields map[interface{}]string) []map[interface{}]string
}

var (
	// Blank strings in RequireFilter and ExcludeFilter must be represented by this placeholder.
	// If for some reason your input contains this text and you need a different representation,
	// it may be overridden in user code.
	BLANK_ENTRY = "<BLANK>"

	filters = make(map[string]Filter)
)

///

// NullFilter remaps fields from a placeholder string into an empty string. For example, many
// data sources use a placeholder of "-" or "n/a" to indicate a missing element. This feature
// may also be used to suppress particular values from records.
//
// Automatically registered as "null_fields"
type NullFilter struct {
	parts map[interface{}]string
}

func (f *NullFilter) Setup(parts map[interface{}]string) error {
	f.parts = parts
	return nil
}

func (f *NullFilter) Apply(fields map[interface{}]string) []map[interface{}]string {
	nnull := len(fields)
	for k, v := range f.parts {
		if v != "" {
			if v2, found := fields[k]; found && v2 == v {
				fields[k] = ""
				nnull -= 1
			}
		}
	}
	if nnull == 0 {
		return nil
	}
	return []map[interface{}]string{fields}
}

///

// SplitFieldFilter splits fields on a delimiter, creating new records for each split. For
// example, a single record with 3="A,B,C" and a delimiter of "," - three records with 3="A",
// 3="B" and 3="C" are emitted. Note that the delimiter "" is not allowed.
//
// Automatically registered as "split_fields"
type SplitFieldFilter struct {
	parts map[interface{}]string
}

func (f *SplitFieldFilter) Setup(parts map[interface{}]string) error {
	f.parts = parts
	return nil
}

func (f *SplitFieldFilter) recApply(parts map[interface{}][]string, ki int, keys []interface{},
	lastmaps []map[interface{}]string) []map[interface{}]string {

	if len(keys) == ki {
		return lastmaps
	}

	k := keys[ki]
	if len(parts[k]) == 1 {
		// cover the easy case with one element
		for _, m := range lastmaps {
			m[k] = parts[k][0]
		}
		return f.recApply(parts, ki+1, keys, lastmaps)
	}

	newmaps := []map[interface{}]string{}
	for _, m := range lastmaps {
		for _, v := range parts[k] {
			m2 := make(map[interface{}]string)
			for k2, v2 := range m {
				m2[k2] = v2
			}
			m2[k] = v
			newmaps = append(newmaps, m2)
		}
	}
	return f.recApply(parts, ki+1, keys, newmaps)
}

func (f *SplitFieldFilter) Apply(fields map[interface{}]string) []map[interface{}]string {
	allparts := make(map[interface{}][]string)
	keys := []interface{}{}

	// split all the fields
	for k, v := range f.parts {
		keys = append(keys, k)

		if v == "" {
			allparts[k] = []string{fields[k]}
		} else {
			if v2, found := fields[k]; found && strings.Contains(v2, v) {
				ss := []string{}
				for _, s := range strings.Split(v2, v) {
					if s != "" {
						ss = append(ss, s)
					}
				}
				allparts[k] = ss
			} else {
				allparts[k] = []string{fields[k]}
			}
		}
	}

	mstart := make(map[interface{}]string)
	return f.recApply(allparts, 0, keys, []map[interface{}]string{mstart})
}

///////

// RequireFilter drops any record that does NOT match ALL of it's field entries. An empty
// string ("") require field is skipped, so if you want to require records with blank
// fields, use the special string BLANK_ENTRY
//
// Automatically registered as "require"
type RequireFilter struct {
	parts map[interface{}]string
}

func (f *RequireFilter) Setup(parts map[interface{}]string) error {
	f.parts = parts
	return nil
}

func (f *RequireFilter) Apply(fields map[interface{}]string) []map[interface{}]string {
	for k, v := range f.parts {
		if v == "" {
			continue
		}

		if v == BLANK_ENTRY {
			v = ""
		}
		if fields[k] != v {
			return nil
		}
	}
	return []map[interface{}]string{fields}
}

///////

// ExcludeFilter drops any record matching at least one of it's field entries. An empty
// string ("") exclude field is skipped, so if you want to exclude records with blank
// fields, use the special string BLANK_ENTRY
//
// To exclude multiple keywords from one field, you will either need to use multiple
// ExcludeFilters or write a new Filter.
//
// Automatically registered as "excludes"
type ExcludeFilter struct {
	parts map[interface{}]string
}

func (f *ExcludeFilter) Setup(parts map[interface{}]string) error {
	f.parts = parts
	return nil
}

func (f *ExcludeFilter) Apply(fields map[interface{}]string) []map[interface{}]string {
	for k, v := range f.parts {
		if v == "" {
			continue
		}

		if v == BLANK_ENTRY {
			v = ""
		}
		if fields[k] == v {
			return nil
		}
	}
	return []map[interface{}]string{fields}
}

///////

// DateFormatFilter parses the field value using an strptime format string, and reformats it
// into a standard representation, UTC time formatted as "2006-01-02 15:04:05". NB not all
// strptime formats are available, see package github.com/pbnjay/strptime for a listing.
//
// Automatically registered as "date_formats"
type DateFormatFilter struct {
	parts map[interface{}]string
}

func (f *DateFormatFilter) Setup(parts map[interface{}]string) error {
	f.parts = parts

	// check date format strings are supported
	for k, v := range f.parts {
		if v == "" {
			continue
		}
		err := strptime.Check(v)
		if err != nil {
			return fmt.Errorf("Error in DateFormatFilter '%s' - %s", v, err.Error())
		}
	}
	return nil
}

func (f *DateFormatFilter) Apply(fields map[interface{}]string) []map[interface{}]string {
	for k, v := range f.parts {
		if v == "" {
			continue
		}

		v2, found := fields[k]
		if !found || v2 == "" {
			continue
		}

		tm := strptime.MustParse(v2, v)
		fields[k] = tm.UTC().Format("2006-01-02 15:04:05")
	}
	return []map[interface{}]string{fields}
}

///////

// FilterSet defines an ordered set of filters that are applied to incoming data records. These
// filters can be use to restrict, reformat, and subdivide data into unique records. Filters
// are applied in the order they are added with Append(), so results are cumulative and early
// restrictions can bypass more expensive field splits.
type FilterSet struct {
	filters []Filter
}

// Append adds a new filter onto the end of the FilterSet chain.
func (fs *FilterSet) Append(ftype string, fields map[interface{}]string) error {
	fltr, err := GetFilter(ftype, fields)
	if err != nil {
		return err
	}

	fs.filters = append(fs.filters, fltr)
	return nil
}

// Apply calls Filter.Apply for each filter in the FilterSet, and accumulates results.
// Restrictive filters (such as Require/Exclude) should be applied as early as possible,
// and expansive filters (such as Split and DateFormat) should be applied as late as
// possible in order to decrease computational times.
func (fs *FilterSet) Apply(fields map[interface{}]string) []map[interface{}]string {
	lastset := []map[interface{}]string{fields}
	for _, fltr := range fs.filters {
		newset := []map[interface{}]string{}
		for _, mf := range lastset {
			for _, nf := range fltr.Apply(mf) {
				if len(nf) > 0 {
					newset = append(newset, nf)
				}
			}
		}
		// short-circuit nulls
		if len(newset) == 0 {
			return nil
		}
		lastset = newset
	}
	return lastset
}

///////

// RegisterFilter adds a new named Filter for discovery by GetFilter or FilterSet.Append.
func RegisterFilter(name string, f Filter) {
	filters[name] = f
}

// GetFilter returns the named filter, initialized using Setup() with the fields parameter.
func GetFilter(name string, fields map[interface{}]string) (Filter, error) {
	f, found := filters[name]

	if !found {
		return nil, fmt.Errorf("No registered filters match '%s'", name)
	}

	err := f.Setup(fields)
	if err != nil {
		return nil, err
	}
	return f, nil
}

func init() {
	RegisterFilter("null_fields", &NullFilter{})
	RegisterFilter("split_fields", &SplitFieldFilter{})
	RegisterFilter("excludes", &ExcludeFilter{})
	RegisterFilter("require", &RequireFilter{})
	RegisterFilter("date_formats", &DateFormatFilter{})
}
<commit_msg>Un-exporting Filters and improving docs and lint.<commit_after>// Package filters provides a data-record filtering mechanism and basic implementations
// for typical use cases. It is intended as a complement to the formats sister package,
// useful for automating unique record extraction from a data file.
//
// A loose naming convention of adding "s" on the end implies that the filter is applied
// independently for each field of the record. Thus the missing "s" on "require" means that
// all supplied fields are required simultaneously. The currently supported filters are:
//
//    "require"      - drops any record that does NOT match ALL of it's field entries. An empty
//                     string ("") require field is skipped, so if you want to require records
//                     with blank fields, use the special string FilterBlankEntry
//
//    "excludes"     - drops any record matching at least one of it's field entries. An empty
//                     string ("") exclude field is skipped, so if you want to exclude records
//                     with blank fields, use the special string FilterBlankEntry
//
//                     To exclude multiple keywords from one field, you will either need to
//                     use multiple excludes or write a new Filter.
//
//    "null_fields"  - remaps fields from a placeholder string into an empty string. For
//                     example, many data sources use a placeholder of "-" or "n/a" to
//                     indicate a missing element. This filter may also be used to suppress
//                     particular values from records.
//
//    "split_fields" - splits fields on a delimiter, creating new records for each split. For
//                     example, a single record with 3="A,B,C" and a delimiter of "," emits
//                     three records with 3="A", 3="B" and 3="C".
//                     Note that the delimiter "" is not allowed.
//
//    "date_formats" - parses the field value using an strptime format string, and reformats
//                     it into a standard representation, of "2006-01-02 15:04:05" in UTC.
//                     Note that not all strptime formats are available, see the package
//                     at github.com/pbnjay/strptime for a listing.
//
// To support new filters, simply implement the Filter interface and call RegisterFilter before
// using GetFilter or FilterSet.Append.
//
package filters

import (
	"fmt"
	"strings"

	"github.com/pbnjay/strptime"
)

// Filter defines an interface that manipulates fields from one record into a new slice of
// records (most often 1-to-1). These manipulations can have optional parameters provided
// by Setup to control them.
type Filter interface {
	// Setup defines the part strings used to apply this filter to new records.
	Setup(parts map[interface{}]string) error
	// Apply takes an input record and applies the Filter to create 0 or more records.
	Apply(fields map[interface{}]string) []map[interface{}]string
}

var (
	// FilterBlankEntry is a placeholder for blank string matching in RequireFilter and
	// ExcludeFilter. If for some reason your input contains this text and you need a
	// different representation, this may be overridden in user code.
	FilterBlankEntry = "<BLANK>"

	filters = make(map[string]Filter)
)

///

type nullFilter struct {
	parts map[interface{}]string
}

func (f *nullFilter) Setup(parts map[interface{}]string) error {
	f.parts = parts
	return nil
}

func (f *nullFilter) Apply(fields map[interface{}]string) []map[interface{}]string {
	nnull := len(fields)
	for k, v := range f.parts {
		if v != "" {
			if v2, found := fields[k]; found && v2 == v {
				fields[k] = ""
				nnull--
			}
		}
	}
	if nnull == 0 {
		return nil
	}
	return []map[interface{}]string{fields}
}

///

type splitFieldFilter struct {
	parts map[interface{}]string
}

func (f *splitFieldFilter) Setup(parts map[interface{}]string) error {
	f.parts = parts
	return nil
}

func (f *splitFieldFilter) recApply(parts map[interface{}][]string, ki int, keys []interface{},
	lastmaps []map[interface{}]string) []map[interface{}]string {

	if len(keys) == ki {
		return lastmaps
	}

	k := keys[ki]
	if len(parts[k]) == 1 {
		// cover the easy case with one element
		for _, m := range lastmaps {
			m[k] = parts[k][0]
		}
		return f.recApply(parts, ki+1, keys, lastmaps)
	}

	newmaps := []map[interface{}]string{}
	for _, m := range lastmaps {
		for _, v := range parts[k] {
			m2 := make(map[interface{}]string)
			for k2, v2 := range m {
				m2[k2] = v2
			}
			m2[k] = v
			newmaps = append(newmaps, m2)
		}
	}
	return f.recApply(parts, ki+1, keys, newmaps)
}

func (f *splitFieldFilter) Apply(fields map[interface{}]string) []map[interface{}]string {
	allparts := make(map[interface{}][]string)
	keys := []interface{}{}

	// split all the fields
	for k, v := range f.parts {
		keys = append(keys, k)

		if v == "" {
			allparts[k] = []string{fields[k]}
		} else {
			if v2, found := fields[k]; found && strings.Contains(v2, v) {
				ss := []string{}
				for _, s := range strings.Split(v2, v) {
					if s != "" {
						ss = append(ss, s)
					}
				}
				allparts[k] = ss
			} else {
				allparts[k] = []string{fields[k]}
			}
		}
	}

	mstart := make(map[interface{}]string)
	return f.recApply(allparts, 0, keys, []map[interface{}]string{mstart})
}

///////

type requireFilter struct {
	parts map[interface{}]string
}

func (f *requireFilter) Setup(parts map[interface{}]string) error {
	f.parts = parts
	return nil
}

func (f *requireFilter) Apply(fields map[interface{}]string) []map[interface{}]string {
	for k, v := range f.parts {
		if v == "" {
			continue
		}

		if v == FilterBlankEntry {
			v = ""
		}
		if fields[k] != v {
			return nil
		}
	}
	return []map[interface{}]string{fields}
}

///////

type excludeFilter struct {
	parts map[interface{}]string
}

func (f *excludeFilter) Setup(parts map[interface{}]string) error {
	f.parts = parts
	return nil
}

func (f *excludeFilter) Apply(fields map[interface{}]string) []map[interface{}]string {
	for k, v := range f.parts {
		if v == "" {
			continue
		}

		if v == FilterBlankEntry {
			v = ""
		}
		if fields[k] == v {
			return nil
		}
	}
	return []map[interface{}]string{fields}
}

///////

type dateFormatFilter struct {
	parts map[interface{}]string
}

func (f *dateFormatFilter) Setup(parts map[interface{}]string) error {
	f.parts = parts

	// check date format strings are supported
	for _, v := range f.parts {
		if v == "" {
			continue
		}
		err := strptime.Check(v)
		if err != nil {
			return fmt.Errorf("error in date format filter '%s' - %s", v, err.Error())
		}
	}
	return nil
}

func (f *dateFormatFilter) Apply(fields map[interface{}]string) []map[interface{}]string {
	for k, v := range f.parts {
		if v == "" {
			continue
		}

		v2, found := fields[k]
		if !found || v2 == "" {
			continue
		}

		tm := strptime.MustParse(v2, v)
		fields[k] = tm.UTC().Format("2006-01-02 15:04:05")
	}
	return []map[interface{}]string{fields}
}

///////

// FilterSet defines an ordered set of filters that are applied to incoming data records. These
// filters can be use to restrict, reformat, and subdivide data into unique records. Filters
// are applied in the order they are added with Append(), so results are cumulative and early
// restrictions can bypass more expensive field splits.
type FilterSet struct {
	filters []Filter
}

// Append adds a new filter onto the end of the FilterSet chain.
func (fs *FilterSet) Append(ftype string, fields map[interface{}]string) error {
	fltr, err := GetFilter(ftype, fields)
	if err != nil {
		return err
	}

	fs.filters = append(fs.filters, fltr)
	return nil
}

// Apply calls Filter.Apply for each filter in the FilterSet, and accumulates results.
// Restrictive filters (such as Require/Exclude) should be applied as early as possible,
// and expansive filters (such as Split and DateFormat) should be applied as late as
// possible in order to decrease computational times.
func (fs *FilterSet) Apply(fields map[interface{}]string) []map[interface{}]string {
	lastset := []map[interface{}]string{fields}
	for _, fltr := range fs.filters {
		newset := []map[interface{}]string{}
		for _, mf := range lastset {
			for _, nf := range fltr.Apply(mf) {
				if len(nf) > 0 {
					newset = append(newset, nf)
				}
			}
		}
		// short-circuit nulls
		if len(newset) == 0 {
			return nil
		}
		lastset = newset
	}
	return lastset
}

///////

// RegisterFilter adds a new named Filter for discovery by GetFilter or FilterSet.Append.
func RegisterFilter(name string, f Filter) {
	filters[name] = f
}

// GetFilter returns the named filter, initialized using Setup() with the fields parameter.
func GetFilter(name string, fields map[interface{}]string) (Filter, error) {
	f, found := filters[name]

	if !found {
		return nil, fmt.Errorf("no registered filters match '%s'", name)
	}

	err := f.Setup(fields)
	if err != nil {
		return nil, err
	}
	return f, nil
}

func init() {
	RegisterFilter("null_fields", &nullFilter{})
	RegisterFilter("split_fields", &splitFieldFilter{})
	RegisterFilter("excludes", &excludeFilter{})
	RegisterFilter("require", &requireFilter{})
	RegisterFilter("date_formats", &dateFormatFilter{})
}
<|endoftext|>


<commit_before>package database

import (
	"fmt"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"mud/utils"
)

func useFmt() { fmt.Printf("") }

type dbError struct {
	message string
}

func (self dbError) Error() string {
	return self.message
}

func newDbError(message string) dbError {
	var err dbError
	err.message = message
	return err
}

type collectionName string

func getCollection(session *mgo.Session, collection collectionName) *mgo.Collection {
	return session.DB("mud").C(string(collection))
}

// Collection names
const (
	cUsers      = collectionName("users")
	cCharacters = collectionName("characters")
	cRooms      = collectionName("rooms")
)

// Field names
const (
	fId          = "_id"
	fName        = "name"
	fCharacters  = "characters"
	fRoom        = "room"
	fTitle       = "title"
	fDescription = "description"
)

func FindUser(session *mgo.Session, name string) (bool, error) {
	c := getCollection(session, cUsers)
	q := c.Find(bson.M{fName: name})

	count, err := q.Count()

	if err != nil {
		return false, err
	}

	return count > 0, nil
}

func FindCharacter(session *mgo.Session, name string) (bool, error) {
	c := getCollection(session, cCharacters)
	q := c.Find(bson.M{fName: name})

	count, err := q.Count()

	if err != nil {
		return false, err
	}

	return count > 0, nil
}

func NewUser(session *mgo.Session, name string) error {

	found, err := FindUser(session, name)

	if err != nil {
		return err
	}

	if found {
		return newDbError("That user already exists")
	}

	c := getCollection(session, cUsers)
	c.Insert(bson.M{fName: name})

	return nil
}

func NewCharacter(session *mgo.Session, user string, character string) error {

	found, err := FindCharacter(session, character)

	if err != nil {
		return err
	}

	if found {
		return newDbError("That character already exists")
	}

	c := getCollection(session, cUsers)
	c.Update(bson.M{fName: user}, bson.M{"$push": bson.M{fCharacters: character}})

	c = getCollection(session, cCharacters)
	c.Insert(bson.M{fName: character})

	return nil
}

func GetCharacterRoom(session *mgo.Session, character string) (Room, error) {
	c := getCollection(session, cCharacters)
	q := c.Find(bson.M{fName: character})

	result := map[string]string{}
	err := q.One(&result)

	var room Room
	if err != nil {
		return room, err
	}

	c = getCollection(session, cRooms)
	q = c.Find(bson.M{fId: result[fRoom]})

	count, err := q.Count()

	if err != nil {
		return room, err
	}

	if count == 0 {
		SetCharacterRoom(session, character, "1")
		q = c.Find(bson.M{fId: "1"})
	}

	err = q.One(&result)

	if err != nil {
		return room, err
	}

	room.Id = result[fId]
	room.Title = result[fTitle]
	room.Description = result[fDescription]

	return room, nil
}

func SetCharacterRoom(session *mgo.Session, character string, roomId string) error {
	c := getCollection(session, cCharacters)
	return c.Update(bson.M{fName: character}, bson.M{"$set": bson.M{fRoom: roomId}})
}

func GetUserCharacters(session *mgo.Session, name string) ([]string, error) {
	c := getCollection(session, cUsers)
	q := c.Find(bson.M{fName: name})

	result := map[string][]string{}
	err := q.One(&result)

	return result[fCharacters], err
}

func DeleteCharacter(session *mgo.Session, user string, character string) error {
	c := getCollection(session, cUsers)
	c.Update(bson.M{fName: user}, bson.M{"$pull": bson.M{fCharacters: utils.Simplify(character)}})

	c = getCollection(session, cCharacters)
	c.Remove(bson.M{fName: character})

	return nil
}

func GenerateDefaultMap(session *mgo.Session) {
	c := getCollection(session, cRooms)
	c.DropCollection()

	c.Insert(bson.M{fId: "1",
		fTitle: "The Void",
		fDescription: "You are floating in the blackness of space. Complete darkness surrounds" +
			"you in all directions. There is no escape, there is no hope, just the emptiness. " +
			"You are likely to be eaten by a grue."})
}

func SetRoomTitle(session *mgo.Session, roomId string, title string) error {
	c := getCollection(session, cRooms)
	return c.Update(bson.M{fId: roomId}, bson.M{"$set": bson.M{fTitle: title}})
}

func SetRoomDescription(session *mgo.Session, roomId string, description string) error {
	c := getCollection(session, cRooms)
	return c.Update(bson.M{fId: roomId}, bson.M{"$set": bson.M{fDescription: description}})
}

// vim: nocindent
<commit_msg>Add missing space<commit_after>package database

import (
	"fmt"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
	"mud/utils"
)

func useFmt() { fmt.Printf("") }

type dbError struct {
	message string
}

func (self dbError) Error() string {
	return self.message
}

func newDbError(message string) dbError {
	var err dbError
	err.message = message
	return err
}

type collectionName string

func getCollection(session *mgo.Session, collection collectionName) *mgo.Collection {
	return session.DB("mud").C(string(collection))
}

// Collection names
const (
	cUsers      = collectionName("users")
	cCharacters = collectionName("characters")
	cRooms      = collectionName("rooms")
)

// Field names
const (
	fId          = "_id"
	fName        = "name"
	fCharacters  = "characters"
	fRoom        = "room"
	fTitle       = "title"
	fDescription = "description"
)

func FindUser(session *mgo.Session, name string) (bool, error) {
	c := getCollection(session, cUsers)
	q := c.Find(bson.M{fName: name})

	count, err := q.Count()

	if err != nil {
		return false, err
	}

	return count > 0, nil
}

func FindCharacter(session *mgo.Session, name string) (bool, error) {
	c := getCollection(session, cCharacters)
	q := c.Find(bson.M{fName: name})

	count, err := q.Count()

	if err != nil {
		return false, err
	}

	return count > 0, nil
}

func NewUser(session *mgo.Session, name string) error {

	found, err := FindUser(session, name)

	if err != nil {
		return err
	}

	if found {
		return newDbError("That user already exists")
	}

	c := getCollection(session, cUsers)
	c.Insert(bson.M{fName: name})

	return nil
}

func NewCharacter(session *mgo.Session, user string, character string) error {

	found, err := FindCharacter(session, character)

	if err != nil {
		return err
	}

	if found {
		return newDbError("That character already exists")
	}

	c := getCollection(session, cUsers)
	c.Update(bson.M{fName: user}, bson.M{"$push": bson.M{fCharacters: character}})

	c = getCollection(session, cCharacters)
	c.Insert(bson.M{fName: character})

	return nil
}

func GetCharacterRoom(session *mgo.Session, character string) (Room, error) {
	c := getCollection(session, cCharacters)
	q := c.Find(bson.M{fName: character})

	result := map[string]string{}
	err := q.One(&result)

	var room Room
	if err != nil {
		return room, err
	}

	c = getCollection(session, cRooms)
	q = c.Find(bson.M{fId: result[fRoom]})

	count, err := q.Count()

	if err != nil {
		return room, err
	}

	if count == 0 {
		SetCharacterRoom(session, character, "1")
		q = c.Find(bson.M{fId: "1"})
	}

	err = q.One(&result)

	if err != nil {
		return room, err
	}

	room.Id = result[fId]
	room.Title = result[fTitle]
	room.Description = result[fDescription]

	return room, nil
}

func SetCharacterRoom(session *mgo.Session, character string, roomId string) error {
	c := getCollection(session, cCharacters)
	return c.Update(bson.M{fName: character}, bson.M{"$set": bson.M{fRoom: roomId}})
}

func GetUserCharacters(session *mgo.Session, name string) ([]string, error) {
	c := getCollection(session, cUsers)
	q := c.Find(bson.M{fName: name})

	result := map[string][]string{}
	err := q.One(&result)

	return result[fCharacters], err
}

func DeleteCharacter(session *mgo.Session, user string, character string) error {
	c := getCollection(session, cUsers)
	c.Update(bson.M{fName: user}, bson.M{"$pull": bson.M{fCharacters: utils.Simplify(character)}})

	c = getCollection(session, cCharacters)
	c.Remove(bson.M{fName: character})

	return nil
}

func GenerateDefaultMap(session *mgo.Session) {
	c := getCollection(session, cRooms)
	c.DropCollection()

	c.Insert(bson.M{fId: "1",
		fTitle: "The Void",
		fDescription: "You are floating in the blackness of space. Complete darkness surrounds " +
			"you in all directions. There is no escape, there is no hope, just the emptiness. " +
			"You are likely to be eaten by a grue."})
}

func SetRoomTitle(session *mgo.Session, roomId string, title string) error {
	c := getCollection(session, cRooms)
	return c.Update(bson.M{fId: roomId}, bson.M{"$set": bson.M{fTitle: title}})
}

func SetRoomDescription(session *mgo.Session, roomId string, description string) error {
	c := getCollection(session, cRooms)
	return c.Update(bson.M{fId: roomId}, bson.M{"$set": bson.M{fDescription: description}})
}

// vim: nocindent
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sync_test

import (
	"fmt"
	"net/http"
	"sync"
)

// This example fetches several URLs concurrently,
// using a WaitGroup to block until all the fetches are complete.
func ExampleWaitGroup() {
	var wg sync.WaitGroup
	var urls = []string{
		"http://www.golang.org/",
		"http://www.google.com/",
		"http://www.somestupidname.com/",
	}
	for _, url := range urls {
		// Increment the WaitGroup counter.
		wg.Add(1)
		// Launch a goroutine to fetch the URL.
		go func(url string) {
			// Fetch the URL.
			http.Get(url)
			// Decrement the counter.
			wg.Done()
		}(url)
	}
	// Wait for all HTTP fetches to complete.
	wg.Wait()
}

func ExampleOnce() {
	var once sync.Once
	onceBody := func() {
		fmt.Printf("Only once\n")
	}
	done := make(chan bool)
	for i := 0; i < 10; i++ {
		go func() {
			once.Do(onceBody)
			done <- true
		}()
	}
	for i := 0; i < 10; i++ {
		<-done
	}
	// Output:
	// Only once
}
<commit_msg>sync: improve WaitGroup example by putting the call to Done in a deferred block. This makes hangs in the waiting code less likely if a goroutine exits abnormally.<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sync_test

import (
	"fmt"
	"net/http"
	"sync"
)

// This example fetches several URLs concurrently,
// using a WaitGroup to block until all the fetches are complete.
func ExampleWaitGroup() {
	var wg sync.WaitGroup
	var urls = []string{
		"http://www.golang.org/",
		"http://www.google.com/",
		"http://www.somestupidname.com/",
	}
	for _, url := range urls {
		// Increment the WaitGroup counter.
		wg.Add(1)
		// Launch a goroutine to fetch the URL.
		go func(url string) {
			// Decrement the counter when the goroutine completes.
			defer wg.Done()
			// Fetch the URL.
			http.Get(url)
		}(url)
	}
	// Wait for all HTTP fetches to complete.
	wg.Wait()
}

func ExampleOnce() {
	var once sync.Once
	onceBody := func() {
		fmt.Println("Only once")
	}
	done := make(chan bool)
	for i := 0; i < 10; i++ {
		go func() {
			once.Do(onceBody)
			done <- true
		}()
	}
	for i := 0; i < 10; i++ {
		<-done
	}
	// Output:
	// Only once
}
<|endoftext|>


<commit_before>package Marshal

import (
	"errors"
	"reflect"
	"strings"

	"github.com/xitongsys/parquet-go/Common"
	"github.com/xitongsys/parquet-go/Layout"
	"github.com/xitongsys/parquet-go/ParquetType"
	"github.com/xitongsys/parquet-go/SchemaHandler"
	"github.com/xitongsys/parquet-go/parquet"
)

type Node struct {
	Val     reflect.Value
	PathMap *SchemaHandler.PathMapType
	RL      int32
	DL      int32
}

//Improve Performance///////////////////////////
//NodeBuf
type NodeBufType struct {
	Index int
	Buf   []*Node
}

func NewNodeBuf(ln int) *NodeBufType {
	nodeBuf := new(NodeBufType)
	nodeBuf.Index = 0
	nodeBuf.Buf = make([]*Node, ln)
	for i := 0; i < ln; i++ {
		nodeBuf.Buf[i] = new(Node)
	}
	return nodeBuf
}

func (self *NodeBufType) GetNode() *Node {
	if self.Index >= len(self.Buf) {
		self.Buf = append(self.Buf, new(Node))
	}
	self.Index++
	return self.Buf[self.Index-1]
}

func (self *NodeBufType) Reset() {
	self.Index = 0
}

////////for improve performance///////////////////////////////////
type Marshaler interface {
	Marshal(node *Node, nodeBuf *NodeBufType) []*Node
}

type ParquetPtr struct{}

func (p *ParquetPtr) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	nodes := make([]*Node, 0)
	if node.Val.IsNil() {
		return nodes
	}
	node.Val = node.Val.Elem()
	node.DL++
	nodes = append(nodes, node)
	return nodes
}

type ParquetStruct struct{}

func (p *ParquetStruct) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	var ok bool

	numField := node.Val.Type().NumField()
	nodes := make([]*Node, 0, numField)
	for j := 0; j < numField; j++ {
		tf := node.Val.Type().Field(j)
		name := tf.Name
		newNode := nodeBuf.GetNode()

		//some ignored item
		if newNode.PathMap, ok = node.PathMap.Children[name]; !ok {
			continue
		}

		newNode.Val = node.Val.Field(j)
		newNode.RL = node.RL
		newNode.DL = node.DL
		nodes = append(nodes, newNode)
	}
	return nodes
}

type ParquetSlice struct {
	schemaHandler *SchemaHandler.SchemaHandler
}

func (p *ParquetSlice) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	nodes := make([]*Node, 0)
	ln := node.Val.Len()
	pathMap := node.PathMap
	path := node.PathMap.Path
	if *p.schemaHandler.SchemaElements[p.schemaHandler.MapIndex[node.PathMap.Path]].RepetitionType != parquet.FieldRepetitionType_REPEATED {
		pathMap = pathMap.Children["list"].Children["element"]
		path += ".list" + ".element"
	}
	if ln <= 0 {
		return nodes
	}

	rlNow, _ := p.schemaHandler.MaxRepetitionLevel(Common.StrToPath(path))
	for j := ln - 1; j >= 0; j-- {
		newNode := nodeBuf.GetNode()
		newNode.PathMap = pathMap
		newNode.Val = node.Val.Index(j)
		if j == 0 {
			newNode.RL = node.RL
		} else {
			newNode.RL = rlNow
		}
		newNode.DL = node.DL + 1
		nodes = append(nodes, newNode)
	}
	return nodes
}

type ParquetMap struct {
	schemaHandler *SchemaHandler.SchemaHandler
}

func (p *ParquetMap) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	nodes := make([]*Node, 0)
	path := node.PathMap.Path + ".key_value"
	keys := node.Val.MapKeys()
	if len(keys) <= 0 {
		return nodes
	}

	rlNow, _ := p.schemaHandler.MaxRepetitionLevel(Common.StrToPath(path))
	for j := len(keys) - 1; j >= 0; j-- {
		key := keys[j]
		value := node.Val.MapIndex(key)
		newNode := nodeBuf.GetNode()
		newNode.PathMap = node.PathMap.Children["key_value"].Children["key"]
		newNode.Val = key
		newNode.DL = node.DL + 1
		if j == 0 {
			newNode.RL = node.RL
		} else {
			newNode.RL = rlNow
		}
		nodes = append(nodes, newNode)

		newNode = nodeBuf.GetNode()
		newNode.PathMap = node.PathMap.Children["key_value"].Children["value"]
		newNode.Val = value
		newNode.DL = node.DL + 1
		if j == 0 {
			newNode.RL = node.RL
		} else {
			newNode.RL = rlNow
		}
		nodes = append(nodes, newNode)
	}
	return nodes
}

//Convert the objects to table map. srcInterface is a slice of objects
func Marshal(srcInterface []interface{}, bgn int, end int, schemaHandler *SchemaHandler.SchemaHandler) (tb *map[string]*Layout.Table, err error) {
	defer func() {
		if r := recover(); r != nil {
			switch x := r.(type) {
			case string:
				err = errors.New(x)
			case error:
				err = x
			default:
				err = errors.New("unkown error")
			}
		}
	}()

	src := reflect.ValueOf(srcInterface)
	res := make(map[string]*Layout.Table)
	pathMap := schemaHandler.PathMap
	nodeBuf := NewNodeBuf(1)

	for i := 0; i < len(schemaHandler.SchemaElements); i++ {
		schema := schemaHandler.SchemaElements[i]
		pathStr := schemaHandler.IndexMap[int32(i)]
		numChildren := schema.GetNumChildren()
		if numChildren == 0 {
			res[pathStr] = Layout.NewEmptyTable()
			res[pathStr].Path = Common.StrToPath(pathStr)
			res[pathStr].MaxDefinitionLevel, _ = schemaHandler.MaxDefinitionLevel(res[pathStr].Path)
			res[pathStr].MaxRepetitionLevel, _ = schemaHandler.MaxRepetitionLevel(res[pathStr].Path)
			res[pathStr].RepetitionType = schema.GetRepetitionType()
			res[pathStr].Type = schemaHandler.SchemaElements[schemaHandler.MapIndex[pathStr]].GetType()
			res[pathStr].Info = schemaHandler.Infos[i]
		}
	}

	stack := make([]*Node, 0, 100)
	for i := bgn; i < end; i++ {
		stack = stack[:0]
		nodeBuf.Reset()

		node := nodeBuf.GetNode()
		node.Val = src.Index(i)
		if src.Index(i).Type().Kind() == reflect.Interface {
			node.Val = src.Index(i).Elem()
		}
		node.PathMap = pathMap
		stack = append(stack, node)

		for len(stack) > 0 {
			ln := len(stack)
			node := stack[ln-1]
			stack = stack[:ln-1]

			tk := node.Val.Type().Kind()
			var m Marshaler

			if tk == reflect.Ptr {
				m = &ParquetPtr{}
			} else if tk == reflect.Struct {
				m = &ParquetStruct{}
			} else if tk == reflect.Slice {
				m = &ParquetSlice{schemaHandler: schemaHandler}
			} else if tk == reflect.Map {
				m = &ParquetMap{schemaHandler: schemaHandler}
			} else {
				table := res[node.PathMap.Path]
				schemaIndex := schemaHandler.MapIndex[node.PathMap.Path]
				sele := schemaHandler.SchemaElements[schemaIndex]
				table.Values = append(table.Values, ParquetType.GoTypeToParquetType(node.Val.Interface(), sele.Type, sele.ConvertedType))
				table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
				table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
				continue
			}

			nodes := m.Marshal(node, nodeBuf)
			if len(nodes) == 0 {
				path := node.PathMap.Path
				index := schemaHandler.MapIndex[path]
				numChildren := schemaHandler.SchemaElements[index].GetNumChildren()
				if numChildren > int32(0) {
					for key, table := range res {
						if strings.HasPrefix(key, path) &&
							(len(key) == len(path) || key[len(path)] == '.') {
							table.Values = append(table.Values, nil)
							table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
							table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
						}
					}
				} else {
					table := res[path]
					table.Values = append(table.Values, nil)
					table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
					table.RepetitionLevels = append(table.RepetitionLevels, node.RL)

				}
			} else {
				for _, node := range nodes {
					stack = append(stack, node)
				}
			}
		}
	}

	return &res, nil
}
<commit_msg>Allow maps to be used as structs<commit_after>package Marshal

import (
	"errors"
	"reflect"
	"strings"

	"github.com/xitongsys/parquet-go/Common"
	"github.com/xitongsys/parquet-go/Layout"
	"github.com/xitongsys/parquet-go/ParquetType"
	"github.com/xitongsys/parquet-go/SchemaHandler"
	"github.com/xitongsys/parquet-go/parquet"
)

type Node struct {
	Val     reflect.Value
	PathMap *SchemaHandler.PathMapType
	RL      int32
	DL      int32
}

//Improve Performance///////////////////////////
//NodeBuf
type NodeBufType struct {
	Index int
	Buf   []*Node
}

func NewNodeBuf(ln int) *NodeBufType {
	nodeBuf := new(NodeBufType)
	nodeBuf.Index = 0
	nodeBuf.Buf = make([]*Node, ln)
	for i := 0; i < ln; i++ {
		nodeBuf.Buf[i] = new(Node)
	}
	return nodeBuf
}

func (self *NodeBufType) GetNode() *Node {
	if self.Index >= len(self.Buf) {
		self.Buf = append(self.Buf, new(Node))
	}
	self.Index++
	return self.Buf[self.Index-1]
}

func (self *NodeBufType) Reset() {
	self.Index = 0
}

////////for improve performance///////////////////////////////////
type Marshaler interface {
	Marshal(node *Node, nodeBuf *NodeBufType) []*Node
}

type ParquetPtr struct{}

func (p *ParquetPtr) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	nodes := make([]*Node, 0)
	if node.Val.IsNil() {
		return nodes
	}
	node.Val = node.Val.Elem()
	node.DL++
	nodes = append(nodes, node)
	return nodes
}

type ParquetStruct struct{}

func (p *ParquetStruct) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	var ok bool

	numField := node.Val.Type().NumField()
	nodes := make([]*Node, 0, numField)
	for j := 0; j < numField; j++ {
		tf := node.Val.Type().Field(j)
		name := tf.Name
		newNode := nodeBuf.GetNode()

		//some ignored item
		if newNode.PathMap, ok = node.PathMap.Children[name]; !ok {
			continue
		}

		newNode.Val = node.Val.Field(j)
		newNode.RL = node.RL
		newNode.DL = node.DL
		nodes = append(nodes, newNode)
	}
	return nodes
}

type ParquetMapStruct struct{}

func (p *ParquetMapStruct) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	var ok bool

	nodes := make([]*Node, 0)
	keys := node.Val.MapKeys()
	if len(keys) <= 0 {
		return nodes
	}

	for j := len(keys) - 1; j >= 0; j-- {
		key := keys[j]
		newNode := nodeBuf.GetNode()

		//some ignored item
		if newNode.PathMap, ok = node.PathMap.Children[key.String()]; !ok {
			continue
		}

		newNode.Val = node.Val.MapIndex(key)
		newNode.RL = node.RL
		newNode.DL = node.DL
		nodes = append(nodes, newNode)
	}
	return nodes
}

type ParquetSlice struct {
	schemaHandler *SchemaHandler.SchemaHandler
}

func (p *ParquetSlice) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	nodes := make([]*Node, 0)
	ln := node.Val.Len()
	pathMap := node.PathMap
	path := node.PathMap.Path
	if *p.schemaHandler.SchemaElements[p.schemaHandler.MapIndex[node.PathMap.Path]].RepetitionType != parquet.FieldRepetitionType_REPEATED {
		pathMap = pathMap.Children["list"].Children["element"]
		path += ".list" + ".element"
	}
	if ln <= 0 {
		return nodes
	}

	rlNow, _ := p.schemaHandler.MaxRepetitionLevel(Common.StrToPath(path))
	for j := ln - 1; j >= 0; j-- {
		newNode := nodeBuf.GetNode()
		newNode.PathMap = pathMap
		newNode.Val = node.Val.Index(j)
		if j == 0 {
			newNode.RL = node.RL
		} else {
			newNode.RL = rlNow
		}
		newNode.DL = node.DL + 1
		nodes = append(nodes, newNode)
	}
	return nodes
}

type ParquetMap struct {
	schemaHandler *SchemaHandler.SchemaHandler
}

func (p *ParquetMap) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	nodes := make([]*Node, 0)
	path := node.PathMap.Path + ".key_value"
	keys := node.Val.MapKeys()
	if len(keys) <= 0 {
		return nodes
	}

	rlNow, _ := p.schemaHandler.MaxRepetitionLevel(Common.StrToPath(path))
	for j := len(keys) - 1; j >= 0; j-- {
		key := keys[j]
		value := node.Val.MapIndex(key)
		newNode := nodeBuf.GetNode()
		newNode.PathMap = node.PathMap.Children["key_value"].Children["key"]
		newNode.Val = key
		newNode.DL = node.DL + 1
		if j == 0 {
			newNode.RL = node.RL
		} else {
			newNode.RL = rlNow
		}
		nodes = append(nodes, newNode)

		newNode = nodeBuf.GetNode()
		newNode.PathMap = node.PathMap.Children["key_value"].Children["value"]
		newNode.Val = value
		newNode.DL = node.DL + 1
		if j == 0 {
			newNode.RL = node.RL
		} else {
			newNode.RL = rlNow
		}
		nodes = append(nodes, newNode)
	}
	return nodes
}

//Convert the objects to table map. srcInterface is a slice of objects
func Marshal(srcInterface []interface{}, bgn int, end int, schemaHandler *SchemaHandler.SchemaHandler) (tb *map[string]*Layout.Table, err error) {
	defer func() {
		if r := recover(); r != nil {
			switch x := r.(type) {
			case string:
				err = errors.New(x)
			case error:
				err = x
			default:
				err = errors.New("unkown error")
			}
		}
	}()

	src := reflect.ValueOf(srcInterface)
	res := make(map[string]*Layout.Table)
	pathMap := schemaHandler.PathMap
	nodeBuf := NewNodeBuf(1)

	for i := 0; i < len(schemaHandler.SchemaElements); i++ {
		schema := schemaHandler.SchemaElements[i]
		pathStr := schemaHandler.IndexMap[int32(i)]
		numChildren := schema.GetNumChildren()
		if numChildren == 0 {
			res[pathStr] = Layout.NewEmptyTable()
			res[pathStr].Path = Common.StrToPath(pathStr)
			res[pathStr].MaxDefinitionLevel, _ = schemaHandler.MaxDefinitionLevel(res[pathStr].Path)
			res[pathStr].MaxRepetitionLevel, _ = schemaHandler.MaxRepetitionLevel(res[pathStr].Path)
			res[pathStr].RepetitionType = schema.GetRepetitionType()
			res[pathStr].Type = schemaHandler.SchemaElements[schemaHandler.MapIndex[pathStr]].GetType()
			res[pathStr].Info = schemaHandler.Infos[i]
		}
	}

	stack := make([]*Node, 0, 100)
	for i := bgn; i < end; i++ {
		stack = stack[:0]
		nodeBuf.Reset()

		node := nodeBuf.GetNode()
		node.Val = src.Index(i)
		if src.Index(i).Type().Kind() == reflect.Interface {
			node.Val = src.Index(i).Elem()
		}
		node.PathMap = pathMap
		stack = append(stack, node)

		for len(stack) > 0 {
			ln := len(stack)
			node := stack[ln-1]
			stack = stack[:ln-1]

			tk := node.Val.Type().Kind()
			var m Marshaler

			if tk == reflect.Ptr {
				m = &ParquetPtr{}
			} else if tk == reflect.Struct {
				m = &ParquetStruct{}
			} else if tk == reflect.Slice {
				m = &ParquetSlice{schemaHandler: schemaHandler}
			} else if tk == reflect.Map {
				schemaIndex := schemaHandler.MapIndex[node.PathMap.Path]
				sele := schemaHandler.SchemaElements[schemaIndex]
				if !sele.IsSetConvertedType() {
					m = &ParquetMapStruct{}
				} else {
					m = &ParquetMap{schemaHandler: schemaHandler}
				}
			} else {
				table := res[node.PathMap.Path]
				schemaIndex := schemaHandler.MapIndex[node.PathMap.Path]
				sele := schemaHandler.SchemaElements[schemaIndex]
				table.Values = append(table.Values, ParquetType.GoTypeToParquetType(node.Val.Interface(), sele.Type, sele.ConvertedType))
				table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
				table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
				continue
			}

			nodes := m.Marshal(node, nodeBuf)
			if len(nodes) == 0 {
				path := node.PathMap.Path
				index := schemaHandler.MapIndex[path]
				numChildren := schemaHandler.SchemaElements[index].GetNumChildren()
				if numChildren > int32(0) {
					for key, table := range res {
						if strings.HasPrefix(key, path) &&
							(len(key) == len(path) || key[len(path)] == '.') {
							table.Values = append(table.Values, nil)
							table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
							table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
						}
					}
				} else {
					table := res[path]
					table.Values = append(table.Values, nil)
					table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
					table.RepetitionLevels = append(table.RepetitionLevels, node.RL)

				}
			} else {
				for _, node := range nodes {
					stack = append(stack, node)
				}
			}
		}
	}

	return &res, nil
}
<|endoftext|>


<commit_before>package ftest

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"

	"github.com/heroku/shaas/pkg"
)

func TestGetFile(t *testing.T) {
	res, err := http.Get(env.fixturesUrl(env.services[ServiceDefault]) + "/a")
	assert.Nil(t, err)
	assert.Equal(t, http.StatusOK, res.StatusCode)

	body, err := ioutil.ReadAll(res.Body)
	assert.Nil(t, err)
	assert.Equal(t, "A\n", string(body))
}

func TestGetFile_NotFound(t *testing.T) {
	res, err := http.Get(env.fixturesUrl(env.services[ServiceDefault]) + "/b")
	assert.Nil(t, err)
	assert.Equal(t, http.StatusNotFound, res.StatusCode)
}

func TestGetDir(t *testing.T) {
	res, err := http.Get(env.fixturesUrl(env.services[ServiceDefault]))
	assert.Nil(t, err)
	assert.Equal(t, http.StatusOK, res.StatusCode)

	body, err := ioutil.ReadAll(res.Body)
	assert.Nil(t, err)
	dir := &map[string]pkg.FileInfoDetails{}
	assert.Nil(t, json.Unmarshal(body, dir))

	a := (*dir)["a"]
	assert.NotNil(t, a)
	assert.Equal(t, "-", a.Type)
	assert.Equal(t, int64(2), a.Size)
	assert.Equal(t, 420, a.Perm)
}

func TestPostFile(t *testing.T) {
	res, err := http.Post(env.baseUrl(env.services[ServiceDefault])+"/usr/bin/factor", "", strings.NewReader("42"))
	assert.Nil(t, err)
	assert.Equal(t, http.StatusOK, res.StatusCode)

	body, err := ioutil.ReadAll(res.Body)
	assert.Nil(t, err)
	assert.Equal(t, "42: 2 3 7\n", string(body))
}

func TestPostFile_NotFound(t *testing.T) {
	res, err := http.Post(env.fixturesUrl(env.services[ServiceDefault])+"/b", "", strings.NewReader(""))
	assert.Nil(t, err)
	assert.Equal(t, http.StatusNotFound, res.StatusCode)
}

func TestPostDir(t *testing.T) {
	res, err := http.Post(env.fixturesUrl(env.services[ServiceDefault]), "", strings.NewReader("pwd"))
	assert.Nil(t, err)
	assert.Equal(t, http.StatusOK, res.StatusCode)

	body, err := ioutil.ReadAll(res.Body)
	assert.Nil(t, err)
	assert.Equal(t, strings.TrimPrefix(env.fixturesUrl(env.services[ServiceDefault]), env.baseUrl(env.services[ServiceDefault]))+"\n", string(body))
}
<commit_msg>Test read only<commit_after>package ftest

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"

	"github.com/heroku/shaas/pkg"
)

func TestGetFile(t *testing.T) {
	res, err := http.Get(env.fixturesUrl(env.services[ServiceDefault]) + "/a")
	assert.Nil(t, err)
	assert.Equal(t, http.StatusOK, res.StatusCode)

	body, err := ioutil.ReadAll(res.Body)
	assert.Nil(t, err)
	assert.Equal(t, "A\n", string(body))
}

func TestGetFile_NotFound(t *testing.T) {
	res, err := http.Get(env.fixturesUrl(env.services[ServiceDefault]) + "/b")
	assert.Nil(t, err)
	assert.Equal(t, http.StatusNotFound, res.StatusCode)
}

func TestGetDir(t *testing.T) {
	res, err := http.Get(env.fixturesUrl(env.services[ServiceDefault]))
	assert.Nil(t, err)
	assert.Equal(t, http.StatusOK, res.StatusCode)

	body, err := ioutil.ReadAll(res.Body)
	assert.Nil(t, err)
	dir := &map[string]pkg.FileInfoDetails{}
	assert.Nil(t, json.Unmarshal(body, dir))

	a := (*dir)["a"]
	assert.NotNil(t, a)
	assert.Equal(t, "-", a.Type)
	assert.Equal(t, int64(2), a.Size)
	assert.Equal(t, 420, a.Perm)
}

func TestPostFile(t *testing.T) {
	res, err := http.Post(env.baseUrl(env.services[ServiceDefault])+"/usr/bin/factor", "", strings.NewReader("42"))
	assert.Nil(t, err)
	assert.Equal(t, http.StatusOK, res.StatusCode)

	body, err := ioutil.ReadAll(res.Body)
	assert.Nil(t, err)
	assert.Equal(t, "42: 2 3 7\n", string(body))
}

func TestPostFile_NotFound(t *testing.T) {
	res, err := http.Post(env.fixturesUrl(env.services[ServiceDefault])+"/b", "", strings.NewReader(""))
	assert.Nil(t, err)
	assert.Equal(t, http.StatusNotFound, res.StatusCode)
}

func TestPostDir(t *testing.T) {
	res, err := http.Post(env.fixturesUrl(env.services[ServiceDefault]), "", strings.NewReader("pwd"))
	assert.Nil(t, err)
	assert.Equal(t, http.StatusOK, res.StatusCode)

	body, err := ioutil.ReadAll(res.Body)
	assert.Nil(t, err)
	assert.Equal(t, strings.TrimPrefix(env.fixturesUrl(env.services[ServiceDefault]), env.baseUrl(env.services[ServiceDefault]))+"\n", string(body))
}

func TestReadonlyAllowsGet(t *testing.T) {
	res, err := http.Get(env.fixturesUrl(env.services[ServiceReadonly]) + "/a")
	assert.Nil(t, err)
	assert.Equal(t, http.StatusOK, res.StatusCode)
}

func TestReadonlyForbidsPost(t *testing.T) {
	res, err := http.Post(env.fixturesUrl(env.services[ServiceReadonly])+"/a", "", nil)
	assert.Nil(t, err)
	assert.Equal(t, http.StatusMethodNotAllowed, res.StatusCode)
}
<|endoftext|>


<commit_before>package hdfs

import (
	"fmt"
	"testing"
)

func TestHdfs(t *testing.T) {
	h, e := NewHdfs(NewHdfsConfig("http://awshdc01:50070", "hdfs"))
	if e != nil {
		t.Fatalf(e.Error())
	}
	h.Config.PoolSize = 100

	fmt.Println(">>>> TEST DELETE FOLDERS <<<<")
	if e = h.Delete("/user/ariefdarmawan", true); e != nil {
		t.Error(e.Error())
	}

	fmt.Println(">>>> TEST CHANGE OWNER <<<<")
	if e = h.SetOwner("/user/ariefdarmawan", "ariefdarmawan", ""); e != nil {
		t.Error(e.Error())
	}

	fmt.Println(">>>> TEST CREATE DIR <<<<")
	es := h.MakeDirs([]string{"/user/ariefdarmawan/inbox", "/user/ariefdarmawan/temp", "/user/ariefdarmawan/outbox"}, "")
	if es != nil {
		for k, v := range es {
			t.Error(fmt.Sprintf("Error when create %v : %v \n", k, v))
		}
	}

	/*
		fmt.Println(">>>> TEST COPY DIR <<<<")
		e, es = h.PutDir("/Users/ariefdarmawan/Temp/ECFZ/TempVisa/JSON", "/user/ariefdarmawan/inbox/ecfz/json")
		if es != nil {
			for k, v := range es {
				t.Error(fmt.Sprintf("Error when create %v : %v \n", k, v))
			}
		}
	*/

	fmt.Println(">>>> TEST PUT FILE <<<<")
	e = h.Put("/Users/ariefdarmawan/Temp/BHPWellReport.pdf", "/user/ariefdarmawan/inbox/wellreport.pdf", "", nil)
	if e != nil {
		t.Error(e.Error())
	}

	fmt.Println(">>>> TEST GET STATUS <<<<")
	hdata, e := h.List("/user/ariefdarmawan")
	if e != nil {
		t.Error(e.Error())
	} else {
		fmt.Printf("Data Processed :\n%v\n", len(hdata.FileStatuses.FileStatus))
	}

	fmt.Println("Test Done\n")
}
<commit_msg>add setowner<commit_after>package hdfs

import (
	"fmt"
	"testing"
)

func TestHdfs(t *testing.T) {
	h, e := NewHdfs(NewHdfsConfig("http://awshdc01:50070", "hdfs"))
	if e != nil {
		t.Fatalf(e.Error())
	}
	h.Config.PoolSize = 100

	fmt.Println(">>>> TEST DELETE FOLDERS <<<<")
	if e = h.Delete("/user/ariefdarmawan", true); e != nil {
		t.Error(e.Error())
	}

	fmt.Println(">>>> TEST CREATE DIR <<<<")
	es := h.MakeDirs([]string{"/user/ariefdarmawan/inbox", "/user/ariefdarmawan/temp", "/user/ariefdarmawan/outbox"}, "")
	if es != nil {
		for k, v := range es {
			t.Error(fmt.Sprintf("Error when create %v : %v \n", k, v))
		}
	}

	fmt.Println(">>>> TEST CHANGE OWNER <<<<")
	if e = h.SetOwner("/user/ariefdarmawan", "ariefdarmawan", ""); e != nil {
		t.Error(e.Error())
	}

	/*
		fmt.Println(">>>> TEST COPY DIR <<<<")
		e, es = h.PutDir("/Users/ariefdarmawan/Temp/ECFZ/TempVisa/JSON", "/user/ariefdarmawan/inbox/ecfz/json")
		if es != nil {
			for k, v := range es {
				t.Error(fmt.Sprintf("Error when create %v : %v \n", k, v))
			}
		}
	*/

	fmt.Println(">>>> TEST PUT FILE <<<<")
	e = h.Put("/Users/ariefdarmawan/Temp/BHPWellReport.pdf", "/user/ariefdarmawan/inbox/wellreport.pdf", "", nil)
	if e != nil {
		t.Error(e.Error())
	}

	fmt.Println(">>>> TEST GET STATUS <<<<")
	hdata, e := h.List("/user/ariefdarmawan")
	if e != nil {
		t.Error(e.Error())
	} else {
		fmt.Printf("Data Processed :\n%v\n", len(hdata.FileStatuses.FileStatus))
	}

	fmt.Println("Test Done\n")
}
<|endoftext|>


<commit_before>package hid

// #cgo pkg-config: libusb-1.0
// #cgo LDFLAGS: -lusb-1.0
// #include <libusb-1.0/libusb.h>
import "C"

import (
	"errors"
	"fmt"
	"reflect"
	"unsafe"
)

type linuxDevice struct {
	handle *C.libusb_device_handle
	info   *DeviceInfo
}

func Init() {
	C.libusb_init(nil)
}

func newDeviceInfo(dev *C.libusb_device) (*DeviceInfo, error) {
	var desc C.struct_libusb_device_descriptor
	if errno := C.libusb_get_device_descriptor(dev, &desc); errno < 0 {
		return nil, usbError(errno)
	}
	// todo: check if libusb_get_string_descriptor can be used to get manufacturer
	return &DeviceInfo{
		Path:          "",
		VendorId:      uint16(desc.idVendor),
		ProductId:     uint16(desc.idProduct),
		VersionNumber: uint16(desc.bcdDevice),
		Manufacturer:  "",
		Product:       "",
	}, nil
}

func Devices() <-chan *DeviceInfo {
	result := make(chan *DeviceInfo)
	go func() {
		var devices **C.libusb_device
		cnt := C.libusb_get_device_list(nil, &devices)
		defer C.libusb_free_device_list(devices, 1)

		var device_list []*C.libusb_device
		*(*reflect.SliceHeader)(unsafe.Pointer(&device_list)) = reflect.SliceHeader{
			Data: uintptr(unsafe.Pointer(devices)),
			Len:  int(cnt),
			Cap:  int(cnt),
		}

		for _, dev := range device_list {
			di, err := newDeviceInfo(dev)
			if err != nil {
				fmt.Printf("ERROR: %s\n", err)
				continue
			}
			result <- di

		}

		close(result)
	}()
	return result
}

func ByPath(path string) (*DeviceInfo, error) {
	for d := range Devices() {
		if d.Path == path {
			return d, nil
		}
	}
	return nil, errors.New("Device not found")
}

func (di *DeviceInfo) Open() (Device, error) {
	// todo: use another mechanism, cause vid/pid isn't uniqqu for multiple devices
	// maybe libusb_get_port_numbers can be sused as path
	dev := &linuxDevice{
		info:   di,
		handle: C.libusb_open_device_with_vid_pid(nil, C.uint16_t(di.VendorId), C.uint16_t(di.ProductId)),
	}
	return dev, nil
}

func (dev *linuxDevice) Close() {
	C.libusb_exit(nil)
}

func (dev *linuxDevice) WriteFeature(data []byte) error {
	if dev.handle == nil {
		return errors.New("No USB device opend before.")
	}
	if len(data) > 0xffff {
		return errors.New("data longer than 65535 bytes, means overflow, isn't supported")
	}
	if len(data) == 0 {
		return nil
	}

	const reportId = 1
	const index = 0
	const timeout = 1000

	err := C.libusb_control_transfer(dev.handle,
		C.uint8_t(ENDPOINT_OUT|REQUEST_TYPE_CLASS|RECIPIENT_DEVICE),
		C.uint8_t(HID_SET_REPORT),
		C.uint16_t(reportId),
		C.uint16_t(index),
		(*C.uchar)(&data[0]),
		C.uint16_t(len(data)),
		C.uint(timeout))

	return usbError(err)
}

func (dev *linuxDevice) Write(data []byte) error {
	return nil
}
<commit_msg>fixed init() method, now everything works<commit_after>package hid

// #cgo pkg-config: libusb-1.0
// #cgo LDFLAGS: -lusb-1.0
// #include <libusb-1.0/libusb.h>
import "C"

import (
	"errors"
	"fmt"
	"reflect"
	"unsafe"
)

type linuxDevice struct {
	handle *C.libusb_device_handle
	info   *DeviceInfo
}

func init() {
	C.libusb_init(nil)
}

func newDeviceInfo(dev *C.libusb_device) (*DeviceInfo, error) {
	var desc C.struct_libusb_device_descriptor
	if errno := C.libusb_get_device_descriptor(dev, &desc); errno < 0 {
		return nil, usbError(errno)
	}
	// todo: check if libusb_get_string_descriptor can be used to get manufacturer
	return &DeviceInfo{
		Path:          "",
		VendorId:      uint16(desc.idVendor),
		ProductId:     uint16(desc.idProduct),
		VersionNumber: uint16(desc.bcdDevice),
		Manufacturer:  "",
		Product:       "",
	}, nil
}

func Devices() <-chan *DeviceInfo {
	result := make(chan *DeviceInfo)
	go func() {
		var devices **C.struct_libusb_device
		cnt := C.libusb_get_device_list(nil, &devices)
		if cnt < 0 {
			close(result)
			return
		}
		defer C.libusb_free_device_list(devices, 1)

		var device_list []*C.libusb_device
		*(*reflect.SliceHeader)(unsafe.Pointer(&device_list)) = reflect.SliceHeader{
			Data: uintptr(unsafe.Pointer(devices)),
			Len:  int(cnt),
			Cap:  int(cnt),
		}

		for _, dev := range device_list {
			di, err := newDeviceInfo(dev)
			if err != nil {
				fmt.Printf("ERROR: %s\n", err)
				continue
			}
			result <- di
		}

		close(result)
	}()
	return result
}

func ByPath(path string) (*DeviceInfo, error) {
	for d := range Devices() {
		if d.Path == path {
			return d, nil
		}
	}
	return nil, errors.New("Device not found")
}

func (di *DeviceInfo) Open() (Device, error) {
	// todo: use another mechanism, cause vid/pid isn't uniqqu for multiple devices
	// maybe libusb_get_port_numbers can be sused as path
	dev := &linuxDevice{
		info:   di,
		handle: C.libusb_open_device_with_vid_pid(nil, C.uint16_t(di.VendorId), C.uint16_t(di.ProductId)),
	}
	return dev, nil
}

func (dev *linuxDevice) Close() {
	C.libusb_exit(nil)
}

func (dev *linuxDevice) WriteFeature(data []byte) error {
	if dev.handle == nil {
		return errors.New("No USB device opend before.")
	}
	if len(data) > 0xffff {
		return errors.New("data longer than 65535 bytes, means overflow, isn't supported")
	}
	if len(data) == 0 {
		return nil
	}

	const reportId = 1
	const index = 0
	const timeout = 1000

	err := C.libusb_control_transfer(dev.handle,
		C.uint8_t(ENDPOINT_OUT|REQUEST_TYPE_CLASS|RECIPIENT_DEVICE),
		C.uint8_t(HID_SET_REPORT),
		C.uint16_t(reportId),
		C.uint16_t(index),
		(*C.uchar)(&data[0]),
		C.uint16_t(len(data)),
		C.uint(timeout))

	return usbError(err)
}

func (dev *linuxDevice) Write(data []byte) error {
	return errors.New("not yet implemented")
}
<|endoftext|>


<commit_before>// Copyright 2014 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ht

import (
	"fmt"
	htmltemplate "html/template"
	"io"
	"io/ioutil"
	"os"
	"path"
	"text/template"
)

// ----------------------------------------------------------------------------
// Status

// Status describes the status of a Test or a Check.
type Status int

const (
	NotRun  Status = iota // Not jet executed
	Skipped               // Omitted deliberately
	Pass                  // That's what we want
	Fail                  // One ore more checks failed
	Error                 // Request or body reading failed (not for checks).
	Bogus                 // Bogus test or check (malformd URL, bad regexp, etc.)
)

func (s Status) String() string {
	return []string{"NotRun", "Skipped", "Pass", "Fail", "Error", "Bogus"}[int(s)]
}

func (s Status) MarshalText() ([]byte, error) {
	if s < 0 || s > Bogus {
		return []byte(""), fmt.Errorf("no such status %d", s)
	}
	return []byte(s.String()), nil
}

// Stats counts the test results of sr.
func (s *Suite) Stats() (notRun int, skipped int, passed int, failed int, errored int, bogus int) {
	for _, tr := range s.AllTests() {
		switch tr.Status {
		case NotRun:
			notRun++
		case Skipped:
			skipped++
		case Pass:
			passed++
		case Fail:
			failed++
		case Error:
			errored++
		case Bogus:
			bogus++
		default:
			panic(fmt.Sprintf("No such Status %d in suite %q test %q",
				tr.Status, s.Name, tr.Name))
		}
	}
	return
}

// ----------------------------------------------------------------------------
// CheckResult

// CheckResult captures the outcom of a single check inside a test.
type CheckResult struct {
	Name     string   // Name of the check as registered.
	JSON     string   // JSON serialization of check.
	Status   Status   // Outcome of check. All status but Error
	Duration Duration // How long the check took.
	Error    error    // For a Status of Bogus or Fail.
}

// ----------------------------------------------------------------------------
// Templates to output

var defaultCheckTmpl = `{{define "CHECK"}}{{printf "%-7s %-15s %s" .Status .Name .JSON}}` +
	`{{if eq .Status 3 5}} {{.Error.Error}}{{end}}{{end}}`

var htmlCheckTmpl = `{{define "CHECK"}}

Check:
      {{ToUpper .Status.String}}
      {{.Name}} ▹

Check:
      {{ToUpper .Status.String}}
      {{.Name}} ▾

Checking took {{.Duration}}

{{.JSON}}

      {{if eq .Status 3 5}}
{{.Error.Error}}
{{end}}

{{end}}
`

var defaultTestTmpl = `{{define "TEST"}}{{ToUpper .Status.String}}: {{.Name}}{{if gt .Tries 1}}
  {{printf "(after %d tries)" .Tries}}{{end}}
  Started: {{.Started}}   Duration: {{.FullDuration}}   Request: {{.Duration}}{{if .Error}}
  Error: {{.Error}}{{end}}
{{if eq .Status 2 3 4 5}}  {{if .CheckResults}}Checks:
{{range $i, $c := .CheckResults}}{{printf "    %2d. " $i}}{{template "CHECK" .}}
{{end}}{{end}}{{end}}{{end}}`

var htmlTestTmpl = `{{define "TEST"}}

{{.SeqNo}}:
      {{ToUpper .Status.String}}
      "{{.Name}}"
      ({{.FullDuration}}) ▹

{{.SeqNo}}:
      {{ToUpper .Status.String}}
      "{{.Name}}"
      ({{.FullDuration}}) ▾

{{.Description}}

	Started: {{.Started}}

	Full Duration: {{.FullDuration}}

        Number of tries: {{.Tries}}

        Request Duration: {{.Duration}}
        {{if .Error}}
Error: {{.Error}}{{end}}

      {{if eq .Status 2 3 4 5}}{{if .CheckResults}}

          {{range $i, $c := .CheckResults}}{{template "CHECK" .}}{{end}}

      {{end}}{{end}}
      {{if .Request.Request}}{{template "REQUEST" .}}{{end}}
      {{if .Response.Response}}{{template "RESPONSE" .}}{{end}}

{{end}}`

var htmlHeaderTmpl = `{{define "HEADER"}}

  {{range $h, $v := .}}
    {{range $v}}
      {{printf "%25s: " $h}} {{.}}

    {{end}}
  {{end}}

{{end}}`

var htmlResponseTmpl = `{{define "RESPONSE"}}

HTTP Response ▾

      {{if .Response.Response}}
        {{template "HEADER" .Response.Response.Header}}
      {{end}}
      {{if .Response.BodyErr}}Error reading body: {{.Response.BodyErr.Error}}
      {{else}}
        Response Body
      {{end}}

HTTP Response ▹

{{end}}
`

var htmlRequestTmpl = `{{define "REQUEST"}}

HTTP Request ▾

      {{.Request.Request.Method}} {{.Request.Request.URL.String}}
      {{template "HEADER" .Request.Request.Header}}

{{.Request.SentBody}}

HTTP Request ▹

{{end}}
`

var defaultSuiteTmpl = `{{Box (printf "%s: %s" (ToUpper .Status.String) .Name) ""}}{{if .Error}}
Error: {{.Error}}{{end}}
Started: {{.Started}}   Duration: {{.Duration}}

{{range .Setup}}{{template "TEST" .}}
{{end}}
{{range .Tests}}{{template "TEST" .}}
{{end}}
{{range .Teardown}}{{template "TEST" .}}
{{end}}
`

var htmlStyleTmpl = `{{define "STYLE"}}

{{end}}`

var htmlJavascriptTmpl = `{{define "JAVASCRIPT"}}

{{end}}`

var htmlSuiteTmpl = `<!DOCTYPE html>

  <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
  {{template "STYLE"}}
  <title>Suite {{.Name}}</title>

Results of Suite {{.Name}}

{{.Description}}

  Status: {{ToUpper .Status.String}}

  Started: {{.Started}}

  Full Duration: {{.Duration}}

{{range .AllTests}}{{template "TEST" .}}{{end}}

{{template "JAVASCRIPT"}}

`

var (
	TestTmpl      *template.Template
	SuiteTmpl     *template.Template
	HtmlSuiteTmpl *htmltemplate.Template
)

func init() {
	fm := make(template.FuncMap)
	fm["Underline"] = Underline
	fm["Box"] = Box
	fm["ToUpper"] = ToUpper

	TestTmpl = template.New("TEST")
	TestTmpl.Funcs(fm)
	TestTmpl = template.Must(TestTmpl.Parse(defaultTestTmpl))
	TestTmpl = template.Must(TestTmpl.Parse(defaultCheckTmpl))

	SuiteTmpl = template.New("SUITE")
	SuiteTmpl.Funcs(fm)
	SuiteTmpl = template.Must(SuiteTmpl.Parse(defaultSuiteTmpl))
	SuiteTmpl = template.Must(SuiteTmpl.Parse(defaultTestTmpl))
	SuiteTmpl = template.Must(SuiteTmpl.Parse(defaultCheckTmpl))

	HtmlSuiteTmpl = htmltemplate.New("SUITE")
	HtmlSuiteTmpl.Funcs(htmltemplate.FuncMap{"ToUpper": ToUpper})
	HtmlSuiteTmpl = htmltemplate.Must(HtmlSuiteTmpl.Parse(htmlSuiteTmpl))
	HtmlSuiteTmpl = htmltemplate.Must(HtmlSuiteTmpl.Parse(htmlTestTmpl))
	HtmlSuiteTmpl = htmltemplate.Must(HtmlSuiteTmpl.Parse(htmlCheckTmpl))
	HtmlSuiteTmpl = htmltemplate.Must(HtmlSuiteTmpl.Parse(htmlResponseTmpl))
	HtmlSuiteTmpl = htmltemplate.Must(HtmlSuiteTmpl.Parse(htmlRequestTmpl))
	HtmlSuiteTmpl = htmltemplate.Must(HtmlSuiteTmpl.Parse(htmlHeaderTmpl))
	HtmlSuiteTmpl = htmltemplate.Must(HtmlSuiteTmpl.Parse(htmlStyleTmpl))
	HtmlSuiteTmpl = htmltemplate.Must(HtmlSuiteTmpl.Parse(htmlJavascriptTmpl))
}

func (t Test) PrintReport(w io.Writer) error {
	return TestTmpl.Execute(os.Stdout, t)
}

func (r Suite) PrintReport(w io.Writer) error {
	return SuiteTmpl.Execute(os.Stdout, r)
}

func (s Suite) HTMLReport(dir string) error {
	report, err := os.Create(path.Join(dir, "Report.html"))
	if err != nil {
		return err
	}
	err = HtmlSuiteTmpl.Execute(report, s)
	if err != nil {
		return err
	}
	for _, tr := range s.AllTests() {
		body := tr.Response.BodyBytes
		err = ioutil.WriteFile(path.Join(dir, tr.SeqNo+".ResponseBody"), body, 0666)
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Fix bug in report printing<commit_after>// Copyright 2014 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package ht

import (
	"fmt"
	htmltemplate "html/template"
	"io"
	"io/ioutil"
	"os"
	"path"
	"text/template"
)

// ----------------------------------------------------------------------------
// Status

// Status describes the status of a Test or a Check.
type Status int

const (
	NotRun  Status = iota // Not jet executed
	Skipped               // Omitted deliberately
	Pass                  // That's what we want
	Fail                  // One ore more checks failed
	Error                 // Request or body reading failed (not for checks).
	Bogus                 // Bogus test or check (malformd URL, bad regexp, etc.)
)

func (s Status) String() string {
	return []string{"NotRun", "Skipped", "Pass", "Fail", "Error", "Bogus"}[int(s)]
}

func (s Status) MarshalText() ([]byte, error) {
	if s < 0 || s > Bogus {
		return []byte(""), fmt.Errorf("no such status %d", s)
	}
	return []byte(s.String()), nil
}

// Stats counts the test results of sr.
func (s *Suite) Stats() (notRun int, skipped int, passed int, failed int, errored int, bogus int) {
	for _, tr := range s.AllTests() {
		switch tr.Status {
		case NotRun:
			notRun++
		case Skipped:
			skipped++
		case Pass:
			passed++
		case Fail:
			failed++
		case Error:
			errored++
		case Bogus:
			bogus++
		default:
			panic(fmt.Sprintf("No such Status %d in suite %q test %q",
				tr.Status, s.Name, tr.Name))
		}
	}
	return
}

// ----------------------------------------------------------------------------
// CheckResult

// CheckResult captures the outcom of a single check inside a test.
type CheckResult struct {
	Name     string   // Name of the check as registered.
	JSON     string   // JSON serialization of check.
	Status   Status   // Outcome of check. All status but Error
	Duration Duration // How long the check took.
	Error    error    // For a Status of Bogus or Fail.
}

// ----------------------------------------------------------------------------
// Templates to output

var defaultCheckTmpl = `{{define "CHECK"}}{{printf "%-7s %-15s %s" .Status .Name .JSON}}` +
	`{{if eq .Status 3 5}} {{.Error.Error}}{{end}}{{end}}`

var htmlCheckTmpl = `{{define "CHECK"}}

Check:
      {{ToUpper .Status.String}}
      {{.Name}} ▹

Check:
      {{ToUpper .Status.String}}
      {{.Name}} ▾

Checking took {{.Duration}}

{{.JSON}}

      {{if eq .Status 3 5}}
{{.Error.Error}}
{{end}}

{{end}}
`

var defaultTestTmpl = `{{define "TEST"}}{{ToUpper .Status.String}}: {{.Name}}{{if gt .Tries 1}}
  {{printf "(after %d tries)" .Tries}}{{end}}
  Started: {{.Started}}   Duration: {{.FullDuration}}   Request: {{.Duration}}{{if .Error}}
  Error: {{.Error}}{{end}}
{{if eq .Status 2 3 4 5}}  {{if .CheckResults}}Checks:
{{range $i, $c := .CheckResults}}{{printf "    %2d. " $i}}{{template "CHECK" .}}
{{end}}{{end}}{{end}}{{end}}`

var htmlTestTmpl = `{{define "TEST"}}

{{.SeqNo}}:
      {{ToUpper .Status.String}}
      "{{.Name}}"
      ({{.FullDuration}}) ▹

{{.SeqNo}}:
      {{ToUpper .Status.String}}
      "{{.Name}}"
      ({{.FullDuration}}) ▾

{{.Description}}

	Started: {{.Started}}

	Full Duration: {{.FullDuration}}

        Number of tries: {{.Tries}}

        Request Duration: {{.Duration}}
        {{if .Error}}
Error: {{.Error}}{{end}}

      {{if eq .Status 2 3 4 5}}{{if .CheckResults}}

          {{range $i, $c := .CheckResults}}{{template "CHECK" .}}{{end}}

      {{end}}{{end}}
      {{if .Request.Request}}{{template "REQUEST" .}}{{end}}
      {{if .Response.Response}}{{template "RESPONSE" .}}{{end}}

{{end}}`

var htmlHeaderTmpl = `{{define "HEADER"}}

  {{range $h, $v := .}}
    {{range $v}}
      {{printf "%25s: " $h}} {{.}}

    {{end}}
  {{end}}

{{end}}`

var htmlResponseTmpl = `{{define "RESPONSE"}}

HTTP Response ▾

      {{if .Response.Response}}
        {{template "HEADER" .Response.Response.Header}}
      {{end}}
      {{if .Response.BodyErr}}Error reading body: {{.Response.BodyErr.Error}}
      {{else}}
        Response Body
      {{end}}

HTTP Response ▹

{{end}}
`

var htmlRequestTmpl = `{{define "REQUEST"}}

HTTP Request ▾

      {{.Request.Request.Method}} {{.Request.Request.URL.String}}
      {{template "HEADER" .Request.Request.Header}}

{{.Request.SentBody}}

HTTP Request ▹

{{end}}
`

var defaultSuiteTmpl = `{{Box (printf "%s: %s" (ToUpper .Status.String) .Name) ""}}{{if .Error}}
Error: {{.Error}}{{end}}
Started: {{.Started}}   Duration: {{.Duration}}

{{range .Setup}}{{template "TEST" .}}
{{end}}
{{range .Tests}}{{template "TEST" .}}
{{end}}
{{range .Teardown}}{{template "TEST" .}}
{{end}}
`

var htmlStyleTmpl = `{{define "STYLE"}}

{{end}}`

var htmlJavascriptTmpl = `{{define "JAVASCRIPT"}}

{{end}}`

var htmlSuiteTmpl = `<!DOCTYPE html>

  <meta http-equiv="content-type" content="text/html; charset=UTF-8" />
  {{template "STYLE"}}
  <title>Suite {{.Name}}</title>

Results of Suite {{.Name}}

{{.Description}}

  Status: {{ToUpper .Status.String}}

  Started: {{.Started}}

  Full Duration: {{.Duration}}

{{range .AllTests}}{{template "TEST" .}}{{end}}

{{template "JAVASCRIPT"}}

`

var (
	TestTmpl      *template.Template
	SuiteTmpl     *template.Template
	HtmlSuiteTmpl *htmltemplate.Template
)

func init() {
	fm := make(template.FuncMap)
	fm["Underline"] = Underline
	fm["Box"] = Box
	fm["ToUpper"] = ToUpper

	TestTmpl = template.New("TEST")
	TestTmpl.Funcs(fm)
	TestTmpl = template.Must(TestTmpl.Parse(defaultTestTmpl))
	TestTmpl = template.Must(TestTmpl.Parse(defaultCheckTmpl))

	SuiteTmpl = template.New("SUITE")
	SuiteTmpl.Funcs(fm)
	SuiteTmpl = template.Must(SuiteTmpl.Parse(defaultSuiteTmpl))
	SuiteTmpl = template.Must(SuiteTmpl.Parse(defaultTestTmpl))
	SuiteTmpl = template.Must(SuiteTmpl.Parse(defaultCheckTmpl))

	HtmlSuiteTmpl = htmltemplate.New("SUITE")
	HtmlSuiteTmpl.Funcs(htmltemplate.FuncMap{"ToUpper": ToUpper})
	HtmlSuiteTmpl = htmltemplate.Must(HtmlSuiteTmpl.Parse(htmlSuiteTmpl))
	HtmlSuiteTmpl = htmltemplate.Must(HtmlSuiteTmpl.Parse(htmlTestTmpl))
	HtmlSuiteTmpl = htmltemplate.Must(HtmlSuiteTmpl.Parse(htmlCheckTmpl))
	HtmlSuiteTmpl = htmltemplate.Must(HtmlSuiteTmpl.Parse(htmlResponseTmpl))
	HtmlSuiteTmpl = htmltemplate.Must(HtmlSuiteTmpl.Parse(htmlRequestTmpl))
	HtmlSuiteTmpl = htmltemplate.Must(HtmlSuiteTmpl.Parse(htmlHeaderTmpl))
	HtmlSuiteTmpl = htmltemplate.Must(HtmlSuiteTmpl.Parse(htmlStyleTmpl))
	HtmlSuiteTmpl = htmltemplate.Must(HtmlSuiteTmpl.Parse(htmlJavascriptTmpl))
}

func (t Test) PrintReport(w io.Writer) error {
	return TestTmpl.Execute(w, t)
}

func (r Suite) PrintReport(w io.Writer) error {
	return SuiteTmpl.Execute(w, r)
}

func (s Suite) HTMLReport(dir string) error {
	report, err := os.Create(path.Join(dir, "Report.html"))
	if err != nil {
		return err
	}
	err = HtmlSuiteTmpl.Execute(report, s)
	if err != nil {
		return err
	}
	for _, tr := range s.AllTests() {
		body := tr.Response.BodyBytes
		err = ioutil.WriteFile(path.Join(dir, tr.SeqNo+".ResponseBody"), body, 0666)
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package http

import (
	"bytes"
	"compress/zlib"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptrace"
	"net/url"
	"sync"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/sirupsen/logrus"
	"github.com/stripe/veneur/trace"
)

var tracer = trace.GlobalTracer

// httpClientTracer is a wrapper around a `net/http/httptrace` that handles
// proper tracing and metric emission.
type httpClientTracer struct {
	prefix      string
	stats       *statsd.Client
	traceClient *trace.Client
	mutex       *sync.Mutex
	ctx         context.Context
	currentSpan *trace.Span
}

// newHTTPClientTracer makes a new HTTPClientTracer with new span created from
// the context.
func newHTTPClientTracer(ctx context.Context, stats *statsd.Client, tc *trace.Client, prefix string) *httpClientTracer {
	span, _ := trace.StartSpanFromContext(ctx, "http.start")
	return &httpClientTracer{
		prefix:      prefix,
		stats:       stats,
		traceClient: tc,
		mutex:       &sync.Mutex{},
		ctx:         ctx,
		currentSpan: span,
	}
}

// getClientTrace is a convenience to return a filled-in `ClientTrace`.
func (hct *httpClientTracer) getClientTrace() *httptrace.ClientTrace {
	return &httptrace.ClientTrace{
		GotConn:              hct.gotConn,
		DNSStart:             hct.dnsStart,
		GotFirstResponseByte: hct.gotFirstResponseByte,
		ConnectStart:         hct.connectStart,
		WroteRequest:         hct.wroteRequest,
	}
}

// startSpan is a convenience that replaces the current span in our
// tracer and flushes the outgoing one.
func (hct *httpClientTracer) startSpan(name string) *trace.Span {
	hct.mutex.Lock()
	defer hct.mutex.Unlock()
	newSpan, _ := trace.StartSpanFromContext(hct.ctx, name)
	hct.currentSpan.ClientFinish(hct.traceClient)

	hct.currentSpan = newSpan
	return newSpan
}

// gotConn signals that the connection — be it new or reused — was fetched
// from the pool.
func (hct *httpClientTracer) gotConn(info httptrace.GotConnInfo) {
	state := "new"
	if info.Reused {
		state = "reused"
	}
	hct.startSpan(fmt.Sprintf("http.gotConnection.%s", state))

	hct.stats.Count(hct.prefix+".connections_used_total", 1, []string{fmt.Sprintf("state:%s", state)}, 1.0)
}

// dnsStart marks the beginning of the DNS lookup
func (hct *httpClientTracer) dnsStart(info httptrace.DNSStartInfo) {
	hct.startSpan("http.resolvingDNS")
}

// gotFirstResponseByte marks us receiving our first byte
func (hct *httpClientTracer) gotFirstResponseByte() {
	hct.startSpan("http.gotFirstByte")
}

// connectStart marks beginning of the connect process
func (hct *httpClientTracer) connectStart(network, addr string) {
	hct.startSpan("http.connecting")
}

// wroteRequest marks the write being completed
func (hct *httpClientTracer) wroteRequest(info httptrace.WroteRequestInfo) {
	hct.startSpan("http.finishedWrite")
}

// finishSpan is called to ensure we're done tracing
func (hct *httpClientTracer) finishSpan() {
	hct.currentSpan.ClientFinish(hct.traceClient)
}

// PostHelper is shared code for POSTing to an endpoint, that consumes JSON, is zlib-
// compressed, that returns 202 on success, that has a small response
// action as a string used for statsd metric names and log messages emitted from
// this function - probably a static string for each callsite
// you can disable compression with compress=false for endpoints that don't
// support it
func PostHelper(ctx context.Context, httpClient *http.Client, stats *statsd.Client, tc *trace.Client, endpoint string, bodyObject interface{}, action string, compress bool, log *logrus.Logger) error {
	span, _ := trace.StartSpanFromContext(ctx, "")
	span.SetTag("action", action)
	defer span.ClientFinish(tc)

	// attach this field to all the logs we generate
	innerLogger := log.WithField("action", action)

	marshalStart := time.Now()
	var (
		bodyBuffer bytes.Buffer
		encoder    *json.Encoder
		compressor *zlib.Writer
	)
	if compress {
		compressor = zlib.NewWriter(&bodyBuffer)
		encoder = json.NewEncoder(compressor)
	} else {
		encoder = json.NewEncoder(&bodyBuffer)
	}
	if err := encoder.Encode(bodyObject); err != nil {
		stats.Count(action+".error_total", 1, []string{"cause:json"}, 1.0)
		innerLogger.WithError(err).Error("Could not render JSON")
		return err
	}
	if compress {
		// don't forget to flush leftover compressed bytes to the buffer
		if err := compressor.Close(); err != nil {
			stats.Count(action+".error_total", 1, []string{"cause:compress"}, 1.0)
			innerLogger.WithError(err).Error("Could not finalize compression")
			return err
		}
	}
	stats.TimeInMilliseconds(action+".duration_ns", float64(time.Since(marshalStart).Nanoseconds()), []string{"part:json"}, 1.0)

	// Len reports the unread length, so we have to record this before the
	// http client consumes it
	bodyLength := bodyBuffer.Len()
	stats.Histogram(action+".content_length_bytes", float64(bodyLength), nil, 1.0)

	req, err := http.NewRequest(http.MethodPost, endpoint, &bodyBuffer)

	if err != nil {
		stats.Count(action+".error_total", 1, []string{"cause:construct"}, 1.0)
		innerLogger.WithError(err).Error("Could not construct request")
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	if compress {
		req.Header.Set("Content-Encoding", "deflate")
	}

	err = tracer.InjectRequest(span.Trace, req)
	if err != nil {
		stats.Count("veneur.opentracing.flush.inject.errors", 1, nil, 1.0)
		innerLogger.WithError(err).Error("Error injecting header")
	}

	// Attach a ClientTrace that emits span for all our HTTP bits.
	hct := newHTTPClientTracer(span.Attach(ctx), stats, tc, action)
	req = req.WithContext(httptrace.WithClientTrace(req.Context(), hct.getClientTrace()))
	defer hct.finishSpan()

	resp, err := httpClient.Do(req)
	if err != nil {
		if urlErr, ok := err.(*url.Error); ok {
			// if the error has the url in it, then retrieve the inner error
			// and ditch the url (which might contain secrets)
			err = urlErr.Err
		}
		stats.Count(action+".error_total", 1, []string{"cause:io"}, 1.0)
		// Log at Warn level instead of Error, because we don't want to create
		// Sentry events for these (they're only important in large numbers, and
		// we already have Datadog metrics for them)
		innerLogger.WithError(err).Warn("Could not execute request")
		return err
	}
	defer resp.Body.Close()

	responseBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		// this error is not fatal, since we only need the body for reporting
		// purposes
		stats.Count(action+".error_total", 1, []string{"cause:readresponse"}, 1.0)
		innerLogger.WithError(err).Error("Could not read response body")
	}
	resultLogger := innerLogger.WithFields(logrus.Fields{
		"endpoint":         endpoint,
		"request_length":   bodyLength,
		"request_headers":  req.Header,
		"status":           resp.Status,
		"response_headers": resp.Header,
		"response":         string(responseBody),
	})

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
		stats.Count(action+".error_total", 1, []string{fmt.Sprintf("cause:%d", resp.StatusCode)}, 1.0)
		resultLogger.WithError(err).Warn("Could not POST")
		return err
	}

	// make sure the error metric isn't sparse
	stats.Count(action+".error_total", 0, nil, 1.0)
	resultLogger.Debug("POSTed successfully")
	return nil
}
<commit_msg>Oops, don't race!<commit_after>package http

import (
	"bytes"
	"compress/zlib"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptrace"
	"net/url"
	"sync"
	"time"

	"github.com/DataDog/datadog-go/statsd"
	"github.com/sirupsen/logrus"
	"github.com/stripe/veneur/trace"
)

var tracer = trace.GlobalTracer

// httpClientTracer is a wrapper around a `net/http/httptrace` that handles
// proper tracing and metric emission.
type httpClientTracer struct {
	prefix      string
	stats       *statsd.Client
	traceClient *trace.Client
	mutex       *sync.Mutex
	ctx         context.Context
	currentSpan *trace.Span
}

// newHTTPClientTracer makes a new HTTPClientTracer with new span created from
// the context.
func newHTTPClientTracer(ctx context.Context, stats *statsd.Client, tc *trace.Client, prefix string) *httpClientTracer {
	span, _ := trace.StartSpanFromContext(ctx, "http.start")
	return &httpClientTracer{
		prefix:      prefix,
		stats:       stats,
		traceClient: tc,
		mutex:       &sync.Mutex{},
		ctx:         ctx,
		currentSpan: span,
	}
}

// getClientTrace is a convenience to return a filled-in `ClientTrace`.
func (hct *httpClientTracer) getClientTrace() *httptrace.ClientTrace {
	return &httptrace.ClientTrace{
		GotConn:              hct.gotConn,
		DNSStart:             hct.dnsStart,
		GotFirstResponseByte: hct.gotFirstResponseByte,
		ConnectStart:         hct.connectStart,
		WroteRequest:         hct.wroteRequest,
	}
}

// startSpan is a convenience that replaces the current span in our
// tracer and flushes the outgoing one.
func (hct *httpClientTracer) startSpan(name string) *trace.Span {
	hct.mutex.Lock()
	defer hct.mutex.Unlock()
	newSpan, _ := trace.StartSpanFromContext(hct.ctx, name)
	hct.currentSpan.ClientFinish(hct.traceClient)

	hct.currentSpan = newSpan
	return newSpan
}

// gotConn signals that the connection — be it new or reused — was fetched
// from the pool.
func (hct *httpClientTracer) gotConn(info httptrace.GotConnInfo) {
	state := "new"
	if info.Reused {
		state = "reused"
	}
	hct.startSpan(fmt.Sprintf("http.gotConnection.%s", state))

	hct.stats.Count(hct.prefix+".connections_used_total", 1, []string{fmt.Sprintf("state:%s", state)}, 1.0)
}

// dnsStart marks the beginning of the DNS lookup
func (hct *httpClientTracer) dnsStart(info httptrace.DNSStartInfo) {
	hct.startSpan("http.resolvingDNS")
}

// gotFirstResponseByte marks us receiving our first byte
func (hct *httpClientTracer) gotFirstResponseByte() {
	hct.startSpan("http.gotFirstByte")
}

// connectStart marks beginning of the connect process
func (hct *httpClientTracer) connectStart(network, addr string) {
	hct.startSpan("http.connecting")
}

// wroteRequest marks the write being completed
func (hct *httpClientTracer) wroteRequest(info httptrace.WroteRequestInfo) {
	hct.startSpan("http.finishedWrite")
}

// finishSpan is called to ensure we're done tracing
func (hct *httpClientTracer) finishSpan() {
	hct.mutex.Lock()
	defer hct.mutex.Unlock()
	hct.currentSpan.ClientFinish(hct.traceClient)
}

// PostHelper is shared code for POSTing to an endpoint, that consumes JSON, is zlib-
// compressed, that returns 202 on success, that has a small response
// action as a string used for statsd metric names and log messages emitted from
// this function - probably a static string for each callsite
// you can disable compression with compress=false for endpoints that don't
// support it
func PostHelper(ctx context.Context, httpClient *http.Client, stats *statsd.Client, tc *trace.Client, endpoint string, bodyObject interface{}, action string, compress bool, log *logrus.Logger) error {
	span, _ := trace.StartSpanFromContext(ctx, "")
	span.SetTag("action", action)
	defer span.ClientFinish(tc)

	// attach this field to all the logs we generate
	innerLogger := log.WithField("action", action)

	marshalStart := time.Now()
	var (
		bodyBuffer bytes.Buffer
		encoder    *json.Encoder
		compressor *zlib.Writer
	)
	if compress {
		compressor = zlib.NewWriter(&bodyBuffer)
		encoder = json.NewEncoder(compressor)
	} else {
		encoder = json.NewEncoder(&bodyBuffer)
	}
	if err := encoder.Encode(bodyObject); err != nil {
		stats.Count(action+".error_total", 1, []string{"cause:json"}, 1.0)
		innerLogger.WithError(err).Error("Could not render JSON")
		return err
	}
	if compress {
		// don't forget to flush leftover compressed bytes to the buffer
		if err := compressor.Close(); err != nil {
			stats.Count(action+".error_total", 1, []string{"cause:compress"}, 1.0)
			innerLogger.WithError(err).Error("Could not finalize compression")
			return err
		}
	}
	stats.TimeInMilliseconds(action+".duration_ns", float64(time.Since(marshalStart).Nanoseconds()), []string{"part:json"}, 1.0)

	// Len reports the unread length, so we have to record this before the
	// http client consumes it
	bodyLength := bodyBuffer.Len()
	stats.Histogram(action+".content_length_bytes", float64(bodyLength), nil, 1.0)

	req, err := http.NewRequest(http.MethodPost, endpoint, &bodyBuffer)

	if err != nil {
		stats.Count(action+".error_total", 1, []string{"cause:construct"}, 1.0)
		innerLogger.WithError(err).Error("Could not construct request")
		return err
	}
	req.Header.Set("Content-Type", "application/json")
	if compress {
		req.Header.Set("Content-Encoding", "deflate")
	}

	err = tracer.InjectRequest(span.Trace, req)
	if err != nil {
		stats.Count("veneur.opentracing.flush.inject.errors", 1, nil, 1.0)
		innerLogger.WithError(err).Error("Error injecting header")
	}

	// Attach a ClientTrace that emits span for all our HTTP bits.
	hct := newHTTPClientTracer(span.Attach(ctx), stats, tc, action)
	req = req.WithContext(httptrace.WithClientTrace(req.Context(), hct.getClientTrace()))
	defer hct.finishSpan()

	resp, err := httpClient.Do(req)
	if err != nil {
		if urlErr, ok := err.(*url.Error); ok {
			// if the error has the url in it, then retrieve the inner error
			// and ditch the url (which might contain secrets)
			err = urlErr.Err
		}
		stats.Count(action+".error_total", 1, []string{"cause:io"}, 1.0)
		// Log at Warn level instead of Error, because we don't want to create
		// Sentry events for these (they're only important in large numbers, and
		// we already have Datadog metrics for them)
		innerLogger.WithError(err).Warn("Could not execute request")
		return err
	}
	defer resp.Body.Close()

	responseBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		// this error is not fatal, since we only need the body for reporting
		// purposes
		stats.Count(action+".error_total", 1, []string{"cause:readresponse"}, 1.0)
		innerLogger.WithError(err).Error("Could not read response body")
	}
	resultLogger := innerLogger.WithFields(logrus.Fields{
		"endpoint":         endpoint,
		"request_length":   bodyLength,
		"request_headers":  req.Header,
		"status":           resp.Status,
		"response_headers": resp.Header,
		"response":         string(responseBody),
	})

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
		stats.Count(action+".error_total", 1, []string{fmt.Sprintf("cause:%d", resp.StatusCode)}, 1.0)
		resultLogger.WithError(err).Warn("Could not POST")
		return err
	}

	// make sure the error metric isn't sparse
	stats.Count(action+".error_total", 0, nil, 1.0)
	resultLogger.Debug("POSTed successfully")
	return nil
}
<|endoftext|>


<commit_before>package github

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/longkai/xiaolongtongxue.com/env"
	"io"
	"io/ioutil"
	"net/http"
)

const (
	endpoint = "https://api.github.com"
)

type markdownRender struct {
	Text    string `json:"text"`
	Mode    string `json:"mode"`
	Context string `json:"context"`

	reader io.Reader
}

func (m *markdownRender) Read(p []byte) (n int, err error) {
	if m.reader == nil {
		bs, err := json.Marshal(m)
		if err != nil {
			return 0, err
		}
		m.reader = bytes.NewReader(bs)
	}
	return m.reader.Read(p)
}

// Render the makrdown to html []byte with Github API.
func Markdown(text string) ([]byte, error) {
	m := &markdownRender{text, "gfm", "github/longkai", nil}
	req, err := http.NewRequest(http.MethodPost, endpoint+"/markdown", m)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Authorization", fmt.Sprintf("token %s", env.Config().AccessToken))

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	// the bytes may http error content
	result, err := ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusOK {
		return result, fmt.Errorf("http StatusCode %d", resp.StatusCode)
	}
	return result, err
}
<commit_msg>Switch api to render a document as plain Markdown.<commit_after>package github

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/longkai/xiaolongtongxue.com/env"
	"io"
	"io/ioutil"
	"net/http"
)

const (
	endpoint = "https://api.github.com"
)

type markdownRender struct {
	Text    string `json:"text"`
	Mode    string `json:"mode"`
	Context string `json:"context"`

	reader io.Reader
}

func (m *markdownRender) Read(p []byte) (n int, err error) {
	if m.reader == nil {
		bs, err := json.Marshal(m)
		if err != nil {
			return 0, err
		}
		m.reader = bytes.NewReader(bs)
	}
	return m.reader.Read(p)
}

// Render the makrdown to html []byte with Github API.
func Markdown(text string) ([]byte, error) {
	m := &markdownRender{text, "markdown", "", nil}
	req, err := http.NewRequest(http.MethodPost, endpoint+"/markdown", m)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Authorization", fmt.Sprintf("token %s", env.Config().AccessToken))

	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	// the bytes may http error content
	result, err := ioutil.ReadAll(resp.Body)
	if resp.StatusCode != http.StatusOK {
		return result, fmt.Errorf("http StatusCode %d", resp.StatusCode)
	}
	return result, err
}
<|endoftext|>